diff --git a/Dockerfile b/Dockerfile index ecdb6f0..88340f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,44 +4,22 @@ FROM python:3.12-slim # Set environment variables ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -ENV DJANGO_SETTINGS_MODULE url_manager.settings # Set work directory WORKDIR /app # Install system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - nodejs \ - npm \ - gettext \ +RUN apt-get update && apt-get install -y \ + build-essential \ + python3-dev \ && rm -rf /var/lib/apt/lists/* # Install Python dependencies -COPY requirements.txt /app/ -RUN pip install --upgrade pip && pip install -r requirements.txt +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt # Copy project -COPY . /app/ +COPY . . -# Install Tailwind CSS dependencies -RUN python manage.py tailwind install - -# Build Tailwind CSS -RUN python manage.py tailwind build - -# Collect static files -RUN python manage.py collectstatic --noinput - -# Compile translation messages -RUN python manage.py compilemessages --locale=zh_Hans - -# Create data directory for SQLite database -RUN mkdir -p /app/data && chmod 777 /app/data - -ENV PYTHONPATH=/app -# Expose port 8000 -EXPOSE 8000 - -# Run the application -CMD ["gunicorn", "--chdir", "/app", "url_manager.wsgi:application", "--bind", "0.0.0.0:8000"] +# Remove the automatic migration +# RUN python manage.py migrate diff --git a/README.md b/README.md index 1097399..54772c9 100644 --- a/README.md +++ b/README.md @@ -1,135 +1,182 @@ # URL Manager -URL Manager is a Django-based web application for managing and tracking short URLs. +A URL management tool that helps you organize and access your links efficiently. ## Features -- Create, edit, and delete short URL links -- Track click statistics for each link -- Search functionality for quick access to links -- Internationalization support (English and Simplified Chinese) -- Responsive design using Tailwind CSS +- Create and manage short links +- Template links with dynamic parameters +- Bookmark pages with automatic title and summary extraction +- Advanced search capabilities +- API access +- Asynchronous page processing -## Prerequisites +## Installation -- Docker and Docker Compose +### Using Docker (Recommended) -## Installation and Setup with Docker +1. Prerequisites: + - Docker + - Docker Compose -1. Clone the repository: - ``` +2. Clone the repository: + ```bash git clone https://github.com/yourusername/url-manager.git cd url-manager ``` -2. Build the Docker image: - ``` - docker build -t url-manager . +3. Build and start services: + ```bash + ./manage-docker.sh build + ./manage-docker.sh start ``` -3. Create a Docker volume for persistent database storage: - ``` - docker volume create url-manager-data - ``` +The application will be available at `http://localhost:8000` -4. Run the Docker container: - ``` - docker run -d --name url-manager-container -p 8000:8000 -v url-manager-data:/app/data url-manager - ``` +### Docker Management Commands -5. Initialize the database and create a superuser: - ``` - docker exec -it url-manager-container python manage.py migrate - docker exec -it url-manager-container python manage.py createsuperuser - ``` +- Start all services: + ```bash + ./manage-docker.sh start + ``` -6. Open your web browser and navigate to `http://localhost:8000` to access the application. +- Stop all services: + ```bash + ./manage-docker.sh stop + ``` -## Usage +- Restart services: + ```bash + ./manage-docker.sh restart + ``` -- To create a new short URL, click on the "New Link" button on the home page. -- To edit or delete a link, use the corresponding buttons in the link list. -- To view detailed statistics for a link, click on the "Details" button. -- Use the search bar in the navigation to quickly find links by alias or original URL. +- View logs: + ```bash + ./manage-docker.sh logs + ``` -## Admin Interface +- Run database migrations: + ```bash + ./manage-docker.sh migrate + ``` -To access the admin interface: +- Create new migrations: + ```bash + ./manage-docker.sh makemigrations + ``` -1. Go to `http://localhost:8000/admin` -2. Log in with the superuser account you created earlier +- Access Django shell: + ```bash + ./manage-docker.sh shell + ``` -## Changing the Language +### Docker Services -To switch between English and Simplified Chinese, use the language selector in the navigation bar. +The application runs the following services: -## Persistent Data +- `web`: Django web server (port 8000) +- `celery_worker`: Processes background tasks +- `celery_beat`: Schedules periodic tasks +- `redis`: Message broker and result backend -The SQLite database is stored in the Docker volume `url-manager-data`. This ensures that your data persists even if the container is stopped or removed. To backup your data, you can copy the contents of this volume. +### Manual Installation -## Contributing +If you prefer not to use Docker: -Contributions are welcome! Please feel free to submit a Pull Request. +1. Install Python 3.12 and Redis -## License - -This project is licensed under the MIT License. - -## Local Development - -To run the URL Manager locally: - -1. Clone the repository: - ``` - git clone https://github.com/yourusername/url-manager.git - cd url-manager - ``` - -2. Create and activate a virtual environment: - ``` - python -m venv venv - source venv/bin/activate # On Windows, use `venv\Scripts\activate` - ``` - -3. Install dependencies: - ``` +2. Install dependencies: + ```bash pip install -r requirements.txt ``` -4. Apply database migrations: - ``` +3. Run migrations: + ```bash python manage.py migrate ``` -5. Create a superuser: - ``` - python manage.py createsuperuser +4. Start the development server: + ```bash + ./run_server.sh ``` -6. Compile translation messages: - ``` - python manage.py compilemessages +5. Start Celery worker: + ```bash + ./worker.sh ``` -7. In one terminal, start the Tailwind CSS build process: - ``` - python manage.py tailwind start +6. Start Celery beat: + ```bash + ./celery_beat.sh ``` -8. In another terminal, run the Django development server: - ``` - python manage.py runserver +## Usage + +### Managing Links + +1. Create a new link: + - Visit `/create/` + - Enter the original URL and desired alias + - For template links, use `{param}` syntax + +2. Access a link: + - Use `http://localhost:8000/your-alias` + - For template links: `http://localhost:8000/your-alias/parameter` + +### Managing Pages + +1. Create a new page: + - Visit `/ui/pages/new/` + - Enter the URL + - Title and summary will be automatically extracted + +2. View all pages: + - Visit `/ui/pages/` + +### API Access + +The application provides a REST API: + +- List pages: `GET /api/pages/` +- Create page: `POST /api/pages/` + +Example: + +## Monitoring Celery Tasks + +The application includes Flower for monitoring Celery tasks. You can access it in two ways: + +1. Direct access: + - Visit `http://localhost:5555` in your browser + +2. Using management script: + ```bash + ./manage-docker.sh flower ``` -9. Open your browser and go to `http://127.0.0.1:8000` +Flower provides: +- Real-time monitoring of Celery tasks +- Task progress and history +- Worker status and statistics +- Error tracking +- Task graphs and charts -Remember to keep both the Tailwind CSS build process and the Django development server running while you're developing. +### Monitoring Features -## Tools +1. View all tasks: + - Active tasks + - Scheduled tasks + - Failed tasks + - Success rate -The URL Manager includes a Tools page with the following features: +2. Worker information: + - Status + - Resource usage + - Queue length -- Export all aliases as a JSON file -- Import aliases from a JSON file - -To access the Tools page, click on the "Tools" link in the navigation bar. +3. Task details: + - Arguments + - Start time + - Runtime + - Result + - Stack traces for failed tasks diff --git a/celery_beat.sh b/celery_beat.sh new file mode 100755 index 0000000..fcb4c02 --- /dev/null +++ b/celery_beat.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -eu +source 3.12/bin/activate +echo "Starting Django development server..." +celery -A url_manager beat -l info diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 0000000..30eeaa0 Binary files /dev/null and b/celerybeat-schedule differ diff --git a/data/db.sqlite3 b/data/db.sqlite3 index b8b2fee..560f287 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cbbbd53 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,89 @@ +version: '3.8' + +services: + web: + build: . + command: > + sh -c "python manage.py migrate && + python manage.py runserver 0.0.0.0:8000" + volumes: + - .:/app + ports: + - "8000:8000" + environment: + - DJANGO_SETTINGS_MODULE=url_manager.settings + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - redis + - tailwind + networks: + - app-network + + celery_worker: + build: . + command: celery -A url_manager worker -l info + volumes: + - .:/app + environment: + - DJANGO_SETTINGS_MODULE=url_manager.settings + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - redis + - web + networks: + - app-network + + celery_beat: + build: . + command: celery -A url_manager beat -l info + volumes: + - .:/app + environment: + - DJANGO_SETTINGS_MODULE=url_manager.settings + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - redis + - web + networks: + - app-network + + redis: + image: redis:alpine + ports: + - "6379:6379" + networks: + - app-network + + tailwind: + build: . + command: python manage.py tailwind start + volumes: + - .:/app + ports: + - "3000:3000" + networks: + - app-network + + flower: + build: . + command: celery -A url_manager flower --port=5555 + volumes: + - .:/app + ports: + - "5555:5555" + environment: + - DJANGO_SETTINGS_MODULE=url_manager.settings + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - redis + - celery_worker + networks: + - app-network + +networks: + app-network: + driver: bridge diff --git a/links/migrations/0011_page_error_message_page_last_retry_at_and_more.py b/links/migrations/0011_page_error_message_page_last_retry_at_and_more.py new file mode 100644 index 0000000..5cd799d --- /dev/null +++ b/links/migrations/0011_page_error_message_page_last_retry_at_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.9 on 2024-11-04 01:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0010_alter_page_title'), + ] + + operations = [ + migrations.AddField( + model_name='page', + name='error_message', + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name='page', + name='last_retry_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='page', + name='process_status', + field=models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed')], default='pending', max_length=20), + ), + migrations.AddField( + model_name='page', + name='retry_count', + field=models.IntegerField(default=0), + ), + ] diff --git a/links/models.py b/links/models.py index bccfcbc..0884907 100644 --- a/links/models.py +++ b/links/models.py @@ -128,6 +128,12 @@ class LinkChangeLog(models.Model): return f"URL changed from {self.old_url} to {self.new_url}" class Page(models.Model): + class ProcessStatus(models.TextChoices): + PENDING = 'pending', _('Pending') + PROCESSING = 'processing', _('Processing') + COMPLETED = 'completed', _('Completed') + FAILED = 'failed', _('Failed') + url = models.URLField(_('URL'), max_length=2000) title = models.CharField(_('Title'), max_length=200, blank=True) summary = models.TextField(_('Summary'), blank=True) @@ -135,13 +141,28 @@ class Page(models.Model): created_at = models.DateTimeField(_('Created at'), default=timezone.now) updated_at = models.DateTimeField(_('Updated at'), auto_now=True) + # New fields for processing status + process_status = models.CharField( + max_length=20, + choices=ProcessStatus.choices, + default=ProcessStatus.PENDING + ) + retry_count = models.IntegerField(default=0) + last_retry_at = models.DateTimeField(null=True, blank=True) + error_message = models.TextField(blank=True) + class Meta: ordering = ['-updated_at'] verbose_name = _('Page') verbose_name_plural = _('Pages') def __str__(self): - return self.title + return self.title or self.url def get_absolute_url(self): return reverse('page-detail', kwargs={'pk': self.pk}) + + def needs_processing(self): + return (not self.title or not self.summary) and \ + self.process_status != self.ProcessStatus.FAILED and \ + self.retry_count < 3 diff --git a/links/page_views.py b/links/page_views.py new file mode 100644 index 0000000..57f3f9c --- /dev/null +++ b/links/page_views.py @@ -0,0 +1,140 @@ +from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView +from django.urls import reverse_lazy +from django.http import JsonResponse +import requests +from bs4 import BeautifulSoup +from urllib.parse import urlparse +import re +from rest_framework import viewsets, status +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination +from .models import Page +from .forms import PageForm +from .serializers import PageSerializer + +class PageListView(ListView): + model = Page + template_name = 'links/page_list.html' + context_object_name = 'pages' + paginate_by = 10 + +class PageDetailView(DetailView): + model = Page + template_name = 'links/page_detail.html' + +class PageCreateView(CreateView): + model = Page + form_class = PageForm + template_name = 'links/page_form.html' + success_url = reverse_lazy('page-list') + + def form_valid(self, form): + # 如果标题和摘要都已经自动填充，则将状态设置为已完成 + if form.instance.title and form.instance.summary: + form.instance.process_status = Page.ProcessStatus.COMPLETED + else: + form.instance.process_status = Page.ProcessStatus.PENDING + + return super().form_valid(form) + +class PageUpdateView(UpdateView): + model = Page + form_class = PageForm + template_name = 'links/page_form.html' + success_url = reverse_lazy('page-list') + +class PageDeleteView(DeleteView): + model = Page + template_name = 'links/page_confirm_delete.html' + success_url = reverse_lazy('page-list') + +def fetch_page_info(request): + url = request.GET.get('url') + if not url: + return JsonResponse({'error': 'URL is required'}, status=400) + + # 移除URL开头可能的@符号 + url = url.lstrip('@') + + try: + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + } + + response = requests.get(url, headers=headers, timeout=10, verify=False) + + if response.encoding == 'ISO-8859-1': + response.encoding = response.apparent_encoding or 'utf-8' + + soup = BeautifulSoup(response.text, 'html.parser') + + # 获取标题 + title = None + if soup.title: + title = soup.title.string + elif soup.find('h1'): + title = soup.find('h1').get_text(strip=True) + elif soup.find('meta', property='og:title'): + title = soup.find('meta', property='og:title').get('content') + + if title: + title = re.sub(r'\s+', ' ', title.strip()) + title = title.replace(' | ', ' - ').replace(' :: ', ' - ') + else: + title = urlparse(url).netloc + + # 获取描述 + description = None + meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'}) + if meta_desc: + description = meta_desc.get('content') + + if not description: + for tag in soup(['script', 'style', 'nav', 'header', 'footer']): + tag.decompose() + + paragraphs = soup.find_all(['p', 'div']) + for p in paragraphs: + text = p.get_text(strip=True) + if len(text) > 100: + description = text + break + + if not description: + description = title + + description = re.sub(r'\s+', ' ', description.strip()) + description = description[:500] + '...' if len(description) > 500 else description + + return JsonResponse({ + 'title': title, + 'summary': description + }) + + except requests.exceptions.RequestException as e: + return JsonResponse({ + 'error': f'Failed to fetch page: {str(e)}' + }, status=400) + except Exception as e: + return JsonResponse({ + 'error': f'Error processing page: {str(e)}' + }, status=400) + +class StandardResultsSetPagination(PageNumberPagination): + page_size = 10 + page_size_query_param = 'page_size' + max_page_size = 100 + +class PageViewSet(viewsets.ModelViewSet): + queryset = Page.objects.all().order_by('-created_at') + serializer_class = PageSerializer + pagination_class = StandardResultsSetPagination + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) diff --git a/links/tasks.py b/links/tasks.py new file mode 100644 index 0000000..953cf4c --- /dev/null +++ b/links/tasks.py @@ -0,0 +1,113 @@ +from celery import shared_task +from django.utils import timezone +from datetime import timedelta +import requests +from bs4 import BeautifulSoup +import logging +from .models import Page + +logger = logging.getLogger(__name__) + +def fibonacci(n): + if n <= 0: + return 0 + elif n == 1: + return 1 + else: + a, b = 0, 1 + for _ in range(2, n + 1): + a, b = b, a + b + return b + +@shared_task(bind=True, max_retries=3) +def process_page(self, page_id): + try: + page = Page.objects.get(id=page_id) + + # Check if page still needs processing + if not page.needs_processing(): + return + + # Update status to processing + page.process_status = Page.ProcessStatus.PROCESSING + page.save() + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + } + + response = requests.get(page.url, headers=headers, timeout=10, verify=False) + response.raise_for_status() + + if response.encoding == 'ISO-8859-1': + response.encoding = response.apparent_encoding or 'utf-8' + + soup = BeautifulSoup(response.text, 'html.parser') + + # Extract title if not present + if not page.title: + title = None + if soup.title: + title = soup.title.string + elif soup.find('h1'): + title = soup.find('h1').get_text(strip=True) + elif soup.find('meta', property='og:title'): + title = soup.find('meta', property='og:title').get('content') + + if title: + page.title = title.strip() + + # Extract summary if not present + if not page.summary: + description = None + meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'}) + if meta_desc: + description = meta_desc.get('content') + + if not description: + for tag in soup(['script', 'style', 'nav', 'header', 'footer']): + tag.decompose() + + paragraphs = soup.find_all(['p', 'div']) + for p in paragraphs: + text = p.get_text(strip=True) + if len(text) > 100: + description = text + break + + if description: + page.summary = description[:500] + + page.process_status = Page.ProcessStatus.COMPLETED + page.save() + + except Exception as exc: + page.retry_count += 1 + page.last_retry_at = timezone.now() + + if page.retry_count >= 3: + page.process_status = Page.ProcessStatus.FAILED + page.error_message = str(exc) + else: + page.process_status = Page.ProcessStatus.PENDING + # Schedule retry with fibonacci backoff + retry_delay = fibonacci(page.retry_count) + self.retry(exc=exc, countdown=retry_delay) + + page.save() + raise exc + +@shared_task +def schedule_pending_pages(): + """Periodic task to schedule processing of pending pages""" + pending_pages = Page.objects.filter( + process_status=Page.ProcessStatus.PENDING, + retry_count__lt=3 + ).exclude( + last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(3)) + ) + + for page in pending_pages: + process_page.delay(page.id) diff --git a/links/templates/links/page_detail.html b/links/templates/links/page_detail.html index 52ec29c..b580008 100644 --- a/links/templates/links/page_detail.html +++ b/links/templates/links/page_detail.html @@ -6,25 +6,108 @@
+
-

{{ page.title }}

+

{{ page.title|default:"Untitled" }}

- + + + + {% trans "Edit" %} - + + + + {% trans "Visit URL" %}
-
- {{ page.content|markdown|safe }} + +
+ {% trans "Status" %}: + {% if page.process_status == 'completed' %} + + {% trans "Completed" %} + + {% elif page.process_status == 'processing' %} + + {% trans "Processing" %} + + {% elif page.process_status == 'failed' %} + + {% trans "Failed" %} + + {% else %} + + {% trans "Pending" %} + + {% endif %} + {% if page.retry_count > 0 %} + + ({% trans "Retries" %}: {{ page.retry_count }}/3) + + {% endif %}
-
- {% trans "Last updated" %}: {{ page.updated_at|date:"Y-m-d H:i" }} + +
+

{% trans "URL" %}

+ + {{ page.url }} +
+ + +
+
+ {% trans "Created" %}: + {{ page.created_at|date:"Y-m-d H:i:s" }} +
+
+ {% trans "Updated" %}: + {{ page.updated_at|date:"Y-m-d H:i:s" }} +
+ {% if page.last_retry_at %} +
+ {% trans "Last Retry" %}: + {{ page.last_retry_at|date:"Y-m-d H:i:s" }} +
+ {% endif %} +
+ + + {% if page.summary %} +
+

{% trans "Summary" %}

+
+ {{ page.summary }} +
+
+ {% endif %} + + + {% if page.content %} +
+

{% trans "Content" %}

+
+ {{ page.content|markdown|safe }} +
+
+ {% endif %} + + + {% if page.error_message %} +
+

{% trans "Error Message" %}

+
+ {{ page.error_message }} +
+
+ {% endif %}
diff --git a/links/views.py b/links/views.py index cf88936..7be5c7d 100644 --- a/links/views.py +++ b/links/views.py @@ -35,6 +35,11 @@ from rest_framework import viewsets, status from rest_framework.response import Response from rest_framework.pagination import PageNumberPagination from .serializers import PageSerializer +from .page_views import ( + PageListView, PageDetailView, PageCreateView, + PageUpdateView, PageDeleteView, fetch_page_info, + PageViewSet +) logger = logging.getLogger(__name__) @@ -440,134 +445,3 @@ def export_database(request): class HelpView(TemplateView): template_name = 'links/help.html' - -class PageListView(ListView): - model = Page - template_name = 'links/page_list.html' - context_object_name = 'pages' - paginate_by = 10 - -class PageDetailView(DetailView): - model = Page - template_name = 'links/page_detail.html' - -class PageCreateView(CreateView): - model = Page - form_class = PageForm - template_name = 'links/page_form.html' - success_url = reverse_lazy('page-list') - -class PageUpdateView(UpdateView): - model = Page - form_class = PageForm - template_name = 'links/page_form.html' - success_url = reverse_lazy('page-list') - -class PageDeleteView(DeleteView): - model = Page - template_name = 'links/page_confirm_delete.html' - success_url = reverse_lazy('page-list') - -def fetch_page_info(request): - url = request.GET.get('url') - if not url: - return JsonResponse({'error': 'URL is required'}, status=400) - - # 移除URL开头可能的@符号 - url = url.lstrip('@') - - try: - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - } - - response = requests.get(url, headers=headers, timeout=10, verify=False) - - # 确保使用正确的编码 - if response.encoding == 'ISO-8859-1': - response.encoding = response.apparent_encoding or 'utf-8' - - soup = BeautifulSoup(response.text, 'html.parser') - - # 获取标题 - 尝试多种方式 - title = None - # 1. 尝试获取title标签 - if soup.title: - title = soup.title.string - # 2. 尝试获取第一个h1 - if not title and soup.find('h1'): - title = soup.find('h1').get_text(strip=True) - # 3. 尝试获取og:title - if not title: - og_title = soup.find('meta', property='og:title') - if og_title: - title = og_title.get('content') - - # 清理标题 - if title: - title = re.sub(r'\s+', ' ', title.strip()) - title = title.replace(' | ', ' - ').replace(' :: ', ' - ') - else: - title = urlparse(url).netloc - - # 获取描述 - 尝试多种方式 - description = None - # 1. 尝试meta description - meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'}) - if meta_desc: - description = meta_desc.get('content') - - # 2. 如果没有meta描述，尝试获取正文内容 - if not description: - # 移除script, style等标签 - for tag in soup(['script', 'style', 'nav', 'header', 'footer']): - tag.decompose() - - # 获取所有段落 - paragraphs = soup.find_all(['p', 'div']) - for p in paragraphs: - text = p.get_text(strip=True) - if len(text) > 100: # 确保段落有足够的内容 - description = text - break - - # 如果还是没有描述，使用标题 - if not description: - description = title - - # 清理描述 - description = re.sub(r'\s+', ' ', description.strip()) - description = description[:500] + '...' if len(description) > 500 else description - - return JsonResponse({ - 'title': title, - 'summary': description - }) - - except requests.exceptions.RequestException as e: - return JsonResponse({ - 'error': f'Failed to fetch page: {str(e)}' - }, status=400) - except Exception as e: - return JsonResponse({ - 'error': f'Error processing page: {str(e)}' - }, status=400) - -class StandardResultsSetPagination(PageNumberPagination): - page_size = 10 - page_size_query_param = 'page_size' - max_page_size = 100 - -class PageViewSet(viewsets.ModelViewSet): - queryset = Page.objects.all().order_by('-created_at') - serializer_class = PageSerializer - pagination_class = StandardResultsSetPagination - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - self.perform_create(serializer) - headers = self.get_success_headers(serializer.data) - return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) diff --git a/manage-docker.sh b/manage-docker.sh new file mode 100755 index 0000000..071c27d --- /dev/null +++ b/manage-docker.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +case "$1" in + "start") + docker-compose up -d + echo "Services started. Access:" + echo "- Web: http://localhost:8000" + echo "- Flower (Celery Monitor): http://localhost:5555" + ;; + "stop") + docker-compose down + ;; + "restart") + docker-compose restart + ;; + "logs") + if [ "$2" ]; then + docker-compose logs -f "$2" + else + docker-compose logs -f + fi + ;; + "build") + docker-compose build + ;; + "shell") + docker-compose exec web python manage.py shell + ;; + "migrate") + docker-compose exec web python manage.py migrate + ;; + "makemigrations") + docker-compose exec web python manage.py makemigrations + ;; + "tailwind-logs") + docker-compose logs -f tailwind + ;; + "tailwind-build") + docker-compose exec tailwind npm run build + ;; + "flower") + echo "Opening Flower dashboard in default browser..." + docker-compose exec flower open http://localhost:5555 + ;; + *) + echo "Usage: $0 {start|stop|restart|logs|build|shell|migrate|makemigrations|tailwind-logs|tailwind-build|flower}" + echo "" + echo "Additional commands:" + echo " logs [service] View logs of all services or specific service" + echo " flower Open Flower dashboard in browser" + exit 1 + ;; +esac diff --git a/package.json b/package.json new file mode 100644 index 0000000..ae70b3a --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "name": "url-manager", + "version": "1.0.0", + "scripts": { + "dev": "tailwindcss -i ./new_theme/static/css/src/styles.css -o ./new_theme/static/css/dist/styles.css --watch", + "build": "tailwindcss -i ./new_theme/static/css/src/styles.css -o ./new_theme/static/css/dist/styles.css --minify" + }, + "dependencies": { + "tailwindcss": "^3.3.0", + "@tailwindcss/typography": "^0.5.9" + } +} diff --git a/requirements.txt b/requirements.txt index cd15e61..f98626d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,6 @@ markdown==3.7 requests==2.32.3 beautifulsoup4==4.12.3 djangorestframework==3.15.2 +celery>=5.3.0 +redis>=4.5.0 +flower>=2.0.0 diff --git a/url_manager/celery.py b/url_manager/celery.py new file mode 100644 index 0000000..d6a0c58 --- /dev/null +++ b/url_manager/celery.py @@ -0,0 +1,8 @@ +import os +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'url_manager.settings') + +app = Celery('url_manager') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/url_manager/settings.py b/url_manager/settings.py index 7835976..de2d442 100644 --- a/url_manager/settings.py +++ b/url_manager/settings.py @@ -192,3 +192,23 @@ REST_FRAMEWORK = { 'PAGE_SIZE': 10, 'UNAUTHENTICATED_USER': None, } + +# Time zone setting +TIME_ZONE = 'UTC' +USE_TZ = True + +# Celery Configuration +CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0') +CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0') +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = TIME_ZONE # Now TIME_ZONE is defined + +# Celery Beat Schedule +CELERY_BEAT_SCHEDULE = { + 'check-pending-pages': { + 'task': 'links.tasks.schedule_pending_pages', + 'schedule': 300.0, # Run every 5 minutes + }, +} diff --git a/worker.sh b/worker.sh new file mode 100755 index 0000000..0db2c06 --- /dev/null +++ b/worker.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -eu +source 3.12/bin/activate +echo "Starting Django development server..." +celery -A url_manager worker -l info