Add pages

Dockerfile

@@ -4,44 +4,22 @@ FROM python:3.12-slim
 # Set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
-ENV DJANGO_SETTINGS_MODULE url_manager.settings
 
 # Set work directory
 WORKDIR /app
 
 # Install system dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc \
-    nodejs \
-    npm \
-    gettext \
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    python3-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Python dependencies
-COPY requirements.txt /app/
-RUN pip install --upgrade pip && pip install -r requirements.txt
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy project
-COPY . /app/
+COPY . .
 
-# Install Tailwind CSS dependencies
-RUN python manage.py tailwind install
-
-# Build Tailwind CSS
-RUN python manage.py tailwind build
-
-# Collect static files
-RUN python manage.py collectstatic --noinput
-
-# Compile translation messages
-RUN python manage.py compilemessages --locale=zh_Hans
-
-# Create data directory for SQLite database
-RUN mkdir -p /app/data && chmod 777 /app/data
-
-ENV PYTHONPATH=/app
-# Expose port 8000
-EXPOSE 8000
-
-# Run the application
-CMD ["gunicorn", "--chdir", "/app", "url_manager.wsgi:application", "--bind", "0.0.0.0:8000"]
+# Remove the automatic migration
+# RUN python manage.py migrate

README.md

@@ -1,135 +1,182 @@
 # URL Manager
 
-URL Manager is a Django-based web application for managing and tracking short URLs.
+A URL management tool that helps you organize and access your links efficiently.
 
 ## Features
 
-- Create, edit, and delete short URL links
-- Track click statistics for each link
-- Search functionality for quick access to links
-- Internationalization support (English and Simplified Chinese)
-- Responsive design using Tailwind CSS
+- Create and manage short links
+- Template links with dynamic parameters
+- Bookmark pages with automatic title and summary extraction
+- Advanced search capabilities
+- API access
+- Asynchronous page processing
 
-## Prerequisites
+## Installation
 
-- Docker and Docker Compose
+### Using Docker (Recommended)
 
-## Installation and Setup with Docker
+1. Prerequisites:
+   - Docker
+   - Docker Compose
 
-1. Clone the repository:
-   ```
+2. Clone the repository:
+   ```bash
    git clone https://github.com/yourusername/url-manager.git
    cd url-manager
    ```
 
-2. Build the Docker image:
-   ```
-   docker build -t url-manager .
+3. Build and start services:
+   ```bash
+   ./manage-docker.sh build
+   ./manage-docker.sh start
    ```
 
-3. Create a Docker volume for persistent database storage:
-   ```
-   docker volume create url-manager-data
-   ```
+The application will be available at `http://localhost:8000`
 
-4. Run the Docker container:
-   ```
-   docker run -d --name url-manager-container -p 8000:8000 -v url-manager-data:/app/data url-manager
-   ```
+### Docker Management Commands
 
-5. Initialize the database and create a superuser:
-   ```
-   docker exec -it url-manager-container python manage.py migrate
-   docker exec -it url-manager-container python manage.py createsuperuser
-   ```
+- Start all services:
+  ```bash
+  ./manage-docker.sh start
+  ```
 
-6. Open your web browser and navigate to `http://localhost:8000` to access the application.
+- Stop all services:
+  ```bash
+  ./manage-docker.sh stop
+  ```
 
-## Usage
+- Restart services:
+  ```bash
+  ./manage-docker.sh restart
+  ```
 
-- To create a new short URL, click on the "New Link" button on the home page.
-- To edit or delete a link, use the corresponding buttons in the link list.
-- To view detailed statistics for a link, click on the "Details" button.
-- Use the search bar in the navigation to quickly find links by alias or original URL.
+- View logs:
+  ```bash
+  ./manage-docker.sh logs
+  ```
 
-## Admin Interface
+- Run database migrations:
+  ```bash
+  ./manage-docker.sh migrate
+  ```
 
-To access the admin interface:
+- Create new migrations:
+  ```bash
+  ./manage-docker.sh makemigrations
+  ```
 
-1. Go to `http://localhost:8000/admin`
-2. Log in with the superuser account you created earlier
+- Access Django shell:
+  ```bash
+  ./manage-docker.sh shell
+  ```
 
-## Changing the Language
+### Docker Services
 
-To switch between English and Simplified Chinese, use the language selector in the navigation bar.
+The application runs the following services:
 
-## Persistent Data
+- `web`: Django web server (port 8000)
+- `celery_worker`: Processes background tasks
+- `celery_beat`: Schedules periodic tasks
+- `redis`: Message broker and result backend
 
-The SQLite database is stored in the Docker volume `url-manager-data`. This ensures that your data persists even if the container is stopped or removed. To backup your data, you can copy the contents of this volume.
+### Manual Installation
 
-## Contributing
+If you prefer not to use Docker:
 
-Contributions are welcome! Please feel free to submit a Pull Request.
+1. Install Python 3.12 and Redis
 
-## License
-
-This project is licensed under the MIT License.
-
-## Local Development
-
-To run the URL Manager locally:
-
-1. Clone the repository:
-   ```
-   git clone https://github.com/yourusername/url-manager.git
-   cd url-manager
-   ```
-
-2. Create and activate a virtual environment:
-   ```
-   python -m venv venv
-   source venv/bin/activate  # On Windows, use `venv\Scripts\activate`
-   ```
-
-3. Install dependencies:
-   ```
+2. Install dependencies:
+   ```bash
    pip install -r requirements.txt
    ```
 
-4. Apply database migrations:
-   ```
+3. Run migrations:
+   ```bash
    python manage.py migrate
    ```
 
-5. Create a superuser:
-   ```
-   python manage.py createsuperuser
+4. Start the development server:
+   ```bash
+   ./run_server.sh
    ```
 
-6. Compile translation messages:
-   ```
-   python manage.py compilemessages
+5. Start Celery worker:
+   ```bash
+   ./worker.sh
    ```
 
-7. In one terminal, start the Tailwind CSS build process:
-   ```
-   python manage.py tailwind start
+6. Start Celery beat:
+   ```bash
+   ./celery_beat.sh
    ```
 
-8. In another terminal, run the Django development server:
-   ```
-   python manage.py runserver
+## Usage
+
+### Managing Links
+
+1. Create a new link:
+   - Visit `/create/`
+   - Enter the original URL and desired alias
+   - For template links, use `{param}` syntax
+
+2. Access a link:
+   - Use `http://localhost:8000/your-alias`
+   - For template links: `http://localhost:8000/your-alias/parameter`
+
+### Managing Pages
+
+1. Create a new page:
+   - Visit `/ui/pages/new/`
+   - Enter the URL
+   - Title and summary will be automatically extracted
+
+2. View all pages:
+   - Visit `/ui/pages/`
+
+### API Access
+
+The application provides a REST API:
+
+- List pages: `GET /api/pages/`
+- Create page: `POST /api/pages/`
+
+Example:
+
+## Monitoring Celery Tasks
+
+The application includes Flower for monitoring Celery tasks. You can access it in two ways:
+
+1. Direct access:
+   - Visit `http://localhost:5555` in your browser
+
+2. Using management script:
+   ```bash
+   ./manage-docker.sh flower
    ```
 
-9. Open your browser and go to `http://127.0.0.1:8000`
+Flower provides:
+- Real-time monitoring of Celery tasks
+- Task progress and history
+- Worker status and statistics
+- Error tracking
+- Task graphs and charts
 
-Remember to keep both the Tailwind CSS build process and the Django development server running while you're developing.
+### Monitoring Features
 
-## Tools
+1. View all tasks:
+   - Active tasks
+   - Scheduled tasks
+   - Failed tasks
+   - Success rate
 
-The URL Manager includes a Tools page with the following features:
+2. Worker information:
+   - Status
+   - Resource usage
+   - Queue length
 
-- Export all aliases as a JSON file
-- Import aliases from a JSON file
-
-To access the Tools page, click on the "Tools" link in the navigation bar.
+3. Task details:
+   - Arguments
+   - Start time
+   - Runtime
+   - Result
+   - Stack traces for failed tasks

celery_beat.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eu
+source 3.12/bin/activate
+echo "Starting Django development server..."
+celery -A url_manager beat -l info

docker-compose.yml

@@ -0,0 +1,89 @@
+version: '3.8'
+
+services:
+  web:
+    build: .
+    command: >
+      sh -c "python manage.py migrate &&
+             python manage.py runserver 0.0.0.0:8000"
+    volumes:
+      - .:/app
+    ports:
+      - "8000:8000"
+    environment:
+      - DJANGO_SETTINGS_MODULE=url_manager.settings
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    depends_on:
+      - redis
+      - tailwind
+    networks:
+      - app-network
+
+  celery_worker:
+    build: .
+    command: celery -A url_manager worker -l info
+    volumes:
+      - .:/app
+    environment:
+      - DJANGO_SETTINGS_MODULE=url_manager.settings
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    depends_on:
+      - redis
+      - web
+    networks:
+      - app-network
+
+  celery_beat:
+    build: .
+    command: celery -A url_manager beat -l info
+    volumes:
+      - .:/app
+    environment:
+      - DJANGO_SETTINGS_MODULE=url_manager.settings
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    depends_on:
+      - redis
+      - web
+    networks:
+      - app-network
+
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379:6379"
+    networks:
+      - app-network
+
+  tailwind:
+    build: .
+    command: python manage.py tailwind start
+    volumes:
+      - .:/app
+    ports:
+      - "3000:3000"
+    networks:
+      - app-network
+
+  flower:
+    build: .
+    command: celery -A url_manager flower --port=5555
+    volumes:
+      - .:/app
+    ports:
+      - "5555:5555"
+    environment:
+      - DJANGO_SETTINGS_MODULE=url_manager.settings
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    depends_on:
+      - redis
+      - celery_worker
+    networks:
+      - app-network
+
+networks:
+  app-network:
+    driver: bridge

links/migrations/0011_page_error_message_page_last_retry_at_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.0.9 on 2024-11-04 01:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('links', '0010_alter_page_title'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='page',
+            name='error_message',
+            field=models.TextField(blank=True),
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='last_retry_at',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='process_status',
+            field=models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('completed', 'Completed'), ('failed', 'Failed')], default='pending', max_length=20),
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='retry_count',
+            field=models.IntegerField(default=0),
+        ),
+    ]

links/models.py

@@ -128,6 +128,12 @@ class LinkChangeLog(models.Model):
         return f"URL changed from {self.old_url} to {self.new_url}"
 
 class Page(models.Model):
+    class ProcessStatus(models.TextChoices):
+        PENDING = 'pending', _('Pending')
+        PROCESSING = 'processing', _('Processing')
+        COMPLETED = 'completed', _('Completed')
+        FAILED = 'failed', _('Failed')
+
     url = models.URLField(_('URL'), max_length=2000)
     title = models.CharField(_('Title'), max_length=200, blank=True)
     summary = models.TextField(_('Summary'), blank=True)
@@ -135,13 +141,28 @@ class Page(models.Model):
     created_at = models.DateTimeField(_('Created at'), default=timezone.now)
     updated_at = models.DateTimeField(_('Updated at'), auto_now=True)
 
+    # New fields for processing status
+    process_status = models.CharField(
+        max_length=20,
+        choices=ProcessStatus.choices,
+        default=ProcessStatus.PENDING
+    )
+    retry_count = models.IntegerField(default=0)
+    last_retry_at = models.DateTimeField(null=True, blank=True)
+    error_message = models.TextField(blank=True)
+
     class Meta:
         ordering = ['-updated_at']
         verbose_name = _('Page')
         verbose_name_plural = _('Pages')
 
     def __str__(self):
-        return self.title
+        return self.title or self.url
 
     def get_absolute_url(self):
         return reverse('page-detail', kwargs={'pk': self.pk})
+
+    def needs_processing(self):
+        return (not self.title or not self.summary) and \
+               self.process_status != self.ProcessStatus.FAILED and \
+               self.retry_count < 3

links/page_views.py

@@ -0,0 +1,140 @@
+from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView
+from django.urls import reverse_lazy
+from django.http import JsonResponse
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse
+import re
+from rest_framework import viewsets, status
+from rest_framework.response import Response
+from rest_framework.pagination import PageNumberPagination
+from .models import Page
+from .forms import PageForm
+from .serializers import PageSerializer
+
+class PageListView(ListView):
+    model = Page
+    template_name = 'links/page_list.html'
+    context_object_name = 'pages'
+    paginate_by = 10
+
+class PageDetailView(DetailView):
+    model = Page
+    template_name = 'links/page_detail.html'
+
+class PageCreateView(CreateView):
+    model = Page
+    form_class = PageForm
+    template_name = 'links/page_form.html'
+    success_url = reverse_lazy('page-list')
+
+    def form_valid(self, form):
+        # 如果标题和摘要都已经自动填充，则将状态设置为已完成
+        if form.instance.title and form.instance.summary:
+            form.instance.process_status = Page.ProcessStatus.COMPLETED
+        else:
+            form.instance.process_status = Page.ProcessStatus.PENDING
+
+        return super().form_valid(form)
+
+class PageUpdateView(UpdateView):
+    model = Page
+    form_class = PageForm
+    template_name = 'links/page_form.html'
+    success_url = reverse_lazy('page-list')
+
+class PageDeleteView(DeleteView):
+    model = Page
+    template_name = 'links/page_confirm_delete.html'
+    success_url = reverse_lazy('page-list')
+
+def fetch_page_info(request):
+    url = request.GET.get('url')
+    if not url:
+        return JsonResponse({'error': 'URL is required'}, status=400)
+
+    # 移除URL开头可能的@符号
+    url = url.lstrip('@')
+
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+        }
+
+        response = requests.get(url, headers=headers, timeout=10, verify=False)
+
+        if response.encoding == 'ISO-8859-1':
+            response.encoding = response.apparent_encoding or 'utf-8'
+
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # 获取标题
+        title = None
+        if soup.title:
+            title = soup.title.string
+        elif soup.find('h1'):
+            title = soup.find('h1').get_text(strip=True)
+        elif soup.find('meta', property='og:title'):
+            title = soup.find('meta', property='og:title').get('content')
+
+        if title:
+            title = re.sub(r'\s+', ' ', title.strip())
+            title = title.replace(' | ', ' - ').replace(' :: ', ' - ')
+        else:
+            title = urlparse(url).netloc
+
+        # 获取描述
+        description = None
+        meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'})
+        if meta_desc:
+            description = meta_desc.get('content')
+
+        if not description:
+            for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
+                tag.decompose()
+
+            paragraphs = soup.find_all(['p', 'div'])
+            for p in paragraphs:
+                text = p.get_text(strip=True)
+                if len(text) > 100:
+                    description = text
+                    break
+
+        if not description:
+            description = title
+
+        description = re.sub(r'\s+', ' ', description.strip())
+        description = description[:500] + '...' if len(description) > 500 else description
+
+        return JsonResponse({
+            'title': title,
+            'summary': description
+        })
+
+    except requests.exceptions.RequestException as e:
+        return JsonResponse({
+            'error': f'Failed to fetch page: {str(e)}'
+        }, status=400)
+    except Exception as e:
+        return JsonResponse({
+            'error': f'Error processing page: {str(e)}'
+        }, status=400)
+
+class StandardResultsSetPagination(PageNumberPagination):
+    page_size = 10
+    page_size_query_param = 'page_size'
+    max_page_size = 100
+
+class PageViewSet(viewsets.ModelViewSet):
+    queryset = Page.objects.all().order_by('-created_at')
+    serializer_class = PageSerializer
+    pagination_class = StandardResultsSetPagination
+
+    def create(self, request, *args, **kwargs):
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        self.perform_create(serializer)
+        headers = self.get_success_headers(serializer.data)
+        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)

links/tasks.py

@@ -0,0 +1,113 @@
+from celery import shared_task
+from django.utils import timezone
+from datetime import timedelta
+import requests
+from bs4 import BeautifulSoup
+import logging
+from .models import Page
+
+logger = logging.getLogger(__name__)
+
+def fibonacci(n):
+    if n <= 0:
+        return 0
+    elif n == 1:
+        return 1
+    else:
+        a, b = 0, 1
+        for _ in range(2, n + 1):
+            a, b = b, a + b
+        return b
+
+@shared_task(bind=True, max_retries=3)
+def process_page(self, page_id):
+    try:
+        page = Page.objects.get(id=page_id)
+
+        # Check if page still needs processing
+        if not page.needs_processing():
+            return
+
+        # Update status to processing
+        page.process_status = Page.ProcessStatus.PROCESSING
+        page.save()
+
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+        }
+
+        response = requests.get(page.url, headers=headers, timeout=10, verify=False)
+        response.raise_for_status()
+
+        if response.encoding == 'ISO-8859-1':
+            response.encoding = response.apparent_encoding or 'utf-8'
+
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # Extract title if not present
+        if not page.title:
+            title = None
+            if soup.title:
+                title = soup.title.string
+            elif soup.find('h1'):
+                title = soup.find('h1').get_text(strip=True)
+            elif soup.find('meta', property='og:title'):
+                title = soup.find('meta', property='og:title').get('content')
+
+            if title:
+                page.title = title.strip()
+
+        # Extract summary if not present
+        if not page.summary:
+            description = None
+            meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'})
+            if meta_desc:
+                description = meta_desc.get('content')
+
+            if not description:
+                for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
+                    tag.decompose()
+
+                paragraphs = soup.find_all(['p', 'div'])
+                for p in paragraphs:
+                    text = p.get_text(strip=True)
+                    if len(text) > 100:
+                        description = text
+                        break
+
+            if description:
+                page.summary = description[:500]
+
+        page.process_status = Page.ProcessStatus.COMPLETED
+        page.save()
+
+    except Exception as exc:
+        page.retry_count += 1
+        page.last_retry_at = timezone.now()
+
+        if page.retry_count >= 3:
+            page.process_status = Page.ProcessStatus.FAILED
+            page.error_message = str(exc)
+        else:
+            page.process_status = Page.ProcessStatus.PENDING
+            # Schedule retry with fibonacci backoff
+            retry_delay = fibonacci(page.retry_count)
+            self.retry(exc=exc, countdown=retry_delay)
+
+        page.save()
+        raise exc
+
+@shared_task
+def schedule_pending_pages():
+    """Periodic task to schedule processing of pending pages"""
+    pending_pages = Page.objects.filter(
+        process_status=Page.ProcessStatus.PENDING,
+        retry_count__lt=3
+    ).exclude(
+        last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(3))
+    )
+
+    for page in pending_pages:
+        process_page.delay(page.id)

links/templates/links/page_detail.html

@@ -6,25 +6,108 @@
 <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
     <div class="bg-white shadow-sm rounded-lg overflow-hidden">
         <div class="px-4 py-5 sm:p-6">
+            <!-- Header with actions -->
             <div class="flex justify-between items-center mb-6">
-                <h1 class="text-3xl font-bold text-gray-900">{{ page.title }}</h1>
+                <h1 class="text-3xl font-bold text-gray-900">{{ page.title|default:"Untitled" }}</h1>
                 <div class="flex space-x-2">
-                    <a href="{% url 'page-update' page.pk %}" class="text-blue-600 hover:text-blue-800">
+                    <a href="{% url 'page-update' page.pk %}" class="inline-flex items-center text-blue-600 hover:text-blue-800">
+                        <svg class="w-5 h-5 mr-1" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"/>
+                        </svg>
                         {% trans "Edit" %}
                     </a>
-                    <a href="{{ page.url }}" target="_blank" rel="noopener noreferrer" class="text-green-600 hover:text-green-800">
+                    <a href="{{ page.url }}" target="_blank" rel="noopener noreferrer" class="inline-flex items-center text-green-600 hover:text-green-800">
+                        <svg class="w-5 h-5 mr-1" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14"/>
+                        </svg>
                         {% trans "Visit URL" %}
                     </a>
                 </div>
             </div>
 
-            <div class="prose prose-sm sm:prose lg:prose-lg xl:prose-xl w-full">
-                {{ page.content|markdown|safe }}
+            <!-- Page Status -->
+            <div class="mb-6 flex items-center">
+                <span class="text-sm font-medium text-gray-500 mr-2">{% trans "Status" %}:</span>
+                {% if page.process_status == 'completed' %}
+                    <span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-green-100 text-green-800">
+                        {% trans "Completed" %}
+                    </span>
+                {% elif page.process_status == 'processing' %}
+                    <span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-yellow-100 text-yellow-800">
+                        {% trans "Processing" %}
+                    </span>
+                {% elif page.process_status == 'failed' %}
+                    <span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-red-100 text-red-800">
+                        {% trans "Failed" %}
+                    </span>
+                {% else %}
+                    <span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium bg-gray-100 text-gray-800">
+                        {% trans "Pending" %}
+                    </span>
+                {% endif %}
+                {% if page.retry_count > 0 %}
+                    <span class="ml-2 text-sm text-gray-500">
+                        ({% trans "Retries" %}: {{ page.retry_count }}/3)
+                    </span>
+                {% endif %}
             </div>
 
-            <div class="mt-6 text-sm text-gray-500">
-                {% trans "Last updated" %}: {{ page.updated_at|date:"Y-m-d H:i" }}
+            <!-- URL -->
+            <div class="mb-6">
+                <h2 class="text-sm font-medium text-gray-500 mb-1">{% trans "URL" %}</h2>
+                <a href="{{ page.url }}" target="_blank" rel="noopener noreferrer"
+                   class="text-blue-600 hover:text-blue-800 break-all">
+                    {{ page.url }}
+                </a>
             </div>
+
+            <!-- Timestamps -->
+            <div class="mb-6 space-y-2">
+                <div class="flex items-center text-sm">
+                    <span class="font-medium text-gray-500 w-24">{% trans "Created" %}:</span>
+                    <span class="text-gray-900">{{ page.created_at|date:"Y-m-d H:i:s" }}</span>
+                </div>
+                <div class="flex items-center text-sm">
+                    <span class="font-medium text-gray-500 w-24">{% trans "Updated" %}:</span>
+                    <span class="text-gray-900">{{ page.updated_at|date:"Y-m-d H:i:s" }}</span>
+                </div>
+                {% if page.last_retry_at %}
+                <div class="flex items-center text-sm">
+                    <span class="font-medium text-gray-500 w-24">{% trans "Last Retry" %}:</span>
+                    <span class="text-gray-900">{{ page.last_retry_at|date:"Y-m-d H:i:s" }}</span>
+                </div>
+                {% endif %}
+            </div>
+
+            <!-- Summary -->
+            {% if page.summary %}
+            <div class="mb-6">
+                <h2 class="text-sm font-medium text-gray-500 mb-1">{% trans "Summary" %}</h2>
+                <div class="bg-gray-50 rounded-lg p-4 text-gray-700">
+                    {{ page.summary }}
+                </div>
+            </div>
+            {% endif %}
+
+            <!-- Content -->
+            {% if page.content %}
+            <div class="mb-6">
+                <h2 class="text-sm font-medium text-gray-500 mb-1">{% trans "Content" %}</h2>
+                <div class="prose prose-sm sm:prose lg:prose-lg xl:prose-xl w-full">
+                    {{ page.content|markdown|safe }}
+                </div>
+            </div>
+            {% endif %}
+
+            <!-- Error Message -->
+            {% if page.error_message %}
+            <div class="mb-6">
+                <h2 class="text-sm font-medium text-red-500 mb-1">{% trans "Error Message" %}</h2>
+                <div class="bg-red-50 text-red-700 p-4 rounded-lg">
+                    {{ page.error_message }}
+                </div>
+            </div>
+            {% endif %}
         </div>
     </div>
 </div>

links/views.py

@@ -35,6 +35,11 @@ from rest_framework import viewsets, status
 from rest_framework.response import Response
 from rest_framework.pagination import PageNumberPagination
 from .serializers import PageSerializer
+from .page_views import (
+    PageListView, PageDetailView, PageCreateView,
+    PageUpdateView, PageDeleteView, fetch_page_info,
+    PageViewSet
+)
 
 logger = logging.getLogger(__name__)
 
@@ -440,134 +445,3 @@ def export_database(request):
 
 class HelpView(TemplateView):
     template_name = 'links/help.html'
-
-class PageListView(ListView):
-    model = Page
-    template_name = 'links/page_list.html'
-    context_object_name = 'pages'
-    paginate_by = 10
-
-class PageDetailView(DetailView):
-    model = Page
-    template_name = 'links/page_detail.html'
-
-class PageCreateView(CreateView):
-    model = Page
-    form_class = PageForm
-    template_name = 'links/page_form.html'
-    success_url = reverse_lazy('page-list')
-
-class PageUpdateView(UpdateView):
-    model = Page
-    form_class = PageForm
-    template_name = 'links/page_form.html'
-    success_url = reverse_lazy('page-list')
-
-class PageDeleteView(DeleteView):
-    model = Page
-    template_name = 'links/page_confirm_delete.html'
-    success_url = reverse_lazy('page-list')
-
-def fetch_page_info(request):
-    url = request.GET.get('url')
-    if not url:
-        return JsonResponse({'error': 'URL is required'}, status=400)
-
-    # 移除URL开头可能的@符号
-    url = url.lstrip('@')
-
-    try:
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5',
-        }
-
-        response = requests.get(url, headers=headers, timeout=10, verify=False)
-
-        # 确保使用正确的编码
-        if response.encoding == 'ISO-8859-1':
-            response.encoding = response.apparent_encoding or 'utf-8'
-
-        soup = BeautifulSoup(response.text, 'html.parser')
-
-        # 获取标题 - 尝试多种方式
-        title = None
-        # 1. 尝试获取title标签
-        if soup.title:
-            title = soup.title.string
-        # 2. 尝试获取第一个h1
-        if not title and soup.find('h1'):
-            title = soup.find('h1').get_text(strip=True)
-        # 3. 尝试获取og:title
-        if not title:
-            og_title = soup.find('meta', property='og:title')
-            if og_title:
-                title = og_title.get('content')
-
-        # 清理标题
-        if title:
-            title = re.sub(r'\s+', ' ', title.strip())
-            title = title.replace(' | ', ' - ').replace(' :: ', ' - ')
-        else:
-            title = urlparse(url).netloc
-
-        # 获取描述 - 尝试多种方式
-        description = None
-        # 1. 尝试meta description
-        meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'})
-        if meta_desc:
-            description = meta_desc.get('content')
-
-        # 2. 如果没有meta描述，尝试获取正文内容
-        if not description:
-            # 移除script, style等标签
-            for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
-                tag.decompose()
-
-            # 获取所有段落
-            paragraphs = soup.find_all(['p', 'div'])
-            for p in paragraphs:
-                text = p.get_text(strip=True)
-                if len(text) > 100:  # 确保段落有足够的内容
-                    description = text
-                    break
-
-        # 如果还是没有描述，使用标题
-        if not description:
-            description = title
-
-        # 清理描述
-        description = re.sub(r'\s+', ' ', description.strip())
-        description = description[:500] + '...' if len(description) > 500 else description
-
-        return JsonResponse({
-            'title': title,
-            'summary': description
-        })
-
-    except requests.exceptions.RequestException as e:
-        return JsonResponse({
-            'error': f'Failed to fetch page: {str(e)}'
-        }, status=400)
-    except Exception as e:
-        return JsonResponse({
-            'error': f'Error processing page: {str(e)}'
-        }, status=400)
-
-class StandardResultsSetPagination(PageNumberPagination):
-    page_size = 10
-    page_size_query_param = 'page_size'
-    max_page_size = 100
-
-class PageViewSet(viewsets.ModelViewSet):
-    queryset = Page.objects.all().order_by('-created_at')
-    serializer_class = PageSerializer
-    pagination_class = StandardResultsSetPagination
-
-    def create(self, request, *args, **kwargs):
-        serializer = self.get_serializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        self.perform_create(serializer)
-        headers = self.get_success_headers(serializer.data)
-        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)

manage-docker.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+case "$1" in
+    "start")
+        docker-compose up -d
+        echo "Services started. Access:"
+        echo "- Web: http://localhost:8000"
+        echo "- Flower (Celery Monitor): http://localhost:5555"
+        ;;
+    "stop")
+        docker-compose down
+        ;;
+    "restart")
+        docker-compose restart
+        ;;
+    "logs")
+        if [ "$2" ]; then
+            docker-compose logs -f "$2"
+        else
+            docker-compose logs -f
+        fi
+        ;;
+    "build")
+        docker-compose build
+        ;;
+    "shell")
+        docker-compose exec web python manage.py shell
+        ;;
+    "migrate")
+        docker-compose exec web python manage.py migrate
+        ;;
+    "makemigrations")
+        docker-compose exec web python manage.py makemigrations
+        ;;
+    "tailwind-logs")
+        docker-compose logs -f tailwind
+        ;;
+    "tailwind-build")
+        docker-compose exec tailwind npm run build
+        ;;
+    "flower")
+        echo "Opening Flower dashboard in default browser..."
+        docker-compose exec flower open http://localhost:5555
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|restart|logs|build|shell|migrate|makemigrations|tailwind-logs|tailwind-build|flower}"
+        echo ""
+        echo "Additional commands:"
+        echo "  logs [service]    View logs of all services or specific service"
+        echo "  flower           Open Flower dashboard in browser"
+        exit 1
+        ;;
+esac

package.json

@@ -0,0 +1,12 @@
+{
+  "name": "url-manager",
+  "version": "1.0.0",
+  "scripts": {
+    "dev": "tailwindcss -i ./new_theme/static/css/src/styles.css -o ./new_theme/static/css/dist/styles.css --watch",
+    "build": "tailwindcss -i ./new_theme/static/css/src/styles.css -o ./new_theme/static/css/dist/styles.css --minify"
+  },
+  "dependencies": {
+    "tailwindcss": "^3.3.0",
+    "@tailwindcss/typography": "^0.5.9"
+  }
+}

requirements.txt

@@ -14,3 +14,6 @@ markdown==3.7
 requests==2.32.3
 beautifulsoup4==4.12.3
 djangorestframework==3.15.2
+celery>=5.3.0
+redis>=4.5.0
+flower>=2.0.0

url_manager/celery.py

@@ -0,0 +1,8 @@
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'url_manager.settings')
+
+app = Celery('url_manager')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()

url_manager/settings.py

@@ -192,3 +192,23 @@ REST_FRAMEWORK = {
     'PAGE_SIZE': 10,
     'UNAUTHENTICATED_USER': None,
 }
+
+# Time zone setting
+TIME_ZONE = 'UTC'
+USE_TZ = True
+
+# Celery Configuration
+CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0')
+CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0')
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TIMEZONE = TIME_ZONE  # Now TIME_ZONE is defined
+
+# Celery Beat Schedule
+CELERY_BEAT_SCHEDULE = {
+    'check-pending-pages': {
+        'task': 'links.tasks.schedule_pending_pages',
+        'schedule': 300.0,  # Run every 5 minutes
+    },
+}

worker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eu
+source 3.12/bin/activate
+echo "Starting Django development server..."
+celery -A url_manager worker -l info