Add craw4ai

.env.example

@@ -5,4 +5,8 @@ R2_ACCESS_KEY_ID=
 R2_SECRET_ACCESS_KEY=
 R2_BUCKET_NAME=home-links
 
+# Crawl4AI Configuration
+CRAWL4AI_API_URL=https://crawl-api.junv.cc
+CRAWL4AI_ENABLED=true
+
 

core/settings.py

@@ -212,6 +212,11 @@ R2_ACCESS_KEY_ID = os.environ.get('R2_ACCESS_KEY_ID')
 R2_SECRET_ACCESS_KEY = os.environ.get('R2_SECRET_ACCESS_KEY')
 R2_BUCKET_NAME = os.environ.get('R2_BUCKET_NAME')
 R2_CUSTOM_DOMAIN = os.environ.get('R2_CUSTOM_DOMAIN')
+
+# Crawl4AI Configuration
+CRAWL4AI_API_URL = os.environ.get('CRAWL4AI_API_URL', 'https://crawl-api.junv.cc')
+CRAWL4AI_ENABLED = os.environ.get('CRAWL4AI_ENABLED', 'True').lower() in ('true', '1', 'yes')
+
 # For debugging
 LOGGING = {
     'version': 1,

k8s/manifest.yaml

@@ -122,6 +122,10 @@ spec:
                 secretKeyRef:
                   name: r2-credentials
                   key: key_id
+            - name: CRAWL4AI_API_URL
+              value: "http://crawl4ai.ai.svc.cluster.local:80"
+            - name: CRAWL4AI_ENABLED
+              value: "true"
           ports:
             - containerPort: 8000
               name: links-port

links/migrations/0035_add_webpage_content_to_page.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.13 on 2025-11-04 11:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('links', '0034_post_summary'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='page',
+            name='webpage_content',
+            field=models.TextField(blank=True, help_text='Markdown content extracted from Crawl4AI', verbose_name='Webpage Content (Markdown)'),
+        ),
+    ]

links/models.py

@@ -176,6 +176,7 @@ class Page(models.Model):
     title = models.CharField(_('Title'), max_length=200, blank=True)
     summary = models.TextField(_('Summary'), blank=True)
     content = models.TextField(_('Content'), blank=True)
+    webpage_content = models.TextField(_('Webpage Content (Markdown)'), blank=True, help_text=_('Markdown content extracted from Crawl4AI'))
     created_at = models.DateTimeField(_('Created at'), default=timezone.now)
     updated_at = models.DateTimeField(_('Updated at'), auto_now=True)
     tags = models.ManyToManyField('Tag', blank=True, related_name='pages')

links/page_views.py

@@ -18,12 +18,47 @@ from bs4 import BeautifulSoup
 from urllib.parse import urlparse, quote
 import re
 import base64
-from .tasks import capture_screenshot, process_page
+from .tasks import capture_screenshot, process_page, fetch_webpage_content_from_crawl4ai
 from pathlib import Path
 from datetime import datetime
 from rest_framework.decorators import action
 from threading import Thread
 
+
+def trigger_page_processing(page):
+    """
+    Trigger background tasks for a newly created page:
+    1. Screenshot capture
+    2. Crawl4AI content extraction
+    3. Page metadata processing
+    
+    This is called whenever a page is created, regardless of the source (UI or API).
+    """
+    from .tasks import capture_screenshot, fetch_webpage_content_from_crawl4ai, process_page
+    
+    # Create screenshot record and trigger capture
+    screenshot = Screenshot.objects.create(
+        page=page,
+        status=Screenshot.Status.PENDING
+    )
+    
+    # Start screenshot capture thread
+    thread_screenshot = Thread(target=capture_screenshot, args=(page.id, screenshot.id))
+    thread_screenshot.daemon = True
+    thread_screenshot.start()
+    
+    # Start Crawl4AI content extraction thread
+    thread_crawl = Thread(target=fetch_webpage_content_from_crawl4ai, args=(page.id,))
+    thread_crawl.daemon = True
+    thread_crawl.start()
+    
+    # Start page metadata processing thread (if needed)
+    if not page.title or not page.summary:
+        thread_process = Thread(target=process_page, args=(page.id,))
+        thread_process.daemon = True
+        thread_process.start()
+
+
 class PageListView(ListView):
     model = Page
     template_name = 'links/page_list.html'
@@ -49,17 +84,9 @@ class PageCreateView(CreateView):
         response = super().form_valid(form)
         page = self.object
 
-        # Create a pending screenshot record
-        screenshot = Screenshot.objects.create(
-            page=page,
-            status=Screenshot.Status.PENDING
-        )
-
-        # Start processing the page
+        # Trigger all background processing tasks
         try:
-            thread = Thread(target=process_page, args=(page.id,))
-            thread.daemon = True
-            thread.start()
+            trigger_page_processing(page)
             messages.success(self.request, _('Page created successfully and processing started.'))
         except Exception as e:
             messages.error(self.request, _('Page created but processing failed to start.'))
@@ -184,18 +211,6 @@ class PageViewSet(viewsets.ModelViewSet):
         serializer.is_valid(raise_exception=True)
         self.perform_create(serializer)
         headers = self.get_success_headers(serializer.data)
-
-        # Trigger screenshot capture if needed
-        from .tasks import capture_screenshot
-        # Create screenshot record first
-        screenshot = Screenshot.objects.create(
-            page=serializer.instance,
-            status=Screenshot.Status.PENDING
-        )
-        thread = Thread(target=capture_screenshot, args=(serializer.instance.id, screenshot.id))
-        thread.daemon = True
-        thread.start()
-
         return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
 
     def perform_create(self, serializer):
@@ -206,6 +221,9 @@ class PageViewSet(viewsets.ModelViewSet):
             instance.process_status = Page.ProcessStatus.PENDING
         instance.save()
         
+        # Trigger all background processing tasks
+        trigger_page_processing(instance)
+
     @action(detail=True, methods=['post'], url_path='take-screenshot')
     def take_screenshot(self, request, pk=None):
         """API endpoint to take a new screenshot of a page"""

links/serializers.py

@@ -4,9 +4,9 @@ from .models import Page, Post, ImageCollection, Image, Tag
 class PageSerializer(serializers.ModelSerializer):
     class Meta:
         model = Page
-        fields = ['id', 'url', 'title', 'summary', 'content', 'screenshot_path',
+        fields = ['id', 'url', 'title', 'summary', 'content', 'webpage_content', 'screenshot_path',
                  'process_status', 'created_at', 'updated_at']
-        read_only_fields = ['id', 'screenshot_path', 'process_status',
+        read_only_fields = ['id', 'screenshot_path', 'process_status', 'webpage_content',
                           'created_at', 'updated_at']
 
 class PostSerializer(serializers.ModelSerializer):

links/tasks.py

@@ -276,3 +276,63 @@ def schedule_pending_pages():
         thread = Thread(target=process_page, args=(page.id,))
         thread.daemon = True
         thread.start()
+
+def fetch_webpage_content_from_crawl4ai(page_id, retry_count=0):
+    """
+    Fetch webpage content using Crawl4AI /md endpoint
+    """
+    from .models import Page
+    
+    if not settings.CRAWL4AI_ENABLED:
+        logger.info("Crawl4AI is disabled, skipping webpage content extraction")
+        return
+    
+    try:
+        page = Page.objects.get(id=page_id)
+        logger.info(f"Fetching webpage content for page {page_id} from Crawl4AI")
+        
+        # Call Crawl4AI /md endpoint
+        api_url = f"{settings.CRAWL4AI_API_URL}/md"
+        payload = {
+            "url": page.url
+        }
+        
+        response = requests.post(
+            api_url,
+            json=payload,
+            timeout=60
+        )
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        # Extract markdown content from response
+        markdown_content = data.get('markdown', '')
+        
+        if markdown_content:
+            page.webpage_content = markdown_content
+            page.save()
+            logger.info(f"Successfully fetched webpage content for page {page_id}")
+        else:
+            logger.warning(f"No markdown content returned for page {page_id}")
+            
+    except requests.RequestException as e:
+        logger.error(f"Failed to fetch webpage content for page {page_id}: {e}")
+        
+        # Retry logic
+        if retry_count < MAX_RETRIES:
+            retry_delay = fibonacci(retry_count + 1)
+            from core.scheduler import scheduler
+            from datetime import datetime, timedelta
+            run_date = datetime.now() + timedelta(seconds=retry_delay)
+            scheduler.add_job(
+                fetch_webpage_content_from_crawl4ai,
+                'date',
+                run_date=run_date,
+                args=[page_id, retry_count + 1],
+                id=f'fetch_crawl4ai_{page_id}_retry_{retry_count + 1}',
+                replace_existing=True
+            )
+            logger.info(f"Scheduled Crawl4AI retry {retry_count + 1} for page {page_id} in {retry_delay} seconds")
+    except Exception as e:
+        logger.error(f"Unexpected error fetching webpage content for page {page_id}: {e}", exc_info=True)

links/templates/links/page_detail.html

@@ -3,6 +3,13 @@
 {% load markdown_extras %}
 {% load static %}
 
+{% block extra_head %}
+<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
+<style>
+    [x-cloak] { display: none !important; }
+</style>
+{% endblock %}
+
 {% block content %}
 <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
     <!-- Back Button -->
@@ -275,6 +282,44 @@
                     {% endif %}
                 {% endwith %}
             </div>
+
+            <!-- Webpage Content from Crawl4AI -->
+            {% if page.webpage_content %}
+            <div class="mt-8">
+                <div class="mb-6" x-data="{ expanded: false }">
+                    <div class="flex items-center justify-between mb-2">
+                        <h3 class="text-lg font-medium text-gray-900">{% trans "Full Webpage Content" %}</h3>
+                        <button 
+                            @click="expanded = !expanded"
+                            class="inline-flex items-center px-3 py-1 text-sm font-medium text-blue-600 hover:text-blue-800 focus:outline-none">
+                            <span x-show="!expanded">{% trans "Expand" %}</span>
+                            <span x-show="expanded" x-cloak>{% trans "Collapse" %}</span>
+                            <svg class="ml-2 w-4 h-4 transform transition-transform" 
+                                 :class="{ 'rotate-180': expanded }"
+                                 fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"/>
+                            </svg>
+                        </button>
+                    </div>
+                    <div x-show="expanded" 
+                         x-transition:enter="transition ease-out duration-200"
+                         x-transition:enter-start="opacity-0 transform scale-95"
+                         x-transition:enter-end="opacity-100 transform scale-100"
+                         x-transition:leave="transition ease-in duration-150"
+                         x-transition:leave-start="opacity-100 transform scale-100"
+                         x-transition:leave-end="opacity-0 transform scale-95"
+                         class="bg-gray-50 rounded-lg p-6 border border-gray-200">
+                        <div class="prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none">
+                            {{ page.webpage_content|markdown|safe }}
+                        </div>
+                    </div>
+                    <div x-show="!expanded" 
+                         class="bg-gray-50 rounded-lg p-4 border border-gray-200">
+                        <p class="text-sm text-gray-600 italic">{% trans "Click 'Expand' to view the full webpage content extracted by Crawl4AI" %}</p>
+                    </div>
+                </div>
+            </div>
+            {% endif %}
         </div>
     </div>
 </div>

templates/base.html

@@ -72,6 +72,7 @@
     </style>
     {{ form.media }}
     {% block extra_css %}{% endblock %}
+    {% block extra_head %}{% endblock %}
     <script src="{% static 'js/common.js' %}"></script>
 </head>
 <body class="bg-gray-100">