Add image imports

2026-08-08 21:04:53 +10:00 · 2026-03-22 16:15:53 +11:00
parent f6f115690b
commit f4d36e6953
11 changed files with 377 additions and 12 deletions
@@ -16,7 +16,7 @@ class CoreConfig(AppConfig):
        Initialize APScheduler when Django starts
        """
        from core.scheduler import scheduler, start_scheduler
-        from links.tasks import schedule_pending_pages, schedule_pending_screenshots
+        from links.tasks import schedule_pending_pages, schedule_pending_screenshots, retry_stuck_image_imports
        from apscheduler.triggers.interval import IntervalTrigger

        # Start the scheduler
@@ -49,3 +49,12 @@ class CoreConfig(AppConfig):
            replace_existing=True,
        )
        logger.info(f"Scheduled periodic task: schedule_pending_screenshots (every {screenshots_interval}s)")
+
+        # Add periodic job for retrying stuck image imports (every 5 minutes)
+        scheduler.add_job(
+            retry_stuck_image_imports,
+            trigger=IntervalTrigger(seconds=300),
+            id='retry_stuck_image_imports',
+            replace_existing=True,
+        )
+        logger.info("Scheduled periodic task: retry_stuck_image_imports (every 300s)")
@@ -7,6 +7,7 @@ from . import file_views

 # Create a router and register our viewsets with it
 router = DefaultRouter(trailing_slash=False)
+router.register('links', api_views.LinkViewSet, basename='api-links')
 router.register('pages', page_views.PageViewSet, basename='api-pages')
 router.register('posts', post_views.PostViewSet, basename='api-posts')
 router.register('music', api_views.MusicViewSet, basename='api-music')
@@ -1,8 +1,9 @@
 from rest_framework import viewsets, status
 from rest_framework.decorators import action
 from rest_framework.response import Response
-from .models import ImageCollection, Image
-from .serializers import ImageCollectionSerializer, ImageSerializer, ImageDescriptionSerializer
+from rest_framework.pagination import PageNumberPagination
+from .models import Link, ImageCollection, Image
+from .serializers import LinkSerializer, ImageCollectionSerializer, ImageSerializer, ImageDescriptionSerializer
 from .storage import R2Storage
 import uuid
 import logging
@@ -12,6 +13,31 @@ import os

 logger = logging.getLogger(__name__)

+
+class StandardResultsSetPagination(PageNumberPagination):
+    page_size = 10
+    page_size_query_param = 'page_size'
+    max_page_size = 100
+
+
+class LinkViewSet(viewsets.ReadOnlyModelViewSet):
+    serializer_class = LinkSerializer
+    pagination_class = StandardResultsSetPagination
+
+    def get_queryset(self):
+        return Link.objects.prefetch_related('tags').order_by('-created_at')
+
+    @action(detail=False, methods=['get'], url_path='most-visited')
+    def most_visited(self, request):
+        """Return links sorted by click_count descending."""
+        queryset = Link.objects.prefetch_related('tags').filter(click_count__gt=0).order_by('-click_count')
+        page = self.paginate_queryset(queryset)
+        if page is not None:
+            serializer = self.get_serializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+        serializer = self.get_serializer(queryset, many=True)
+        return Response(serializer.data)
+
 class ImageCollectionViewSet(viewsets.ModelViewSet):
    queryset = ImageCollection.objects.all()
    serializer_class = ImageCollectionSerializer
@@ -217,16 +217,27 @@ def import_image_view(request, image_url):
      3. Immediately redirects to the original URL so the image is visible right away.
    On subsequent requests, once the file is saved locally, the view redirects to
    the stored public URL instead — no more dependency on the original host.
+
+    If the background download fails, the stub record is deleted automatically so no
+    size=0 ghost appears in the file list. The next visit to the same URL will create
+    a fresh stub and retry. A periodic APScheduler task (`retry_stuck_image_imports`)
+    also reschedules any stubs left behind by killed threads.
    """
    import hashlib
    import posixpath
    from threading import Thread
    from .tasks import download_and_save_image

-    # Build the canonical source URL (https preferred)
+    # Build the canonical source URL, preserving query string.
+    # Django's <path:> converter only captures the path component; query params
+    # like ?format=jpg&name=900x900 end up in QUERY_STRING and must be re-attached.
+    query_string = request.META.get('QUERY_STRING', '')
    source_url = f'https://{image_url}'
+    if query_string:
+        source_url = f'{source_url}?{query_string}'

    url_hash = hashlib.sha256(source_url.encode()).hexdigest()[:20]
+    # Strip query string for filename derivation
    filename = posixpath.basename(image_url.split('?')[0]) or f'image_{url_hash}'

    # Derive extension from filename; fall back to .jpg for bare names
@@ -1,5 +1,18 @@
 from rest_framework import serializers
-from .models import Page, Post, ImageCollection, Image, Tag, FileUpload
+from .models import Link, Page, Post, ImageCollection, Image, Tag, FileUpload
+
+
+class LinkSerializer(serializers.ModelSerializer):
+    tags = serializers.SerializerMethodField()
+
+    class Meta:
+        model = Link
+        fields = ['id', 'alias', 'original_url', 'description', 'link_type',
+                  'click_count', 'tags', 'created_at', 'updated_at']
+        read_only_fields = ['id', 'click_count', 'created_at', 'updated_at']
+
+    def get_tags(self, obj):
+        return [{'id': tag.id, 'name': tag.name, 'slug': tag.slug} for tag in obj.tags.all()]

 class PageSerializer(serializers.ModelSerializer):
    class Meta:
@@ -500,3 +500,44 @@ def download_and_save_image(file_upload_id):
                os.remove(dest_path)
            except OSError:
                pass
+        # Remove the stub record so a size=0 ghost doesn't appear in the file list.
+        # The next import request for the same URL will create a fresh stub and retry.
+        try:
+            FileUpload.objects.filter(pk=file_upload_id, size=0).delete()
+            logger.info(f"download_and_save_image: deleted stub FileUpload {file_upload_id} after failed download")
+        except Exception as del_exc:
+            logger.error(f"download_and_save_image: could not delete stub {file_upload_id}: {del_exc}")
+
+
+def retry_stuck_image_imports():
+    """Periodic task: retry imported images that are stuck with size=0 and no file on disk.
+
+    A stub FileUpload (size=0, source_url set) can be left behind when the background
+    download thread is killed mid-flight (e.g. server restart). This task finds those
+    orphaned stubs and re-kicks the download, provided the stub is old enough that we
+    are confident it is not a currently in-progress download (>5 minutes since last update).
+    """
+    from .models import FileUpload
+
+    threshold = timezone.now() - timedelta(minutes=5)
+    stuck = FileUpload.objects.filter(
+        size=0,
+        source_url__isnull=False,
+        updated_at__lt=threshold,
+    ).exclude(source_url='')
+
+    count = stuck.count()
+    if count:
+        logger.info(f"retry_stuck_image_imports: found {count} stuck import stub(s) — retrying")
+
+    for record in stuck:
+        if os.path.exists(record.file_path):
+            # File landed on disk but DB wasn't updated — fix it now
+            size = os.path.getsize(record.file_path)
+            FileUpload.objects.filter(pk=record.pk, size=0).update(size=size)
+            logger.info(f"retry_stuck_image_imports: fixed size for FileUpload {record.pk} ({size} bytes)")
+            continue
+
+        logger.info(f"retry_stuck_image_imports: re-queuing download for FileUpload {record.pk} ({record.source_url})")
+        thread = Thread(target=download_and_save_image, args=(str(record.pk),), daemon=True)
+        thread.start()
@@ -147,10 +147,19 @@
                        {{ scheduled_jobs|length }}
                    </span>
                </a>
+                <a href="?tab=image_imports&status=all"
+                   class="px-4 py-2 text-sm font-medium rounded-t-md border-b-2 transition-colors
+                       {% if tab == 'image_imports' %}border-blue-500 text-blue-600 bg-white{% else %}border-transparent text-gray-500 hover:text-gray-700 hover:bg-gray-50{% endif %}">
+                    {% trans "Image Imports" %}
+                    <span class="ml-1 px-1.5 py-0.5 text-xs rounded-full
+                        {% if tab == 'image_imports' %}bg-blue-100 text-blue-600{% else %}bg-gray-100 text-gray-500{% endif %}">
+                        {{ stats.image_imports.total }}
+                    </span>
+                </a>
            </div>

            <!-- Status filter pills (only for data tabs) -->
-            {% if tab != 'scheduler' %}
+            {% if tab == 'screenshots' or tab == 'pages' %}
            <div class="flex gap-1.5 pb-1">
                {% for s, label, pill_active_style, pill_inactive_style in filter_options %}
                <a href="?tab={{ tab }}&status={{ s }}"
@@ -159,6 +168,15 @@
                </a>
                {% endfor %}
            </div>
+            {% elif tab == 'image_imports' %}
+            <div class="flex gap-1.5 pb-1">
+                {% for s, label, pill_active_style, pill_inactive_style in import_filter_options %}
+                <a href="?tab={{ tab }}&status={{ s }}"
+                   style="{% if status_filter == s %}{{ pill_active_style }}{% else %}{{ pill_inactive_style }}{% endif %}padding:.3rem .8rem;border-radius:9999px;font-size:.8rem;font-weight:500;text-decoration:none;display:inline-block;white-space:nowrap;">
+                    {{ label }}
+                </a>
+                {% endfor %}
+            </div>
            {% endif %}
        </div>
    </div>
@@ -176,7 +194,7 @@
                    style="background:#1d4ed8;color:#fff;padding:.3rem .85rem;border-radius:.25rem;font-size:.8rem;cursor:pointer;font-weight:500;">
                    ↺ {% trans "Retry" %}
                </button>
-                <button @click="bulkAction('fail')"
+                <button @click="bulkAction('fail')" x-show="tab !== 'image_imports'"
                    style="background:#d97706;color:#fff;padding:.3rem .85rem;border-radius:.25rem;font-size:.8rem;cursor:pointer;font-weight:500;">
                    ✕ {% trans "Mark Failed" %}
                </button>
@@ -367,6 +385,71 @@
        {% else %}
        <div class="px-6 py-12 text-center text-gray-400 text-sm">{% trans "No pages match this filter." %}</div>
        {% endif %}
+
+        {% elif tab == 'image_imports' %}
+        <!-- Image Imports Table -->
+        {% if page_obj.object_list %}
+        <div class="overflow-x-auto">
+            <table class="min-w-full text-sm">
+                <thead class="bg-gray-50 text-xs text-gray-500 uppercase tracking-wide">
+                    <tr>
+                        <th class="px-3 py-3 w-8">
+                            <input type="checkbox" class="rounded text-blue-600"
+                                   @change="toggleAll($event.target.checked, allIds)"
+                                   :checked="allIds.length > 0 && selectedIds.length === allIds.length">
+                        </th>
+                        <th class="px-3 py-3 text-left">ID</th>
+                        <th class="px-3 py-3 text-left">{% trans "Filename" %}</th>
+                        <th class="px-3 py-3 text-left">{% trans "Source URL" %}</th>
+                        <th class="px-3 py-3 text-left">{% trans "Status" %}</th>
+                        <th class="px-3 py-3 text-left">{% trans "Size" %}</th>
+                        <th class="px-3 py-3 text-left">{% trans "Updated" %}</th>
+                    </tr>
+                </thead>
+                <tbody class="divide-y divide-gray-100">
+                    {% for imp in page_obj.object_list %}
+                    <tr class="hover:bg-gray-50" :class="{'bg-blue-50': selectedIds.includes('{{ imp.id }}') }">
+                        <td class="px-3 py-2">
+                            <input type="checkbox" class="rounded text-blue-600"
+                                   value="{{ imp.id }}"
+                                   @change="toggle('{{ imp.id }}')"
+                                   :checked="selectedIds.includes('{{ imp.id }}')">
+                        </td>
+                        <td class="px-3 py-2 font-mono text-gray-400" style="font-size:.7rem;">{{ imp.id|truncatechars:12 }}</td>
+                        <td class="px-3 py-2 max-w-xs">
+                            <a href="{{ imp.download_url }}" class="text-blue-600 hover:underline block truncate max-w-xs" style="font-size:.85rem;" title="{{ imp.name }}">
+                                {{ imp.name|truncatechars:40 }}
+                            </a>
+                        </td>
+                        <td class="px-3 py-2 max-w-xs">
+                            <a href="{{ imp.source_url }}" target="_blank" rel="noopener"
+                               class="text-gray-400 hover:text-blue-500 block truncate max-w-xs" style="font-size:.8rem;" title="{{ imp.source_url }}">
+                                {{ imp.source_url|truncatechars:50 }}
+                            </a>
+                        </td>
+                        <td class="px-3 py-2">
+                            {% if imp.size > 0 %}
+                            <span style="display:inline-flex;align-items:center;padding:.15rem .55rem;border-radius:9999px;font-size:.75rem;font-weight:500;background:#dcfce7;color:#166534;">
+                                {% trans "done" %}
+                            </span>
+                            {% else %}
+                            <span style="display:inline-flex;align-items:center;padding:.15rem .55rem;border-radius:9999px;font-size:.75rem;font-weight:500;background:#fef9c3;color:#854d0e;">
+                                {% trans "pending" %}
+                            </span>
+                            {% endif %}
+                        </td>
+                        <td class="px-3 py-2 text-gray-500" style="font-size:.85rem;">
+                            {% if imp.size > 0 %}{{ imp.formatted_size }}{% else %}<span class="text-gray-300">—</span>{% endif %}
+                        </td>
+                        <td class="px-3 py-2 text-gray-400 whitespace-nowrap" style="font-size:.8rem;">{{ imp.updated_at|timesince }} {% trans "ago" %}</td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+        {% else %}
+        <div class="px-6 py-12 text-center text-gray-400 text-sm">{% trans "No image imports match this filter." %}</div>
+        {% endif %}
        {% endif %}

        <!-- Pagination -->
@@ -461,9 +544,13 @@ function jobsManager() {
            if (this.selectedIds.length === 0) return;
            const tab = this.tab;
            const actionMap = {
-                retry:  tab === 'screenshots' ? 'bulk_retry_screenshots' : 'bulk_retry_pages',
-                fail:   tab === 'screenshots' ? 'bulk_fail_screenshots'  : 'bulk_fail_pages',
-                delete: tab === 'screenshots' ? 'bulk_delete_screenshots': 'bulk_delete_pages',
+                retry:  tab === 'screenshots' ? 'bulk_retry_screenshots'
+                      : tab === 'image_imports' ? 'bulk_retry_image_imports'
+                      : 'bulk_retry_pages',
+                fail:   tab === 'screenshots' ? 'bulk_fail_screenshots' : 'bulk_fail_pages',
+                delete: tab === 'screenshots' ? 'bulk_delete_screenshots'
+                      : tab === 'image_imports' ? 'bulk_delete_image_imports'
+                      : 'bulk_delete_pages',
            };
            if (type === 'delete' && !confirm(`Delete ${this.selectedIds.length} item(s)?`)) return;
            document.getElementById('bulk-action-input').value = actionMap[type];
@@ -586,7 +586,8 @@ class JobsView(View):
    PAGE_SIZE = 50

    def _get_stats(self):
-        from .models import Screenshot, Page
+        from .models import Screenshot, Page, FileUpload
+        imports_qs = FileUpload.objects.filter(source_url__isnull=False).exclude(source_url='')
        return {
            'screenshots': {
                'total': Screenshot.objects.count(),
@@ -602,10 +603,15 @@ class JobsView(View):
                'completed': Page.objects.filter(process_status=Page.ProcessStatus.COMPLETED).count(),
                'failed': Page.objects.filter(process_status=Page.ProcessStatus.FAILED).count(),
            },
+            'image_imports': {
+                'total': imports_qs.count(),
+                'pending': imports_qs.filter(size=0).count(),
+                'completed': imports_qs.filter(size__gt=0).count(),
+            },
        }

    def get(self, request):
-        from .models import Screenshot, Page, SiteSettings as SS
+        from .models import Screenshot, Page, FileUpload, SiteSettings as SS
        from core.scheduler import scheduler
        from django.core.paginator import Paginator

@@ -623,6 +629,15 @@ class JobsView(View):
        if status_filter != 'all':
            pg_qs = pg_qs.filter(process_status=status_filter)

+        # Build image imports queryset
+        import_qs = FileUpload.objects.filter(
+            source_url__isnull=False,
+        ).exclude(source_url='').order_by('-updated_at')
+        if status_filter == 'pending':
+            import_qs = import_qs.filter(size=0)
+        elif status_filter == 'completed':
+            import_qs = import_qs.filter(size__gt=0)
+
        # Paginate the active tab's queryset
        if tab == 'screenshots':
            paginator = Paginator(ss_qs, self.PAGE_SIZE)
@@ -632,6 +647,10 @@ class JobsView(View):
            paginator = Paginator(pg_qs, self.PAGE_SIZE)
            page_obj = paginator.get_page(page_num)
            all_ids = [str(obj.id) for obj in page_obj.object_list]
+        elif tab == 'image_imports':
+            paginator = Paginator(import_qs, self.PAGE_SIZE)
+            page_obj = paginator.get_page(page_num)
+            all_ids = [str(obj.id) for obj in page_obj.object_list]
        else:
            page_obj = None
            all_ids = []
@@ -655,6 +674,11 @@ class JobsView(View):
            ('completed',  'Completed',  'background:#16a34a;color:#fff;',      'background:#f0fdf4;color:#166534;'),
            ('failed',     'Failed',     'background:#dc2626;color:#fff;',      'background:#fff1f2;color:#991b1b;'),
        ]
+        import_filter_options = [
+            ('all',       'All',       'background:#1f2937;color:#fff;',  'background:#f3f4f6;color:#374151;'),
+            ('pending',   'Pending',   'background:#d97706;color:#fff;',  'background:#fef3c7;color:#92400e;'),
+            ('completed', 'Done',      'background:#16a34a;color:#fff;',  'background:#f0fdf4;color:#166534;'),
+        ]

        return render(request, self.template_name, {
            'stats': self._get_stats(),
@@ -666,6 +690,7 @@ class JobsView(View):
            'scheduler_running': scheduler.running,
            'site_settings': SS.get(),
            'filter_options': filter_options,
+            'import_filter_options': import_filter_options,
        })

    def post(self, request):
@@ -708,6 +733,33 @@ class JobsView(View):
            n = qs.delete()[0]
            messages.success(request, _(f'Deleted {n} page(s).'))

+        # ── Bulk image import actions ─────────────────────────────────────
+        elif action == 'bulk_retry_image_imports':
+            from .models import FileUpload
+            from .tasks import download_and_save_image
+            from threading import Thread as _Thread
+            qs = FileUpload.objects.filter(pk__in=ids) if ids else FileUpload.objects.none()
+            n = 0
+            for record in qs:
+                _Thread(target=download_and_save_image, args=(str(record.pk),), daemon=True).start()
+                n += 1
+            messages.success(request, _(f'Retrying {n} image import(s).'))
+
+        elif action == 'bulk_delete_image_imports':
+            from .models import FileUpload
+            import os as _os
+            qs = FileUpload.objects.filter(pk__in=ids) if ids else FileUpload.objects.none()
+            n = 0
+            for record in qs:
+                if _os.path.exists(record.file_path):
+                    try:
+                        _os.remove(record.file_path)
+                    except OSError:
+                        pass
+                record.delete()
+                n += 1
+            messages.success(request, _(f'Deleted {n} image import record(s).'))
+
        else:
            messages.error(request, _('Unknown action.'))

@@ -4,6 +4,8 @@

 ## What you can do with this API

+- **List short links** (`/api/links`) — GET a paginated list of all short links with their aliases, target URLs, tags, and click counts
+- **Most visited links** (`/api/links/most-visited`) — GET links sorted by visit count descending (only links with at least one click)
 - **Bookmark pages** (`/api/pages`) — POST a URL and the server auto-fetches the title, description and screenshot in the background
 - **Write blog posts** (`/api/posts`) — Create Markdown posts with tag categorisation
 - **Manage image collections** (`/api/collections`) — Group images into named albums; upload multiple images at once
@@ -43,6 +45,8 @@ No authentication is currently required.

 | operationId | Method | Path | Description |
 |---|---|---|---|
+| listLinks | GET | /api/links | List all short links |
+| listMostVisitedLinks | GET | /api/links/most-visited | Links sorted by visit count |
 | listPages | GET | /api/pages | List bookmarked pages |
 | createPage | POST | /api/pages | Bookmark a new URL |
 | listPosts | GET | /api/posts | List blog posts |
@@ -50,6 +50,8 @@ servers:
    description: Local development server

 tags:
+  - name: Links
+    description: Short links with click tracking. Supports listing all links and retrieving the most visited ones sorted by click count.
  - name: Pages
    description: |
      Bookmarked web pages. Creating a page triggers an async background job that fetches
@@ -69,6 +71,62 @@ tags:
      Files can be made public/private and given an expiry date.

 paths:
+  /api/links:
+    get:
+      operationId: listLinks
+      tags: [Links]
+      summary: List all links
+      description: Retrieve a paginated list of all short links ordered by creation date (newest first).
+      parameters:
+        - in: query
+          name: page
+          schema:
+            type: integer
+            default: 1
+          description: Page number (1-based).
+        - in: query
+          name: page_size
+          schema:
+            type: integer
+            default: 10
+            maximum: 100
+          description: Number of results per page.
+      responses:
+        '200':
+          description: Paginated list of links.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/LinkList'
+
+  /api/links/most-visited:
+    get:
+      operationId: listMostVisitedLinks
+      tags: [Links]
+      summary: Most visited links
+      description: Return links that have at least one click, sorted by click count descending.
+      parameters:
+        - in: query
+          name: page
+          schema:
+            type: integer
+            default: 1
+          description: Page number (1-based).
+        - in: query
+          name: page_size
+          schema:
+            type: integer
+            default: 10
+            maximum: 100
+          description: Number of results per page.
+      responses:
+        '200':
+          description: Paginated list of links sorted by click_count descending.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/LinkList'
+
  /api/pages:
    get:
      operationId: listPages
@@ -969,3 +1027,66 @@ components:
          enum: [success]
        message:
          type: string
+
+    TagSummary:
+      type: object
+      properties:
+        id:
+          type: integer
+        name:
+          type: string
+        slug:
+          type: string
+
+    Link:
+      type: object
+      properties:
+        id:
+          type: integer
+          readOnly: true
+        alias:
+          type: string
+          description: Unique slug used as the short-link identifier (e.g. /gh → github.com).
+        original_url:
+          type: string
+          description: Target URL. May contain template parameters like `{param,default=value}`.
+        description:
+          type: string
+          nullable: true
+        link_type:
+          type: string
+          enum: [LINK, CUSTOM]
+          description: LINK redirects to original_url; CUSTOM renders Markdown content.
+        click_count:
+          type: integer
+          readOnly: true
+          description: Number of times the short link has been visited.
+        tags:
+          type: array
+          readOnly: true
+          items:
+            $ref: '#/components/schemas/TagSummary'
+        created_at:
+          type: string
+          format: date-time
+          readOnly: true
+        updated_at:
+          type: string
+          format: date-time
+          readOnly: true
+
+    LinkList:
+      type: object
+      properties:
+        count:
+          type: integer
+        next:
+          type: string
+          nullable: true
+        previous:
+          type: string
+          nullable: true
+        results:
+          type: array
+          items:
+            $ref: '#/components/schemas/Link'