diff --git a/core/apps.py b/core/apps.py index eceb15f..0125765 100644 --- a/core/apps.py +++ b/core/apps.py @@ -16,7 +16,7 @@ class CoreConfig(AppConfig): Initialize APScheduler when Django starts """ from core.scheduler import scheduler, start_scheduler - from links.tasks import schedule_pending_pages, schedule_pending_screenshots + from links.tasks import schedule_pending_pages, schedule_pending_screenshots, retry_stuck_image_imports from apscheduler.triggers.interval import IntervalTrigger # Start the scheduler @@ -49,3 +49,12 @@ class CoreConfig(AppConfig): replace_existing=True, ) logger.info(f"Scheduled periodic task: schedule_pending_screenshots (every {screenshots_interval}s)") + + # Add periodic job for retrying stuck image imports (every 5 minutes) + scheduler.add_job( + retry_stuck_image_imports, + trigger=IntervalTrigger(seconds=300), + id='retry_stuck_image_imports', + replace_existing=True, + ) + logger.info("Scheduled periodic task: retry_stuck_image_imports (every 300s)") diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 1df67b1..093307b 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/links/api_urls.py b/links/api_urls.py index faffe54..4896d45 100644 --- a/links/api_urls.py +++ b/links/api_urls.py @@ -7,6 +7,7 @@ from . import file_views # Create a router and register our viewsets with it router = DefaultRouter(trailing_slash=False) +router.register('links', api_views.LinkViewSet, basename='api-links') router.register('pages', page_views.PageViewSet, basename='api-pages') router.register('posts', post_views.PostViewSet, basename='api-posts') router.register('music', api_views.MusicViewSet, basename='api-music') diff --git a/links/api_views.py b/links/api_views.py index 41f9940..a7f7e79 100644 --- a/links/api_views.py +++ b/links/api_views.py @@ -1,8 +1,9 @@ from rest_framework import viewsets, status from rest_framework.decorators import action from rest_framework.response import Response -from .models import ImageCollection, Image -from .serializers import ImageCollectionSerializer, ImageSerializer, ImageDescriptionSerializer +from rest_framework.pagination import PageNumberPagination +from .models import Link, ImageCollection, Image +from .serializers import LinkSerializer, ImageCollectionSerializer, ImageSerializer, ImageDescriptionSerializer from .storage import R2Storage import uuid import logging @@ -12,6 +13,31 @@ import os logger = logging.getLogger(__name__) + +class StandardResultsSetPagination(PageNumberPagination): + page_size = 10 + page_size_query_param = 'page_size' + max_page_size = 100 + + +class LinkViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = LinkSerializer + pagination_class = StandardResultsSetPagination + + def get_queryset(self): + return Link.objects.prefetch_related('tags').order_by('-created_at') + + @action(detail=False, methods=['get'], url_path='most-visited') + def most_visited(self, request): + """Return links sorted by click_count descending.""" + queryset = Link.objects.prefetch_related('tags').filter(click_count__gt=0).order_by('-click_count') + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + class ImageCollectionViewSet(viewsets.ModelViewSet): queryset = ImageCollection.objects.all() serializer_class = ImageCollectionSerializer diff --git a/links/file_views.py b/links/file_views.py index 5ddc232..0dd180f 100644 --- a/links/file_views.py +++ b/links/file_views.py @@ -217,16 +217,27 @@ def import_image_view(request, image_url): 3. Immediately redirects to the original URL so the image is visible right away. On subsequent requests, once the file is saved locally, the view redirects to the stored public URL instead — no more dependency on the original host. + + If the background download fails, the stub record is deleted automatically so no + size=0 ghost appears in the file list. The next visit to the same URL will create + a fresh stub and retry. A periodic APScheduler task (`retry_stuck_image_imports`) + also reschedules any stubs left behind by killed threads. """ import hashlib import posixpath from threading import Thread from .tasks import download_and_save_image - # Build the canonical source URL (https preferred) + # Build the canonical source URL, preserving query string. + # Django's converter only captures the path component; query params + # like ?format=jpg&name=900x900 end up in QUERY_STRING and must be re-attached. + query_string = request.META.get('QUERY_STRING', '') source_url = f'https://{image_url}' + if query_string: + source_url = f'{source_url}?{query_string}' url_hash = hashlib.sha256(source_url.encode()).hexdigest()[:20] + # Strip query string for filename derivation filename = posixpath.basename(image_url.split('?')[0]) or f'image_{url_hash}' # Derive extension from filename; fall back to .jpg for bare names diff --git a/links/serializers.py b/links/serializers.py index b41ded0..b41098c 100644 --- a/links/serializers.py +++ b/links/serializers.py @@ -1,5 +1,18 @@ from rest_framework import serializers -from .models import Page, Post, ImageCollection, Image, Tag, FileUpload +from .models import Link, Page, Post, ImageCollection, Image, Tag, FileUpload + + +class LinkSerializer(serializers.ModelSerializer): + tags = serializers.SerializerMethodField() + + class Meta: + model = Link + fields = ['id', 'alias', 'original_url', 'description', 'link_type', + 'click_count', 'tags', 'created_at', 'updated_at'] + read_only_fields = ['id', 'click_count', 'created_at', 'updated_at'] + + def get_tags(self, obj): + return [{'id': tag.id, 'name': tag.name, 'slug': tag.slug} for tag in obj.tags.all()] class PageSerializer(serializers.ModelSerializer): class Meta: diff --git a/links/tasks.py b/links/tasks.py index de934d3..26e24d2 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -500,3 +500,44 @@ def download_and_save_image(file_upload_id): os.remove(dest_path) except OSError: pass + # Remove the stub record so a size=0 ghost doesn't appear in the file list. + # The next import request for the same URL will create a fresh stub and retry. + try: + FileUpload.objects.filter(pk=file_upload_id, size=0).delete() + logger.info(f"download_and_save_image: deleted stub FileUpload {file_upload_id} after failed download") + except Exception as del_exc: + logger.error(f"download_and_save_image: could not delete stub {file_upload_id}: {del_exc}") + + +def retry_stuck_image_imports(): + """Periodic task: retry imported images that are stuck with size=0 and no file on disk. + + A stub FileUpload (size=0, source_url set) can be left behind when the background + download thread is killed mid-flight (e.g. server restart). This task finds those + orphaned stubs and re-kicks the download, provided the stub is old enough that we + are confident it is not a currently in-progress download (>5 minutes since last update). + """ + from .models import FileUpload + + threshold = timezone.now() - timedelta(minutes=5) + stuck = FileUpload.objects.filter( + size=0, + source_url__isnull=False, + updated_at__lt=threshold, + ).exclude(source_url='') + + count = stuck.count() + if count: + logger.info(f"retry_stuck_image_imports: found {count} stuck import stub(s) — retrying") + + for record in stuck: + if os.path.exists(record.file_path): + # File landed on disk but DB wasn't updated — fix it now + size = os.path.getsize(record.file_path) + FileUpload.objects.filter(pk=record.pk, size=0).update(size=size) + logger.info(f"retry_stuck_image_imports: fixed size for FileUpload {record.pk} ({size} bytes)") + continue + + logger.info(f"retry_stuck_image_imports: re-queuing download for FileUpload {record.pk} ({record.source_url})") + thread = Thread(target=download_and_save_image, args=(str(record.pk),), daemon=True) + thread.start() diff --git a/links/templates/links/jobs.html b/links/templates/links/jobs.html index 6285383..e52dd06 100644 --- a/links/templates/links/jobs.html +++ b/links/templates/links/jobs.html @@ -147,10 +147,19 @@ {{ scheduled_jobs|length }} + + {% trans "Image Imports" %} + + {{ stats.image_imports.total }} + + - {% if tab != 'scheduler' %} + {% if tab == 'screenshots' or tab == 'pages' %}
{% for s, label, pill_active_style, pill_inactive_style in filter_options %} {% endfor %}
+ {% elif tab == 'image_imports' %} +
+ {% for s, label, pill_active_style, pill_inactive_style in import_filter_options %} + + {{ label }} + + {% endfor %} +
{% endif %} @@ -176,7 +194,7 @@ style="background:#1d4ed8;color:#fff;padding:.3rem .85rem;border-radius:.25rem;font-size:.8rem;cursor:pointer;font-weight:500;"> ↺ {% trans "Retry" %} - @@ -367,6 +385,71 @@ {% else %}
{% trans "No pages match this filter." %}
{% endif %} + + {% elif tab == 'image_imports' %} + + {% if page_obj.object_list %} +
+ + + + + + + + + + + + + + {% for imp in page_obj.object_list %} + + + + + + + + + + {% endfor %} + +
+ + ID{% trans "Filename" %}{% trans "Source URL" %}{% trans "Status" %}{% trans "Size" %}{% trans "Updated" %}
+ + {{ imp.id|truncatechars:12 }} + + {{ imp.name|truncatechars:40 }} + + + + {{ imp.source_url|truncatechars:50 }} + + + {% if imp.size > 0 %} + + {% trans "done" %} + + {% else %} + + {% trans "pending" %} + + {% endif %} + + {% if imp.size > 0 %}{{ imp.formatted_size }}{% else %}—{% endif %} + {{ imp.updated_at|timesince }} {% trans "ago" %}
+
+ {% else %} +
{% trans "No image imports match this filter." %}
+ {% endif %} {% endif %} @@ -461,9 +544,13 @@ function jobsManager() { if (this.selectedIds.length === 0) return; const tab = this.tab; const actionMap = { - retry: tab === 'screenshots' ? 'bulk_retry_screenshots' : 'bulk_retry_pages', - fail: tab === 'screenshots' ? 'bulk_fail_screenshots' : 'bulk_fail_pages', - delete: tab === 'screenshots' ? 'bulk_delete_screenshots': 'bulk_delete_pages', + retry: tab === 'screenshots' ? 'bulk_retry_screenshots' + : tab === 'image_imports' ? 'bulk_retry_image_imports' + : 'bulk_retry_pages', + fail: tab === 'screenshots' ? 'bulk_fail_screenshots' : 'bulk_fail_pages', + delete: tab === 'screenshots' ? 'bulk_delete_screenshots' + : tab === 'image_imports' ? 'bulk_delete_image_imports' + : 'bulk_delete_pages', }; if (type === 'delete' && !confirm(`Delete ${this.selectedIds.length} item(s)?`)) return; document.getElementById('bulk-action-input').value = actionMap[type]; diff --git a/links/views.py b/links/views.py index 1ff1280..1a5fe0c 100644 --- a/links/views.py +++ b/links/views.py @@ -586,7 +586,8 @@ class JobsView(View): PAGE_SIZE = 50 def _get_stats(self): - from .models import Screenshot, Page + from .models import Screenshot, Page, FileUpload + imports_qs = FileUpload.objects.filter(source_url__isnull=False).exclude(source_url='') return { 'screenshots': { 'total': Screenshot.objects.count(), @@ -602,10 +603,15 @@ class JobsView(View): 'completed': Page.objects.filter(process_status=Page.ProcessStatus.COMPLETED).count(), 'failed': Page.objects.filter(process_status=Page.ProcessStatus.FAILED).count(), }, + 'image_imports': { + 'total': imports_qs.count(), + 'pending': imports_qs.filter(size=0).count(), + 'completed': imports_qs.filter(size__gt=0).count(), + }, } def get(self, request): - from .models import Screenshot, Page, SiteSettings as SS + from .models import Screenshot, Page, FileUpload, SiteSettings as SS from core.scheduler import scheduler from django.core.paginator import Paginator @@ -623,6 +629,15 @@ class JobsView(View): if status_filter != 'all': pg_qs = pg_qs.filter(process_status=status_filter) + # Build image imports queryset + import_qs = FileUpload.objects.filter( + source_url__isnull=False, + ).exclude(source_url='').order_by('-updated_at') + if status_filter == 'pending': + import_qs = import_qs.filter(size=0) + elif status_filter == 'completed': + import_qs = import_qs.filter(size__gt=0) + # Paginate the active tab's queryset if tab == 'screenshots': paginator = Paginator(ss_qs, self.PAGE_SIZE) @@ -632,6 +647,10 @@ class JobsView(View): paginator = Paginator(pg_qs, self.PAGE_SIZE) page_obj = paginator.get_page(page_num) all_ids = [str(obj.id) for obj in page_obj.object_list] + elif tab == 'image_imports': + paginator = Paginator(import_qs, self.PAGE_SIZE) + page_obj = paginator.get_page(page_num) + all_ids = [str(obj.id) for obj in page_obj.object_list] else: page_obj = None all_ids = [] @@ -655,6 +674,11 @@ class JobsView(View): ('completed', 'Completed', 'background:#16a34a;color:#fff;', 'background:#f0fdf4;color:#166534;'), ('failed', 'Failed', 'background:#dc2626;color:#fff;', 'background:#fff1f2;color:#991b1b;'), ] + import_filter_options = [ + ('all', 'All', 'background:#1f2937;color:#fff;', 'background:#f3f4f6;color:#374151;'), + ('pending', 'Pending', 'background:#d97706;color:#fff;', 'background:#fef3c7;color:#92400e;'), + ('completed', 'Done', 'background:#16a34a;color:#fff;', 'background:#f0fdf4;color:#166534;'), + ] return render(request, self.template_name, { 'stats': self._get_stats(), @@ -666,6 +690,7 @@ class JobsView(View): 'scheduler_running': scheduler.running, 'site_settings': SS.get(), 'filter_options': filter_options, + 'import_filter_options': import_filter_options, }) def post(self, request): @@ -708,6 +733,33 @@ class JobsView(View): n = qs.delete()[0] messages.success(request, _(f'Deleted {n} page(s).')) + # ── Bulk image import actions ───────────────────────────────────── + elif action == 'bulk_retry_image_imports': + from .models import FileUpload + from .tasks import download_and_save_image + from threading import Thread as _Thread + qs = FileUpload.objects.filter(pk__in=ids) if ids else FileUpload.objects.none() + n = 0 + for record in qs: + _Thread(target=download_and_save_image, args=(str(record.pk),), daemon=True).start() + n += 1 + messages.success(request, _(f'Retrying {n} image import(s).')) + + elif action == 'bulk_delete_image_imports': + from .models import FileUpload + import os as _os + qs = FileUpload.objects.filter(pk__in=ids) if ids else FileUpload.objects.none() + n = 0 + for record in qs: + if _os.path.exists(record.file_path): + try: + _os.remove(record.file_path) + except OSError: + pass + record.delete() + n += 1 + messages.success(request, _(f'Deleted {n} image import record(s).')) + else: messages.error(request, _('Unknown action.')) diff --git a/static/llms.txt b/static/llms.txt index e344a46..b3e9cd1 100644 --- a/static/llms.txt +++ b/static/llms.txt @@ -4,6 +4,8 @@ ## What you can do with this API +- **List short links** (`/api/links`) — GET a paginated list of all short links with their aliases, target URLs, tags, and click counts +- **Most visited links** (`/api/links/most-visited`) — GET links sorted by visit count descending (only links with at least one click) - **Bookmark pages** (`/api/pages`) — POST a URL and the server auto-fetches the title, description and screenshot in the background - **Write blog posts** (`/api/posts`) — Create Markdown posts with tag categorisation - **Manage image collections** (`/api/collections`) — Group images into named albums; upload multiple images at once @@ -43,6 +45,8 @@ No authentication is currently required. | operationId | Method | Path | Description | |---|---|---|---| +| listLinks | GET | /api/links | List all short links | +| listMostVisitedLinks | GET | /api/links/most-visited | Links sorted by visit count | | listPages | GET | /api/pages | List bookmarked pages | | createPage | POST | /api/pages | Bookmark a new URL | | listPosts | GET | /api/posts | List blog posts | diff --git a/static/openapi.yaml b/static/openapi.yaml index f84d7d4..e267e35 100644 --- a/static/openapi.yaml +++ b/static/openapi.yaml @@ -50,6 +50,8 @@ servers: description: Local development server tags: + - name: Links + description: Short links with click tracking. Supports listing all links and retrieving the most visited ones sorted by click count. - name: Pages description: | Bookmarked web pages. Creating a page triggers an async background job that fetches @@ -69,6 +71,62 @@ tags: Files can be made public/private and given an expiry date. paths: + /api/links: + get: + operationId: listLinks + tags: [Links] + summary: List all links + description: Retrieve a paginated list of all short links ordered by creation date (newest first). + parameters: + - in: query + name: page + schema: + type: integer + default: 1 + description: Page number (1-based). + - in: query + name: page_size + schema: + type: integer + default: 10 + maximum: 100 + description: Number of results per page. + responses: + '200': + description: Paginated list of links. + content: + application/json: + schema: + $ref: '#/components/schemas/LinkList' + + /api/links/most-visited: + get: + operationId: listMostVisitedLinks + tags: [Links] + summary: Most visited links + description: Return links that have at least one click, sorted by click count descending. + parameters: + - in: query + name: page + schema: + type: integer + default: 1 + description: Page number (1-based). + - in: query + name: page_size + schema: + type: integer + default: 10 + maximum: 100 + description: Number of results per page. + responses: + '200': + description: Paginated list of links sorted by click_count descending. + content: + application/json: + schema: + $ref: '#/components/schemas/LinkList' + /api/pages: get: operationId: listPages @@ -969,3 +1027,66 @@ components: enum: [success] message: type: string + + TagSummary: + type: object + properties: + id: + type: integer + name: + type: string + slug: + type: string + + Link: + type: object + properties: + id: + type: integer + readOnly: true + alias: + type: string + description: Unique slug used as the short-link identifier (e.g. /gh → github.com). + original_url: + type: string + description: Target URL. May contain template parameters like `{param,default=value}`. + description: + type: string + nullable: true + link_type: + type: string + enum: [LINK, CUSTOM] + description: LINK redirects to original_url; CUSTOM renders Markdown content. + click_count: + type: integer + readOnly: true + description: Number of times the short link has been visited. + tags: + type: array + readOnly: true + items: + $ref: '#/components/schemas/TagSummary' + created_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + + LinkList: + type: object + properties: + count: + type: integer + next: + type: string + nullable: true + previous: + type: string + nullable: true + results: + type: array + items: + $ref: '#/components/schemas/Link'