Add redis caching

core/apps.py

@@ -16,7 +16,10 @@ class CoreConfig(AppConfig):
         Initialize APScheduler when Django starts
         """
         from core.scheduler import scheduler, start_scheduler
-        from links.tasks import schedule_pending_pages, schedule_pending_screenshots, retry_stuck_image_imports
+        from links.tasks import (
+            schedule_pending_pages, schedule_pending_screenshots,
+            retry_stuck_image_imports, flush_click_buffer,
+        )
         from apscheduler.triggers.interval import IntervalTrigger
 
         # Start the scheduler
@@ -58,3 +61,12 @@ class CoreConfig(AppConfig):
             replace_existing=True,
         )
         logger.info("Scheduled periodic task: retry_stuck_image_imports (every 300s)")
+
+        # Flush Redis-buffered click counts to the database (every 60 seconds)
+        scheduler.add_job(
+            flush_click_buffer,
+            trigger=IntervalTrigger(seconds=60),
+            id='flush_click_buffer',
+            replace_existing=True,
+        )
+        logger.info("Scheduled periodic task: flush_click_buffer (every 60s)")

core/settings.py

@@ -104,6 +104,24 @@ STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
 
 CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS', 'http://localhost:8000').split(',')
 
+# Redis cache
+REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': REDIS_URL,
+        'OPTIONS': {
+            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
+        },
+        'TIMEOUT': 300,
+    }
+}
+
+# Store sessions in Redis instead of the database
+SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
+SESSION_CACHE_ALIAS = 'default'
+
 # SimpleMDE 配置
 SIMPLEMDE_OPTIONS = {
     'placeholder': 'Type here...',

k8s/manifest.yaml

@@ -158,6 +158,8 @@ spec:
                 secretKeyRef:
                   name: r2-credentials
                   key: key_id
+            - name: REDIS_URL
+              value: "redis://redis.db.svc.cluster.local:6379/0"
             - name: CRAWL4AI_API_URL
               value: "http://crawl4ai.ai.svc.cluster.local:80"
             - name: CRAWL4AI_ENABLED

links/page_views.py

@@ -6,6 +6,7 @@ from django.contrib import messages
 from django.utils.translation import gettext_lazy as _
 from django.core.files.base import ContentFile
 from django.conf import settings
+from django.core.cache import cache
 from rest_framework import viewsets, status
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -142,6 +143,13 @@ def fetch_page_info(request):
 
     url = url.lstrip('@')
 
+    # Return cached metadata if available (24-hour TTL)
+    import hashlib as _hashlib
+    cache_key = 'pageinfo:' + _hashlib.md5(url.encode()).hexdigest()
+    cached = cache.get(cache_key)
+    if cached is not None:
+        return JsonResponse(cached)
+
     try:
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
@@ -193,10 +201,9 @@ def fetch_page_info(request):
         description = re.sub(r'\s+', ' ', description.strip())
         description = description[:500] + '...' if len(description) > 500 else description
 
-        return JsonResponse({
-            'title': title,
-            'summary': description
-        })
+        result = {'title': title, 'summary': description}
+        cache.set(cache_key, result, timeout=86400)
+        return JsonResponse(result)
 
     except Exception as e:
         return JsonResponse({

links/search_views.py

@@ -10,6 +10,8 @@ from .models import Link, Page, Post
 from .search_backend import search_backend
 from django.views.decorators.http import require_http_methods
 from django.views.decorators.csrf import csrf_exempt
+from django.core.cache import cache
+import hashlib
 import json
 import requests
 import os
@@ -62,7 +64,7 @@ def search_vector_api(request):
                 'score': hit.score,
                 'tags': p.get('tags', [])
             })
-        
+
         return JsonResponse({'results': results, 'total': len(results)})
     except Exception as e:
         logger.error(f"Vector search error: {e}")
@@ -154,6 +156,14 @@ def search_api_v2(request):
     per_page = int(request.GET.get('per_page', 20))
     if not query:
         return JsonResponse({'results': [], 'total': 0, 'page': page, 'per_page': per_page, 'has_next': False, 'has_prev': False})
+
+    cache_key = 'search:v2:' + hashlib.md5(
+        f'{query}:{type_filter}:{sort_by}:{page}:{per_page}'.encode()
+    ).hexdigest()
+    cached = cache.get(cache_key)
+    if cached is not None:
+        return JsonResponse(cached)
+
     try:
         search_results = search_backend.search(query_string=query, model_type=type_filter if type_filter else None, page=page, per_page=per_page, sort_by=sort_by)
         enriched_results = []
@@ -173,7 +183,9 @@ def search_api_v2(request):
             except Exception as e:
                 logger.error(f"Error enriching {model_type} {model_id}: {e}")
                 continue
-        return JsonResponse({'results': enriched_results, 'total': search_results['total'], 'page': search_results['page'], 'per_page': search_results['per_page'], 'has_next': search_results['has_next'], 'has_prev': search_results['has_prev']})
+        result_payload = {'results': enriched_results, 'total': search_results['total'], 'page': search_results['page'], 'per_page': search_results['per_page'], 'has_next': search_results['has_next'], 'has_prev': search_results['has_prev']}
+        cache.set(cache_key, result_payload, timeout=300)
+        return JsonResponse(result_payload)
     except Exception as e:
         logger.error(f"Search API error: {e}", exc_info=True)
         return JsonResponse({'error': str(e), 'results': [], 'total': 0}, status=500)

links/tasks.py

@@ -295,7 +295,7 @@ def process_page(page_id, retry_count=0):
         else:
             page.process_status = Page.ProcessStatus.PENDING
             page.save()
-            
+
             # Schedule retry using APScheduler
             retry_delay = fibonacci(retry_count)
             from core.scheduler import scheduler
@@ -390,43 +390,43 @@ def fetch_webpage_content_from_crawl4ai(page_id, retry_count=0):
     Fetch webpage content using Crawl4AI /md endpoint
     """
     from .models import Page
-    
+
     if not settings.CRAWL4AI_ENABLED:
         logger.info("Crawl4AI is disabled, skipping webpage content extraction")
         return
-    
+
     try:
         page = Page.objects.get(id=page_id)
         logger.info(f"Fetching webpage content for page {page_id} from Crawl4AI")
-        
+
         # Call Crawl4AI /md endpoint
         api_url = f"{settings.CRAWL4AI_API_URL}/md"
         payload = {
             "url": page.url
         }
-        
+
         response = requests.post(
             api_url,
             json=payload,
             timeout=60
         )
         response.raise_for_status()
-        
+
         data = response.json()
-        
+
         # Extract markdown content from response
         markdown_content = data.get('markdown', '')
-        
+
         if markdown_content:
             page.content = markdown_content
             page.save()
             logger.info(f"Successfully fetched webpage content for page {page_id}")
         else:
             logger.warning(f"No markdown content returned for page {page_id}")
-            
+
     except requests.RequestException as e:
         logger.error(f"Failed to fetch webpage content for page {page_id}: {e}")
-        
+
         # Retry logic
         if retry_count < MAX_RETRIES:
             retry_delay = fibonacci(retry_count + 1)
@@ -509,6 +509,86 @@ def download_and_save_image(file_upload_id):
             logger.error(f"download_and_save_image: could not delete stub {file_upload_id}: {del_exc}")
 
 
+def flush_click_buffer():
+    """Periodic task: flush Redis-buffered click counts into the database.
+
+    redirect_to_original increments Redis keys of the form 'clicks:{link_id}:{date}'
+    instead of writing a ClickLog row per hit. This task drains those keys every ~60s
+    and bulk-inserts the ClickLog records plus updates Link.click_count in one shot.
+    """
+    from .models import Link, ClickLog
+    from django_redis import get_redis_connection
+
+    try:
+        redis_conn = get_redis_connection('default')
+    except Exception as e:
+        logger.error(f'flush_click_buffer: cannot get Redis connection: {e}')
+        return
+
+    cursor = 0
+    pattern = 'clicks:*'
+    # django-redis stores keys with a prefix like ':1:' — match it broadly
+    prefix = redis_conn.connection_pool.connection_kwargs.get('db', 0)
+    all_keys = []
+    while True:
+        cursor, keys = redis_conn.scan(cursor, match=f'*clicks:*', count=200)
+        all_keys.extend(keys)
+        if cursor == 0:
+            break
+
+    if not all_keys:
+        return
+
+    logs_to_create = []
+    link_count_deltas = {}  # {link_id: total_clicks}
+
+    pipeline = redis_conn.pipeline()
+    for key in all_keys:
+        pipeline.getdel(key)
+    counts = pipeline.execute()
+
+    for raw_key, count_bytes in zip(all_keys, counts):
+        if not count_bytes:
+            continue
+        count = int(count_bytes)
+        if count <= 0:
+            continue
+        try:
+            # Key format: '<cache-prefix>clicks:{link_id}:{date}'
+            key_str = raw_key.decode() if isinstance(raw_key, bytes) else raw_key
+            # Strip any cache key prefix (':1:clicks:...' → 'clicks:...')
+            parts = key_str.rsplit('clicks:', 1)
+            if len(parts) != 2:
+                continue
+            remainder = parts[1]  # '{link_id}:{date}'
+            link_id_str, date_str = remainder.split(':', 1)
+            link_id = int(link_id_str)
+        except (ValueError, IndexError):
+            logger.warning(f'flush_click_buffer: unrecognised key format: {raw_key}')
+            continue
+
+        link_count_deltas[link_id] = link_count_deltas.get(link_id, 0) + count
+        try:
+            click_date = timezone.datetime.strptime(date_str, '%Y-%m-%d').replace(
+                tzinfo=timezone.get_current_timezone()
+            )
+        except ValueError:
+            click_date = timezone.now()
+
+        for _ in range(count):
+            logs_to_create.append(ClickLog(link_id=link_id, clicked_at=click_date))
+
+    if logs_to_create:
+        ClickLog.objects.bulk_create(logs_to_create, ignore_conflicts=True)
+        logger.info(f'flush_click_buffer: inserted {len(logs_to_create)} ClickLog rows')
+
+    for link_id, delta in link_count_deltas.items():
+        Link.objects.filter(pk=link_id).update(click_count=F('click_count') + delta)
+
+    if link_count_deltas:
+        logger.info(f'flush_click_buffer: updated click_count for {len(link_count_deltas)} link(s)')
+
+
 def retry_stuck_image_imports():
     """Periodic task: retry imported images that are stuck with size=0 and no file on disk.
 

links/views.py

@@ -6,6 +6,7 @@ from django.db.models import F, Count, Q, Case, When, Value, IntegerField
 from django.db.models.functions import TruncDate
 from .models import Link, ClickLog, LinkChangeLog, Page, Post, SiteSettings
 from .forms import LinkForm, PageForm
+from django.core.cache import cache
 import json
 from django.core.serializers.json import DjangoJSONEncoder
 from django.contrib import messages
@@ -195,6 +196,9 @@ class LinkUpdateView(UpdateView):
 
         response = super().form_valid(form)
 
+        # Invalidate the alias cache so the redirect hot-path sees the updated link
+        cache.delete(f"link:alias:{form.instance.alias.lower()}")
+
         new_url = form.cleaned_data['original_url']
         if self.original_url != new_url:
             logger.info(f"URL changed from {self.original_url} to {new_url}")
@@ -225,6 +229,10 @@ class LinkDeleteView(DeleteView):
     template_name = 'links/link_confirm_delete.html'
     success_url = reverse_lazy('link_list')
 
+    def delete(self, request, *args, **kwargs):
+        cache.delete(f"link:alias:{self.get_object().alias.lower()}")
+        return super().delete(request, *args, **kwargs)
+
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context['object'] = self.get_object()
@@ -240,10 +248,21 @@ def redirect_to_original(request, alias, param=None):
     processed_alias = ''.join(e for e in alias.lower() if e.isalnum())
 
     try:
-        link = Link.objects.get(alias=processed_alias)
-        link.click_count = F('click_count') + 1
-        link.save()
-        ClickLog.objects.create(link=link)
+        cache_key = f'link:alias:{processed_alias}'
+        link = cache.get(cache_key)
+        if link is None:
+            link = Link.objects.get(alias=processed_alias)
+            cache.set(cache_key, link, timeout=3600)
+
+        # Buffer click in Redis (atomic increment). A periodic task flushes to DB.
+        from django.utils import timezone
+        date_str = timezone.now().strftime('%Y-%m-%d')
+        click_key = f'clicks:{link.id}:{date_str}'
+        try:
+            cache.incr(click_key)
+        except ValueError:
+            # Key doesn't exist yet — set it then give it a 48-hour safety-net TTL
+            cache.set(click_key, 1, timeout=172800)
 
         try:
             # 如果是模板 URL 并且提供了参数
@@ -271,6 +290,8 @@ def redirect_to_original(request, alias, param=None):
             return redirect('link_list')
 
     except Link.DoesNotExist:
+        # Also clear any stale cache entry that might be causing the miss
+        cache.delete(f'link:alias:{processed_alias}')
         messages.warning(request, _("The alias '{}' doesn't exist. Do you want to create a new one?").format(processed_alias))
         return redirect(reverse('link_create') + f'?alias={processed_alias}')
 
@@ -329,56 +350,63 @@ class LinkDetailView(DetailView):
             period_name = "3 Months"
             interval_days = 1  # Daily
 
-        # Get click stats from database
-        click_stats = ClickLog.objects.filter(
-            link=self.object,
-            clicked_at__date__gte=start_date,
-            clicked_at__date__lte=end_date
-        ).annotate(
-            date=TruncDate('clicked_at')
-        ).values('date').annotate(count=Count('id')).order_by('date')
+        # Get click stats — cache per link+period to avoid repeated ClickLog full-scans
+        stats_cache_key = f'link:stats:{self.object.pk}:{period}'
+        click_stats_list = cache.get(stats_cache_key)
 
-        # Convert to dictionary for easy lookup
-        click_dict = {item['date']: item['count'] for item in click_stats}
+        if click_stats_list is None:
+            # Get click stats from database
+            click_stats = ClickLog.objects.filter(
+                link=self.object,
+                clicked_at__date__gte=start_date,
+                clicked_at__date__lte=end_date
+            ).annotate(
+                date=TruncDate('clicked_at')
+            ).values('date').annotate(count=Count('id')).order_by('date')
 
-        # Create complete dataset with appropriate intervals
-        click_stats_list = []
-        current_date = start_date
+            # Convert to dictionary for easy lookup
+            click_dict = {item['date']: item['count'] for item in click_stats}
 
-        if interval_days == 1:
-            # Daily intervals
-            while current_date <= end_date:
-                click_stats_list.append({
-                    'date': current_date.strftime('%Y-%m-%d'),
-                    'count': click_dict.get(current_date, 0)
-                })
-                current_date += timedelta(days=1)
-        else:
-            # Weekly, bi-weekly, or monthly intervals
-            while current_date <= end_date:
-                interval_end = min(current_date + timedelta(days=interval_days - 1), end_date)
+            # Create complete dataset with appropriate intervals
+            click_stats_list = []
+            current_date = start_date
 
-                # Sum clicks for this interval
-                interval_count = 0
-                temp_date = current_date
-                while temp_date <= interval_end:
-                    interval_count += click_dict.get(temp_date, 0)
-                    temp_date += timedelta(days=1)
+            if interval_days == 1:
+                # Daily intervals
+                while current_date <= end_date:
+                    click_stats_list.append({
+                        'date': current_date.strftime('%Y-%m-%d'),
+                        'count': click_dict.get(current_date, 0)
+                    })
+                    current_date += timedelta(days=1)
+            else:
+                # Weekly, bi-weekly, or monthly intervals
+                while current_date <= end_date:
+                    interval_end = min(current_date + timedelta(days=interval_days - 1), end_date)
 
-                # Format label based on interval
-                if interval_days == 7:  # Weekly
-                    label = f"{current_date.strftime('%m/%d')}"
-                elif interval_days == 14:  # Bi-weekly
-                    label = f"{current_date.strftime('%m/%d')}"
-                else:  # Monthly
-                    label = f"{calendar.month_abbr[current_date.month]} {current_date.year}"
+                    # Sum clicks for this interval
+                    interval_count = 0
+                    temp_date = current_date
+                    while temp_date <= interval_end:
+                        interval_count += click_dict.get(temp_date, 0)
+                        temp_date += timedelta(days=1)
 
-                click_stats_list.append({
-                    'date': current_date.strftime('%Y-%m-%d'),
-                    'count': interval_count,
-                    'label': label
-                })
-                current_date += timedelta(days=interval_days)
+                    # Format label based on interval
+                    if interval_days == 7:  # Weekly
+                        label = f"{current_date.strftime('%m/%d')}"
+                    elif interval_days == 14:  # Bi-weekly
+                        label = f"{current_date.strftime('%m/%d')}"
+                    else:  # Monthly
+                        label = f"{calendar.month_abbr[current_date.month]} {current_date.year}"
+
+                    click_stats_list.append({
+                        'date': current_date.strftime('%Y-%m-%d'),
+                        'count': interval_count,
+                        'label': label
+                    })
+                    current_date += timedelta(days=interval_days)
+
+            cache.set(stats_cache_key, click_stats_list, timeout=600)
 
         context['click_stats'] = json.dumps(click_stats_list, cls=DjangoJSONEncoder)
         context['current_period'] = period

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "python-magic>=0.4.27",
     "pillow~=12.1.1",
     "whoosh==2.7.4",
+    "django-redis>=5.4.0",
     "qdrant-client>=1.13.2",
     "cryptography>=42.0.0",
 ]

tests/conftest.py

@@ -47,3 +47,14 @@ def use_tmp_upload_dir(settings, tmp_path):
     """Redirect FILE_UPLOADS_FOLDER to a temp dir for each test."""
     settings.FILE_UPLOADS_FOLDER = str(tmp_path)
     yield
+
+
+@pytest.fixture(autouse=True)
+def use_locmem_cache(settings):
+    """Override Redis cache with in-process LocMemCache so tests don't need a Redis server."""
+    settings.CACHES = {
+        'default': {
+            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        }
+    }
+    yield

uv.lock

@@ -311,6 +311,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4e/32/4b144e125678efccf5d5b61581de1c4088d6b0286e46096e3b8de0d556c8/django-5.2.12-py3-none-any.whl", hash = "sha256:4853482f395c3a151937f6991272540fcbf531464f254a347bf7c89f53c8cff7", size = 8310245, upload-time = "2026-03-03T13:56:01.174Z" },
 ]
 
+[[package]]
+name = "django-redis"
+version = "6.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "django" },
+    { name = "redis" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/08/53/dbcfa1e528e0d6c39947092625b2c89274b5d88f14d357cee53c4d6dbbd4/django_redis-6.0.0.tar.gz", hash = "sha256:2d9cb12a20424a4c4dde082c6122f486628bae2d9c2bee4c0126a4de7fda00dd", size = 56904, upload-time = "2025-06-17T18:15:46.376Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/79/055dfcc508cfe9f439d9f453741188d633efa9eab90fc78a67b0ab50b137/django_redis-6.0.0-py3-none-any.whl", hash = "sha256:20bf0063a8abee567eb5f77f375143c32810c8700c0674ced34737f8de4e36c0", size = 33687, upload-time = "2025-06-17T18:15:34.165Z" },
+]
+
 [[package]]
 name = "django-simplemde"
 version = "0.1.4"
@@ -587,6 +600,7 @@ dependencies = [
     { name = "boto3" },
     { name = "cryptography" },
     { name = "django" },
+    { name = "django-redis" },
     { name = "django-simplemde" },
     { name = "django-tailwind" },
     { name = "django-widget-tweaks" },
@@ -628,6 +642,7 @@ requires-dist = [
     { name = "boto3", specifier = ">=1.35.0" },
     { name = "cryptography", specifier = ">=42.0.0" },
     { name = "django", specifier = ">=5.2.9" },
+    { name = "django-redis", specifier = ">=5.4.0" },
     { name = "django-simplemde", specifier = "==0.1.4" },
     { name = "django-tailwind", specifier = "==3.8.0" },
     { name = "django-widget-tweaks", specifier = "==1.5.0" },
@@ -1131,6 +1146,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/08/13/8ce16f808297e16968269de44a14f4fef19b64d9766be1d6ba5ba78b579d/qdrant_client-1.16.2-py3-none-any.whl", hash = "sha256:442c7ef32ae0f005e88b5d3c0783c63d4912b97ae756eb5e052523be682f17d3", size = 377186, upload-time = "2025-12-12T10:58:29.282Z" },
 ]
 
+[[package]]
+name = "redis"
+version = "7.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/3759b1d0d72b7c92f0d70ffd9dc962b7b7b5ee74e135f9d7d8ab06b8a318/redis-7.4.0.tar.gz", hash = "sha256:64a6ea7bf567ad43c964d2c30d82853f8df927c5c9017766c55a1d1ed95d18ad", size = 4943913, upload-time = "2026-03-24T09:14:37.53Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" },
+]
+
 [[package]]
 name = "requests"
 version = "2.33.0"