From c26c620f78baea459d8d1565d4215f1fde9264bf Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Sun, 29 Mar 2026 10:23:42 +1100 Subject: [PATCH] Add redis caching --- core/apps.py | 14 ++++- core/settings.py | 18 ++++++ k8s/manifest.yaml | 2 + links/page_views.py | 15 +++-- links/search_views.py | 16 +++++- links/tasks.py | 100 ++++++++++++++++++++++++++++++---- links/views.py | 124 ++++++++++++++++++++++++++---------------- pyproject.toml | 1 + tests/conftest.py | 11 ++++ uv.lock | 24 ++++++++ 10 files changed, 260 insertions(+), 65 deletions(-) diff --git a/core/apps.py b/core/apps.py index 0125765..ea6bcce 100644 --- a/core/apps.py +++ b/core/apps.py @@ -16,7 +16,10 @@ class CoreConfig(AppConfig): Initialize APScheduler when Django starts """ from core.scheduler import scheduler, start_scheduler - from links.tasks import schedule_pending_pages, schedule_pending_screenshots, retry_stuck_image_imports + from links.tasks import ( + schedule_pending_pages, schedule_pending_screenshots, + retry_stuck_image_imports, flush_click_buffer, + ) from apscheduler.triggers.interval import IntervalTrigger # Start the scheduler @@ -58,3 +61,12 @@ class CoreConfig(AppConfig): replace_existing=True, ) logger.info("Scheduled periodic task: retry_stuck_image_imports (every 300s)") + + # Flush Redis-buffered click counts to the database (every 60 seconds) + scheduler.add_job( + flush_click_buffer, + trigger=IntervalTrigger(seconds=60), + id='flush_click_buffer', + replace_existing=True, + ) + logger.info("Scheduled periodic task: flush_click_buffer (every 60s)") diff --git a/core/settings.py b/core/settings.py index 54281b7..0d18a76 100644 --- a/core/settings.py +++ b/core/settings.py @@ -104,6 +104,24 @@ STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS', 'http://localhost:8000').split(',') +# Redis cache +REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0') + +CACHES = { + 'default': { + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': REDIS_URL, + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + }, + 'TIMEOUT': 300, + } +} + +# Store sessions in Redis instead of the database +SESSION_ENGINE = 'django.contrib.sessions.backends.cache' +SESSION_CACHE_ALIAS = 'default' + # SimpleMDE 配置 SIMPLEMDE_OPTIONS = { 'placeholder': 'Type here...', diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index a6afb2f..84e6c27 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -158,6 +158,8 @@ spec: secretKeyRef: name: r2-credentials key: key_id + - name: REDIS_URL + value: "redis://redis.db.svc.cluster.local:6379/0" - name: CRAWL4AI_API_URL value: "http://crawl4ai.ai.svc.cluster.local:80" - name: CRAWL4AI_ENABLED diff --git a/links/page_views.py b/links/page_views.py index 27b08aa..50eb334 100644 --- a/links/page_views.py +++ b/links/page_views.py @@ -6,6 +6,7 @@ from django.contrib import messages from django.utils.translation import gettext_lazy as _ from django.core.files.base import ContentFile from django.conf import settings +from django.core.cache import cache from rest_framework import viewsets, status from rest_framework.decorators import action from rest_framework.response import Response @@ -142,6 +143,13 @@ def fetch_page_info(request): url = url.lstrip('@') + # Return cached metadata if available (24-hour TTL) + import hashlib as _hashlib + cache_key = 'pageinfo:' + _hashlib.md5(url.encode()).hexdigest() + cached = cache.get(cache_key) + if cached is not None: + return JsonResponse(cached) + try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', @@ -193,10 +201,9 @@ def fetch_page_info(request): description = re.sub(r'\s+', ' ', description.strip()) description = description[:500] + '...' if len(description) > 500 else description - return JsonResponse({ - 'title': title, - 'summary': description - }) + result = {'title': title, 'summary': description} + cache.set(cache_key, result, timeout=86400) + return JsonResponse(result) except Exception as e: return JsonResponse({ diff --git a/links/search_views.py b/links/search_views.py index 13c06fe..1a18c9f 100644 --- a/links/search_views.py +++ b/links/search_views.py @@ -10,6 +10,8 @@ from .models import Link, Page, Post from .search_backend import search_backend from django.views.decorators.http import require_http_methods from django.views.decorators.csrf import csrf_exempt +from django.core.cache import cache +import hashlib import json import requests import os @@ -62,7 +64,7 @@ def search_vector_api(request): 'score': hit.score, 'tags': p.get('tags', []) }) - + return JsonResponse({'results': results, 'total': len(results)}) except Exception as e: logger.error(f"Vector search error: {e}") @@ -154,6 +156,14 @@ def search_api_v2(request): per_page = int(request.GET.get('per_page', 20)) if not query: return JsonResponse({'results': [], 'total': 0, 'page': page, 'per_page': per_page, 'has_next': False, 'has_prev': False}) + + cache_key = 'search:v2:' + hashlib.md5( + f'{query}:{type_filter}:{sort_by}:{page}:{per_page}'.encode() + ).hexdigest() + cached = cache.get(cache_key) + if cached is not None: + return JsonResponse(cached) + try: search_results = search_backend.search(query_string=query, model_type=type_filter if type_filter else None, page=page, per_page=per_page, sort_by=sort_by) enriched_results = [] @@ -173,7 +183,9 @@ def search_api_v2(request): except Exception as e: logger.error(f"Error enriching {model_type} {model_id}: {e}") continue - return JsonResponse({'results': enriched_results, 'total': search_results['total'], 'page': search_results['page'], 'per_page': search_results['per_page'], 'has_next': search_results['has_next'], 'has_prev': search_results['has_prev']}) + result_payload = {'results': enriched_results, 'total': search_results['total'], 'page': search_results['page'], 'per_page': search_results['per_page'], 'has_next': search_results['has_next'], 'has_prev': search_results['has_prev']} + cache.set(cache_key, result_payload, timeout=300) + return JsonResponse(result_payload) except Exception as e: logger.error(f"Search API error: {e}", exc_info=True) return JsonResponse({'error': str(e), 'results': [], 'total': 0}, status=500) diff --git a/links/tasks.py b/links/tasks.py index 26e24d2..9e6130b 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -295,7 +295,7 @@ def process_page(page_id, retry_count=0): else: page.process_status = Page.ProcessStatus.PENDING page.save() - + # Schedule retry using APScheduler retry_delay = fibonacci(retry_count) from core.scheduler import scheduler @@ -390,43 +390,43 @@ def fetch_webpage_content_from_crawl4ai(page_id, retry_count=0): Fetch webpage content using Crawl4AI /md endpoint """ from .models import Page - + if not settings.CRAWL4AI_ENABLED: logger.info("Crawl4AI is disabled, skipping webpage content extraction") return - + try: page = Page.objects.get(id=page_id) logger.info(f"Fetching webpage content for page {page_id} from Crawl4AI") - + # Call Crawl4AI /md endpoint api_url = f"{settings.CRAWL4AI_API_URL}/md" payload = { "url": page.url } - + response = requests.post( api_url, json=payload, timeout=60 ) response.raise_for_status() - + data = response.json() - + # Extract markdown content from response markdown_content = data.get('markdown', '') - + if markdown_content: page.content = markdown_content page.save() logger.info(f"Successfully fetched webpage content for page {page_id}") else: logger.warning(f"No markdown content returned for page {page_id}") - + except requests.RequestException as e: logger.error(f"Failed to fetch webpage content for page {page_id}: {e}") - + # Retry logic if retry_count < MAX_RETRIES: retry_delay = fibonacci(retry_count + 1) @@ -509,6 +509,86 @@ def download_and_save_image(file_upload_id): logger.error(f"download_and_save_image: could not delete stub {file_upload_id}: {del_exc}") +def flush_click_buffer(): + """Periodic task: flush Redis-buffered click counts into the database. + + redirect_to_original increments Redis keys of the form 'clicks:{link_id}:{date}' + instead of writing a ClickLog row per hit. This task drains those keys every ~60s + and bulk-inserts the ClickLog records plus updates Link.click_count in one shot. + """ + from .models import Link, ClickLog + from django_redis import get_redis_connection + + try: + redis_conn = get_redis_connection('default') + except Exception as e: + logger.error(f'flush_click_buffer: cannot get Redis connection: {e}') + return + + cursor = 0 + pattern = 'clicks:*' + # django-redis stores keys with a prefix like ':1:' — match it broadly + prefix = redis_conn.connection_pool.connection_kwargs.get('db', 0) + all_keys = [] + while True: + cursor, keys = redis_conn.scan(cursor, match=f'*clicks:*', count=200) + all_keys.extend(keys) + if cursor == 0: + break + + if not all_keys: + return + + logs_to_create = [] + link_count_deltas = {} # {link_id: total_clicks} + + pipeline = redis_conn.pipeline() + for key in all_keys: + pipeline.getdel(key) + counts = pipeline.execute() + + for raw_key, count_bytes in zip(all_keys, counts): + if not count_bytes: + continue + count = int(count_bytes) + if count <= 0: + continue + try: + # Key format: 'clicks:{link_id}:{date}' + key_str = raw_key.decode() if isinstance(raw_key, bytes) else raw_key + # Strip any cache key prefix (':1:clicks:...' → 'clicks:...') + parts = key_str.rsplit('clicks:', 1) + if len(parts) != 2: + continue + remainder = parts[1] # '{link_id}:{date}' + link_id_str, date_str = remainder.split(':', 1) + link_id = int(link_id_str) + except (ValueError, IndexError): + logger.warning(f'flush_click_buffer: unrecognised key format: {raw_key}') + continue + + link_count_deltas[link_id] = link_count_deltas.get(link_id, 0) + count + try: + click_date = timezone.datetime.strptime(date_str, '%Y-%m-%d').replace( + tzinfo=timezone.get_current_timezone() + ) + except ValueError: + click_date = timezone.now() + + for _ in range(count): + logs_to_create.append(ClickLog(link_id=link_id, clicked_at=click_date)) + + if logs_to_create: + ClickLog.objects.bulk_create(logs_to_create, ignore_conflicts=True) + logger.info(f'flush_click_buffer: inserted {len(logs_to_create)} ClickLog rows') + + for link_id, delta in link_count_deltas.items(): + Link.objects.filter(pk=link_id).update(click_count=F('click_count') + delta) + + if link_count_deltas: + logger.info(f'flush_click_buffer: updated click_count for {len(link_count_deltas)} link(s)') + + def retry_stuck_image_imports(): """Periodic task: retry imported images that are stuck with size=0 and no file on disk. diff --git a/links/views.py b/links/views.py index 69b7d90..f4418ac 100644 --- a/links/views.py +++ b/links/views.py @@ -6,6 +6,7 @@ from django.db.models import F, Count, Q, Case, When, Value, IntegerField from django.db.models.functions import TruncDate from .models import Link, ClickLog, LinkChangeLog, Page, Post, SiteSettings from .forms import LinkForm, PageForm +from django.core.cache import cache import json from django.core.serializers.json import DjangoJSONEncoder from django.contrib import messages @@ -195,6 +196,9 @@ class LinkUpdateView(UpdateView): response = super().form_valid(form) + # Invalidate the alias cache so the redirect hot-path sees the updated link + cache.delete(f"link:alias:{form.instance.alias.lower()}") + new_url = form.cleaned_data['original_url'] if self.original_url != new_url: logger.info(f"URL changed from {self.original_url} to {new_url}") @@ -225,6 +229,10 @@ class LinkDeleteView(DeleteView): template_name = 'links/link_confirm_delete.html' success_url = reverse_lazy('link_list') + def delete(self, request, *args, **kwargs): + cache.delete(f"link:alias:{self.get_object().alias.lower()}") + return super().delete(request, *args, **kwargs) + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['object'] = self.get_object() @@ -240,10 +248,21 @@ def redirect_to_original(request, alias, param=None): processed_alias = ''.join(e for e in alias.lower() if e.isalnum()) try: - link = Link.objects.get(alias=processed_alias) - link.click_count = F('click_count') + 1 - link.save() - ClickLog.objects.create(link=link) + cache_key = f'link:alias:{processed_alias}' + link = cache.get(cache_key) + if link is None: + link = Link.objects.get(alias=processed_alias) + cache.set(cache_key, link, timeout=3600) + + # Buffer click in Redis (atomic increment). A periodic task flushes to DB. + from django.utils import timezone + date_str = timezone.now().strftime('%Y-%m-%d') + click_key = f'clicks:{link.id}:{date_str}' + try: + cache.incr(click_key) + except ValueError: + # Key doesn't exist yet — set it then give it a 48-hour safety-net TTL + cache.set(click_key, 1, timeout=172800) try: # 如果是模板 URL 并且提供了参数 @@ -271,6 +290,8 @@ def redirect_to_original(request, alias, param=None): return redirect('link_list') except Link.DoesNotExist: + # Also clear any stale cache entry that might be causing the miss + cache.delete(f'link:alias:{processed_alias}') messages.warning(request, _("The alias '{}' doesn't exist. Do you want to create a new one?").format(processed_alias)) return redirect(reverse('link_create') + f'?alias={processed_alias}') @@ -329,56 +350,63 @@ class LinkDetailView(DetailView): period_name = "3 Months" interval_days = 1 # Daily - # Get click stats from database - click_stats = ClickLog.objects.filter( - link=self.object, - clicked_at__date__gte=start_date, - clicked_at__date__lte=end_date - ).annotate( - date=TruncDate('clicked_at') - ).values('date').annotate(count=Count('id')).order_by('date') + # Get click stats — cache per link+period to avoid repeated ClickLog full-scans + stats_cache_key = f'link:stats:{self.object.pk}:{period}' + click_stats_list = cache.get(stats_cache_key) - # Convert to dictionary for easy lookup - click_dict = {item['date']: item['count'] for item in click_stats} + if click_stats_list is None: + # Get click stats from database + click_stats = ClickLog.objects.filter( + link=self.object, + clicked_at__date__gte=start_date, + clicked_at__date__lte=end_date + ).annotate( + date=TruncDate('clicked_at') + ).values('date').annotate(count=Count('id')).order_by('date') - # Create complete dataset with appropriate intervals - click_stats_list = [] - current_date = start_date + # Convert to dictionary for easy lookup + click_dict = {item['date']: item['count'] for item in click_stats} - if interval_days == 1: - # Daily intervals - while current_date <= end_date: - click_stats_list.append({ - 'date': current_date.strftime('%Y-%m-%d'), - 'count': click_dict.get(current_date, 0) - }) - current_date += timedelta(days=1) - else: - # Weekly, bi-weekly, or monthly intervals - while current_date <= end_date: - interval_end = min(current_date + timedelta(days=interval_days - 1), end_date) + # Create complete dataset with appropriate intervals + click_stats_list = [] + current_date = start_date - # Sum clicks for this interval - interval_count = 0 - temp_date = current_date - while temp_date <= interval_end: - interval_count += click_dict.get(temp_date, 0) - temp_date += timedelta(days=1) + if interval_days == 1: + # Daily intervals + while current_date <= end_date: + click_stats_list.append({ + 'date': current_date.strftime('%Y-%m-%d'), + 'count': click_dict.get(current_date, 0) + }) + current_date += timedelta(days=1) + else: + # Weekly, bi-weekly, or monthly intervals + while current_date <= end_date: + interval_end = min(current_date + timedelta(days=interval_days - 1), end_date) - # Format label based on interval - if interval_days == 7: # Weekly - label = f"{current_date.strftime('%m/%d')}" - elif interval_days == 14: # Bi-weekly - label = f"{current_date.strftime('%m/%d')}" - else: # Monthly - label = f"{calendar.month_abbr[current_date.month]} {current_date.year}" + # Sum clicks for this interval + interval_count = 0 + temp_date = current_date + while temp_date <= interval_end: + interval_count += click_dict.get(temp_date, 0) + temp_date += timedelta(days=1) - click_stats_list.append({ - 'date': current_date.strftime('%Y-%m-%d'), - 'count': interval_count, - 'label': label - }) - current_date += timedelta(days=interval_days) + # Format label based on interval + if interval_days == 7: # Weekly + label = f"{current_date.strftime('%m/%d')}" + elif interval_days == 14: # Bi-weekly + label = f"{current_date.strftime('%m/%d')}" + else: # Monthly + label = f"{calendar.month_abbr[current_date.month]} {current_date.year}" + + click_stats_list.append({ + 'date': current_date.strftime('%Y-%m-%d'), + 'count': interval_count, + 'label': label + }) + current_date += timedelta(days=interval_days) + + cache.set(stats_cache_key, click_stats_list, timeout=600) context['click_stats'] = json.dumps(click_stats_list, cls=DjangoJSONEncoder) context['current_period'] = period diff --git a/pyproject.toml b/pyproject.toml index 2fa5dc5..ab74c7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "python-magic>=0.4.27", "pillow~=12.1.1", "whoosh==2.7.4", + "django-redis>=5.4.0", "qdrant-client>=1.13.2", "cryptography>=42.0.0", ] diff --git a/tests/conftest.py b/tests/conftest.py index 23b6c19..fa7dbe4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,3 +47,14 @@ def use_tmp_upload_dir(settings, tmp_path): """Redirect FILE_UPLOADS_FOLDER to a temp dir for each test.""" settings.FILE_UPLOADS_FOLDER = str(tmp_path) yield + + +@pytest.fixture(autouse=True) +def use_locmem_cache(settings): + """Override Redis cache with in-process LocMemCache so tests don't need a Redis server.""" + settings.CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } + } + yield diff --git a/uv.lock b/uv.lock index 35f30a2..c590c98 100644 --- a/uv.lock +++ b/uv.lock @@ -311,6 +311,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/32/4b144e125678efccf5d5b61581de1c4088d6b0286e46096e3b8de0d556c8/django-5.2.12-py3-none-any.whl", hash = "sha256:4853482f395c3a151937f6991272540fcbf531464f254a347bf7c89f53c8cff7", size = 8310245, upload-time = "2026-03-03T13:56:01.174Z" }, ] +[[package]] +name = "django-redis" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, + { name = "redis" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/53/dbcfa1e528e0d6c39947092625b2c89274b5d88f14d357cee53c4d6dbbd4/django_redis-6.0.0.tar.gz", hash = "sha256:2d9cb12a20424a4c4dde082c6122f486628bae2d9c2bee4c0126a4de7fda00dd", size = 56904, upload-time = "2025-06-17T18:15:46.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/79/055dfcc508cfe9f439d9f453741188d633efa9eab90fc78a67b0ab50b137/django_redis-6.0.0-py3-none-any.whl", hash = "sha256:20bf0063a8abee567eb5f77f375143c32810c8700c0674ced34737f8de4e36c0", size = 33687, upload-time = "2025-06-17T18:15:34.165Z" }, +] + [[package]] name = "django-simplemde" version = "0.1.4" @@ -587,6 +600,7 @@ dependencies = [ { name = "boto3" }, { name = "cryptography" }, { name = "django" }, + { name = "django-redis" }, { name = "django-simplemde" }, { name = "django-tailwind" }, { name = "django-widget-tweaks" }, @@ -628,6 +642,7 @@ requires-dist = [ { name = "boto3", specifier = ">=1.35.0" }, { name = "cryptography", specifier = ">=42.0.0" }, { name = "django", specifier = ">=5.2.9" }, + { name = "django-redis", specifier = ">=5.4.0" }, { name = "django-simplemde", specifier = "==0.1.4" }, { name = "django-tailwind", specifier = "==3.8.0" }, { name = "django-widget-tweaks", specifier = "==1.5.0" }, @@ -1131,6 +1146,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/08/13/8ce16f808297e16968269de44a14f4fef19b64d9766be1d6ba5ba78b579d/qdrant_client-1.16.2-py3-none-any.whl", hash = "sha256:442c7ef32ae0f005e88b5d3c0783c63d4912b97ae756eb5e052523be682f17d3", size = 377186, upload-time = "2025-12-12T10:58:29.282Z" }, ] +[[package]] +name = "redis" +version = "7.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/3759b1d0d72b7c92f0d70ffd9dc962b7b7b5ee74e135f9d7d8ab06b8a318/redis-7.4.0.tar.gz", hash = "sha256:64a6ea7bf567ad43c964d2c30d82853f8df927c5c9017766c55a1d1ed95d18ad", size = 4943913, upload-time = "2026-03-24T09:14:37.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" }, +] + [[package]] name = "requests" version = "2.33.0"