diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 852b63b..3b3aae4 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/links/apps.py b/links/apps.py new file mode 100644 index 0000000..e133061 --- /dev/null +++ b/links/apps.py @@ -0,0 +1,10 @@ +from django.apps import AppConfig + + +class LinksConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'links' + + def ready(self): + # Import signals to register them + import links.signals diff --git a/links/management/__init__.py b/links/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/links/management/commands/__init__.py b/links/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/links/management/commands/rebuild_search_index.py b/links/management/commands/rebuild_search_index.py new file mode 100644 index 0000000..11b36fe --- /dev/null +++ b/links/management/commands/rebuild_search_index.py @@ -0,0 +1,14 @@ +from django.core.management.base import BaseCommand +from links.search_backend import search_backend + + +class Command(BaseCommand): + help = 'Rebuild the full-text search index' + + def handle(self, *args, **options): + self.stdout.write('Rebuilding search index...') + try: + search_backend.rebuild_index() + self.stdout.write(self.style.SUCCESS('Successfully rebuilt search index')) + except Exception as e: + self.stdout.write(self.style.ERROR(f'Failed to rebuild index: {e}')) diff --git a/links/migrations/0035_remove_webpage_content.py b/links/migrations/0035_remove_webpage_content.py new file mode 100644 index 0000000..1718fd3 --- /dev/null +++ b/links/migrations/0035_remove_webpage_content.py @@ -0,0 +1,58 @@ +# Generated by Django 5.1.13 on 2025-11-05 10:14 + +from django.db import migrations + + +def remove_webpage_content_column(apps, schema_editor): + """Remove webpage_content column if it exists""" + with schema_editor.connection.cursor() as cursor: + # Check if column exists + cursor.execute("PRAGMA table_info(links_page)") + columns = [row[1] for row in cursor.fetchall()] + + if 'webpage_content' in columns: + # SQLite doesn't support DROP COLUMN directly, need to recreate table + cursor.execute(""" + CREATE TABLE links_page_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url varchar(2000) NOT NULL, + title varchar(200) NOT NULL, + summary TEXT NOT NULL, + content TEXT NOT NULL, + created_at datetime NOT NULL, + updated_at datetime NOT NULL, + error_message TEXT NOT NULL, + last_retry_at datetime, + process_status varchar(20) NOT NULL, + retry_count INTEGER NOT NULL, + screenshot_error TEXT, + screenshot_last_attempt datetime, + screenshot_path varchar(255) NOT NULL + ) + """) + + # Copy data + cursor.execute(""" + INSERT INTO links_page_new + SELECT id, url, title, summary, content, created_at, updated_at, + error_message, last_retry_at, process_status, retry_count, + screenshot_error, screenshot_last_attempt, screenshot_path + FROM links_page + """) + + # Drop old table + cursor.execute("DROP TABLE links_page") + + # Rename new table + cursor.execute("ALTER TABLE links_page_new RENAME TO links_page") + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0034_post_summary'), + ] + + operations = [ + migrations.RunPython(remove_webpage_content_column, migrations.RunPython.noop), + ] diff --git a/links/search_backend.py b/links/search_backend.py new file mode 100644 index 0000000..fa49896 --- /dev/null +++ b/links/search_backend.py @@ -0,0 +1,266 @@ +""" +Full-text search backend using Whoosh +""" +import os +import logging +from django.conf import settings +from whoosh import index +from whoosh.fields import Schema, TEXT, ID, DATETIME, KEYWORD, NUMERIC +from whoosh.qparser import MultifieldParser, OrGroup +from whoosh.analysis import StemmingAnalyzer +from .models import Link, Page, Post + +logger = logging.getLogger(__name__) + +# Define index directory +INDEX_DIR = os.path.join(settings.BASE_DIR, 'data', 'search_index') + +# Define schema for search index +SCHEMA = Schema( + id=ID(stored=True, unique=True), + model_type=ID(stored=True), + title=TEXT(stored=True, analyzer=StemmingAnalyzer()), + content=TEXT(analyzer=StemmingAnalyzer()), + url=TEXT(stored=True), + alias=TEXT(stored=True), + description=TEXT(stored=True, analyzer=StemmingAnalyzer()), + summary=TEXT(stored=True, analyzer=StemmingAnalyzer()), + tags=KEYWORD(stored=True, commas=True, scorable=True), + created_at=DATETIME(stored=True, sortable=True), + click_count=NUMERIC(stored=True, sortable=True) +) + + +class SearchBackend: + """ + Search backend powered by Whoosh for full-text indexing and searching + """ + + def __init__(self): + self.index_dir = INDEX_DIR + self.ensure_index_exists() + + def ensure_index_exists(self): + """Create index directory and index if they don't exist""" + if not os.path.exists(self.index_dir): + os.makedirs(self.index_dir, exist_ok=True) + index.create_in(self.index_dir, SCHEMA) + logger.info(f"Created new search index at {self.index_dir}") + + def get_index(self): + """Get or create the search index""" + if not index.exists_in(self.index_dir): + return index.create_in(self.index_dir, SCHEMA) + return index.open_dir(self.index_dir) + + def index_link(self, link): + """Index a single link""" + ix = self.get_index() + writer = ix.writer() + + try: + # Get tags as comma-separated string + tags = ','.join([tag.name for tag in link.tags.all()]) + + writer.update_document( + id=f"link_{link.id}", + model_type="link", + title=link.alias or "", + content=link.text or "", + url=link.original_url or "", + alias=link.alias or "", + description=link.description or "", + tags=tags, + created_at=link.created_at, + click_count=link.click_count + ) + writer.commit() + logger.debug(f"Indexed link: {link.id}") + except Exception as e: + writer.cancel() + logger.error(f"Error indexing link {link.id}: {e}", exc_info=True) + + def index_page(self, page): + """Index a single page""" + ix = self.get_index() + writer = ix.writer() + + try: + # Get tags as comma-separated string + tags = ','.join([tag.name for tag in page.tags.all()]) + + writer.update_document( + id=f"page_{page.id}", + model_type="page", + title=page.title or "", + content=page.content or "", + url=page.url or "", + summary=page.summary or "", + tags=tags, + created_at=page.created_at + ) + writer.commit() + logger.debug(f"Indexed page: {page.id}") + except Exception as e: + writer.cancel() + logger.error(f"Error indexing page {page.id}: {e}", exc_info=True) + + def index_post(self, post): + """Index a single post""" + ix = self.get_index() + writer = ix.writer() + + try: + # Get tags as comma-separated string + tags = ','.join([tag.name for tag in post.tags.all()]) + + writer.update_document( + id=f"post_{post.id}", + model_type="post", + title=post.title or "", + content=post.content or "", + summary=post.summary or "", + tags=tags, + created_at=post.created_at + ) + writer.commit() + logger.debug(f"Indexed post: {post.id}") + except Exception as e: + writer.cancel() + logger.error(f"Error indexing post {post.id}: {e}", exc_info=True) + + def remove_from_index(self, model_type, model_id): + """Remove a document from the index""" + ix = self.get_index() + writer = ix.writer() + + try: + writer.delete_by_term('id', f"{model_type}_{model_id}") + writer.commit() + logger.debug(f"Removed {model_type} {model_id} from index") + except Exception as e: + writer.cancel() + logger.error(f"Error removing {model_type} {model_id} from index: {e}", exc_info=True) + + def rebuild_index(self): + """Rebuild the entire search index from scratch""" + logger.info("Starting search index rebuild...") + + # Delete and recreate index + ix = index.create_in(self.index_dir, SCHEMA) + writer = ix.writer() + + try: + # Index all links + for link in Link.objects.all().prefetch_related('tags'): + tags = ','.join([tag.name for tag in link.tags.all()]) + writer.add_document( + id=f"link_{link.id}", + model_type="link", + title=link.alias or "", + content=link.text or "", + url=link.original_url or "", + alias=link.alias or "", + description=link.description or "", + tags=tags, + created_at=link.created_at, + click_count=link.click_count + ) + + # Index all pages + for page in Page.objects.all().prefetch_related('tags'): + tags = ','.join([tag.name for tag in page.tags.all()]) + writer.add_document( + id=f"page_{page.id}", + model_type="page", + title=page.title or "", + content=page.content or "", + url=page.url or "", + summary=page.summary or "", + tags=tags, + created_at=page.created_at + ) + + # Index all posts + for post in Post.objects.all().prefetch_related('tags'): + tags = ','.join([tag.name for tag in post.tags.all()]) + writer.add_document( + id=f"post_{post.id}", + model_type="post", + title=post.title or "", + content=post.content or "", + summary=post.summary or "", + tags=tags, + created_at=post.created_at + ) + + writer.commit() + logger.info("Search index rebuild completed successfully") + except Exception as e: + writer.cancel() + logger.error(f"Error rebuilding index: {e}", exc_info=True) + raise + + def search(self, query_string, model_type=None, limit=100, page=1, per_page=20): + """ + Search the index with the given query + + Args: + query_string: Search query + model_type: Filter by model type (link, page, post) or None for all + limit: Maximum number of results to return + page: Page number for pagination + per_page: Results per page + + Returns: + dict with results and metadata + """ + if not query_string.strip(): + return {'results': [], 'total': 0, 'page': page, 'per_page': per_page} + + ix = self.get_index() + + with ix.searcher() as searcher: + # Create multifield parser - search across multiple fields + fields = ['title', 'content', 'url', 'alias', 'description', 'summary', 'tags'] + parser = MultifieldParser(fields, schema=ix.schema, group=OrGroup) + + try: + query = parser.parse(query_string) + except Exception as e: + logger.error(f"Error parsing query '{query_string}': {e}") + return {'results': [], 'total': 0, 'page': page, 'per_page': per_page} + + # Apply model type filter if specified + if model_type: + from whoosh.query import And, Term + query = And([query, Term('model_type', model_type)]) + + # Execute search + results = searcher.search(query, limit=limit) + + # Extract results with pagination + start = (page - 1) * per_page + end = start + per_page + total = len(results) + + result_list = [] + for hit in results[start:end]: + result_dict = dict(hit) + # Extract model ID from composite ID + model_id = result_dict['id'].split('_')[1] + result_dict['model_id'] = int(model_id) + result_list.append(result_dict) + + return { + 'results': result_list, + 'total': total, + 'page': page, + 'per_page': per_page, + 'has_next': end < total, + 'has_prev': page > 1 + } + + +# Global search backend instance +search_backend = SearchBackend() diff --git a/links/search_urls.py b/links/search_urls.py index a3ce020..314ec95 100644 --- a/links/search_urls.py +++ b/links/search_urls.py @@ -1,6 +1,8 @@ from django.urls import path -from . import views +from .search_views import SearchView, search, search_aliases urlpatterns = [ - path('', views.search_aliases, name='search_aliases'), + path('', SearchView.as_view(), name='search'), + path('api/', search, name='search_api'), + path('aliases/', search_aliases, name='search_aliases'), ] diff --git a/links/search_views.py b/links/search_views.py index 6f190c7..2b666f6 100644 --- a/links/search_views.py +++ b/links/search_views.py @@ -6,6 +6,7 @@ from django.db.models import Q, Value, CharField from django.db import models from django.utils.text import slugify from .models import Link, Page, Post +from .search_backend import search_backend logger = logging.getLogger(__name__) @@ -17,77 +18,86 @@ class SearchView(ListView): def get_queryset(self): query = self.request.GET.get('q', '').strip() type_filter = self.request.GET.get('type', '') - sort_order = self.request.GET.get('sort', 'newest') + page_num = self.request.GET.get('page', 1) + + try: + page_num = int(page_num) + except (ValueError, TypeError): + page_num = 1 if query: - results = [] - - # Apply type filter - if type_filter: - if type_filter == 'link': - results = self._get_link_results(query) - elif type_filter == 'page': - results = self._get_page_results(query) - elif type_filter == 'post': - results = self._get_post_results(query) - else: - # Get all results - results = self._get_all_results(query) - - # Apply sorting - results = self._sort_results(results, sort_order) - - return results + # Use Whoosh search backend for full-text search + search_results = search_backend.search( + query_string=query, + model_type=type_filter if type_filter else None, + page=page_num, + per_page=self.paginate_by + ) + + # Enrich results with actual model data + enriched_results = [] + for result in search_results['results']: + enriched_result = self._enrich_result(result) + if enriched_result: + enriched_results.append(enriched_result) + + # Store search metadata for pagination + self.search_total = search_results['total'] + self.search_has_next = search_results['has_next'] + self.search_has_prev = search_results['has_prev'] + + return enriched_results return [] - - def _get_link_results(self, query): - return Link.objects.filter( - Q(alias__icontains=query) | - Q(original_url__icontains=query) | - Q(description__icontains=query) - ).values('id', 'alias', 'original_url', 'click_count', 'created_at').annotate( - model_type=Value('link', output_field=CharField()) - ) - - def _get_page_results(self, query): - return Page.objects.filter( - Q(url__icontains=query) | - Q(title__icontains=query) | - Q(summary__icontains=query) | - Q(content__icontains=query) - ).values('id', 'url', 'title', 'summary', 'created_at').annotate( - model_type=Value('page', output_field=CharField()) - ) - - def _get_post_results(self, query): - return Post.objects.filter( - Q(title__icontains=query) | - Q(content__icontains=query) - ).values('id', 'title', 'created_at').annotate( - model_type=Value('post', output_field=CharField()) - ) - - def _get_all_results(self, query): - links = self._get_link_results(query) - pages = self._get_page_results(query) - posts = self._get_post_results(query) - return list(chain(links, pages, posts)) - - def _sort_results(self, results, sort_order): - if sort_order == 'newest': - return sorted(results, key=lambda x: x.get('created_at', ''), reverse=True) - elif sort_order == 'oldest': - return sorted(results, key=lambda x: x.get('created_at', '')) - elif sort_order == 'type': - return sorted(results, key=lambda x: (x.get('model_type', ''), x.get('created_at', '')), reverse=True) - return results + + def _enrich_result(self, result): + """Fetch full model data for search result""" + model_type = result['model_type'] + model_id = result['model_id'] + + try: + if model_type == 'link': + link = Link.objects.get(id=model_id) + return { + 'id': link.id, + 'model_type': 'link', + 'alias': link.alias, + 'original_url': link.original_url, + 'description': link.description, + 'click_count': link.click_count, + 'created_at': link.created_at, + 'tags': list(link.tags.all()) + } + elif model_type == 'page': + page = Page.objects.get(id=model_id) + return { + 'id': page.id, + 'model_type': 'page', + 'title': page.title, + 'url': page.url, + 'summary': page.summary, + 'created_at': page.created_at, + 'tags': list(page.tags.all()) + } + elif model_type == 'post': + post = Post.objects.get(id=model_id) + return { + 'id': post.id, + 'model_type': 'post', + 'title': post.title, + 'summary': post.summary, + 'created_at': post.created_at, + 'tags': list(post.tags.all()) + } + except Exception as e: + logger.error(f"Error enriching result {model_type} {model_id}: {e}") + return None def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ 'query': self.request.GET.get('q', ''), 'selected_type': self.request.GET.get('type', ''), - 'sort_order': self.request.GET.get('sort', 'newest'), + 'total_results': getattr(self, 'search_total', 0), }) return context diff --git a/links/signals.py b/links/signals.py new file mode 100644 index 0000000..2a62cf1 --- /dev/null +++ b/links/signals.py @@ -0,0 +1,94 @@ +""" +Django signals for search index maintenance +""" +from django.db.models.signals import post_save, post_delete, m2m_changed +from django.dispatch import receiver +from .models import Link, Page, Post +from .search_backend import search_backend +import logging + +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=Link) +def update_link_index(sender, instance, created, **kwargs): + """Update search index when a link is saved""" + try: + search_backend.index_link(instance) + except Exception as e: + logger.error(f"Error indexing link {instance.id}: {e}") + + +@receiver(post_save, sender=Page) +def update_page_index(sender, instance, created, **kwargs): + """Update search index when a page is saved""" + try: + search_backend.index_page(instance) + except Exception as e: + logger.error(f"Error indexing page {instance.id}: {e}") + + +@receiver(post_save, sender=Post) +def update_post_index(sender, instance, created, **kwargs): + """Update search index when a post is saved""" + try: + search_backend.index_post(instance) + except Exception as e: + logger.error(f"Error indexing post {instance.id}: {e}") + + +@receiver(post_delete, sender=Link) +def remove_link_from_index(sender, instance, **kwargs): + """Remove link from search index when deleted""" + try: + search_backend.remove_from_index('link', instance.id) + except Exception as e: + logger.error(f"Error removing link {instance.id} from index: {e}") + + +@receiver(post_delete, sender=Page) +def remove_page_from_index(sender, instance, **kwargs): + """Remove page from search index when deleted""" + try: + search_backend.remove_from_index('page', instance.id) + except Exception as e: + logger.error(f"Error removing page {instance.id} from index: {e}") + + +@receiver(post_delete, sender=Post) +def remove_post_from_index(sender, instance, **kwargs): + """Remove post from search index when deleted""" + try: + search_backend.remove_from_index('post', instance.id) + except Exception as e: + logger.error(f"Error removing post {instance.id} from index: {e}") + + +@receiver(m2m_changed, sender=Link.tags.through) +def update_link_tags_index(sender, instance, action, **kwargs): + """Update link index when tags change""" + if action in ['post_add', 'post_remove', 'post_clear']: + try: + search_backend.index_link(instance) + except Exception as e: + logger.error(f"Error updating link {instance.id} tags in index: {e}") + + +@receiver(m2m_changed, sender=Page.tags.through) +def update_page_tags_index(sender, instance, action, **kwargs): + """Update page index when tags change""" + if action in ['post_add', 'post_remove', 'post_clear']: + try: + search_backend.index_page(instance) + except Exception as e: + logger.error(f"Error updating page {instance.id} tags in index: {e}") + + +@receiver(m2m_changed, sender=Post.tags.through) +def update_post_tags_index(sender, instance, action, **kwargs): + """Update post index when tags change""" + if action in ['post_add', 'post_remove', 'post_clear']: + try: + search_backend.index_post(instance) + except Exception as e: + logger.error(f"Error updating post {instance.id} tags in index: {e}") diff --git a/links/templates/links/search.html b/links/templates/links/search.html index ec5861b..9c746a8 100644 --- a/links/templates/links/search.html +++ b/links/templates/links/search.html @@ -79,6 +79,11 @@
+ {% trans "Found" %} {{ total_results }} {% trans "results" %} +
+ {% endif %} {% if results %} @@ -127,7 +132,7 @@ {{ result.title|default:"Untitled" }} - + {% trans "Page" %} @@ -138,6 +143,15 @@ {% if result.summary %}{{ result.summary }}
{% endif %} + {% if result.tags %} +