diff --git a/SELENIUM_TO_PLAYWRIGHT_MIGRATION.md b/SELENIUM_TO_PLAYWRIGHT_MIGRATION.md deleted file mode 100644 index 3fc604b..0000000 --- a/SELENIUM_TO_PLAYWRIGHT_MIGRATION.md +++ /dev/null @@ -1,260 +0,0 @@ -# Selenium to Playwright Migration - -## Summary - -Successfully migrated from Selenium to Playwright for web page screenshot capture and processing. - -## Changes Made - -### 1. Dependencies (`pyproject.toml`) -```diff -- "selenium>=4.0.0", -+ "playwright>=1.40.0", -``` - -### 2. Code Changes (`links/tasks.py`) - -**Before** (Selenium): ~180 lines -- Complex Chrome options setup (15+ arguments) -- Manual scrolling and waiting -- ChromeDriver management -- Complex error handling - -**After** (Playwright): ~70 lines -- Simple async function -- Built-in auto-wait -- No driver management -- Clear timeout handling - -**Key improvements**: -- ✅ 60% less code -- ✅ Built-in `full_page=True` screenshot -- ✅ Automatic network idle detection -- ✅ Better timeout handling -- ✅ No manual scrolling needed - -### 3. Dockerfile Updates - -**Removed**: -```dockerfile -chromium -chromium-driver -``` - -**Added**: -```dockerfile -# Playwright runtime dependencies -libglib2.0-0, libnss3, libnspr4, etc. - -# Install Playwright browsers in builder -RUN uv run playwright install chromium --with-deps - -# Copy Playwright cache to production -COPY --from=builder /root/.cache/ms-playwright /home/appuser/.cache/ms-playwright -``` - -**Result**: ~80 MB smaller Docker image - -### 4. View Updates (`links/page_views.py`) - -**Removed**: -- `take_screenshot()` function (unused) -- Selenium imports - -**Kept**: -- All other functionality unchanged -- Still uses threading for async execution - -## New Screenshot Function - -```python -async def _capture_screenshot_async(page_url, full_path): - """Async function to capture screenshot using Playwright""" - async with async_playwright() as p: - browser = await p.chromium.launch( - headless=True, - args=['--no-sandbox', '--disable-setuid-sandbox'] - ) - - context = await browser.new_context( - viewport={'width': 1366, 'height': 768}, - locale='zh-CN', - ignore_https_errors=True, - ) - - page = await context.new_page() - - # Navigate and wait for network idle - await page.goto(page_url, wait_until='networkidle', timeout=60000) - - # Take full-page screenshot - await page.screenshot(path=full_path, full_page=True) - - await context.close() - await browser.close() -``` - -## Benefits - -### Performance -- ⚡ **40-50% faster** screenshot capture -- ⚡ **80% fewer timeouts** (auto-wait) -- ⚡ **25% less memory** usage - -### Code Quality -- 📉 **60% less code** (180 → 70 lines) -- ✅ **Simpler maintenance** -- ✅ **Better error messages** -- ✅ **Built-in retry logic** - -### Infrastructure -- 📦 **~80 MB smaller** Docker image -- �� **No driver version matching** needed -- 🚀 **Easier deployment** - -### Reliability -- ✅ **Network idle detection** (knows when page is loaded) -- ✅ **Auto-wait for elements** -- ✅ **Better handling of modern SPAs** -- ✅ **Clear timeout errors** - -## Migration Steps Completed - -1. ✅ Updated `pyproject.toml` dependencies -2. ✅ Rewrote `capture_screenshot()` function -3. ✅ Added `_capture_screenshot_async()` helper -4. ✅ Updated Dockerfile to install Playwright -5. ✅ Removed Chromium/ChromeDriver from Dockerfile -6. ✅ Added Playwright runtime dependencies -7. ✅ Removed Selenium imports from `page_views.py` -8. ✅ Removed unused `take_screenshot()` function - -## Testing - -### Install Dependencies -```bash -source .venv/bin/activate -uv sync -uv run playwright install chromium -``` - -### Test Locally -```bash -python manage.py runserver -# Navigate to a page and trigger screenshot -``` - -### Docker Build -```bash -DOCKER_BUILDKIT=1 docker build -t links:playwright . -``` - -### Expected Results -- ✅ Faster screenshot capture (8-12s vs 15-20s) -- ✅ Fewer timeout errors -- ✅ Cleaner error messages -- ✅ Same visual quality - -## Compatibility - -### APScheduler -- ✅ Works perfectly with APScheduler -- ✅ Uses `asyncio.run()` to run async code in sync context -- ✅ Threading still works as before - -### Django -- ✅ No Django changes needed -- ✅ Models unchanged -- ✅ Views unchanged -- ✅ API unchanged - -### Storage -- ✅ R2/S3 storage still works -- ✅ Local storage still works -- ✅ Screenshot format unchanged (PNG) - -## Configuration - -### Viewport Size -Default: 1366x768 (configurable in code) -```python -viewport={'width': 1366, 'height': 768} -``` - -### Timeout -Default: 60 seconds -```python -timeout=60000 # milliseconds -``` - -### Locale -Default: zh-CN (Chinese) -```python -locale='zh-CN' -``` - -### Wait Strategy -Default: networkidle (waits for network requests to finish) -```python -wait_until='networkidle' -``` - -## Troubleshooting - -### Playwright not found -```bash -uv run playwright install chromium -``` - -### Missing dependencies in Docker -Already included in Dockerfile: -- libglib2.0-0 -- libnss3 -- libnspr4 -- etc. - -### Timeout issues -Increase timeout in code: -```python -await page.goto(url, timeout=120000) # 2 minutes -``` - -### Screenshot quality -Adjust viewport or use different options: -```python -await page.screenshot(path=path, full_page=True, quality=90) -``` - -## Rollback Plan - -If needed to rollback: -1. Revert `pyproject.toml` (restore Selenium) -2. Revert `links/tasks.py` to Selenium version -3. Revert Dockerfile changes -4. Run `uv sync` - -## Future Enhancements - -Possible with Playwright (not implemented yet): -- [ ] PDF generation -- [ ] Video recording -- [ ] Network request interception -- [ ] Mobile device emulation -- [ ] Geolocation spoofing -- [ ] Custom JavaScript injection -- [ ] HAR file generation - -## Resources - -- Playwright Docs: https://playwright.dev/python/ -- Playwright API: https://playwright.dev/python/docs/api/class-page -- Migration Guide: https://playwright.dev/python/docs/selenium - -## Notes - -- Playwright uses its own bundled Chromium -- No need to manage ChromeDriver versions -- Auto-updates with `playwright install` -- Works offline after initial install -- Supports Firefox and WebKit too (if needed) - diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 3b3aae4..1e57492 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/links/migrations/0036_link_task_states_page_task_states_post_task_states.py b/links/migrations/0036_link_task_states_page_task_states_post_task_states.py new file mode 100644 index 0000000..64b8c45 --- /dev/null +++ b/links/migrations/0036_link_task_states_page_task_states_post_task_states.py @@ -0,0 +1,28 @@ +# Generated by Django 5.2.9 on 2026-01-17 22:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0035_remove_webpage_content'), + ] + + operations = [ + migrations.AddField( + model_name='link', + name='task_states', + field=models.JSONField(blank=True, default=dict, help_text='Task checkbox states as {task_hash: bool} mapping'), + ), + migrations.AddField( + model_name='page', + name='task_states', + field=models.JSONField(blank=True, default=dict, help_text='Task checkbox states as {task_hash: bool} mapping'), + ), + migrations.AddField( + model_name='post', + name='task_states', + field=models.JSONField(blank=True, default=dict, help_text='Task checkbox states as {task_hash: bool} mapping'), + ), + ] diff --git a/links/migrations/0037_linkchangelog_change_type_linkchangelog_metadata_and_more.py b/links/migrations/0037_linkchangelog_change_type_linkchangelog_metadata_and_more.py new file mode 100644 index 0000000..838ae93 --- /dev/null +++ b/links/migrations/0037_linkchangelog_change_type_linkchangelog_metadata_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.9 on 2026-01-17 22:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0036_link_task_states_page_task_states_post_task_states'), + ] + + operations = [ + migrations.AddField( + model_name='linkchangelog', + name='change_type', + field=models.CharField(choices=[('url_change', 'URL Change'), ('task_toggle', 'Task Toggle'), ('content_edit', 'Content Edit')], default='url_change', max_length=20, verbose_name='Change Type'), + ), + migrations.AddField( + model_name='linkchangelog', + name='metadata', + field=models.JSONField(blank=True, default=dict, help_text='Additional change data as JSON', verbose_name='Metadata'), + ), + migrations.AlterField( + model_name='linkchangelog', + name='new_url', + field=models.URLField(blank=True, max_length=2000, null=True, verbose_name='New URL'), + ), + migrations.AlterField( + model_name='linkchangelog', + name='old_url', + field=models.URLField(blank=True, max_length=2000, null=True, verbose_name='Old URL'), + ), + ] diff --git a/links/models.py b/links/models.py index 996a6d6..e79f0b8 100644 --- a/links/models.py +++ b/links/models.py @@ -33,6 +33,7 @@ class Link(models.Model): updated_at = models.DateTimeField(auto_now=True) description = models.TextField(blank=True, null=True, verbose_name=_("Description")) tags = models.ManyToManyField('Tag', blank=True, related_name='links') + task_states = models.JSONField(default=dict, blank=True, help_text=_("Task checkbox states as {task_hash: bool} mapping")) def get_template_parameters(self): """Extract template parameters and their default values from original_url""" @@ -124,16 +125,37 @@ class ClickLog(models.Model): ordering = ['-clicked_at'] class LinkChangeLog(models.Model): + class ChangeType(models.TextChoices): + URL_CHANGE = 'url_change', _('URL Change') + TASK_TOGGLE = 'task_toggle', _('Task Toggle') + CONTENT_EDIT = 'content_edit', _('Content Edit') + link = models.ForeignKey(Link, on_delete=models.CASCADE, related_name='change_logs') - old_url = models.URLField(_("Old URL"), max_length=2000) - new_url = models.URLField(_("New URL"), max_length=2000) + change_type = models.CharField( + _("Change Type"), + max_length=20, + choices=ChangeType.choices, + default=ChangeType.URL_CHANGE + ) + # For URL changes + old_url = models.URLField(_("Old URL"), max_length=2000, blank=True, null=True) + new_url = models.URLField(_("New URL"), max_length=2000, blank=True, null=True) + # For task toggles and other changes + metadata = models.JSONField(_("Metadata"), default=dict, blank=True, help_text=_("Additional change data as JSON")) changed_at = models.DateTimeField(_("Changed at"), auto_now_add=True) class Meta: ordering = ['-changed_at'] def __str__(self): - return f"URL changed from {self.old_url} to {self.new_url}" + if self.change_type == self.ChangeType.URL_CHANGE: + return f"URL changed from {self.old_url} to {self.new_url}" + elif self.change_type == self.ChangeType.TASK_TOGGLE: + task_hash = self.metadata.get('task_hash', 'unknown') + new_state = self.metadata.get('new_state', 'unknown') + return f"Task {task_hash[:8]} toggled to {new_state}" + else: + return f"{self.get_change_type_display()} at {self.changed_at}" def generate_random_color(): colors = [ @@ -188,6 +210,7 @@ class Page(models.Model): ) retry_count = models.IntegerField(default=0) last_retry_at = models.DateTimeField(null=True, blank=True) + task_states = models.JSONField(default=dict, blank=True, help_text=_("Task checkbox states as {task_hash: bool} mapping")) error_message = models.TextField(blank=True) # New field for screenshot @@ -251,6 +274,7 @@ class Post(models.Model): created_at = models.DateTimeField(_('Created at'), auto_now_add=True) updated_at = models.DateTimeField(_('Updated at'), auto_now=True) tags = models.ManyToManyField('Tag', blank=True, related_name='posts') + task_states = models.JSONField(default=dict, blank=True, help_text=_("Task checkbox states as {task_hash: bool} mapping")) class Meta: ordering = ['-created_at'] diff --git a/links/page_views.py b/links/page_views.py index 7895037..27b08aa 100644 --- a/links/page_views.py +++ b/links/page_views.py @@ -7,7 +7,9 @@ from django.utils.translation import gettext_lazy as _ from django.core.files.base import ContentFile from django.conf import settings from rest_framework import viewsets, status +from rest_framework.decorators import action from rest_framework.response import Response +from .templatetags.tasklist_markdown import update_task_in_markdown from rest_framework.pagination import PageNumberPagination from .models import Page, Screenshot, Tag from .forms import PageForm @@ -31,27 +33,27 @@ def trigger_page_processing(page): 1. Screenshot capture 2. Crawl4AI content extraction 3. Page metadata processing - + This is called whenever a page is created, regardless of the source (UI or API). """ from .tasks import capture_screenshot, fetch_webpage_content_from_crawl4ai, process_page - + # Create screenshot record and trigger capture screenshot = Screenshot.objects.create( page=page, status=Screenshot.Status.PENDING ) - + # Start screenshot capture thread thread_screenshot = Thread(target=capture_screenshot, args=(page.id, screenshot.id)) thread_screenshot.daemon = True thread_screenshot.start() - + # Start Crawl4AI content extraction thread thread_crawl = Thread(target=fetch_webpage_content_from_crawl4ai, args=(page.id,)) thread_crawl.daemon = True thread_crawl.start() - + # Start page metadata processing thread (if needed) if not page.title or not page.summary: thread_process = Thread(target=process_page, args=(page.id,)) @@ -101,6 +103,14 @@ class PageUpdateView(UpdateView): template_name = 'links/page_form.html' success_url = reverse_lazy('page-list') + def dispatch(self, request, *args, **kwargs): + response = super().dispatch(request, *args, **kwargs) + # Prevent caching to ensure fresh markdown content after task toggles + response['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response['Pragma'] = 'no-cache' + response['Expires'] = '0' + return response + def form_valid(self, form): response = super().form_valid(form) @@ -220,7 +230,7 @@ class PageViewSet(viewsets.ModelViewSet): else: instance.process_status = Page.ProcessStatus.PENDING instance.save() - + # Trigger all background processing tasks trigger_page_processing(instance) @@ -288,6 +298,40 @@ class PageViewSet(viewsets.ModelViewSet): status=status.HTTP_500_INTERNAL_SERVER_ERROR ) + @action(detail=True, methods=['post']) + def toggle_task(self, request, pk=None): + """ + Toggle the checked state of a task in a page. + + POST /api/pages/{id}/toggle_task/ + Body: {"task_hash": "abc123"} + """ + page = self.get_object() + task_hash = request.data.get('task_hash') + + if not task_hash: + return Response( + {'error': 'task_hash is required'}, + status=status.HTTP_400_BAD_REQUEST + ) + + # Get current task states or initialize empty dict + task_states = page.task_states if page.task_states else {} + + # Toggle the state (default to False if not set, then toggle) + current_state = task_states.get(task_hash, False) + new_state = not current_state + task_states[task_hash] = new_state + + # Save only task_states, leave markdown content unchanged + page.task_states = task_states + page.save(update_fields=['task_states', 'updated_at']) + + return Response({ + 'task_hash': task_hash, + 'checked': new_state + }) + class ScreenshotGalleryView(TemplateView): template_name = 'links/screenshot_gallery.html' diff --git a/links/post_views.py b/links/post_views.py index b6a87c7..5ee6d20 100644 --- a/links/post_views.py +++ b/links/post_views.py @@ -3,11 +3,13 @@ from django.urls import reverse_lazy from django.utils.translation import gettext_lazy as _ from django.http import Http404 from rest_framework import viewsets, status +from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.pagination import PageNumberPagination from django.utils import timezone from datetime import datetime from .models import Post +from .templatetags.tasklist_markdown import update_task_in_markdown from .forms import PostForm from .serializers import PostSerializer from .templatetags import markdown_extras, think_markdown @@ -20,7 +22,7 @@ class PostListView(ListView): def get_queryset(self): queryset = Post.objects.all().order_by('-created_at') - + # Date range filter date_from = self.request.GET.get('date_from') date_to = self.request.GET.get('date_to') @@ -43,7 +45,7 @@ class PostListView(ListView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - + context['date_from'] = self.request.GET.get('date_from', '') context['date_to'] = self.request.GET.get('date_to', '') @@ -71,6 +73,14 @@ class PostUpdateView(UpdateView): template_name = 'links/post_form.html' success_url = reverse_lazy('post-list') + def dispatch(self, request, *args, **kwargs): + response = super().dispatch(request, *args, **kwargs) + # Prevent caching to ensure fresh markdown content after task toggles + response['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response['Pragma'] = 'no-cache' + response['Expires'] = '0' + return response + class PostDeleteView(DeleteView): model = Post template_name = 'links/post_confirm_delete.html' @@ -120,3 +130,37 @@ class PostViewSet(viewsets.ModelViewSet): serializer = self.get_serializer(queryset, many=True) return Response(serializer.data) + + @action(detail=True, methods=['post']) + def toggle_task(self, request, pk=None): + """ + Toggle the checked state of a task in a post. + + POST /api/posts/{id}/toggle_task/ + Body: {"task_hash": "abc123"} + """ + post = self.get_object() + task_hash = request.data.get('task_hash') + + if not task_hash: + return Response( + {'error': 'task_hash is required'}, + status=status.HTTP_400_BAD_REQUEST + ) + + # Get current task states or initialize empty dict + task_states = post.task_states if post.task_states else {} + + # Toggle the state (default to False if not set, then toggle) + current_state = task_states.get(task_hash, False) + new_state = not current_state + task_states[task_hash] = new_state + + # Save only task_states, leave markdown content unchanged + post.task_states = task_states + post.save(update_fields=['task_states', 'content', 'updated_at']) + + return Response({ + 'task_hash': task_hash, + 'checked': new_state + }) diff --git a/links/templates/links/custom_link.html b/links/templates/links/custom_link.html index ae9d361..bec902d 100644 --- a/links/templates/links/custom_link.html +++ b/links/templates/links/custom_link.html @@ -8,6 +8,25 @@ +