From 921fd1e0cc8cc0af26ca5652a427054a5708f7c0 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Thu, 30 Jul 2026 15:56:19 +1000 Subject: [PATCH] Update links --- README.md | 74 +++++++++++++++++++ justfile | 4 + links/bookmark_views.py | 59 +++++++++++++-- .../management/commands/import_x_bookmarks.py | 15 +++- .../0051_add_bookmark_import_dedup.py | 23 ++++++ links/models.py | 8 ++ links/serializers.py | 66 ++++++++++++++--- tests/test_bookmarks_api.py | 39 +++++++++- 8 files changed, 267 insertions(+), 21 deletions(-) create mode 100644 links/migrations/0051_add_bookmark_import_dedup.py diff --git a/README.md b/README.md index ed2c335..e6b448d 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,80 @@ The app will be available at **http://localhost:8000** 2. Screenshots are captured automatically in the background 3. View all bookmarks at `/ui/pages/` +### X (Twitter) Bookmarks + +The app includes an offline archive of X.com bookmarks. Export your bookmarks from X, then import them for full-text search, filtering, and browsing — no further API calls needed. + +#### 1. Export bookmarks from X + +Use [x-bookmarks-exporter](https://github.com/nickcam/x-bookmarks-exporter) to download your X bookmarks as JSON files: + +```bash +# Install +git clone https://github.com/nickcam/x-bookmarks-exporter.git +cd x-bookmarks-exporter +npm install + +# Configure — copy .env.example to .env and fill in your X credentials +cp .env.example .env + +# Export (produces individual .json files in data/bookmarks/) +npm run export +``` + +The exporter produces either: +- A directory of `.json` files (one per bookmark), or +- A `bookmarks_index.json` file containing all bookmarks in a `{"bookmarks": [...]}` wrapper. + +Both formats are supported. + +#### 2. Import into URL Manager + +**Via management command:** + +```bash +# Import a directory of JSON files +just import-bookmarks data/bookmarks/ + +# Import an index file +just import-bookmarks data/bookmarks_index.json + +# Import a single file +just import-bookmarks data/single_bookmark.json + +# Wipe existing bookmarks first, then re-import +just import-bookmarks data/bookmarks/ --clear +``` + +The command reports `created`, `updated`, `unchanged`, and `skipped` counts. Re-importing the same data is safe — unchanged items are detected via content hash and skipped. + +**Via REST API:** + +```bash +# Bulk import (export-format payload) +curl -X POST http://localhost:8000/api/bookmarks/import/ \ + -H "Content-Type: application/json" \ + -d '{"bookmarks": [{...}, {...}]}' + +# Bulk upsert (flat array, either format) +curl -X POST http://localhost:8000/api/bookmarks/bulk/ \ + -H "Content-Type: application/json" \ + -d '[{"tweet_id": "123", "text": "hello"}, ...]' +``` + +#### 3. Browse imported bookmarks + +- **Web UI**: visit `/ui/bookmarks/` — filter by media type, search by text/author, view full details +- **API**: `GET /api/bookmarks/` with filters like `?q=search&has_video=true&author=screen_name` +- **API docs**: `/ui/api-docs/` (Bookmarks section) + +#### Dedup behaviour + +Each bookmark is keyed by `tweet_id`. On re-import: +- **New tweet_id** → created +- **Existing tweet_id, content changed** → updated (engagement counts, media, etc. refreshed) +- **Existing tweet_id, content identical** → `unchanged` (only `last_imported_at` bumped; no save/signal overhead) + ### Network Scanner (NetScan) Available at `/ui/netscan/`. Create a scan profile to automatically monitor your home network for security issues (router exposure, DNS, TLS certs, camera access). diff --git a/justfile b/justfile index 208f02e..5050c42 100644 --- a/justfile +++ b/justfile @@ -64,6 +64,10 @@ migrate: makemigrations: source .venv/bin/activate && uv run manage.py makemigrations +# Import X (Twitter) bookmarks from x-bookmarks-exporter JSON files +import-bookmarks source: + source .venv/bin/activate && uv run manage.py import_x_bookmarks {{ source }} + # Open Django shell shell: source .venv/bin/activate && uv run manage.py shell diff --git a/links/bookmark_views.py b/links/bookmark_views.py index 62a877d..7fe6134 100644 --- a/links/bookmark_views.py +++ b/links/bookmark_views.py @@ -3,6 +3,7 @@ from collections import OrderedDict from django.views.generic import ListView, DetailView from django.db import models +from django.utils import timezone from rest_framework import viewsets, status, filters as drf_filters from rest_framework.decorators import action from rest_framework.response import Response @@ -12,6 +13,7 @@ from .models import Bookmark from .serializers import ( BookmarkSerializer, BookmarkImportSerializer, + compute_bookmark_content_hash, ) @@ -90,23 +92,31 @@ class BookmarkViewSet(viewsets.ModelViewSet): if not isinstance(items, list): return Response({'detail': 'expected a JSON array'}, status=status.HTTP_400_BAD_REQUEST) - created = updated = skipped = 0 + created = updated = unchanged = skipped = 0 for raw in items: tid = str(raw.get('tweet_id') or raw.get('id') or '') if not tid: skipped += 1 continue + payload = _to_db_payload(raw) + new_hash = payload.get('content_hash', '') obj = Bookmark.objects.filter(tweet_id=tid).first() if obj is None: - BookmarkSerializer().create(_to_db_payload(raw)) + BookmarkSerializer().create(payload) created += 1 + elif obj.content_hash == new_hash: + # Content unchanged — only bump import timestamp. + obj.last_imported_at = timezone.now() + obj.save(update_fields=['last_imported_at']) + unchanged += 1 else: - ser = BookmarkSerializer(obj, data=_to_db_payload(raw), partial=True) + ser = BookmarkSerializer(obj, data=payload, partial=True) ser.is_valid(raise_exception=True) ser.save() updated += 1 return Response({'created': created, 'updated': updated, - 'skipped': skipped, 'total': Bookmark.objects.count()}, + 'unchanged': unchanged, 'skipped': skipped, + 'total': Bookmark.objects.count()}, status=status.HTTP_200_OK) @action(detail=False, methods=['get'], url_path='stats') @@ -168,9 +178,44 @@ def _to_db_payload(raw): 'card_data': raw.get('card'), 'quoted_data': raw.get('quoted'), 'data': raw, + 'content_hash': compute_bookmark_content_hash(raw), + 'last_imported_at': timezone.now(), } - # Already flat/serializer-shaped. - return raw + # Already flat/serializer-shaped — still stamp content_hash + last_imported_at + # so the dedup comparison in bulk_upsert works for both formats. + out = dict(raw) + if 'content_hash' not in out or not out['content_hash']: + out['content_hash'] = _compute_flat_content_hash(raw) + if 'last_imported_at' not in out or not out.get('last_imported_at'): + out['last_imported_at'] = timezone.now() + return out + + +def _compute_flat_content_hash(flat: dict) -> str: + """Compute a SHA-256 hash from a flat/normalized bookmark dict for dedup.""" + import hashlib, json + content = { + 'text': flat.get('text', ''), + 'url': flat.get('url', ''), + 'author_screen_name': flat.get('author_screen_name', ''), + 'author_name': flat.get('author_name', ''), + 'media': flat.get('media', []), + 'card_data': flat.get('card_data'), + 'quoted_data': flat.get('quoted_data'), + 'has_video': flat.get('has_video', False), + 'has_photo': flat.get('has_photo', False), + 'has_card': flat.get('has_card', False), + 'is_quote': flat.get('is_quote', False), + 'is_retweet': flat.get('is_retweet', False), + 'view_count': flat.get('view_count', 0), + 'favorite_count': flat.get('favorite_count', 0), + 'retweet_count': flat.get('retweet_count', 0), + 'reply_count': flat.get('reply_count', 0), + 'bookmark_count': flat.get('bookmark_count', 0), + 'quote_count': flat.get('quote_count', 0), + } + canonical = json.dumps(content, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(canonical.encode()).hexdigest() # ---------- UI views ---------- @@ -227,4 +272,4 @@ class BookmarkListView(ListView): class BookmarkDetailView(DetailView): model = Bookmark template_name = 'links/bookmark_detail.html' - context_object_name = 'bookmark' \ No newline at end of file + context_object_name = 'bookmark' diff --git a/links/management/commands/import_x_bookmarks.py b/links/management/commands/import_x_bookmarks.py index 71c05d7..9308d0b 100644 --- a/links/management/commands/import_x_bookmarks.py +++ b/links/management/commands/import_x_bookmarks.py @@ -10,8 +10,9 @@ import os from pathlib import Path from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone from links.models import Bookmark -from links.serializers import _norm_bookmark_for_import +from links.serializers import _norm_bookmark_for_import, compute_bookmark_content_hash class Command(BaseCommand): @@ -44,7 +45,7 @@ class Command(BaseCommand): Bookmark.objects.all().delete() self.stdout.write(f'cleared {n} existing bookmarks') - created = updated = skipped = 0 + created = updated = unchanged = skipped = 0 for fp in files: try: with open(fp, 'r', encoding='utf-8') as f: @@ -76,7 +77,14 @@ class Command(BaseCommand): if not tid: skipped += 1 continue + new_hash = compute_bookmark_content_hash(raw) obj = Bookmark.objects.filter(tweet_id=tid).first() + if obj is not None and obj.content_hash == new_hash: + # Content unchanged — only bump import timestamp. + obj.last_imported_at = timezone.now() + obj.save(update_fields=['last_imported_at']) + unchanged += 1 + continue try: b = _norm_bookmark_for_import(raw, obj) b.save() @@ -91,4 +99,5 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS( f'import done: created={created} updated={updated} ' - f'skipped={skipped} total_in_db={Bookmark.objects.count()}')) \ No newline at end of file + f'unchanged={unchanged} skipped={skipped} ' + f'total_in_db={Bookmark.objects.count()}')) diff --git a/links/migrations/0051_add_bookmark_import_dedup.py b/links/migrations/0051_add_bookmark_import_dedup.py new file mode 100644 index 0000000..ed801d6 --- /dev/null +++ b/links/migrations/0051_add_bookmark_import_dedup.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.16 on 2026-07-30 05:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0050_bookmark'), + ] + + operations = [ + migrations.AddField( + model_name='bookmark', + name='content_hash', + field=models.CharField(blank=True, default='', help_text='SHA-256 hash of the import payload; used to skip unchanged re-imports', max_length=64), + ), + migrations.AddField( + model_name='bookmark', + name='last_imported_at', + field=models.DateTimeField(blank=True, help_text='When this bookmark was last imported/refreshed from an external source', null=True), + ), + ] diff --git a/links/models.py b/links/models.py index e801545..33ce6e3 100644 --- a/links/models.py +++ b/links/models.py @@ -315,6 +315,14 @@ class Bookmark(models.Model): # the complete normalised export object (for detail view / round-tripping) data = models.JSONField(default=dict, blank=True) + # import dedup tracking + content_hash = models.CharField( + max_length=64, blank=True, default='', + help_text='SHA-256 hash of the import payload; used to skip unchanged re-imports') + last_imported_at = models.DateTimeField( + null=True, blank=True, + help_text='When this bookmark was last imported/refreshed from an external source') + created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) diff --git a/links/serializers.py b/links/serializers.py index 64bef89..db5e5a5 100644 --- a/links/serializers.py +++ b/links/serializers.py @@ -1,5 +1,33 @@ from rest_framework import serializers from .models import Link, Page, Post, ImageCollection, Image, Tag, FileUpload, Bookmark +import hashlib, json +from datetime import datetime +from django.utils import timezone + + +def compute_bookmark_content_hash(raw: dict) -> str: + """Compute a stable SHA-256 hash of the key content fields from a raw + import payload. Two payloads with the same meaningful content produce the + same hash, even if JSON key ordering differs.""" + # Only hash the fields that represent actual content (not metadata like + # timestamps that may drift between exports). + content = { + 'text': raw.get('text', ''), + 'url': raw.get('url', ''), + 'media': raw.get('media', []), + 'card': raw.get('card'), + 'quoted': raw.get('quoted'), + 'retweet': raw.get('retweet'), + 'author': raw.get('author', {}), + 'view_count': raw.get('view_count', 0), + 'favorite_count': raw.get('favorite_count', 0), + 'retweet_count': raw.get('retweet_count', 0), + 'reply_count': raw.get('reply_count', 0), + 'bookmark_count': raw.get('bookmark_count', 0), + 'quote_count': raw.get('quote_count', 0), + } + canonical = json.dumps(content, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(canonical.encode()).hexdigest() class LinkSerializer(serializers.ModelSerializer): @@ -141,9 +169,11 @@ class BookmarkSerializer(serializers.ModelSerializer): 'has_video', 'has_photo', 'has_card', 'is_quote', 'is_retweet', 'media', 'card_data', 'quoted_data', 'data', + 'content_hash', 'last_imported_at', 'created_at', 'updated_at', ] - read_only_fields = ['id', 'created_at', 'updated_at'] + read_only_fields = ['id', 'content_hash', 'last_imported_at', + 'created_at', 'updated_at'] def _norm_bookmark_for_import(raw, obj=None): @@ -169,7 +199,6 @@ def _norm_bookmark_for_import(raw, obj=None): # parse tweet created_at ("Wed Jul 30 12:00:00 +0000 2025" style) — leave # None if not parseable, view layer handles null. - from datetime import datetime tca = raw.get('created_at') if isinstance(tca, str) and tca: try: @@ -206,6 +235,11 @@ def _norm_bookmark_for_import(raw, obj=None): b.card_data = raw.get('card') b.quoted_data = raw.get('quoted') b.data = raw + + # dedup: record content hash + import timestamp + b.content_hash = compute_bookmark_content_hash(raw) + b.last_imported_at = timezone.now() + return b @@ -217,7 +251,13 @@ class BookmarkImportItemSerializer(serializers.Serializer): def create(self, validated): tweet_id = str(validated.get('id') or validated.get('tweet_id')) + new_hash = compute_bookmark_content_hash(validated) obj = Bookmark.objects.filter(tweet_id=tweet_id).first() + if obj is not None and obj.content_hash == new_hash: + # Content unchanged — only bump the import timestamp. + obj.last_imported_at = timezone.now() + obj.save(update_fields=['last_imported_at']) + return obj b = _norm_bookmark_for_import(validated, obj) b.save() return b @@ -232,16 +272,24 @@ class BookmarkImportSerializer(serializers.Serializer): def create(self, validated): items = validated.get('bookmarks', []) - created = updated = skipped = 0 + created = updated = unchanged = skipped = 0 for raw in items: tweet_id = str(raw.get('id') or raw.get('tweet_id') or '') if not tweet_id: skipped += 1 continue - existed = Bookmark.objects.filter(tweet_id=tweet_id).exists() - BookmarkImportItemSerializer().create(raw) - if existed: - updated += 1 - else: + new_hash = compute_bookmark_content_hash(raw) + existed = Bookmark.objects.filter(tweet_id=tweet_id).first() + if existed is None: + BookmarkImportItemSerializer().create(raw) created += 1 - return {'created': created, 'updated': updated, 'skipped': skipped} + elif existed.content_hash == new_hash: + # Content unchanged — only bump import timestamp. + existed.last_imported_at = timezone.now() + existed.save(update_fields=['last_imported_at']) + unchanged += 1 + else: + BookmarkImportItemSerializer().create(raw) + updated += 1 + return {'created': created, 'updated': updated, + 'unchanged': unchanged, 'skipped': skipped} diff --git a/tests/test_bookmarks_api.py b/tests/test_bookmarks_api.py index 14cbe2e..f68c85f 100644 --- a/tests/test_bookmarks_api.py +++ b/tests/test_bookmarks_api.py @@ -80,7 +80,7 @@ class TestBookmarksAPI: format="json") assert r.status_code == 200, r.content assert r.json()["created"] == 1 - # second import updates, not duplicates + # second import with changed content → update r2 = api_client.post(f"{self.BASE}/import", {"bookmarks": [ {**SAMPLE_EXPORT, "text": "updated"}]}, @@ -91,6 +91,41 @@ class TestBookmarksAPI: assert Bookmark.objects.count() == 1 assert Bookmark.objects.first().text == "updated" + def test_bulk_import_unchanged_skips_save(self, api_client): + """Re-importing identical content should count as 'unchanged', not 'updated'.""" + # first import + r1 = api_client.post(f"{self.BASE}/import", + {"bookmarks": [SAMPLE_EXPORT]}, + format="json") + assert r1.json()["created"] == 1 + # second import with identical content + r2 = api_client.post(f"{self.BASE}/import", + {"bookmarks": [SAMPLE_EXPORT]}, + format="json") + assert r2.status_code == 200, r2.content + assert r2.json()["unchanged"] == 1 + assert r2.json()["updated"] == 0 + assert r2.json()["created"] == 0 + from links.models import Bookmark + bm = Bookmark.objects.first() + # last_imported_at should be set even when unchanged + assert bm.last_imported_at is not None + # content_hash should be populated + assert bm.content_hash != '' + + def test_bulk_upsert_array_unchanged(self, api_client): + """Bulk array endpoint should also detect unchanged content.""" + r1 = api_client.post(f"{self.BASE}/bulk", + [{"tweet_id": "ub1", "text": "same"}], + format="json") + assert r1.json()["created"] == 1 + # re-import identical + r2 = api_client.post(f"{self.BASE}/bulk", + [{"tweet_id": "ub1", "text": "same"}], + format="json") + assert r2.json()["unchanged"] == 1 + assert r2.json()["updated"] == 0 + def test_bulk_upsert_array(self, api_client): r = api_client.post(f"{self.BASE}/bulk", [{"tweet_id": "b1", "text": "one"}, @@ -113,4 +148,4 @@ class TestBookmarksAPI: s = r.json() assert s["total"] == 2 assert s["has_video"] == 1 - assert s["has_photo"] == 1 \ No newline at end of file + assert s["has_photo"] == 1