Update links

README.md

@@ -88,6 +88,80 @@ The app will be available at **http://localhost:8000**
 2. Screenshots are captured automatically in the background
 3. View all bookmarks at `/ui/pages/`
 
+### X (Twitter) Bookmarks
+
+The app includes an offline archive of X.com bookmarks. Export your bookmarks from X, then import them for full-text search, filtering, and browsing — no further API calls needed.
+
+#### 1. Export bookmarks from X
+
+Use [x-bookmarks-exporter](https://github.com/nickcam/x-bookmarks-exporter) to download your X bookmarks as JSON files:
+
+```bash
+# Install
+git clone https://github.com/nickcam/x-bookmarks-exporter.git
+cd x-bookmarks-exporter
+npm install
+
+# Configure — copy .env.example to .env and fill in your X credentials
+cp .env.example .env
+
+# Export (produces individual <tweet_id>.json files in data/bookmarks/)
+npm run export
+```
+
+The exporter produces either:
+- A directory of `<tweet_id>.json` files (one per bookmark), or
+- A `bookmarks_index.json` file containing all bookmarks in a `{"bookmarks": [...]}` wrapper.
+
+Both formats are supported.
+
+#### 2. Import into URL Manager
+
+**Via management command:**
+
+```bash
+# Import a directory of JSON files
+just import-bookmarks data/bookmarks/
+
+# Import an index file
+just import-bookmarks data/bookmarks_index.json
+
+# Import a single file
+just import-bookmarks data/single_bookmark.json
+
+# Wipe existing bookmarks first, then re-import
+just import-bookmarks data/bookmarks/ --clear
+```
+
+The command reports `created`, `updated`, `unchanged`, and `skipped` counts. Re-importing the same data is safe — unchanged items are detected via content hash and skipped.
+
+**Via REST API:**
+
+```bash
+# Bulk import (export-format payload)
+curl -X POST http://localhost:8000/api/bookmarks/import/ \
+  -H "Content-Type: application/json" \
+  -d '{"bookmarks": [{...}, {...}]}'
+
+# Bulk upsert (flat array, either format)
+curl -X POST http://localhost:8000/api/bookmarks/bulk/ \
+  -H "Content-Type: application/json" \
+  -d '[{"tweet_id": "123", "text": "hello"}, ...]'
+```
+
+#### 3. Browse imported bookmarks
+
+- **Web UI**: visit `/ui/bookmarks/` — filter by media type, search by text/author, view full details
+- **API**: `GET /api/bookmarks/` with filters like `?q=search&has_video=true&author=screen_name`
+- **API docs**: `/ui/api-docs/` (Bookmarks section)
+
+#### Dedup behaviour
+
+Each bookmark is keyed by `tweet_id`. On re-import:
+- **New tweet_id** → created
+- **Existing tweet_id, content changed** → updated (engagement counts, media, etc. refreshed)
+- **Existing tweet_id, content identical** → `unchanged` (only `last_imported_at` bumped; no save/signal overhead)
+
 ### Network Scanner (NetScan)
 
 Available at `/ui/netscan/`. Create a scan profile to automatically monitor your home network for security issues (router exposure, DNS, TLS certs, camera access).

justfile

@@ -64,6 +64,10 @@ migrate:
 makemigrations:
     source .venv/bin/activate && uv run manage.py makemigrations
 
+# Import X (Twitter) bookmarks from x-bookmarks-exporter JSON files
+import-bookmarks source:
+    source .venv/bin/activate && uv run manage.py import_x_bookmarks {{ source }}
+
 # Open Django shell
 shell:
     source .venv/bin/activate && uv run manage.py shell

links/bookmark_views.py

@@ -3,6 +3,7 @@ from collections import OrderedDict
 
 from django.views.generic import ListView, DetailView
 from django.db import models
+from django.utils import timezone
 from rest_framework import viewsets, status, filters as drf_filters
 from rest_framework.decorators import action
 from rest_framework.response import Response
@@ -12,6 +13,7 @@ from .models import Bookmark
 from .serializers import (
     BookmarkSerializer,
     BookmarkImportSerializer,
+    compute_bookmark_content_hash,
 )
 
 
@@ -90,23 +92,31 @@ class BookmarkViewSet(viewsets.ModelViewSet):
         if not isinstance(items, list):
             return Response({'detail': 'expected a JSON array'},
                             status=status.HTTP_400_BAD_REQUEST)
-        created = updated = skipped = 0
+        created = updated = unchanged = skipped = 0
         for raw in items:
             tid = str(raw.get('tweet_id') or raw.get('id') or '')
             if not tid:
                 skipped += 1
                 continue
+            payload = _to_db_payload(raw)
+            new_hash = payload.get('content_hash', '')
             obj = Bookmark.objects.filter(tweet_id=tid).first()
             if obj is None:
-                BookmarkSerializer().create(_to_db_payload(raw))
+                BookmarkSerializer().create(payload)
                 created += 1
+            elif obj.content_hash == new_hash:
+                # Content unchanged — only bump import timestamp.
+                obj.last_imported_at = timezone.now()
+                obj.save(update_fields=['last_imported_at'])
+                unchanged += 1
             else:
-                ser = BookmarkSerializer(obj, data=_to_db_payload(raw), partial=True)
+                ser = BookmarkSerializer(obj, data=payload, partial=True)
                 ser.is_valid(raise_exception=True)
                 ser.save()
                 updated += 1
         return Response({'created': created, 'updated': updated,
-                         'skipped': skipped, 'total': Bookmark.objects.count()},
+                         'unchanged': unchanged, 'skipped': skipped,
+                         'total': Bookmark.objects.count()},
                         status=status.HTTP_200_OK)
 
     @action(detail=False, methods=['get'], url_path='stats')
@@ -168,9 +178,44 @@ def _to_db_payload(raw):
             'card_data': raw.get('card'),
             'quoted_data': raw.get('quoted'),
             'data': raw,
+            'content_hash': compute_bookmark_content_hash(raw),
+            'last_imported_at': timezone.now(),
         }
-    # Already flat/serializer-shaped.
+    # Already flat/serializer-shaped — still stamp content_hash + last_imported_at
-    return raw
+    # so the dedup comparison in bulk_upsert works for both formats.
+    out = dict(raw)
+    if 'content_hash' not in out or not out['content_hash']:
+        out['content_hash'] = _compute_flat_content_hash(raw)
+    if 'last_imported_at' not in out or not out.get('last_imported_at'):
+        out['last_imported_at'] = timezone.now()
+    return out
+
+
+def _compute_flat_content_hash(flat: dict) -> str:
+    """Compute a SHA-256 hash from a flat/normalized bookmark dict for dedup."""
+    import hashlib, json
+    content = {
+        'text': flat.get('text', ''),
+        'url': flat.get('url', ''),
+        'author_screen_name': flat.get('author_screen_name', ''),
+        'author_name': flat.get('author_name', ''),
+        'media': flat.get('media', []),
+        'card_data': flat.get('card_data'),
+        'quoted_data': flat.get('quoted_data'),
+        'has_video': flat.get('has_video', False),
+        'has_photo': flat.get('has_photo', False),
+        'has_card': flat.get('has_card', False),
+        'is_quote': flat.get('is_quote', False),
+        'is_retweet': flat.get('is_retweet', False),
+        'view_count': flat.get('view_count', 0),
+        'favorite_count': flat.get('favorite_count', 0),
+        'retweet_count': flat.get('retweet_count', 0),
+        'reply_count': flat.get('reply_count', 0),
+        'bookmark_count': flat.get('bookmark_count', 0),
+        'quote_count': flat.get('quote_count', 0),
+    }
+    canonical = json.dumps(content, sort_keys=True, ensure_ascii=False, default=str)
+    return hashlib.sha256(canonical.encode()).hexdigest()
 
 
 # ---------- UI views ----------

links/management/commands/import_x_bookmarks.py

@@ -10,8 +10,9 @@ import os
 from pathlib import Path
 
 from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
 from links.models import Bookmark
-from links.serializers import _norm_bookmark_for_import
+from links.serializers import _norm_bookmark_for_import, compute_bookmark_content_hash
 
 
 class Command(BaseCommand):
@@ -44,7 +45,7 @@ class Command(BaseCommand):
             Bookmark.objects.all().delete()
             self.stdout.write(f'cleared {n} existing bookmarks')
 
-        created = updated = skipped = 0
+        created = updated = unchanged = skipped = 0
         for fp in files:
             try:
                 with open(fp, 'r', encoding='utf-8') as f:
@@ -76,7 +77,14 @@ class Command(BaseCommand):
                 if not tid:
                     skipped += 1
                     continue
+                new_hash = compute_bookmark_content_hash(raw)
                 obj = Bookmark.objects.filter(tweet_id=tid).first()
+                if obj is not None and obj.content_hash == new_hash:
+                    # Content unchanged — only bump import timestamp.
+                    obj.last_imported_at = timezone.now()
+                    obj.save(update_fields=['last_imported_at'])
+                    unchanged += 1
+                    continue
                 try:
                     b = _norm_bookmark_for_import(raw, obj)
                     b.save()
@@ -91,4 +99,5 @@ class Command(BaseCommand):
 
         self.stdout.write(self.style.SUCCESS(
             f'import done: created={created} updated={updated} '
-            f'skipped={skipped} total_in_db={Bookmark.objects.count()}'))
+            f'unchanged={unchanged} skipped={skipped} '
+            f'total_in_db={Bookmark.objects.count()}'))

links/migrations/0051_add_bookmark_import_dedup.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.16 on 2026-07-30 05:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('links', '0050_bookmark'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='bookmark',
+            name='content_hash',
+            field=models.CharField(blank=True, default='', help_text='SHA-256 hash of the import payload; used to skip unchanged re-imports', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='bookmark',
+            name='last_imported_at',
+            field=models.DateTimeField(blank=True, help_text='When this bookmark was last imported/refreshed from an external source', null=True),
+        ),
+    ]

links/models.py

@@ -315,6 +315,14 @@ class Bookmark(models.Model):
     # the complete normalised export object (for detail view / round-tripping)
     data = models.JSONField(default=dict, blank=True)
 
+    # import dedup tracking
+    content_hash = models.CharField(
+        max_length=64, blank=True, default='',
+        help_text='SHA-256 hash of the import payload; used to skip unchanged re-imports')
+    last_imported_at = models.DateTimeField(
+        null=True, blank=True,
+        help_text='When this bookmark was last imported/refreshed from an external source')
+
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
 

links/serializers.py

@@ -1,5 +1,33 @@
 from rest_framework import serializers
 from .models import Link, Page, Post, ImageCollection, Image, Tag, FileUpload, Bookmark
+import hashlib, json
+from datetime import datetime
+from django.utils import timezone
+
+
+def compute_bookmark_content_hash(raw: dict) -> str:
+    """Compute a stable SHA-256 hash of the key content fields from a raw
+    import payload. Two payloads with the same meaningful content produce the
+    same hash, even if JSON key ordering differs."""
+    # Only hash the fields that represent actual content (not metadata like
+    # timestamps that may drift between exports).
+    content = {
+        'text': raw.get('text', ''),
+        'url': raw.get('url', ''),
+        'media': raw.get('media', []),
+        'card': raw.get('card'),
+        'quoted': raw.get('quoted'),
+        'retweet': raw.get('retweet'),
+        'author': raw.get('author', {}),
+        'view_count': raw.get('view_count', 0),
+        'favorite_count': raw.get('favorite_count', 0),
+        'retweet_count': raw.get('retweet_count', 0),
+        'reply_count': raw.get('reply_count', 0),
+        'bookmark_count': raw.get('bookmark_count', 0),
+        'quote_count': raw.get('quote_count', 0),
+    }
+    canonical = json.dumps(content, sort_keys=True, ensure_ascii=False, default=str)
+    return hashlib.sha256(canonical.encode()).hexdigest()
 
 
 class LinkSerializer(serializers.ModelSerializer):
@@ -141,9 +169,11 @@ class BookmarkSerializer(serializers.ModelSerializer):
             'has_video', 'has_photo', 'has_card', 'is_quote', 'is_retweet',
             'media', 'card_data', 'quoted_data',
             'data',
+            'content_hash', 'last_imported_at',
             'created_at', 'updated_at',
         ]
-        read_only_fields = ['id', 'created_at', 'updated_at']
+        read_only_fields = ['id', 'content_hash', 'last_imported_at',
+                           'created_at', 'updated_at']
 
 
 def _norm_bookmark_for_import(raw, obj=None):
@@ -169,7 +199,6 @@ def _norm_bookmark_for_import(raw, obj=None):
 
     # parse tweet created_at ("Wed Jul 30 12:00:00 +0000 2025" style) — leave
     # None if not parseable, view layer handles null.
-    from datetime import datetime
     tca = raw.get('created_at')
     if isinstance(tca, str) and tca:
         try:
@@ -206,6 +235,11 @@ def _norm_bookmark_for_import(raw, obj=None):
     b.card_data = raw.get('card')
     b.quoted_data = raw.get('quoted')
     b.data = raw
+
+    # dedup: record content hash + import timestamp
+    b.content_hash = compute_bookmark_content_hash(raw)
+    b.last_imported_at = timezone.now()
+
     return b
 
 
@@ -217,7 +251,13 @@ class BookmarkImportItemSerializer(serializers.Serializer):
 
     def create(self, validated):
         tweet_id = str(validated.get('id') or validated.get('tweet_id'))
+        new_hash = compute_bookmark_content_hash(validated)
         obj = Bookmark.objects.filter(tweet_id=tweet_id).first()
+        if obj is not None and obj.content_hash == new_hash:
+            # Content unchanged — only bump the import timestamp.
+            obj.last_imported_at = timezone.now()
+            obj.save(update_fields=['last_imported_at'])
+            return obj
         b = _norm_bookmark_for_import(validated, obj)
         b.save()
         return b
@@ -232,16 +272,24 @@ class BookmarkImportSerializer(serializers.Serializer):
 
     def create(self, validated):
         items = validated.get('bookmarks', [])
-        created = updated = skipped = 0
+        created = updated = unchanged = skipped = 0
         for raw in items:
             tweet_id = str(raw.get('id') or raw.get('tweet_id') or '')
             if not tweet_id:
                 skipped += 1
                 continue
-            existed = Bookmark.objects.filter(tweet_id=tweet_id).exists()
+            new_hash = compute_bookmark_content_hash(raw)
-            BookmarkImportItemSerializer().create(raw)
+            existed = Bookmark.objects.filter(tweet_id=tweet_id).first()
-            if existed:
+            if existed is None:
-                updated += 1
+                BookmarkImportItemSerializer().create(raw)
-            else:
                 created += 1
-        return {'created': created, 'updated': updated, 'skipped': skipped}
+            elif existed.content_hash == new_hash:
+                # Content unchanged — only bump import timestamp.
+                existed.last_imported_at = timezone.now()
+                existed.save(update_fields=['last_imported_at'])
+                unchanged += 1
+            else:
+                BookmarkImportItemSerializer().create(raw)
+                updated += 1
+        return {'created': created, 'updated': updated,
+                'unchanged': unchanged, 'skipped': skipped}

tests/test_bookmarks_api.py

@@ -80,7 +80,7 @@ class TestBookmarksAPI:
                             format="json")
         assert r.status_code == 200, r.content
         assert r.json()["created"] == 1
-        # second import updates, not duplicates
+        # second import with changed content → update
         r2 = api_client.post(f"{self.BASE}/import",
                             {"bookmarks": [
                                 {**SAMPLE_EXPORT, "text": "updated"}]},
@@ -91,6 +91,41 @@ class TestBookmarksAPI:
         assert Bookmark.objects.count() == 1
         assert Bookmark.objects.first().text == "updated"
 
+    def test_bulk_import_unchanged_skips_save(self, api_client):
+        """Re-importing identical content should count as 'unchanged', not 'updated'."""
+        # first import
+        r1 = api_client.post(f"{self.BASE}/import",
+                             {"bookmarks": [SAMPLE_EXPORT]},
+                             format="json")
+        assert r1.json()["created"] == 1
+        # second import with identical content
+        r2 = api_client.post(f"{self.BASE}/import",
+                             {"bookmarks": [SAMPLE_EXPORT]},
+                             format="json")
+        assert r2.status_code == 200, r2.content
+        assert r2.json()["unchanged"] == 1
+        assert r2.json()["updated"] == 0
+        assert r2.json()["created"] == 0
+        from links.models import Bookmark
+        bm = Bookmark.objects.first()
+        # last_imported_at should be set even when unchanged
+        assert bm.last_imported_at is not None
+        # content_hash should be populated
+        assert bm.content_hash != ''
+
+    def test_bulk_upsert_array_unchanged(self, api_client):
+        """Bulk array endpoint should also detect unchanged content."""
+        r1 = api_client.post(f"{self.BASE}/bulk",
+                             [{"tweet_id": "ub1", "text": "same"}],
+                             format="json")
+        assert r1.json()["created"] == 1
+        # re-import identical
+        r2 = api_client.post(f"{self.BASE}/bulk",
+                             [{"tweet_id": "ub1", "text": "same"}],
+                             format="json")
+        assert r2.json()["unchanged"] == 1
+        assert r2.json()["updated"] == 0
+
     def test_bulk_upsert_array(self, api_client):
         r = api_client.post(f"{self.BASE}/bulk",
                             [{"tweet_id": "b1", "text": "one"},