Add graph

links/knowledge_graph_views.py

@@ -3,9 +3,11 @@ Views for the Knowledge Graph feature.
 """
 import json
 import logging
+from datetime import timedelta
 
 from django.http import JsonResponse
 from django.shortcuts import render
+from django.utils import timezone
 from django.utils.decorators import method_decorator
 from django.views import View
 from django.views.decorators.csrf import csrf_exempt
@@ -14,6 +16,9 @@ from .models import KnowledgeGraphSnapshot, SiteSettings
 
 logger = logging.getLogger(__name__)
 
+# A BUILDING snapshot with no progress after this many minutes is considered stuck
+STALE_BUILD_MINUTES = 5
+
 
 class KnowledgeGraphPageView(View):
     template_name = "links/knowledge_graph.html"
@@ -46,6 +51,20 @@ class KnowledgeGraphBuildView(View):
         except json.JSONDecodeError:
             body = {}
 
+        # Cancel any stuck BUILDING snapshots before starting a new one
+        stale_cutoff = timezone.now() - timedelta(minutes=STALE_BUILD_MINUTES)
+        stuck = KnowledgeGraphSnapshot.objects.filter(
+            status=KnowledgeGraphSnapshot.Status.BUILDING,
+            created_at__lt=stale_cutoff,
+        )
+        if stuck.exists():
+            stuck.update(
+                status=KnowledgeGraphSnapshot.Status.FAILED,
+                error_message="Build was cancelled (process was killed or server restarted)",
+                completed_at=timezone.now(),
+            )
+            logger.info("Marked %d stale BUILDING snapshot(s) as FAILED", stuck.count())
+
         ss = SiteSettings.get()
         use_llm_requested = body.get("use_llm", False)
         # Only use LLM if a provider is configured
@@ -78,14 +97,30 @@ class KnowledgeGraphStatusView(View):
         snapshot = KnowledgeGraphSnapshot.objects.first()
         if not snapshot:
             return JsonResponse({"status": "none"})
+
+        # Auto-detect stale BUILDING snapshots (e.g. server restarted, thread was killed)
+        status = snapshot.status
+        if status == KnowledgeGraphSnapshot.Status.BUILDING:
+            progress = snapshot.progress_data or {}
+            age_s = (timezone.now() - snapshot.created_at).total_seconds()
+            last_pct = progress.get("pct", 0)
+            # Mark stuck if: no progress at all after 5 min, or no change for 10 min
+            if age_s > STALE_BUILD_MINUTES * 60 and last_pct == 0:
+                snapshot.status = KnowledgeGraphSnapshot.Status.FAILED
+                snapshot.error_message = "Build timed out — the worker thread was likely killed (server restart). Click Build Graph to try again."
+                snapshot.completed_at = timezone.now()
+                snapshot.save(update_fields=["status", "error_message", "completed_at"])
+                status = KnowledgeGraphSnapshot.Status.FAILED
+
         return JsonResponse({
             "snapshot_id": snapshot.pk,
-            "status": snapshot.status,
+            "status": status,
             "node_count": snapshot.node_count,
             "edge_count": snapshot.edge_count,
             "used_llm": snapshot.used_llm,
             "build_duration_ms": snapshot.build_duration_ms,
             "error_message": snapshot.error_message,
+            "progress_data": snapshot.progress_data or {},
             "created_at": snapshot.created_at.isoformat() if snapshot.created_at else None,
             "completed_at": snapshot.completed_at.isoformat() if snapshot.completed_at else None,
         })

links/migrations/0046_kg_progress_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.12 on 2026-03-30 07:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('links', '0045_knowledge_graph'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='knowledgegraphsnapshot',
+            name='progress_data',
+            field=models.JSONField(blank=True, default=dict, help_text='Live build progress: {pct, step, logs}', verbose_name='Progress Data'),
+        ),
+    ]

links/models.py

@@ -550,6 +550,12 @@ class KnowledgeGraphSnapshot(models.Model):
     used_llm = models.BooleanField(_('Used LLM'), default=False)
     error_message = models.TextField(_('Error Message'), blank=True)
     build_duration_ms = models.IntegerField(_('Build Duration (ms)'), default=0)
+    progress_data = models.JSONField(
+        _('Progress Data'),
+        default=dict,
+        blank=True,
+        help_text=_('Live build progress: {pct, step, logs}'),
+    )
     created_at = models.DateTimeField(_('Created At'), auto_now_add=True)
     completed_at = models.DateTimeField(_('Completed At'), null=True, blank=True)
 

links/tasks.py

@@ -666,13 +666,29 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
 
     snapshot.status = KnowledgeGraphSnapshot.Status.BUILDING
     snapshot.used_llm = use_llm
-    snapshot.save(update_fields=["status", "used_llm"])
+    snapshot.progress_data = {"pct": 0, "step": "Starting…", "logs": []}
+    snapshot.save(update_fields=["status", "used_llm", "progress_data"])
 
     try:
         ss = SiteSettings.get()
         nodes = []
         edges = []
         edge_key_counter = [0]
+        _log_entries: list[dict] = []
+
+        def _elapsed_s() -> float:
+            return round((time.time() * 1000 - start_ms) / 1000, 1)
+
+        def emit(pct: int, msg: str) -> None:
+            """Save a progress update to the snapshot so the UI can poll it."""
+            _log_entries.append({"elapsed_s": _elapsed_s(), "msg": msg})
+            snapshot.progress_data = {
+                "pct": pct,
+                "step": msg,
+                "logs": list(_log_entries),
+            }
+            snapshot.save(update_fields=["progress_data"])
+            logger.info(f"build_knowledge_graph [{pct}%]: {msg}")
 
         def next_edge_key():
             edge_key_counter[0] += 1
@@ -681,10 +697,13 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
         def rand_pos():
             return round(random.uniform(0, 100), 2)
 
+        emit(2, "Build started")
+
         # ── Collect nodes ────────────────────────────────────────────────
         # Links
         if ss.kg_include_links:
-            for link in Link.objects.prefetch_related("tags").all():
+            link_qs = list(Link.objects.prefetch_related("tags").all())
+            for link in link_qs:
                 size = max(6, min(20, 6 + link.click_count // 5))
                 nodes.append({
                     "key": f"link-{link.id}",
@@ -702,10 +721,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                         "y": rand_pos(),
                     },
                 })
+            emit(15, f"Collected {len(link_qs)} links")
 
         # Pages
         if ss.kg_include_pages:
-            for page in Page.objects.prefetch_related("tags").all():
+            page_qs = list(Page.objects.prefetch_related("tags").all())
+            for page in page_qs:
                 nodes.append({
                     "key": f"page-{page.id}",
                     "attributes": {
@@ -722,10 +743,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                         "y": rand_pos(),
                     },
                 })
+            emit(28, f"Collected {len(page_qs)} pages")
 
         # Posts
         if ss.kg_include_posts:
-            for post in Post.objects.prefetch_related("tags").all():
+            post_qs = list(Post.objects.prefetch_related("tags").all())
+            for post in post_qs:
                 nodes.append({
                     "key": f"post-{post.id}",
                     "attributes": {
@@ -742,10 +765,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                         "y": rand_pos(),
                     },
                 })
+            emit(38, f"Collected {len(post_qs)} posts")
 
         # Tags
         if ss.kg_include_tags:
-            for tag in Tag.objects.all():
+            tag_qs = list(Tag.objects.all())
+            for tag in tag_qs:
                 nodes.append({
                     "key": f"tag-{tag.id}",
                     "attributes": {
@@ -761,11 +786,13 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                         "y": rand_pos(),
                     },
                 })
+            emit(45, f"Collected {len(tag_qs)} tags — {len(nodes)} total nodes")
 
         # Build a set of existing node keys for fast membership checks
         node_keys = {n["key"] for n in nodes}
 
         # ── Structural edges ─────────────────────────────────────────────
+        emit(48, "Building tag edges…")
         for node in nodes:
             ntype = node["attributes"]["node_type"]
             nkey = node["key"]
@@ -788,7 +815,10 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
             if ntype == "tag":
                 pass  # Tag has no parent field; hierarchy edges skipped
 
+        emit(55, f"Built {len(edges)} tag edges")
+
         # ── Domain edges ─────────────────────────────────────────────────
+        emit(57, "Building domain edges…")
         domain_map: dict[str, list[str]] = {}
         for node in nodes:
             url = node["attributes"].get("original_url", "")
@@ -823,17 +853,21 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                 if domain_edge_count >= 100:
                     break
 
+        emit(65, f"Built {domain_edge_count} domain edges across {len(domain_map)} domains")
+
         # ── Semantic edges (LLM) ─────────────────────────────────────────
         if use_llm and ss.llm_provider != "none":
             from .llm_client import LLMClient, LLMClientError, cosine_similarity
             client = LLMClient.from_settings()
             threshold = ss.kg_semantic_threshold
 
-            # Build text corpus and compute embeddings
             embed_nodes = [n for n in nodes if n["attributes"]["node_type"] != "tag"]
-            embeddings: dict[str, list[float]] = {}
+            total_embed = len(embed_nodes)
+            emit(67, f"Computing embeddings for {total_embed} nodes via {ss.llm_provider}…")
 
-            for node in embed_nodes:
+            embeddings: dict[str, list[float]] = {}
+            emit_every = max(1, total_embed // 10)  # emit ~10 progress steps
+            for idx, node in enumerate(embed_nodes):
                 text_parts = [node["attributes"].get("label", "")]
                 desc = node["attributes"].get("description", "")
                 if desc:
@@ -843,8 +877,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                     embeddings[node["key"]] = client.get_embedding(text)
                 except LLMClientError as exc:
                     logger.warning(f"build_knowledge_graph: embedding failed for {node['key']}: {exc}")
+                if (idx + 1) % emit_every == 0 or (idx + 1) == total_embed:
+                    pct = 67 + int(18 * (idx + 1) / total_embed)
+                    emit(pct, f"Embedded {idx + 1}/{total_embed} nodes…")
 
-            # Compare all pairs
+            emit(85, f"Got {len(embeddings)} embeddings — computing pairwise similarity…")
+            semantic_count = 0
             keys_with_embeds = list(embeddings.keys())
             for i in range(len(keys_with_embeds)):
                 for j in range(i + 1, len(keys_with_embeds)):
@@ -862,8 +900,14 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
                                 "similarity": round(sim, 3),
                             },
                         })
+                        semantic_count += 1
+
+            emit(90, f"Built {semantic_count} semantic edges (threshold={threshold})")
+        else:
+            emit(65, "Skipping LLM semantic edges (no provider configured)")
 
         # ── Save snapshot ─────────────────────────────────────────────────
+        emit(92, f"Saving snapshot — {len(nodes)} nodes, {len(edges)} edges…")
         graph_data = {
             "attributes": {"title": "Knowledge Graph"},
             "nodes": nodes,
@@ -871,12 +915,18 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
         }
         elapsed_ms = int(time.time() * 1000) - start_ms
 
+        _log_entries.append({"elapsed_s": _elapsed_s(), "msg": f"Done in {elapsed_ms / 1000:.1f}s"})
         snapshot.graph_data = graph_data
         snapshot.node_count = len(nodes)
         snapshot.edge_count = len(edges)
         snapshot.status = KnowledgeGraphSnapshot.Status.READY
         snapshot.build_duration_ms = elapsed_ms
         snapshot.completed_at = tz.now()
+        snapshot.progress_data = {
+            "pct": 100,
+            "step": f"Done — {len(nodes)} nodes, {len(edges)} edges in {elapsed_ms / 1000:.1f}s",
+            "logs": list(_log_entries),
+        }
         snapshot.save()
         logger.info(
             f"build_knowledge_graph: done — {len(nodes)} nodes, {len(edges)} edges "
@@ -888,7 +938,7 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False)
         snapshot.status = KnowledgeGraphSnapshot.Status.FAILED
         snapshot.error_message = str(exc)
         snapshot.completed_at = tz.now()
-        snapshot.save(update_fields=["status", "error_message", "completed_at"])
+        snapshot.save(update_fields=["status", "error_message", "completed_at", "progress_data"])
 
 
 def schedule_kg_build():

links/templates/links/knowledge_graph.html

@@ -182,15 +182,64 @@
     </button>
   </div>
 
-  <!-- Building spinner -->
+  <!-- Building progress panel -->
   <div x-show="building"
-       class="absolute inset-0 flex flex-col items-center justify-center gap-3 text-gray-500">
-    <svg class="w-8 h-8 animate-spin text-indigo-500" fill="none" viewBox="0 0 24 24">
-      <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"/>
-      <path class="opacity-75" fill="currentColor"
-        d="M4 12a8 8 0 018-8v8H4z"/>
-    </svg>
-    <p class="text-sm font-medium">{% trans "Building graph…" %}</p>
+       class="absolute inset-0 flex items-center justify-center p-6">
+    <div class="bg-white rounded-xl shadow-lg border border-gray-100 w-full max-w-lg p-6 space-y-4">
+
+      <!-- Header -->
+      <div class="flex items-center gap-3">
+        <svg class="w-5 h-5 animate-spin text-blue-500 flex-shrink-0" fill="none" viewBox="0 0 24 24">
+          <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"/>
+          <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8v8H4z"/>
+        </svg>
+        <span class="font-semibold text-gray-800 text-sm">{% trans "Building Knowledge Graph" %}</span>
+      </div>
+
+      <!-- Progress bar -->
+      <div>
+        <div class="flex justify-between text-xs text-gray-500 mb-1">
+          <span x-text="progress.step || '{% trans "Working…" %}'" class="truncate max-w-xs"></span>
+          <span class="flex-shrink-0 ml-2 font-medium" x-text="(progress.pct || 0) + '%'"></span>
+        </div>
+        <div class="h-2 bg-gray-100 rounded-full overflow-hidden">
+          <div class="h-full bg-blue-500 rounded-full transition-all duration-500"
+               :style="'width:' + (progress.pct || 0) + '%'"></div>
+        </div>
+      </div>
+
+      <!-- Timing row -->
+      <div class="flex gap-4 text-xs text-gray-500">
+        <span>⏱ {% trans "Elapsed" %}: <span class="font-medium text-gray-700" x-text="fmtSecs(elapsedSecs)"></span></span>
+        <span x-show="progress.pct > 5 && progress.pct < 100">
+          {% trans "Est. remaining" %}: <span class="font-medium text-gray-700" x-text="fmtSecs(estRemaining)"></span>
+        </span>
+      </div>
+
+      <!-- Foldable logs -->
+      <div>
+        <button @click="logsExpanded = !logsExpanded"
+          class="flex items-center gap-1.5 text-xs text-gray-400 hover:text-gray-700 transition">
+          <svg class="w-3.5 h-3.5 transition-transform" :class="logsExpanded ? 'rotate-90' : ''"
+               fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5l7 7-7 7"/>
+          </svg>
+          {% trans "Logs" %} (<span x-text="buildLogs.length"></span>)
+        </button>
+        <div x-show="logsExpanded" x-transition
+             class="mt-2 max-h-40 overflow-y-auto rounded-lg bg-gray-900 p-3 space-y-0.5 font-mono">
+          <template x-for="(log, i) in buildLogs" :key="i">
+            <div class="flex gap-2 text-xs leading-5">
+              <span class="text-gray-500 flex-shrink-0"
+                    x-text="'+' + log.elapsed_s.toFixed(1) + 's'"></span>
+              <span class="text-green-400" x-text="log.msg"></span>
+            </div>
+          </template>
+          <div x-show="buildLogs.length === 0" class="text-xs text-gray-500">{% trans "No log entries yet…" %}</div>
+        </div>
+      </div>
+
+    </div>
   </div>
 
   <!-- Build modal -->
@@ -254,6 +303,13 @@ function kgApp() {
     connectedNodes: [],
     statusText: '{% trans "No graph" %}',
     pollTimer: null,
+    progress: { pct: 0, step: '', logs: [] },
+    buildLogs: [],
+    logsExpanded: false,
+    elapsedSecs: 0,
+    estRemaining: 0,
+    _buildStartMs: null,
+    _elapsedTimer: null,
     filters: [
       { type: 'link', label: '{% trans "Links" %}',  color: '#4B9CD3', active: true, count: 0 },
       { type: 'page', label: '{% trans "Pages" %}',  color: '#22C55E', active: true, count: 0 },
@@ -435,6 +491,12 @@ function kgApp() {
     async startBuild() {
       this.showBuildModal = false;
       this.building = true;
+      this.progress = { pct: 0, step: '{% trans "Starting…" %}', logs: [] };
+      this.buildLogs = [];
+      this.elapsedSecs = 0;
+      this.estRemaining = 0;
+      this._buildStartMs = Date.now();
+      this._startElapsedTimer(this._buildStartMs);
       this.statusText = '{% trans "Building…" %}';
       try {
         const resp = await fetch('/api/knowledge-graph/build/', {
@@ -447,10 +509,29 @@ function kgApp() {
         }
       } catch(e) {
         this.building = false;
+        clearInterval(this._elapsedTimer);
         this.statusText = '{% trans "Build failed" %}';
       }
     },
 
+    // ── progress helpers ───────────────────────────────────────────────
+    fmtSecs(s) {
+      if (s < 60) return Math.round(s) + 's';
+      return Math.floor(s / 60) + 'm ' + (Math.round(s) % 60) + 's';
+    },
+
+    _startElapsedTimer(startMs) {
+      this._buildStartMs = startMs;
+      clearInterval(this._elapsedTimer);
+      this._elapsedTimer = setInterval(() => {
+        this.elapsedSecs = (Date.now() - this._buildStartMs) / 1000;
+        const pct = this.progress.pct || 0;
+        if (pct > 5 && pct < 100) {
+          this.estRemaining = (this.elapsedSecs / pct) * (100 - pct);
+        }
+      }, 500);
+    },
+
     // ── status polling ─────────────────────────────────────────────────
     pollStatus() {
       clearInterval(this.pollTimer);
@@ -458,12 +539,32 @@ function kgApp() {
         try {
           const resp = await fetch('/api/knowledge-graph/status/');
           const data = await resp.json();
+
+          // Update live progress
+          if (data.progress_data && data.progress_data.pct !== undefined) {
+            this.progress = data.progress_data;
+            this.buildLogs = data.progress_data.logs || [];
+            if (this.logsExpanded && this.buildLogs.length) {
+              this.$nextTick(() => {
+                const el = document.querySelector('.font-mono.overflow-y-auto');
+                if (el) el.scrollTop = el.scrollHeight;
+              });
+            }
+          }
+
+          // Start elapsed timer from server-reported created_at
+          if (data.status === 'building' && data.created_at && !this._buildStartMs) {
+            this._startElapsedTimer(new Date(data.created_at).getTime());
+          }
+
           if (data.status === 'ready') {
             clearInterval(this.pollTimer);
+            clearInterval(this._elapsedTimer);
             this.building = false;
             await this.loadGraph();
           } else if (data.status === 'failed') {
             clearInterval(this.pollTimer);
+            clearInterval(this._elapsedTimer);
             this.building = false;
             this.statusText = '{% trans "Build failed" %}: ' + (data.error_message || '?');
           } else if (data.status === 'building') {

templates/base.html

@@ -178,20 +178,12 @@
                             </div>
                         </a>
 
-                        <a href="{% url 'site-settings' %}" class="block px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
-                            <div class="flex items-center">
-                                <svg class="w-5 h-5 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
-                                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 6V4m0 2a2 2 0 100 4m0-4a2 2 0 110 4m-6 8a2 2 0 100-4m0 4a2 2 0 110-4m0 4v2m0-6V4m6 6v10m6-2a2 2 0 100-4m0 4a2 2 0 110-4m0 4v2m0-6V4"/>
-                                </svg>
-                                {% trans "Settings" %}
-                            </div>
-                        </a>
 
                         <a href="{% url 'knowledge-graph' %}" class="block px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
                             <div class="flex items-center">
                                 <svg class="w-5 h-5 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
                                     <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                                          d="M4 6a2 2 0 100-4 2 2 0 000 4zm16 0a2 2 0 100-4 2 2 0 000 4zM4 20a2 2 0 100-4 2 2 0 000 4zm16 0a2 2 0 100-4 2 2 0 000 4zm-8-8a2 2 0 100-4 2 2 0 000 4zM8.5 8.5l-3 3m13-3l-3 3m-7 0l3 3m1 0l3-3"/>
+                                    d="M4 6a2 2 0 100-4 2 2 0 000 4zm16 0a2 2 0 100-4 2 2 0 000 4zM4 20a2 2 0 100-4 2 2 0 000 4zm16 0a2 2 0 100-4 2 2 0 000 4zm-8-8a2 2 0 100-4 2 2 0 000 4zM8.5 8.5l-3 3m13-3l-3 3m-7 0l3 3m1 0l3-3"/>
                                 </svg>
                                 {% trans "Knowledge Graph" %}
                             </div>
@@ -201,11 +193,19 @@
                             <div class="flex items-center">
                                 <svg class="w-5 h-5 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
                                     <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                                          d="M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2m-3 7h3m-3 4h3m-6-4h.01M9 16h.01"/>
+                                    d="M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2m-3 7h3m-3 4h3m-6-4h.01M9 16h.01"/>
                                 </svg>
                                 {% trans "Jobs" %}
                             </div>
                         </a>
+                        <a href="{% url 'site-settings' %}" class="block px-4 py-2 text-sm text-gray-700 hover:bg-gray-100">
+                            <div class="flex items-center">
+                                <svg class="w-5 h-5 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 6V4m0 2a2 2 0 100 4m0-4a2 2 0 110 4m-6 8a2 2 0 100-4m0 4a2 2 0 110-4m0 4v2m0-6V4m6 6v10m6-2a2 2 0 100-4m0 4a2 2 0 110-4m0 4v2m0-6V4"/>
+                                </svg>
+                                {% trans "Settings" %}
+                            </div>
+                        </a>
                     </div>
                 </div>
                 <!-- 语言选择器 -->