diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 7ffae66..ffedfdd 100644 Binary files a/data/db.sqlite3 and b/data/db.sqlite3 differ diff --git a/links/knowledge_graph_views.py b/links/knowledge_graph_views.py index f305e5f..b873744 100644 --- a/links/knowledge_graph_views.py +++ b/links/knowledge_graph_views.py @@ -3,9 +3,11 @@ Views for the Knowledge Graph feature. """ import json import logging +from datetime import timedelta from django.http import JsonResponse from django.shortcuts import render +from django.utils import timezone from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.csrf import csrf_exempt @@ -14,6 +16,9 @@ from .models import KnowledgeGraphSnapshot, SiteSettings logger = logging.getLogger(__name__) +# A BUILDING snapshot with no progress after this many minutes is considered stuck +STALE_BUILD_MINUTES = 5 + class KnowledgeGraphPageView(View): template_name = "links/knowledge_graph.html" @@ -46,6 +51,20 @@ class KnowledgeGraphBuildView(View): except json.JSONDecodeError: body = {} + # Cancel any stuck BUILDING snapshots before starting a new one + stale_cutoff = timezone.now() - timedelta(minutes=STALE_BUILD_MINUTES) + stuck = KnowledgeGraphSnapshot.objects.filter( + status=KnowledgeGraphSnapshot.Status.BUILDING, + created_at__lt=stale_cutoff, + ) + if stuck.exists(): + stuck.update( + status=KnowledgeGraphSnapshot.Status.FAILED, + error_message="Build was cancelled (process was killed or server restarted)", + completed_at=timezone.now(), + ) + logger.info("Marked %d stale BUILDING snapshot(s) as FAILED", stuck.count()) + ss = SiteSettings.get() use_llm_requested = body.get("use_llm", False) # Only use LLM if a provider is configured @@ -78,14 +97,30 @@ class KnowledgeGraphStatusView(View): snapshot = KnowledgeGraphSnapshot.objects.first() if not snapshot: return JsonResponse({"status": "none"}) + + # Auto-detect stale BUILDING snapshots (e.g. server restarted, thread was killed) + status = snapshot.status + if status == KnowledgeGraphSnapshot.Status.BUILDING: + progress = snapshot.progress_data or {} + age_s = (timezone.now() - snapshot.created_at).total_seconds() + last_pct = progress.get("pct", 0) + # Mark stuck if: no progress at all after 5 min, or no change for 10 min + if age_s > STALE_BUILD_MINUTES * 60 and last_pct == 0: + snapshot.status = KnowledgeGraphSnapshot.Status.FAILED + snapshot.error_message = "Build timed out — the worker thread was likely killed (server restart). Click Build Graph to try again." + snapshot.completed_at = timezone.now() + snapshot.save(update_fields=["status", "error_message", "completed_at"]) + status = KnowledgeGraphSnapshot.Status.FAILED + return JsonResponse({ "snapshot_id": snapshot.pk, - "status": snapshot.status, + "status": status, "node_count": snapshot.node_count, "edge_count": snapshot.edge_count, "used_llm": snapshot.used_llm, "build_duration_ms": snapshot.build_duration_ms, "error_message": snapshot.error_message, + "progress_data": snapshot.progress_data or {}, "created_at": snapshot.created_at.isoformat() if snapshot.created_at else None, "completed_at": snapshot.completed_at.isoformat() if snapshot.completed_at else None, }) diff --git a/links/migrations/0046_kg_progress_data.py b/links/migrations/0046_kg_progress_data.py new file mode 100644 index 0000000..cca3948 --- /dev/null +++ b/links/migrations/0046_kg_progress_data.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.12 on 2026-03-30 07:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0045_knowledge_graph'), + ] + + operations = [ + migrations.AddField( + model_name='knowledgegraphsnapshot', + name='progress_data', + field=models.JSONField(blank=True, default=dict, help_text='Live build progress: {pct, step, logs}', verbose_name='Progress Data'), + ), + ] diff --git a/links/models.py b/links/models.py index 8e0594d..4d8b632 100644 --- a/links/models.py +++ b/links/models.py @@ -550,6 +550,12 @@ class KnowledgeGraphSnapshot(models.Model): used_llm = models.BooleanField(_('Used LLM'), default=False) error_message = models.TextField(_('Error Message'), blank=True) build_duration_ms = models.IntegerField(_('Build Duration (ms)'), default=0) + progress_data = models.JSONField( + _('Progress Data'), + default=dict, + blank=True, + help_text=_('Live build progress: {pct, step, logs}'), + ) created_at = models.DateTimeField(_('Created At'), auto_now_add=True) completed_at = models.DateTimeField(_('Completed At'), null=True, blank=True) diff --git a/links/tasks.py b/links/tasks.py index 2879a98..95b625e 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -666,13 +666,29 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) snapshot.status = KnowledgeGraphSnapshot.Status.BUILDING snapshot.used_llm = use_llm - snapshot.save(update_fields=["status", "used_llm"]) + snapshot.progress_data = {"pct": 0, "step": "Starting…", "logs": []} + snapshot.save(update_fields=["status", "used_llm", "progress_data"]) try: ss = SiteSettings.get() nodes = [] edges = [] edge_key_counter = [0] + _log_entries: list[dict] = [] + + def _elapsed_s() -> float: + return round((time.time() * 1000 - start_ms) / 1000, 1) + + def emit(pct: int, msg: str) -> None: + """Save a progress update to the snapshot so the UI can poll it.""" + _log_entries.append({"elapsed_s": _elapsed_s(), "msg": msg}) + snapshot.progress_data = { + "pct": pct, + "step": msg, + "logs": list(_log_entries), + } + snapshot.save(update_fields=["progress_data"]) + logger.info(f"build_knowledge_graph [{pct}%]: {msg}") def next_edge_key(): edge_key_counter[0] += 1 @@ -681,10 +697,13 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) def rand_pos(): return round(random.uniform(0, 100), 2) + emit(2, "Build started") + # ── Collect nodes ──────────────────────────────────────────────── # Links if ss.kg_include_links: - for link in Link.objects.prefetch_related("tags").all(): + link_qs = list(Link.objects.prefetch_related("tags").all()) + for link in link_qs: size = max(6, min(20, 6 + link.click_count // 5)) nodes.append({ "key": f"link-{link.id}", @@ -702,10 +721,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) "y": rand_pos(), }, }) + emit(15, f"Collected {len(link_qs)} links") # Pages if ss.kg_include_pages: - for page in Page.objects.prefetch_related("tags").all(): + page_qs = list(Page.objects.prefetch_related("tags").all()) + for page in page_qs: nodes.append({ "key": f"page-{page.id}", "attributes": { @@ -722,10 +743,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) "y": rand_pos(), }, }) + emit(28, f"Collected {len(page_qs)} pages") # Posts if ss.kg_include_posts: - for post in Post.objects.prefetch_related("tags").all(): + post_qs = list(Post.objects.prefetch_related("tags").all()) + for post in post_qs: nodes.append({ "key": f"post-{post.id}", "attributes": { @@ -742,10 +765,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) "y": rand_pos(), }, }) + emit(38, f"Collected {len(post_qs)} posts") # Tags if ss.kg_include_tags: - for tag in Tag.objects.all(): + tag_qs = list(Tag.objects.all()) + for tag in tag_qs: nodes.append({ "key": f"tag-{tag.id}", "attributes": { @@ -761,11 +786,13 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) "y": rand_pos(), }, }) + emit(45, f"Collected {len(tag_qs)} tags — {len(nodes)} total nodes") # Build a set of existing node keys for fast membership checks node_keys = {n["key"] for n in nodes} # ── Structural edges ───────────────────────────────────────────── + emit(48, "Building tag edges…") for node in nodes: ntype = node["attributes"]["node_type"] nkey = node["key"] @@ -788,7 +815,10 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) if ntype == "tag": pass # Tag has no parent field; hierarchy edges skipped + emit(55, f"Built {len(edges)} tag edges") + # ── Domain edges ───────────────────────────────────────────────── + emit(57, "Building domain edges…") domain_map: dict[str, list[str]] = {} for node in nodes: url = node["attributes"].get("original_url", "") @@ -823,17 +853,21 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) if domain_edge_count >= 100: break + emit(65, f"Built {domain_edge_count} domain edges across {len(domain_map)} domains") + # ── Semantic edges (LLM) ───────────────────────────────────────── if use_llm and ss.llm_provider != "none": from .llm_client import LLMClient, LLMClientError, cosine_similarity client = LLMClient.from_settings() threshold = ss.kg_semantic_threshold - # Build text corpus and compute embeddings embed_nodes = [n for n in nodes if n["attributes"]["node_type"] != "tag"] - embeddings: dict[str, list[float]] = {} + total_embed = len(embed_nodes) + emit(67, f"Computing embeddings for {total_embed} nodes via {ss.llm_provider}…") - for node in embed_nodes: + embeddings: dict[str, list[float]] = {} + emit_every = max(1, total_embed // 10) # emit ~10 progress steps + for idx, node in enumerate(embed_nodes): text_parts = [node["attributes"].get("label", "")] desc = node["attributes"].get("description", "") if desc: @@ -843,8 +877,12 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) embeddings[node["key"]] = client.get_embedding(text) except LLMClientError as exc: logger.warning(f"build_knowledge_graph: embedding failed for {node['key']}: {exc}") + if (idx + 1) % emit_every == 0 or (idx + 1) == total_embed: + pct = 67 + int(18 * (idx + 1) / total_embed) + emit(pct, f"Embedded {idx + 1}/{total_embed} nodes…") - # Compare all pairs + emit(85, f"Got {len(embeddings)} embeddings — computing pairwise similarity…") + semantic_count = 0 keys_with_embeds = list(embeddings.keys()) for i in range(len(keys_with_embeds)): for j in range(i + 1, len(keys_with_embeds)): @@ -862,8 +900,14 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) "similarity": round(sim, 3), }, }) + semantic_count += 1 + + emit(90, f"Built {semantic_count} semantic edges (threshold={threshold})") + else: + emit(65, "Skipping LLM semantic edges (no provider configured)") # ── Save snapshot ───────────────────────────────────────────────── + emit(92, f"Saving snapshot — {len(nodes)} nodes, {len(edges)} edges…") graph_data = { "attributes": {"title": "Knowledge Graph"}, "nodes": nodes, @@ -871,12 +915,18 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) } elapsed_ms = int(time.time() * 1000) - start_ms + _log_entries.append({"elapsed_s": _elapsed_s(), "msg": f"Done in {elapsed_ms / 1000:.1f}s"}) snapshot.graph_data = graph_data snapshot.node_count = len(nodes) snapshot.edge_count = len(edges) snapshot.status = KnowledgeGraphSnapshot.Status.READY snapshot.build_duration_ms = elapsed_ms snapshot.completed_at = tz.now() + snapshot.progress_data = { + "pct": 100, + "step": f"Done — {len(nodes)} nodes, {len(edges)} edges in {elapsed_ms / 1000:.1f}s", + "logs": list(_log_entries), + } snapshot.save() logger.info( f"build_knowledge_graph: done — {len(nodes)} nodes, {len(edges)} edges " @@ -888,7 +938,7 @@ def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False) snapshot.status = KnowledgeGraphSnapshot.Status.FAILED snapshot.error_message = str(exc) snapshot.completed_at = tz.now() - snapshot.save(update_fields=["status", "error_message", "completed_at"]) + snapshot.save(update_fields=["status", "error_message", "completed_at", "progress_data"]) def schedule_kg_build(): diff --git a/links/templates/links/knowledge_graph.html b/links/templates/links/knowledge_graph.html index 753cd1e..b925cc0 100644 --- a/links/templates/links/knowledge_graph.html +++ b/links/templates/links/knowledge_graph.html @@ -182,15 +182,64 @@ - +
- - - - -

{% trans "Building graph…" %}

+ class="absolute inset-0 flex items-center justify-center p-6"> +
+ + +
+ + + + + {% trans "Building Knowledge Graph" %} +
+ + +
+
+ + +
+
+
+
+
+ + +
+ ⏱ {% trans "Elapsed" %}: + + {% trans "Est. remaining" %}: + +
+ + +
+ +
+ +
{% trans "No log entries yet…" %}
+
+
+ +
@@ -254,6 +303,13 @@ function kgApp() { connectedNodes: [], statusText: '{% trans "No graph" %}', pollTimer: null, + progress: { pct: 0, step: '', logs: [] }, + buildLogs: [], + logsExpanded: false, + elapsedSecs: 0, + estRemaining: 0, + _buildStartMs: null, + _elapsedTimer: null, filters: [ { type: 'link', label: '{% trans "Links" %}', color: '#4B9CD3', active: true, count: 0 }, { type: 'page', label: '{% trans "Pages" %}', color: '#22C55E', active: true, count: 0 }, @@ -435,6 +491,12 @@ function kgApp() { async startBuild() { this.showBuildModal = false; this.building = true; + this.progress = { pct: 0, step: '{% trans "Starting…" %}', logs: [] }; + this.buildLogs = []; + this.elapsedSecs = 0; + this.estRemaining = 0; + this._buildStartMs = Date.now(); + this._startElapsedTimer(this._buildStartMs); this.statusText = '{% trans "Building…" %}'; try { const resp = await fetch('/api/knowledge-graph/build/', { @@ -447,10 +509,29 @@ function kgApp() { } } catch(e) { this.building = false; + clearInterval(this._elapsedTimer); this.statusText = '{% trans "Build failed" %}'; } }, + // ── progress helpers ─────────────────────────────────────────────── + fmtSecs(s) { + if (s < 60) return Math.round(s) + 's'; + return Math.floor(s / 60) + 'm ' + (Math.round(s) % 60) + 's'; + }, + + _startElapsedTimer(startMs) { + this._buildStartMs = startMs; + clearInterval(this._elapsedTimer); + this._elapsedTimer = setInterval(() => { + this.elapsedSecs = (Date.now() - this._buildStartMs) / 1000; + const pct = this.progress.pct || 0; + if (pct > 5 && pct < 100) { + this.estRemaining = (this.elapsedSecs / pct) * (100 - pct); + } + }, 500); + }, + // ── status polling ───────────────────────────────────────────────── pollStatus() { clearInterval(this.pollTimer); @@ -458,12 +539,32 @@ function kgApp() { try { const resp = await fetch('/api/knowledge-graph/status/'); const data = await resp.json(); + + // Update live progress + if (data.progress_data && data.progress_data.pct !== undefined) { + this.progress = data.progress_data; + this.buildLogs = data.progress_data.logs || []; + if (this.logsExpanded && this.buildLogs.length) { + this.$nextTick(() => { + const el = document.querySelector('.font-mono.overflow-y-auto'); + if (el) el.scrollTop = el.scrollHeight; + }); + } + } + + // Start elapsed timer from server-reported created_at + if (data.status === 'building' && data.created_at && !this._buildStartMs) { + this._startElapsedTimer(new Date(data.created_at).getTime()); + } + if (data.status === 'ready') { clearInterval(this.pollTimer); + clearInterval(this._elapsedTimer); this.building = false; await this.loadGraph(); } else if (data.status === 'failed') { clearInterval(this.pollTimer); + clearInterval(this._elapsedTimer); this.building = false; this.statusText = '{% trans "Build failed" %}: ' + (data.error_message || '?'); } else if (data.status === 'building') { diff --git a/templates/base.html b/templates/base.html index 25499d7..a0823c5 100644 --- a/templates/base.html +++ b/templates/base.html @@ -178,20 +178,12 @@ - -
- - - - {% trans "Settings" %} -
-
+ d="M4 6a2 2 0 100-4 2 2 0 000 4zm16 0a2 2 0 100-4 2 2 0 000 4zM4 20a2 2 0 100-4 2 2 0 000 4zm16 0a2 2 0 100-4 2 2 0 000 4zm-8-8a2 2 0 100-4 2 2 0 000 4zM8.5 8.5l-3 3m13-3l-3 3m-7 0l3 3m1 0l3-3"/> {% trans "Knowledge Graph" %}
@@ -201,11 +193,19 @@
+ d="M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2m-3 7h3m-3 4h3m-6-4h.01M9 16h.01"/> {% trans "Jobs" %}
+ +
+ + + + {% trans "Settings" %} +
+