diff --git a/core/apps.py b/core/apps.py index 46757f1..ea6bcce 100644 --- a/core/apps.py +++ b/core/apps.py @@ -18,7 +18,7 @@ class CoreConfig(AppConfig): from core.scheduler import scheduler, start_scheduler from links.tasks import ( schedule_pending_pages, schedule_pending_screenshots, - retry_stuck_image_imports, flush_click_buffer, schedule_kg_build, + retry_stuck_image_imports, flush_click_buffer, ) from apscheduler.triggers.interval import IntervalTrigger @@ -31,11 +31,9 @@ class CoreConfig(AppConfig): ss = SiteSettings.get() pages_interval = ss.schedule_pending_pages_interval or 120 screenshots_interval = ss.schedule_pending_screenshots_interval or 120 - kg_interval = ss.kg_auto_schedule_interval or 3600 except Exception: pages_interval = 120 screenshots_interval = 120 - kg_interval = 3600 # Add periodic job for checking pending pages scheduler.add_job( @@ -72,12 +70,3 @@ class CoreConfig(AppConfig): replace_existing=True, ) logger.info("Scheduled periodic task: flush_click_buffer (every 60s)") - - # Add periodic job for knowledge graph auto-build - scheduler.add_job( - schedule_kg_build, - trigger=IntervalTrigger(seconds=kg_interval), - id='schedule_kg_build', - replace_existing=True, - ) - logger.info(f"Scheduled periodic task: schedule_kg_build (every {kg_interval}s)") diff --git a/links/apps.py b/links/apps.py index e4581f1..c1bca6b 100644 --- a/links/apps.py +++ b/links/apps.py @@ -181,75 +181,3 @@ class LinksConfig(AppConfig): }, }) - # ── Knowledge Graph Snapshots ──────────────────────────────────── - from .models import KnowledgeGraphSnapshot - - def kg_stats(): - return { - 'total': KnowledgeGraphSnapshot.objects.count(), - 'processing': KnowledgeGraphSnapshot.objects.filter( - status=KnowledgeGraphSnapshot.Status.BUILDING - ).count(), - 'completed': KnowledgeGraphSnapshot.objects.filter( - status=KnowledgeGraphSnapshot.Status.READY - ).count(), - 'failed': KnowledgeGraphSnapshot.objects.filter( - status=KnowledgeGraphSnapshot.Status.FAILED - ).count(), - } - - def kg_queryset(sf): - qs = KnowledgeGraphSnapshot.objects.order_by('-created_at') - if sf == 'processing': - return qs.filter(status=KnowledgeGraphSnapshot.Status.BUILDING) - if sf == 'completed': - return qs.filter(status=KnowledgeGraphSnapshot.Status.READY) - if sf == 'failed': - return qs.filter(status=KnowledgeGraphSnapshot.Status.FAILED) - return qs - - def kg_serialize(obj): - if obj.status == KnowledgeGraphSnapshot.Status.BUILDING: - pct = obj.progress_data.get('pct', 0) if isinstance(obj.progress_data, dict) else 0 - title = f'Building… {pct}%' - else: - title = f'{obj.node_count} nodes / {obj.edge_count} edges' - if obj.used_llm: - title += ' (LLM)' - return { - 'id': str(obj.id), - 'title': title, - 'detail_url': reverse('knowledge-graph'), - 'status': obj.status, - 'retry': None, - 'retry_max': None, - 'error': obj.error_message or '', - 'updated_at': obj.completed_at or obj.created_at, - 'extra': { - 'duration': ( - f'{obj.build_duration_ms // 1000}s' if obj.build_duration_ms else '' - ), - }, - } - - job_registry.register({ - 'id': 'knowledge_graph', - 'label': 'Knowledge Graph', - 'icon_color': 'text-pink-500', - 'icon_path': ( - 'M13.828 10.172a4 4 0 00-5.656 0l-4 4a4 4 0 105.656 5.656l1.102-1.101' - 'm-.758-4.899a4 4 0 005.656 0l4-4a4 4 0 00-5.656-5.656l-1.1 1.1' - ), - 'title_label': 'Snapshot', - 'status_choices': [ - ('all', 'All'), ('processing', 'Building'), ('completed', 'Ready'), - ('failed', 'Failed'), - ], - 'columns': ['id', 'title', 'status', 'error', 'updated'], - 'get_stats': kg_stats, - 'get_queryset': kg_queryset, - 'serialize': kg_serialize, - 'bulk_actions': { - 'delete': 'bulk_delete_knowledge_graph_snapshots', - }, - }) diff --git a/links/knowledge_graph_urls.py b/links/knowledge_graph_urls.py deleted file mode 100644 index 0190944..0000000 --- a/links/knowledge_graph_urls.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.urls import path -from . import knowledge_graph_views - -urlpatterns = [ - path( - "ui/knowledge-graph/", - knowledge_graph_views.KnowledgeGraphPageView.as_view(), - name="knowledge-graph", - ), - path( - "api/knowledge-graph/data/", - knowledge_graph_views.KnowledgeGraphDataView.as_view(), - name="knowledge-graph-data", - ), - path( - "api/knowledge-graph/build/", - knowledge_graph_views.KnowledgeGraphBuildView.as_view(), - name="knowledge-graph-build", - ), - path( - "api/knowledge-graph/status/", - knowledge_graph_views.KnowledgeGraphStatusView.as_view(), - name="knowledge-graph-status", - ), - path( - "api/knowledge-graph/test-llm/", - knowledge_graph_views.LLMTestView.as_view(), - name="knowledge-graph-test-llm", - ), -] diff --git a/links/knowledge_graph_views.py b/links/knowledge_graph_views.py deleted file mode 100644 index b873744..0000000 --- a/links/knowledge_graph_views.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -Views for the Knowledge Graph feature. -""" -import json -import logging -from datetime import timedelta - -from django.http import JsonResponse -from django.shortcuts import render -from django.utils import timezone -from django.utils.decorators import method_decorator -from django.views import View -from django.views.decorators.csrf import csrf_exempt - -from .models import KnowledgeGraphSnapshot, SiteSettings - -logger = logging.getLogger(__name__) - -# A BUILDING snapshot with no progress after this many minutes is considered stuck -STALE_BUILD_MINUTES = 5 - - -class KnowledgeGraphPageView(View): - template_name = "links/knowledge_graph.html" - - def get(self, request): - snapshot = KnowledgeGraphSnapshot.get_latest() - latest_building = KnowledgeGraphSnapshot.objects.filter( - status=KnowledgeGraphSnapshot.Status.BUILDING - ).first() - ss = SiteSettings.get() - return render(request, self.template_name, { - "snapshot": snapshot, - "latest_building": latest_building, - "site_settings": ss, - }) - - -class KnowledgeGraphDataView(View): - def get(self, request): - snapshot = KnowledgeGraphSnapshot.get_latest() - if not snapshot: - return JsonResponse({"error": "No ready snapshot found."}, status=404) - return JsonResponse(snapshot.graph_data, safe=False) - - -class KnowledgeGraphBuildView(View): - def post(self, request): - try: - body = json.loads(request.body or "{}") - except json.JSONDecodeError: - body = {} - - # Cancel any stuck BUILDING snapshots before starting a new one - stale_cutoff = timezone.now() - timedelta(minutes=STALE_BUILD_MINUTES) - stuck = KnowledgeGraphSnapshot.objects.filter( - status=KnowledgeGraphSnapshot.Status.BUILDING, - created_at__lt=stale_cutoff, - ) - if stuck.exists(): - stuck.update( - status=KnowledgeGraphSnapshot.Status.FAILED, - error_message="Build was cancelled (process was killed or server restarted)", - completed_at=timezone.now(), - ) - logger.info("Marked %d stale BUILDING snapshot(s) as FAILED", stuck.count()) - - ss = SiteSettings.get() - use_llm_requested = body.get("use_llm", False) - # Only use LLM if a provider is configured - use_llm = use_llm_requested and ss.llm_provider not in ("none", "") - - snapshot = KnowledgeGraphSnapshot.objects.create( - status=KnowledgeGraphSnapshot.Status.BUILDING, - used_llm=use_llm, - ) - - from threading import Thread - from .tasks import build_knowledge_graph - thread = Thread( - target=build_knowledge_graph, - kwargs={"snapshot_id": snapshot.pk, "use_llm": use_llm}, - daemon=True, - ) - thread.start() - - return JsonResponse({ - "snapshot_id": snapshot.pk, - "status": snapshot.status, - "use_llm": use_llm, - }) - - -class KnowledgeGraphStatusView(View): - def get(self, request): - # Latest snapshot regardless of status (so the UI can poll while building) - snapshot = KnowledgeGraphSnapshot.objects.first() - if not snapshot: - return JsonResponse({"status": "none"}) - - # Auto-detect stale BUILDING snapshots (e.g. server restarted, thread was killed) - status = snapshot.status - if status == KnowledgeGraphSnapshot.Status.BUILDING: - progress = snapshot.progress_data or {} - age_s = (timezone.now() - snapshot.created_at).total_seconds() - last_pct = progress.get("pct", 0) - # Mark stuck if: no progress at all after 5 min, or no change for 10 min - if age_s > STALE_BUILD_MINUTES * 60 and last_pct == 0: - snapshot.status = KnowledgeGraphSnapshot.Status.FAILED - snapshot.error_message = "Build timed out — the worker thread was likely killed (server restart). Click Build Graph to try again." - snapshot.completed_at = timezone.now() - snapshot.save(update_fields=["status", "error_message", "completed_at"]) - status = KnowledgeGraphSnapshot.Status.FAILED - - return JsonResponse({ - "snapshot_id": snapshot.pk, - "status": status, - "node_count": snapshot.node_count, - "edge_count": snapshot.edge_count, - "used_llm": snapshot.used_llm, - "build_duration_ms": snapshot.build_duration_ms, - "error_message": snapshot.error_message, - "progress_data": snapshot.progress_data or {}, - "created_at": snapshot.created_at.isoformat() if snapshot.created_at else None, - "completed_at": snapshot.completed_at.isoformat() if snapshot.completed_at else None, - }) - - -class LLMTestView(View): - def post(self, request): - try: - body = json.loads(request.body or "{}") - except json.JSONDecodeError: - body = {} - - # Accept inline params from the UI test modal, or fall back to SiteSettings - from .llm_client import LLMClient - provider = body.get("provider") or SiteSettings.get().llm_provider - base_url = body.get("base_url") or SiteSettings.get().llm_base_url - model = body.get("model") or SiteSettings.get().llm_model - api_key = body.get("api_key") or SiteSettings.get().llm_api_key - - if provider in ("none", ""): - return JsonResponse({"success": False, "error": "No LLM provider configured."}) - - client = LLMClient(provider=provider, base_url=base_url, model=model, api_key=api_key) - success, error = client.test_connection() - return JsonResponse({"success": success, "error": error}) diff --git a/links/llm_client.py b/links/llm_client.py deleted file mode 100644 index 26f034b..0000000 --- a/links/llm_client.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -LLM / embedding client for the knowledge graph builder. - -Supports: - - Ollama (local, free — recommended for manual builds) - - OpenRouter (cloud, requires API key) -""" -import logging -import math -import requests - -logger = logging.getLogger(__name__) - -EMBED_TIMEOUT = 15 # seconds per request - - -class LLMClientError(Exception): - pass - - -class LLMClient: - """Thin wrapper for embedding generation via Ollama or OpenRouter.""" - - def __init__(self, provider: str, base_url: str, model: str, api_key: str = ""): - self.provider = provider - self.base_url = base_url.rstrip("/") - self.model = model - self.api_key = api_key - - @classmethod - def from_settings(cls) -> "LLMClient": - from links.models import SiteSettings - ss = SiteSettings.get() - return cls( - provider=ss.llm_provider, - base_url=ss.llm_base_url, - model=ss.llm_model, - api_key=ss.llm_api_key, - ) - - def get_embedding(self, text: str) -> list[float]: - """Return a float vector for *text*. Raises LLMClientError on failure.""" - if not text or not text.strip(): - raise LLMClientError("Empty text passed to get_embedding") - - if self.provider == "ollama": - return self._ollama_embed(text) - elif self.provider == "openrouter": - return self._openrouter_embed(text) - else: - raise LLMClientError(f"Unsupported provider: {self.provider}") - - def _ollama_embed(self, text: str) -> list[float]: - url = f"{self.base_url}/api/embeddings" - try: - resp = requests.post( - url, - json={"model": self.model, "prompt": text}, - timeout=EMBED_TIMEOUT, - ) - resp.raise_for_status() - data = resp.json() - vec = data.get("embedding") - if not vec: - raise LLMClientError(f"Ollama returned no embedding. Response: {data}") - return vec - except requests.RequestException as exc: - raise LLMClientError(f"Ollama request failed: {exc}") from exc - - def _openrouter_embed(self, text: str) -> list[float]: - url = "https://openrouter.ai/api/v1/embeddings" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - } - try: - resp = requests.post( - url, - headers=headers, - json={"model": self.model, "input": text}, - timeout=EMBED_TIMEOUT, - ) - resp.raise_for_status() - data = resp.json() - try: - vec = data["data"][0]["embedding"] - except (KeyError, IndexError) as exc: - raise LLMClientError(f"Unexpected OpenRouter response shape: {data}") from exc - return vec - except requests.RequestException as exc: - raise LLMClientError(f"OpenRouter request failed: {exc}") from exc - - def test_connection(self) -> tuple[bool, str]: - """Returns (success: bool, error_message: str).""" - try: - vec = self.get_embedding("ping") - if not vec: - return False, "Got empty embedding vector" - return True, "" - except LLMClientError as exc: - return False, str(exc) - - -def cosine_similarity(a: list[float], b: list[float]) -> float: - """Compute cosine similarity between two vectors.""" - dot = sum(x * y for x, y in zip(a, b)) - mag_a = math.sqrt(sum(x * x for x in a)) - mag_b = math.sqrt(sum(x * x for x in b)) - if mag_a == 0 or mag_b == 0: - return 0.0 - return dot / (mag_a * mag_b) diff --git a/links/migrations/0048_remove_knowledge_graph.py b/links/migrations/0048_remove_knowledge_graph.py new file mode 100644 index 0000000..caf2d13 --- /dev/null +++ b/links/migrations/0048_remove_knowledge_graph.py @@ -0,0 +1,58 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0047_add_is_public_to_post'), + ] + + operations = [ + migrations.DeleteModel( + name='KnowledgeGraphSnapshot', + ), + migrations.RemoveField( + model_name='sitesettings', + name='llm_provider', + ), + migrations.RemoveField( + model_name='sitesettings', + name='llm_base_url', + ), + migrations.RemoveField( + model_name='sitesettings', + name='llm_model', + ), + migrations.RemoveField( + model_name='sitesettings', + name='llm_api_key', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_auto_schedule_enabled', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_auto_schedule_interval', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_semantic_threshold', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_include_links', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_include_pages', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_include_posts', + ), + migrations.RemoveField( + model_name='sitesettings', + name='kg_include_tags', + ), + ] diff --git a/links/models.py b/links/models.py index 36b0537..f2b6a7f 100644 --- a/links/models.py +++ b/links/models.py @@ -417,11 +417,6 @@ class SiteSettings(models.Model): Always use SiteSettings.get() to retrieve the instance. """ - class LLMProvider(models.TextChoices): - NONE = 'none', _('None (no LLM)') - OLLAMA = 'ollama', _('Ollama (local)') - OPENROUTER = 'openrouter', _('OpenRouter') - public_sharing_domain = models.CharField( _('Public Sharing Domain'), max_length=255, @@ -459,57 +454,6 @@ class SiteSettings(models.Model): ), ) - # ── Knowledge Graph: LLM settings ──────────────────────────────────── - llm_provider = models.CharField( - _('LLM Provider'), - max_length=20, - choices=LLMProvider.choices, - default=LLMProvider.NONE, - help_text=_('LLM/embedding provider used to generate semantic edges in the knowledge graph.'), - ) - llm_base_url = models.CharField( - _('LLM Base URL'), - max_length=255, - blank=True, - default='http://localhost:11434', - help_text=_('Base URL for the Ollama server (e.g. http://192.168.1.2:11434).'), - ) - llm_model = models.CharField( - _('LLM Embedding Model'), - max_length=100, - blank=True, - default='nomic-embed-text', - help_text=_('Model name used for embeddings (e.g. nomic-embed-text for Ollama).'), - ) - llm_api_key = models.CharField( - _('LLM API Key'), - max_length=255, - blank=True, - default='', - help_text=_('API key for OpenRouter (not needed for Ollama).'), - ) - - # ── Knowledge Graph: schedule & build options ───────────────────────── - kg_auto_schedule_enabled = models.BooleanField( - _('Auto-rebuild Knowledge Graph'), - default=False, - help_text=_('When enabled, the knowledge graph is rebuilt on the configured interval.'), - ) - kg_auto_schedule_interval = models.IntegerField( - _('Knowledge Graph Rebuild Interval (seconds)'), - default=3600, - help_text=_('How often (in seconds) to auto-rebuild the knowledge graph. Minimum 60.'), - ) - kg_semantic_threshold = models.FloatField( - _('Semantic Similarity Threshold'), - default=0.70, - help_text=_('Minimum cosine similarity (0.0–1.0) required to draw a semantic edge between two items.'), - ) - kg_include_links = models.BooleanField(_('Include Links in Graph'), default=True) - kg_include_pages = models.BooleanField(_('Include Pages in Graph'), default=True) - kg_include_posts = models.BooleanField(_('Include Posts in Graph'), default=True) - kg_include_tags = models.BooleanField(_('Include Tags in Graph'), default=True) - class Meta: verbose_name = _('Site Settings') @@ -524,50 +468,3 @@ class SiteSettings(models.Model): def save(self, *args, **kwargs): self.pk = 1 super().save(*args, **kwargs) - - -class KnowledgeGraphSnapshot(models.Model): - """Stores a point-in-time serialized knowledge graph for immediate front-end consumption.""" - - class Status(models.TextChoices): - BUILDING = 'building', _('Building') - READY = 'ready', _('Ready') - FAILED = 'failed', _('Failed') - - status = models.CharField( - _('Status'), - max_length=20, - choices=Status.choices, - default=Status.BUILDING, - db_index=True, - ) - graph_data = models.JSONField( - _('Graph Data'), - default=dict, - help_text=_('Graphology-compatible serialization: {nodes: [...], edges: [...]}'), - ) - node_count = models.IntegerField(_('Node Count'), default=0) - edge_count = models.IntegerField(_('Edge Count'), default=0) - used_llm = models.BooleanField(_('Used LLM'), default=False) - error_message = models.TextField(_('Error Message'), blank=True) - build_duration_ms = models.IntegerField(_('Build Duration (ms)'), default=0) - progress_data = models.JSONField( - _('Progress Data'), - default=dict, - blank=True, - help_text=_('Live build progress: {pct, step, logs}'), - ) - created_at = models.DateTimeField(_('Created At'), auto_now_add=True) - completed_at = models.DateTimeField(_('Completed At'), null=True, blank=True) - - class Meta: - ordering = ['-created_at'] - verbose_name = _('Knowledge Graph Snapshot') - verbose_name_plural = _('Knowledge Graph Snapshots') - - def __str__(self): - return f"KG Snapshot [{self.status}] {self.node_count} nodes / {self.edge_count} edges @ {self.created_at}" - - @classmethod - def get_latest(cls): - return cls.objects.filter(status=cls.Status.READY).first() diff --git a/links/tasks.py b/links/tasks.py index 95b625e..1bfd6b9 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -622,341 +622,3 @@ def retry_stuck_image_imports(): logger.info(f"retry_stuck_image_imports: re-queuing download for FileUpload {record.pk} ({record.source_url})") thread = Thread(target=download_and_save_image, args=(str(record.pk),), daemon=True) thread.start() - - -# ── Knowledge Graph ─────────────────────────────────────────────────────────── - -def build_knowledge_graph(snapshot_id: int | None = None, use_llm: bool = False): - """ - Build a graphology-compatible knowledge graph from all Links, Pages, Posts and Tags. - - Nodes are coloured by type: - Link → #4B9CD3 (blue) - Page → #22C55E (green) - Post → #F97316 (orange) - Tag → #8B5CF6 (purple) - - Edges (always): - has_tag item → tag (#94A3B8, structural) - same_domain items sharing netloc (#FCD34D, domain) - - Edges (with LLM): - semantic cosine-sim ≥ threshold (#F472B6) - """ - import time - import random - from urllib.parse import urlparse - from django.utils import timezone as tz - from .models import KnowledgeGraphSnapshot, SiteSettings, Link, Page, Post, Tag - - start_ms = int(time.time() * 1000) - - # Resolve / create the snapshot record - if snapshot_id: - try: - snapshot = KnowledgeGraphSnapshot.objects.get(pk=snapshot_id) - except KnowledgeGraphSnapshot.DoesNotExist: - logger.error(f"build_knowledge_graph: snapshot {snapshot_id} not found") - return - else: - snapshot = KnowledgeGraphSnapshot.objects.create( - status=KnowledgeGraphSnapshot.Status.BUILDING, - used_llm=use_llm, - ) - - snapshot.status = KnowledgeGraphSnapshot.Status.BUILDING - snapshot.used_llm = use_llm - snapshot.progress_data = {"pct": 0, "step": "Starting…", "logs": []} - snapshot.save(update_fields=["status", "used_llm", "progress_data"]) - - try: - ss = SiteSettings.get() - nodes = [] - edges = [] - edge_key_counter = [0] - _log_entries: list[dict] = [] - - def _elapsed_s() -> float: - return round((time.time() * 1000 - start_ms) / 1000, 1) - - def emit(pct: int, msg: str) -> None: - """Save a progress update to the snapshot so the UI can poll it.""" - _log_entries.append({"elapsed_s": _elapsed_s(), "msg": msg}) - snapshot.progress_data = { - "pct": pct, - "step": msg, - "logs": list(_log_entries), - } - snapshot.save(update_fields=["progress_data"]) - logger.info(f"build_knowledge_graph [{pct}%]: {msg}") - - def next_edge_key(): - edge_key_counter[0] += 1 - return f"e{edge_key_counter[0]}" - - def rand_pos(): - return round(random.uniform(0, 100), 2) - - emit(2, "Build started") - - # ── Collect nodes ──────────────────────────────────────────────── - # Links - if ss.kg_include_links: - link_qs = list(Link.objects.prefetch_related("tags").all()) - for link in link_qs: - size = max(6, min(20, 6 + link.click_count // 5)) - nodes.append({ - "key": f"link-{link.id}", - "attributes": { - "label": link.alias, - "color": "#4B9CD3", - "size": size, - "node_type": "link", - "item_id": link.id, - "item_url": f"/detail/{link.pk}/", - "original_url": link.original_url or "", - "description": link.description or "", - "tag_ids": [t.id for t in link.tags.all()], - "x": rand_pos(), - "y": rand_pos(), - }, - }) - emit(15, f"Collected {len(link_qs)} links") - - # Pages - if ss.kg_include_pages: - page_qs = list(Page.objects.prefetch_related("tags").all()) - for page in page_qs: - nodes.append({ - "key": f"page-{page.id}", - "attributes": { - "label": (page.title or page.url)[:80], - "color": "#22C55E", - "size": 8, - "node_type": "page", - "item_id": page.id, - "item_url": f"/ui/pages/{page.pk}/", - "original_url": page.url, - "description": page.summary or "", - "tag_ids": [t.id for t in page.tags.all()], - "x": rand_pos(), - "y": rand_pos(), - }, - }) - emit(28, f"Collected {len(page_qs)} pages") - - # Posts - if ss.kg_include_posts: - post_qs = list(Post.objects.prefetch_related("tags").all()) - for post in post_qs: - nodes.append({ - "key": f"post-{post.id}", - "attributes": { - "label": post.title[:80], - "color": "#F97316", - "size": 8, - "node_type": "post", - "item_id": post.id, - "item_url": f"/ui/posts/{post.pk}/", - "original_url": "", - "description": post.summary or "", - "tag_ids": [t.id for t in post.tags.all()], - "x": rand_pos(), - "y": rand_pos(), - }, - }) - emit(38, f"Collected {len(post_qs)} posts") - - # Tags - if ss.kg_include_tags: - tag_qs = list(Tag.objects.all()) - for tag in tag_qs: - nodes.append({ - "key": f"tag-{tag.id}", - "attributes": { - "label": tag.name, - "color": "#8B5CF6", - "size": 14, - "node_type": "tag", - "item_id": tag.id, - "item_url": f"/ui/tags/{tag.slug}/", - "original_url": "", - "description": tag.description or "", - "x": rand_pos(), - "y": rand_pos(), - }, - }) - emit(45, f"Collected {len(tag_qs)} tags — {len(nodes)} total nodes") - - # Build a set of existing node keys for fast membership checks - node_keys = {n["key"] for n in nodes} - - # ── Structural edges ───────────────────────────────────────────── - emit(48, "Building tag edges…") - for node in nodes: - ntype = node["attributes"]["node_type"] - nkey = node["key"] - - if ntype in ("link", "page", "post") and ss.kg_include_tags: - for tag_id in node["attributes"].get("tag_ids", []): - tag_key = f"tag-{tag_id}" - if tag_key in node_keys: - edges.append({ - "key": next_edge_key(), - "source": nkey, - "target": tag_key, - "attributes": { - "edge_type": "has_tag", - "color": "#94A3B8", - "size": 1, - }, - }) - - if ntype == "tag": - pass # Tag has no parent field; hierarchy edges skipped - - emit(55, f"Built {len(edges)} tag edges") - - # ── Domain edges ───────────────────────────────────────────────── - emit(57, "Building domain edges…") - domain_map: dict[str, list[str]] = {} - for node in nodes: - url = node["attributes"].get("original_url", "") - if url: - try: - netloc = urlparse(url).netloc - if netloc: - domain_map.setdefault(netloc, []).append(node["key"]) - except Exception: - pass - - domain_edge_count = 0 - for netloc, keys in domain_map.items(): - if len(keys) < 2: - continue - for i in range(len(keys)): - for j in range(i + 1, len(keys)): - if domain_edge_count >= 100: - break - edges.append({ - "key": next_edge_key(), - "source": keys[i], - "target": keys[j], - "attributes": { - "edge_type": "same_domain", - "color": "#FCD34D", - "size": 0.5, - "domain": netloc, - }, - }) - domain_edge_count += 1 - if domain_edge_count >= 100: - break - - emit(65, f"Built {domain_edge_count} domain edges across {len(domain_map)} domains") - - # ── Semantic edges (LLM) ───────────────────────────────────────── - if use_llm and ss.llm_provider != "none": - from .llm_client import LLMClient, LLMClientError, cosine_similarity - client = LLMClient.from_settings() - threshold = ss.kg_semantic_threshold - - embed_nodes = [n for n in nodes if n["attributes"]["node_type"] != "tag"] - total_embed = len(embed_nodes) - emit(67, f"Computing embeddings for {total_embed} nodes via {ss.llm_provider}…") - - embeddings: dict[str, list[float]] = {} - emit_every = max(1, total_embed // 10) # emit ~10 progress steps - for idx, node in enumerate(embed_nodes): - text_parts = [node["attributes"].get("label", "")] - desc = node["attributes"].get("description", "") - if desc: - text_parts.append(desc[:500]) - text = " ".join(text_parts).strip() - try: - embeddings[node["key"]] = client.get_embedding(text) - except LLMClientError as exc: - logger.warning(f"build_knowledge_graph: embedding failed for {node['key']}: {exc}") - if (idx + 1) % emit_every == 0 or (idx + 1) == total_embed: - pct = 67 + int(18 * (idx + 1) / total_embed) - emit(pct, f"Embedded {idx + 1}/{total_embed} nodes…") - - emit(85, f"Got {len(embeddings)} embeddings — computing pairwise similarity…") - semantic_count = 0 - keys_with_embeds = list(embeddings.keys()) - for i in range(len(keys_with_embeds)): - for j in range(i + 1, len(keys_with_embeds)): - ka, kb = keys_with_embeds[i], keys_with_embeds[j] - sim = cosine_similarity(embeddings[ka], embeddings[kb]) - if sim >= threshold: - edges.append({ - "key": next_edge_key(), - "source": ka, - "target": kb, - "attributes": { - "edge_type": "semantic", - "color": "#F472B6", - "size": round(sim, 3), - "similarity": round(sim, 3), - }, - }) - semantic_count += 1 - - emit(90, f"Built {semantic_count} semantic edges (threshold={threshold})") - else: - emit(65, "Skipping LLM semantic edges (no provider configured)") - - # ── Save snapshot ───────────────────────────────────────────────── - emit(92, f"Saving snapshot — {len(nodes)} nodes, {len(edges)} edges…") - graph_data = { - "attributes": {"title": "Knowledge Graph"}, - "nodes": nodes, - "edges": edges, - } - elapsed_ms = int(time.time() * 1000) - start_ms - - _log_entries.append({"elapsed_s": _elapsed_s(), "msg": f"Done in {elapsed_ms / 1000:.1f}s"}) - snapshot.graph_data = graph_data - snapshot.node_count = len(nodes) - snapshot.edge_count = len(edges) - snapshot.status = KnowledgeGraphSnapshot.Status.READY - snapshot.build_duration_ms = elapsed_ms - snapshot.completed_at = tz.now() - snapshot.progress_data = { - "pct": 100, - "step": f"Done — {len(nodes)} nodes, {len(edges)} edges in {elapsed_ms / 1000:.1f}s", - "logs": list(_log_entries), - } - snapshot.save() - logger.info( - f"build_knowledge_graph: done — {len(nodes)} nodes, {len(edges)} edges " - f"(LLM={use_llm}) in {elapsed_ms}ms" - ) - - except Exception as exc: - logger.error(f"build_knowledge_graph: failed — {exc}", exc_info=True) - snapshot.status = KnowledgeGraphSnapshot.Status.FAILED - snapshot.error_message = str(exc) - snapshot.completed_at = tz.now() - snapshot.save(update_fields=["status", "error_message", "completed_at", "progress_data"]) - - -def schedule_kg_build(): - """Periodic task: trigger a knowledge graph rebuild if auto-schedule is enabled.""" - from .models import SiteSettings, KnowledgeGraphSnapshot - - ss = SiteSettings.get() - if not ss.kg_auto_schedule_enabled: - return - - # Don't start a new build if one is already running - if KnowledgeGraphSnapshot.objects.filter( - status=KnowledgeGraphSnapshot.Status.BUILDING - ).exists(): - logger.debug("schedule_kg_build: skipped — a build is already in progress") - return - - logger.info("schedule_kg_build: starting auto knowledge graph build") - use_llm = ss.llm_provider != "none" - thread = Thread(target=build_knowledge_graph, kwargs={"use_llm": use_llm}, daemon=True) - thread.start() diff --git a/links/templates/links/knowledge_graph.html b/links/templates/links/knowledge_graph.html deleted file mode 100644 index da0e6c9..0000000 --- a/links/templates/links/knowledge_graph.html +++ /dev/null @@ -1,847 +0,0 @@ -{% extends 'base.html' %} -{% load i18n %} -{% load static %} - -{% block extra_css %} - -{% endblock %} - -{% block content %} -
- - -
- - - - - {% if snapshot %}{% with built_at=snapshot.completed_at|default:snapshot.created_at %} - - {% endwith %}{% endif %} -
- - -
- - -
- - - - -

-

{% trans "Computing force-directed layout…" %}

-
- - -
- - - - -
- - -
- - - -
- - -
-
-

{% trans "Node Details" %}

- -
-
- - - - -

-

-
-

{% trans "URL" %}

- -
- - - - - {% trans "View Item" %} - -
-

- {% trans "Connected" %} () -

-
    - -
  • - + {% trans "more" %} -
  • -
-
-
-
- - -
- - - -

{% trans "No knowledge graph yet." %}

- -
- - -
-
-
- - - - - {% trans "Building Knowledge Graph" %} -
-
-
- - -
-
-
-
-
-
- ⏱ {% trans "Elapsed" %}: - - {% trans "Est. remaining" %}: - -
-
- -
- -
{% trans "No log entries yet…" %}
-
-
-
-
- - -
-
-

{% trans "Build Knowledge Graph" %}

-
- -

- {% trans "Semantic edges (pink) connect items with similar meaning using embeddings. Without LLM, the graph uses tag and domain edges only." %} -

-
-
- - -
-
-
- -
-{% endblock %} - -{% block extra_js %} - - - - - - -{% endblock %} diff --git a/links/templates/links/settings.html b/links/templates/links/settings.html index dae48d7..76650e6 100644 --- a/links/templates/links/settings.html +++ b/links/templates/links/settings.html @@ -106,155 +106,6 @@

{% trans "Default: 120. Range: 10–3600. Changes take effect immediately." %}

- -
-

{% trans "Knowledge Graph — LLM Provider" %}

-

- {% trans "Configure an LLM/embedding provider to generate semantic similarity edges in the knowledge graph. Select Ollama to use a local model for free." %} -

- -
- -
- - -
- - -
- - -

{% trans "E.g. http://localhost:11434 or your LAN Ollama address." %}

-
- - -
- - -

- {% trans "Ollama: qwen3-embedding:0.6b, mxbai-embed-large, etc." %} •  - {% trans "OpenRouter: any embedding model slug." %} -

-
- - -
- - -
- - -
- - -
-
-
- - -
-

{% trans "Knowledge Graph — Auto Rebuild" %}

-

- {% trans "Optionally rebuild the knowledge graph on a schedule. You can also trigger a manual build from the " %} - {% trans "Knowledge Graph page" %}. -

- -
- - - - -
- -
- - {% trans "seconds" %} -
-

{% trans "Minimum 60 (1 minute). Default 3600 (1 hour)." %}

-
- - -
- - -
- 0.50 ({% trans "more edges" %}) - 0.95 ({% trans "fewer, tighter edges" %}) -
-
- - -
-

{% trans "Include in graph:" %}

-
- - - - -
-
-
-
-
@@ -369,39 +220,5 @@ function sharedPostsMgr() { }, }; } - -function llmSettingsHelper() { - return { - provider: '{{ site_settings.llm_provider }}', - testing: false, - testResult: null, - testMessage: '', - async testConnection() { - this.testing = true; - this.testResult = null; - const provider = document.getElementById('llm_provider').value; - const base_url = document.getElementById('llm_base_url')?.value || ''; - const model = document.getElementById('llm_model')?.value || ''; - const api_key = document.getElementById('llm_api_key')?.value || ''; - try { - const resp = await fetch('/api/knowledge-graph/test-llm/', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-CSRFToken': document.cookie.match(/csrftoken=([^;]+)/)?.[1] || '', - }, - body: JSON.stringify({ provider, base_url, model, api_key }), - }); - const data = await resp.json(); - this.testResult = data.success; - this.testMessage = data.success ? '✓ {% trans "Connection OK" %}' : '✗ ' + (data.error || '{% trans "Failed" %}'); - } catch(e) { - this.testResult = false; - this.testMessage = '✗ ' + e.message; - } - this.testing = false; - }, - }; -} {% endblock %} diff --git a/links/urls.py b/links/urls.py index 11e7aba..a499227 100644 --- a/links/urls.py +++ b/links/urls.py @@ -74,8 +74,6 @@ urlpatterns = [ path('', include('links.collection_urls')), # Include tag URLs path('', include('links.tag_urls')), - # Include knowledge graph URLs - path('', include('links.knowledge_graph_urls')), # Aliases - these should always be last path('/', views.redirect_to_original, name='redirect_to_original'), diff --git a/links/views.py b/links/views.py index 1fa2082..c7d2538 100644 --- a/links/views.py +++ b/links/views.py @@ -594,32 +594,6 @@ class SiteSettingsView(View): site_settings.schedule_pending_screenshots_interval = max(10, min(screenshots_interval, 3600)) except (ValueError, TypeError): pass - - # ── Knowledge Graph LLM settings ────────────────────────────────── - llm_provider = request.POST.get('llm_provider', 'none').strip() - if llm_provider in ('none', 'ollama', 'openrouter'): - site_settings.llm_provider = llm_provider - site_settings.llm_base_url = request.POST.get('llm_base_url', '').strip() or 'http://localhost:11434' - site_settings.llm_model = request.POST.get('llm_model', '').strip() or 'qwen3-embedding:0.6b' - site_settings.llm_api_key = request.POST.get('llm_api_key', '').strip() - - # ── Knowledge Graph schedule settings ───────────────────────────── - site_settings.kg_auto_schedule_enabled = bool(request.POST.get('kg_auto_schedule_enabled')) - try: - kg_interval = int(request.POST.get('kg_auto_schedule_interval', 3600)) - site_settings.kg_auto_schedule_interval = max(60, min(kg_interval, 86400)) - except (ValueError, TypeError): - pass - try: - threshold = float(request.POST.get('kg_semantic_threshold', 0.70)) - site_settings.kg_semantic_threshold = max(0.0, min(threshold, 1.0)) - except (ValueError, TypeError): - pass - site_settings.kg_include_links = bool(request.POST.get('kg_include_links')) - site_settings.kg_include_pages = bool(request.POST.get('kg_include_pages')) - site_settings.kg_include_posts = bool(request.POST.get('kg_include_posts')) - site_settings.kg_include_tags = bool(request.POST.get('kg_include_tags')) - site_settings.save() # Reschedule periodic jobs with the new intervals @@ -632,11 +606,6 @@ class SiteSettingsView(View): 'schedule_pending_screenshots', trigger=IntervalTrigger(seconds=site_settings.schedule_pending_screenshots_interval), ) - # Reschedule knowledge graph job - scheduler.reschedule_job( - 'schedule_kg_build', - trigger=IntervalTrigger(seconds=site_settings.kg_auto_schedule_interval), - ) except Exception: pass # Scheduler may not be running in test/CLI context @@ -817,12 +786,6 @@ class JobsView(View): messages.success(request, _(f'Deleted {n} netscan run(s).')) elif action == 'bulk_delete_knowledge_graph_snapshots': - from .models import KnowledgeGraphSnapshot - qs = KnowledgeGraphSnapshot.objects.filter(pk__in=ids) if ids else KnowledgeGraphSnapshot.objects.none() - n = qs.delete()[0] - messages.success(request, _(f'Deleted {n} knowledge graph snapshot(s).')) - - else: messages.error(request, _('Unknown action.')) # Preserve tab/status after POST diff --git a/templates/base.html b/templates/base.html index 28313da..f081645 100644 --- a/templates/base.html +++ b/templates/base.html @@ -191,16 +191,6 @@ - -
- - - - {% trans "Knowledge Graph" %} -
-
-