feat: implement Qdrant sync script and k8s CronJob

k8s/qdrant-sync-cronjob.yaml

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: links-qdrant-sync
+  namespace: links
+spec:
+  schedule: "0 0 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sync-worker
+            image: ghcr.io/your-username/links:latest  # Note: Requires correct image path
+            command: ["python", "qdrant_sync.py"]
+            env:
+            - name: DJANGO_API
+              value: "http://links-api-service.links.svc.cluster.local/api/links/"
+            - name: OLLAMA_URL
+              value: "http://ollama-service.ollama.svc.cluster.local:11434"
+            - name: QDRANT_HOST
+              value: "192.168.1.2"
+            - name: OLLAMA_MODEL
+              value: "llama3"
+            resources:
+              limits:
+                memory: "256Mi"
+                cpu: "200m"
+              requests:
+                memory: "128Mi"
+                cpu: "100m"
+          restartPolicy: OnFailure

qdrant_sync.py

@@ -0,0 +1,97 @@
+import requests
+from qdrant_client import QdrantClient
+from qdrant_client.models import Distance, VectorParams, PointStruct
+import os
+
+# Configuration from environment variables
+DJANGO_API = os.getenv("DJANGO_API", "http://192.168.1.1:8000/api/links/")
+OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.1:11434")
+OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3")
+QDRANT_HOST = os.getenv("QDRANT_HOST", "192.168.1.2")
+QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
+COLLECTION_NAME = os.getenv("COLLECTION_NAME", "links")
+
+def get_embedding(text):
+    """Generate embedding using Ollama"""
+    response = requests.post(
+        f"{OLLAMA_URL}/api/embeddings",
+        json={"model": OLLAMA_MODEL, "prompt": text}
+    )
+    response.raise_for_status()
+    return response.json()["embedding"]
+
+def fetch_links():
+    """Fetch links from Django API"""
+    # Assuming the API uses pagination or returns a list
+    response = requests.get(DJANGO_API)
+    response.raise_for_status()
+    data = response.json()
+    if isinstance(data, dict) and "results" in data:
+        return data["results"]
+    return data
+
+def sync_to_qdrant():
+    """Sync links from Django API to Qdrant"""
+    print(f"Connecting to Qdrant at {QDRANT_HOST}:{QDRANT_PORT}")
+    client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
+    
+    print(f"Fetching links from {DJANGO_API}")
+    links = fetch_links()
+    
+    if not links:
+        print("No links to sync")
+        return
+    
+    print(f"Found {len(links)} links. Preparing for vectorization...")
+    
+    # Get embedding dimension
+    sample_text = f"{links[0].get('title', '')} {links[0].get('url', '')}"
+    sample_embedding = get_embedding(sample_text)
+    vector_size = len(sample_embedding)
+    
+    # Ensure collection exists
+    collections = client.get_collections().collections
+    exists = any(c.name == COLLECTION_NAME for c in collections)
+    
+    if not exists:
+        print(f"Creating collection: {COLLECTION_NAME}")
+        client.create_collection(
+            collection_name=COLLECTION_NAME,
+            vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE)
+        )
+    
+    # Prepare points
+    points = []
+    for idx, link in enumerate(links):
+        # Combine text fields for embedding
+        text = f"{link.get('title', '')} {link.get('description', '') or ''} {link.get('url', '')}"
+        try:
+            embedding = get_embedding(text)
+            
+            # Use link id if available, otherwise index
+            point_id = link.get("id", idx)
+            
+            point = PointStruct(
+                id=point_id,
+                vector=embedding,
+                payload={
+                    "id": link.get("id"),
+                    "title": link.get("title"),
+                    "url": link.get("url"),
+                    "description": link.get("description"),
+                    "tags": link.get("tags", [])
+                }
+            )
+            points.append(point)
+            if (idx + 1) % 10 == 0:
+                print(f"Processed {idx + 1}/{len(links)} links")
+        except Exception as e:
+            print(f"Error processing link {idx}: {e}")
+    
+    # Upload to Qdrant
+    if points:
+        client.upsert(collection_name=COLLECTION_NAME, points=points)
+        print(f"Successfully synced {len(points)} links to Qdrant")
+
+if __name__ == "__main__":
+    sync_to_qdrant()