Merge pull request #53 from wahyd4/feat/final-vector-integration

feat: final vector integration

qdrant_sync.py

@@ -2,96 +2,64 @@ import requests
 from qdrant_client import QdrantClient
 from qdrant_client.models import Distance, VectorParams, PointStruct
 import os
+import logging
 
-# Configuration from environment variables
-DJANGO_API = os.getenv("DJANGO_API", "http://links.apps.svc.cluster.local/api/posts/")
-OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.1:11434")
-OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3")
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Configuration
+DJANGO_BASE_URL = os.getenv("DJANGO_API_BASE", "http://links.apps.svc.cluster.local")
+OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.2:11434")
+OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "nomic-embed-text")
 QDRANT_HOST = os.getenv("QDRANT_HOST", "192.168.1.2")
 QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "sss@&9Mnef7#Yd0a")
 COLLECTION_NAME = os.getenv("COLLECTION_NAME", "links")
 
 def get_embedding(text):
-    """Generate embedding using Ollama"""
-    response = requests.post(
-        f"{OLLAMA_URL}/api/embeddings",
-        json={"model": OLLAMA_MODEL, "prompt": text}
-    )
+    response = requests.post(f"{OLLAMA_URL}/api/embeddings", json={"model": OLLAMA_MODEL, "prompt": text})
     response.raise_for_status()
     return response.json()["embedding"]
 
-def fetch_links():
-    """Fetch links from Django API"""
-    # Assuming the API uses pagination or returns a list
-    response = requests.get(DJANGO_API)
-    response.raise_for_status()
-    data = response.json()
-    if isinstance(data, dict) and "results" in data:
-        return data["results"]
-    return data
+def fetch_data(endpoint):
+    url = f"{DJANGO_BASE_URL}/api/{endpoint}/"
+    logger.info(f"Fetching from {url}")
+    r = requests.get(url)
+    r.raise_for_status()
+    return r.json()
 
-def sync_to_qdrant():
-    """Sync links from Django API to Qdrant"""
-    print(f"Connecting to Qdrant at {QDRANT_HOST}:{QDRANT_PORT}")
-    client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
+def sync():
+    client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT, api_key=QDRANT_API_KEY, https=False)
     
-    print(f"Fetching links from {DJANGO_API}")
-    links = fetch_links()
+    # Check Ollama & Get Dimension
+    sample_vec = get_embedding("test")
+    dim = len(sample_vec)
     
-    if not links:
-        print("No links to sync")
-        return
-    
-    print(f"Found {len(links)} links. Preparing for vectorization...")
-    
-    # Get embedding dimension
-    sample_text = f"{links[0].get('title', '')} {links[0].get('url', '')}"
-    sample_embedding = get_embedding(sample_text)
-    vector_size = len(sample_embedding)
-    
-    # Ensure collection exists
-    collections = client.get_collections().collections
-    exists = any(c.name == COLLECTION_NAME for c in collections)
-    
-    if not exists:
-        print(f"Creating collection: {COLLECTION_NAME}")
+    if not any(c.name == COLLECTION_NAME for c in client.get_collections().collections):
         client.create_collection(
             collection_name=COLLECTION_NAME,
-            vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE)
+            vectors_config=VectorParams(size=dim, distance=Distance.COSINE)
         )
-    
-    # Prepare points
+
     points = []
-    for idx, link in enumerate(links):
-        # Combine text fields for embedding
-        text = f"{link.get('title', '')} {link.get('description', '') or ''} {link.get('url', '')}"
-        try:
-            embedding = get_embedding(text)
-            
-            # Use link id if available, otherwise index
-            point_id = link.get("id", idx)
-            
-            point = PointStruct(
-                id=point_id,
-                vector=embedding,
-                payload={
-                    "id": link.get("id"),
-                    "title": link.get("title"),
-                    "url": link.get("url"),
-                    "description": link.get("description"),
-                    "tags": link.get("tags", [])
-                }
-            )
-            points.append(point)
-            if (idx + 1) % 10 == 0:
-                print(f"Processed {idx + 1}/{len(links)} links")
-        except Exception as e:
-            print(f"Error processing link {idx}: {e}")
     
-    # Upload to Qdrant
+    # Sync Posts
+    posts = fetch_data("posts")
+    for p in (posts if isinstance(posts, list) else posts.get('results', [])):
+        text = f"{p.get('title')} {p.get('summary')}"
+        points.append(PointStruct(id=f"post-{p['id']}", vector=get_embedding(text), 
+                                 payload={"id": p['id'], "type": "post", "title": p['title'], "summary": p.get('summary', '')}))
+
+    # Sync Pages
+    pages = fetch_data("pages")
+    for p in (pages if isinstance(pages, list) else pages.get('results', [])):
+        text = f"{p.get('title')} {p.get('summary')}"
+        points.append(PointStruct(id=f"page-{p['id']}", vector=get_embedding(text), 
+                                 payload={"id": p['id'], "type": "page", "title": p.get('title', 'No Title'), "summary": p.get('summary', '')}))
+
     if points:
         client.upsert(collection_name=COLLECTION_NAME, points=points)
-        print(f"Successfully synced {len(points)} links to Qdrant")
+        logger.info(f"Successfully synced {len(points)} items.")
 
 if __name__ == "__main__":
-    sync_to_qdrant()
+    sync()