diff --git a/qdrant_sync.py b/qdrant_sync.py index 28dbbb4..4d5ed32 100644 --- a/qdrant_sync.py +++ b/qdrant_sync.py @@ -2,96 +2,64 @@ import requests from qdrant_client import QdrantClient from qdrant_client.models import Distance, VectorParams, PointStruct import os +import logging -# Configuration from environment variables -DJANGO_API = os.getenv("DJANGO_API", "http://links.apps.svc.cluster.local/api/posts/") -OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.1:11434") -OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3") +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Configuration +DJANGO_BASE_URL = os.getenv("DJANGO_API_BASE", "http://links.apps.svc.cluster.local") +OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.2:11434") +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "nomic-embed-text") QDRANT_HOST = os.getenv("QDRANT_HOST", "192.168.1.2") QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) +QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "sss@&9Mnef7#Yd0a") COLLECTION_NAME = os.getenv("COLLECTION_NAME", "links") def get_embedding(text): - """Generate embedding using Ollama""" - response = requests.post( - f"{OLLAMA_URL}/api/embeddings", - json={"model": OLLAMA_MODEL, "prompt": text} - ) + response = requests.post(f"{OLLAMA_URL}/api/embeddings", json={"model": OLLAMA_MODEL, "prompt": text}) response.raise_for_status() return response.json()["embedding"] -def fetch_links(): - """Fetch links from Django API""" - # Assuming the API uses pagination or returns a list - response = requests.get(DJANGO_API) - response.raise_for_status() - data = response.json() - if isinstance(data, dict) and "results" in data: - return data["results"] - return data +def fetch_data(endpoint): + url = f"{DJANGO_BASE_URL}/api/{endpoint}/" + logger.info(f"Fetching from {url}") + r = requests.get(url) + r.raise_for_status() + return r.json() -def sync_to_qdrant(): - """Sync links from Django API to Qdrant""" - print(f"Connecting to Qdrant at {QDRANT_HOST}:{QDRANT_PORT}") - client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT) +def sync(): + client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT, api_key=QDRANT_API_KEY, https=False) - print(f"Fetching links from {DJANGO_API}") - links = fetch_links() + # Check Ollama & Get Dimension + sample_vec = get_embedding("test") + dim = len(sample_vec) - if not links: - print("No links to sync") - return - - print(f"Found {len(links)} links. Preparing for vectorization...") - - # Get embedding dimension - sample_text = f"{links[0].get('title', '')} {links[0].get('url', '')}" - sample_embedding = get_embedding(sample_text) - vector_size = len(sample_embedding) - - # Ensure collection exists - collections = client.get_collections().collections - exists = any(c.name == COLLECTION_NAME for c in collections) - - if not exists: - print(f"Creating collection: {COLLECTION_NAME}") + if not any(c.name == COLLECTION_NAME for c in client.get_collections().collections): client.create_collection( collection_name=COLLECTION_NAME, - vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE) + vectors_config=VectorParams(size=dim, distance=Distance.COSINE) ) - - # Prepare points + points = [] - for idx, link in enumerate(links): - # Combine text fields for embedding - text = f"{link.get('title', '')} {link.get('description', '') or ''} {link.get('url', '')}" - try: - embedding = get_embedding(text) - - # Use link id if available, otherwise index - point_id = link.get("id", idx) - - point = PointStruct( - id=point_id, - vector=embedding, - payload={ - "id": link.get("id"), - "title": link.get("title"), - "url": link.get("url"), - "description": link.get("description"), - "tags": link.get("tags", []) - } - ) - points.append(point) - if (idx + 1) % 10 == 0: - print(f"Processed {idx + 1}/{len(links)} links") - except Exception as e: - print(f"Error processing link {idx}: {e}") - # Upload to Qdrant + # Sync Posts + posts = fetch_data("posts") + for p in (posts if isinstance(posts, list) else posts.get('results', [])): + text = f"{p.get('title')} {p.get('summary')}" + points.append(PointStruct(id=f"post-{p['id']}", vector=get_embedding(text), + payload={"id": p['id'], "type": "post", "title": p['title'], "summary": p.get('summary', '')})) + + # Sync Pages + pages = fetch_data("pages") + for p in (pages if isinstance(pages, list) else pages.get('results', [])): + text = f"{p.get('title')} {p.get('summary')}" + points.append(PointStruct(id=f"page-{p['id']}", vector=get_embedding(text), + payload={"id": p['id'], "type": "page", "title": p.get('title', 'No Title'), "summary": p.get('summary', '')})) + if points: client.upsert(collection_name=COLLECTION_NAME, points=points) - print(f"Successfully synced {len(points)} links to Qdrant") + logger.info(f"Successfully synced {len(points)} items.") if __name__ == "__main__": - sync_to_qdrant() + sync()