import requests from qdrant_client import QdrantClient from qdrant_client.models import Distance, VectorParams, PointStruct import os import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Configuration DJANGO_BASE_URL = os.getenv("DJANGO_API_BASE", "http://links.apps.svc.cluster.local") OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.2:11434") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3-embedding:0.6b") QDRANT_HOST = os.getenv("QDRANT_HOST", "192.168.1.2") QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "sss@&9Mnef7#Yd0a") COLLECTION_NAME = os.getenv("COLLECTION_NAME", "links") def get_embedding(text): response = requests.post(f"{OLLAMA_URL}/api/embeddings", json={"model": OLLAMA_MODEL, "prompt": text}) response.raise_for_status() return response.json()["embedding"] def fetch_data(endpoint): url = f"{DJANGO_BASE_URL}/api/{endpoint}/" logger.info(f"Fetching from {url}") r = requests.get(url) r.raise_for_status() return r.json() def sync(): client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT, api_key=QDRANT_API_KEY, https=False) # Check Ollama & Get Dimension sample_vec = get_embedding("test") dim = len(sample_vec) if not any(c.name == COLLECTION_NAME for c in client.get_collections().collections): client.create_collection( collection_name=COLLECTION_NAME, vectors_config=VectorParams(size=dim, distance=Distance.COSINE) ) points = [] # Sync Posts posts = fetch_data("posts") for p in (posts if isinstance(posts, list) else posts.get('results', [])): text = f"{p.get('title')} {p.get('summary')}" points.append(PointStruct(id=f"post-{p['id']}", vector=get_embedding(text), payload={"id": p['id'], "type": "post", "title": p['title'], "summary": p.get('summary', '')})) # Sync Pages pages = fetch_data("pages") for p in (pages if isinstance(pages, list) else pages.get('results', [])): text = f"{p.get('title')} {p.get('summary')}" points.append(PointStruct(id=f"page-{p['id']}", vector=get_embedding(text), payload={"id": p['id'], "type": "page", "title": p.get('title', 'No Title'), "summary": p.get('summary', '')})) if points: client.upsert(collection_name=COLLECTION_NAME, points=points) logger.info(f"Successfully synced {len(points)} items.") if __name__ == "__main__": sync()