diff --git a/k8s/qdrant-sync-cronjob.yaml b/k8s/qdrant-sync-cronjob.yaml new file mode 100644 index 0000000..c0b1900 --- /dev/null +++ b/k8s/qdrant-sync-cronjob.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: links-qdrant-sync + namespace: links +spec: + schedule: "0 0 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: sync-worker + image: ghcr.io/your-username/links:latest # Note: Requires correct image path + command: ["python", "qdrant_sync.py"] + env: + - name: DJANGO_API + value: "http://links-api-service.links.svc.cluster.local/api/links/" + - name: OLLAMA_URL + value: "http://ollama-service.ollama.svc.cluster.local:11434" + - name: QDRANT_HOST + value: "192.168.1.2" + - name: OLLAMA_MODEL + value: "llama3" + resources: + limits: + memory: "256Mi" + cpu: "200m" + requests: + memory: "128Mi" + cpu: "100m" + restartPolicy: OnFailure diff --git a/qdrant_sync.py b/qdrant_sync.py new file mode 100644 index 0000000..41702e8 --- /dev/null +++ b/qdrant_sync.py @@ -0,0 +1,97 @@ +import requests +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct +import os + +# Configuration from environment variables +DJANGO_API = os.getenv("DJANGO_API", "http://192.168.1.1:8000/api/links/") +OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.1:11434") +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3") +QDRANT_HOST = os.getenv("QDRANT_HOST", "192.168.1.2") +QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) +COLLECTION_NAME = os.getenv("COLLECTION_NAME", "links") + +def get_embedding(text): + """Generate embedding using Ollama""" + response = requests.post( + f"{OLLAMA_URL}/api/embeddings", + json={"model": OLLAMA_MODEL, "prompt": text} + ) + response.raise_for_status() + return response.json()["embedding"] + +def fetch_links(): + """Fetch links from Django API""" + # Assuming the API uses pagination or returns a list + response = requests.get(DJANGO_API) + response.raise_for_status() + data = response.json() + if isinstance(data, dict) and "results" in data: + return data["results"] + return data + +def sync_to_qdrant(): + """Sync links from Django API to Qdrant""" + print(f"Connecting to Qdrant at {QDRANT_HOST}:{QDRANT_PORT}") + client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT) + + print(f"Fetching links from {DJANGO_API}") + links = fetch_links() + + if not links: + print("No links to sync") + return + + print(f"Found {len(links)} links. Preparing for vectorization...") + + # Get embedding dimension + sample_text = f"{links[0].get('title', '')} {links[0].get('url', '')}" + sample_embedding = get_embedding(sample_text) + vector_size = len(sample_embedding) + + # Ensure collection exists + collections = client.get_collections().collections + exists = any(c.name == COLLECTION_NAME for c in collections) + + if not exists: + print(f"Creating collection: {COLLECTION_NAME}") + client.create_collection( + collection_name=COLLECTION_NAME, + vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE) + ) + + # Prepare points + points = [] + for idx, link in enumerate(links): + # Combine text fields for embedding + text = f"{link.get('title', '')} {link.get('description', '') or ''} {link.get('url', '')}" + try: + embedding = get_embedding(text) + + # Use link id if available, otherwise index + point_id = link.get("id", idx) + + point = PointStruct( + id=point_id, + vector=embedding, + payload={ + "id": link.get("id"), + "title": link.get("title"), + "url": link.get("url"), + "description": link.get("description"), + "tags": link.get("tags", []) + } + ) + points.append(point) + if (idx + 1) % 10 == 0: + print(f"Processed {idx + 1}/{len(links)} links") + except Exception as e: + print(f"Error processing link {idx}: {e}") + + # Upload to Qdrant + if points: + client.upsert(collection_name=COLLECTION_NAME, points=points) + print(f"Successfully synced {len(points)} links to Qdrant") + +if __name__ == "__main__": + sync_to_qdrant()