mirror of
https://github.com/wahyd4/links.git
synced 2026-08-09 05:06:16 +10:00
66 lines
2.5 KiB
Python
66 lines
2.5 KiB
Python
import requests
|
|
from qdrant_client import QdrantClient
|
|
from qdrant_client.models import Distance, VectorParams, PointStruct
|
|
import os
|
|
import logging
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Configuration
|
|
DJANGO_BASE_URL = os.getenv("DJANGO_API_BASE", "http://links.apps.svc.cluster.local")
|
|
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.1.2:11434")
|
|
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3-embedding:0.6b")
|
|
QDRANT_HOST = os.getenv("QDRANT_HOST", "192.168.1.2")
|
|
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
|
|
QDRANT_API_KEY = os.getenv("QDRANT_API_KEY", "sss@&9Mnef7#Yd0a")
|
|
COLLECTION_NAME = os.getenv("COLLECTION_NAME", "links")
|
|
|
|
def get_embedding(text):
|
|
response = requests.post(f"{OLLAMA_URL}/api/embeddings", json={"model": OLLAMA_MODEL, "prompt": text})
|
|
response.raise_for_status()
|
|
return response.json()["embedding"]
|
|
|
|
def fetch_data(endpoint):
|
|
url = f"{DJANGO_BASE_URL}/api/{endpoint}/"
|
|
logger.info(f"Fetching from {url}")
|
|
r = requests.get(url)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
def sync():
|
|
client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT, api_key=QDRANT_API_KEY, https=False)
|
|
|
|
# Check Ollama & Get Dimension
|
|
sample_vec = get_embedding("test")
|
|
dim = len(sample_vec)
|
|
|
|
if not any(c.name == COLLECTION_NAME for c in client.get_collections().collections):
|
|
client.create_collection(
|
|
collection_name=COLLECTION_NAME,
|
|
vectors_config=VectorParams(size=dim, distance=Distance.COSINE)
|
|
)
|
|
|
|
points = []
|
|
|
|
# Sync Posts
|
|
posts = fetch_data("posts")
|
|
for p in (posts if isinstance(posts, list) else posts.get('results', [])):
|
|
text = f"{p.get('title')} {p.get('summary')}"
|
|
points.append(PointStruct(id=f"post-{p['id']}", vector=get_embedding(text),
|
|
payload={"id": p['id'], "type": "post", "title": p['title'], "summary": p.get('summary', '')}))
|
|
|
|
# Sync Pages
|
|
pages = fetch_data("pages")
|
|
for p in (pages if isinstance(pages, list) else pages.get('results', [])):
|
|
text = f"{p.get('title')} {p.get('summary')}"
|
|
points.append(PointStruct(id=f"page-{p['id']}", vector=get_embedding(text),
|
|
payload={"id": p['id'], "type": "page", "title": p.get('title', 'No Title'), "summary": p.get('summary', '')}))
|
|
|
|
if points:
|
|
client.upsert(collection_name=COLLECTION_NAME, points=points)
|
|
logger.info(f"Successfully synced {len(points)} items.")
|
|
|
|
if __name__ == "__main__":
|
|
sync()
|