update apis

This commit is contained in:
2026-02-23 21:43:34 +11:00
parent 0f97bc3566
commit 50593289ac
12 changed files with 182 additions and 21 deletions
+2 -1
View File
@@ -72,9 +72,10 @@ async def api_search(
page: int = Query(1, ge=1, le=50, description="Page number"),
engines: str | None = Query(None, description="Comma-separated engine names to use (e.g. 'google,bing')"),
image_size: Literal["", "large", "medium", "small"] = Query("", description="Filter images by size (images category only)"),
sort: Literal["default", "date_asc", "date_desc"] = Query("default", description="Sort results by publish date"),
):
engine_list = [e.strip() for e in engines.split(",")] if engines else None
result = await search(q, category=category, page=page, engines=engine_list, image_size=image_size)
result = await search(q, category=category, page=page, engines=engine_list, image_size=image_size, sort=sort)
origin_ip = request.client.host if request.client else ""
user_agent = request.headers.get("user-agent", "")
_stats.record_search(
+10 -2
View File
@@ -10,6 +10,7 @@ from lxml import html as lxml_html
from app.models import WebResult, ImageResult
from app.engines.base import SearchEngine, get_http_client
from app.engines.date_utils import parse_date_from_text
logger = logging.getLogger(__name__)
@@ -148,7 +149,13 @@ class BingEngine(SearchEngine):
if len(txt) > len(content):
content = txt
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
# Try news_dt span (present on news/recent results), fall back to content
news_dt = el.xpath('.//span[contains(@class, "news_dt")]')
published_date = (
parse_date_from_text(news_dt[0].text_content().strip())
if news_dt else parse_date_from_text(content)
)
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=published_date))
# Rate limit detection: Bing silently resets to page 1
if page > 1 and results:
@@ -215,6 +222,7 @@ class BingEngine(SearchEngine):
engine=self.name,
width=int(data.get("mw", 0) or 0),
height=int(data.get("mh", 0) or 0),
published_date=parse_date_from_text(str(data.get("datePublished", "") or data.get("age", ""))),
)
)
except _json.JSONDecodeError:
@@ -260,7 +268,7 @@ class BingEngine(SearchEngine):
desc_els = el.xpath('.//div[contains(@class, "compText")]//p') or el.xpath(".//p")
content = desc_els[0].text_content().strip() if desc_els else ""
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=parse_date_from_text(content)))
if not results:
logger.warning("Yahoo fallback returned no results for '%s'", query)
+10 -1
View File
@@ -9,6 +9,7 @@ from lxml import html as lxml_html
from app.models import WebResult, ImageResult
from app.engines.base import SearchEngine, get_http_client
from app.engines.date_utils import parse_date_from_text
logger = logging.getLogger(__name__)
@@ -48,7 +49,15 @@ class BraveEngine(SearchEngine):
title = title_els[0].text_content().strip()
content_els = el.xpath(".//div[contains(concat(' ', @class, ' '), ' content ')]")
content = content_els[0].text_content().strip() if content_els else ""
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
# Try to extract date from dedicated age element, fall back to content text
date_els = el.xpath(".//div[contains(@class, 'snippet-description')]//time/@datetime") \
or el.xpath(".//*[contains(@class, 'age')]")
if date_els and isinstance(date_els[0], str):
published_date = date_els[0][:10]
else:
age_text = date_els[0].text_content().strip() if date_els else ""
published_date = parse_date_from_text(age_text) or parse_date_from_text(content)
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=published_date))
return results
+73
View File
@@ -0,0 +1,73 @@
"""Date extraction utilities for search engine result snippets."""
from __future__ import annotations
import re
from datetime import datetime, timezone, timedelta
_MONTHS = {
"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
"jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
}
def parse_date_from_text(text: str) -> str:
"""
Try to extract a publication date from the start of a snippet text.
Returns an ISO 8601 date string (YYYY-MM-DD) or empty string if not found.
Handles:
- Relative: "3 hours ago", "2 days ago", "1 week ago", "5 months ago"
- MDY: "Jan 15, 2024" or "January 15, 2024"
- DMY: "15 Jan 2024" or "15 January 2024"
- ISO: "2024-01-15"
"""
if not text:
return ""
text = text.strip()
now = datetime.now(timezone.utc)
# Relative: "X unit(s) ago"
m = re.match(r"^(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago", text, re.IGNORECASE)
if m:
n = int(m.group(1))
unit = m.group(2).lower()
deltas: dict[str, timedelta] = {
"second": timedelta(seconds=n),
"minute": timedelta(minutes=n),
"hour": timedelta(hours=n),
"day": timedelta(days=n),
"week": timedelta(weeks=n),
"month": timedelta(days=n * 30),
"year": timedelta(days=n * 365),
}
return (now - deltas[unit]).date().isoformat()
# MDY: "Jan 15, 2024" or "January 15, 2024"
m = re.match(r"^([A-Za-z]{3,9})\s+(\d{1,2}),?\s+(\d{4})", text)
if m:
mon = m.group(1).lower()[:3]
if mon in _MONTHS:
try:
return datetime(int(m.group(3)), _MONTHS[mon], int(m.group(2)), tzinfo=timezone.utc).date().isoformat()
except ValueError:
pass
# DMY: "15 Jan 2024" or "15 January 2024"
m = re.match(r"^(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})", text)
if m:
mon = m.group(2).lower()[:3]
if mon in _MONTHS:
try:
return datetime(int(m.group(3)), _MONTHS[mon], int(m.group(1)), tzinfo=timezone.utc).date().isoformat()
except ValueError:
pass
# ISO: "2024-01-15"
m = re.match(r"^(\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01]))", text)
if m:
return m.group(1)
return ""
+3 -1
View File
@@ -10,6 +10,7 @@ from lxml import html as lxml_html
from app.models import WebResult, ImageResult
from app.engines.base import SearchEngine, get_http_client
from app.engines.date_utils import parse_date_from_text
logger = logging.getLogger(__name__)
@@ -50,7 +51,7 @@ class DuckDuckGoEngine(SearchEngine):
url = url_els[0]
content_els = div.xpath('.//a[contains(@class, "result__snippet")]')
content = content_els[0].text_content().strip() if content_els else ""
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=parse_date_from_text(content)))
return results
@@ -106,6 +107,7 @@ class DuckDuckGoEngine(SearchEngine):
engine=self.name,
width=int(item.get("width", 0) or 0),
height=int(item.get("height", 0) or 0),
published_date=parse_date_from_text(str(item.get("age", ""))),
))
return results
+2 -1
View File
@@ -14,6 +14,7 @@ from lxml import html as lxml_html
from app.models import WebResult, ImageResult
from app.engines.base import SearchEngine, get_http_client
from app.engines.date_utils import parse_date_from_text
logger = logging.getLogger(__name__)
@@ -133,7 +134,7 @@ class GoogleEngine(SearchEngine):
content = txt
break
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=parse_date_from_text(content)))
if not results:
logger.warning("Google returned no parseable results for '%s'", query)
+2
View File
@@ -16,6 +16,7 @@ class WebResult(BaseModel):
content: str = ""
engine: str = ""
rank: int = 0
published_date: str = "" # ISO 8601 date (YYYY-MM-DD) or empty
class ImageResult(BaseModel):
@@ -30,6 +31,7 @@ class ImageResult(BaseModel):
rank: int = 0
width: int = 0
height: int = 0
published_date: str = "" # ISO 8601 date (YYYY-MM-DD) or empty
class EngineError(BaseModel):
+19 -2
View File
@@ -16,6 +16,18 @@ from app import cache
logger = logging.getLogger(__name__)
SortOrder = str # "default" | "date_asc" | "date_desc"
def _apply_sort(results: list, sort: SortOrder) -> None:
"""Sort results in-place by published_date. Undated results go last."""
if sort not in ("date_asc", "date_desc"):
return
with_date = [r for r in results if r.published_date]
no_date = [r for r in results if not r.published_date]
with_date.sort(key=lambda r: r.published_date, reverse=(sort == "date_desc"))
results[:] = with_date + no_date
# Retry config: 2 retries with exponential backoff (0.5s, 1s)
RETRY_DECORATOR = retry(
stop=stop_after_attempt(2),
@@ -60,15 +72,17 @@ async def search(
page: int = 1,
engines: list[str] | None = None,
image_size: str = "",
sort: SortOrder = "default",
) -> SearchResponse:
"""Search across all enabled engines concurrently, with optional Redis caching."""
engines_key = ",".join(sorted(engines)) if engines else ""
# Check cache first
# Check cache first (cache stores default-order results; sort applied after)
cached_data = await cache.get_cached(query, category, page, image_size, engines_key)
if cached_data is not None:
resp = SearchResponse(**cached_data)
resp.cached = True
_apply_sort(resp.results, sort)
return resp
enabled_engines = registry.get_enabled_engines()
@@ -139,10 +153,13 @@ async def search(
has_next=any(s.result_count > 0 for s in all_stats if s.status == "ok"),
)
# Store in cache (only if we got results)
# Store in cache before sorting (cache always holds default-order results)
if unique_results:
await cache.set_cached(query, category, page, image_size, engines_key, response.model_dump())
# Apply requested sort order
_apply_sort(response.results, sort)
return response
+50 -12
View File
@@ -15,6 +15,7 @@ import { cn } from "@/lib/utils";
type Category = "web" | "images";
type ImageSize = "" | "large" | "medium" | "small";
type SortOrder = "default" | "date_desc" | "date_asc";
const IMAGE_SIZE_OPTIONS: { value: ImageSize; label: string }[] = [
{ value: "", label: "All sizes" },
@@ -23,7 +24,13 @@ const IMAGE_SIZE_OPTIONS: { value: ImageSize; label: string }[] = [
{ value: "small", label: "Small" },
];
function parseUrlState(): { q: string; cat: Category; page: number; imageSize: ImageSize; engines: string } {
const SORT_OPTIONS: { value: SortOrder; label: string }[] = [
{ value: "default", label: "Default" },
{ value: "date_desc", label: "Newest" },
{ value: "date_asc", label: "Oldest" },
];
function parseUrlState(): { q: string; cat: Category; page: number; imageSize: ImageSize; engines: string; sort: SortOrder } {
const params = new URLSearchParams(window.location.search);
const q = params.get("q") ?? "";
const cat = params.get("category") === "images" ? "images" : "web";
@@ -31,16 +38,19 @@ function parseUrlState(): { q: string; cat: Category; page: number; imageSize: I
const rawSize = params.get("image_size") ?? "";
const imageSize: ImageSize = (["large", "medium", "small"].includes(rawSize) ? rawSize : "") as ImageSize;
const engines = params.get("engines") ?? "";
return { q, cat, page, imageSize, engines };
const rawSort = params.get("sort") ?? "";
const sort: SortOrder = (["date_desc", "date_asc"].includes(rawSort) ? rawSort : "default") as SortOrder;
return { q, cat, page, imageSize, engines, sort };
}
function pushUrl(q: string, cat: Category, page: number, imageSize: ImageSize = "", engines: string = "") {
function pushUrl(q: string, cat: Category, page: number, imageSize: ImageSize = "", engines: string = "", sort: SortOrder = "default") {
const params = new URLSearchParams();
params.set("q", q);
if (cat !== "web") params.set("category", cat);
if (page > 1) params.set("page", String(page));
if (imageSize) params.set("image_size", imageSize);
if (engines) params.set("engines", engines);
if (sort !== "default") params.set("sort", sort);
const url = `/?${params.toString()}`;
if (window.location.pathname + window.location.search !== url) {
window.history.pushState(null, "", url);
@@ -53,6 +63,7 @@ function App() {
const [category, setCategory] = useState<Category>(initial.cat);
const [page, setPage] = useState(initial.page);
const [imageSize, setImageSize] = useState<ImageSize>(initial.imageSize);
const [sortOrder, setSortOrder] = useState<SortOrder>(initial.sort);
const [response, setResponse] = useState<SearchResponse | null>(null);
const [loading, setLoading] = useState(false);
const [showSettings, setShowSettings] = useState(false);
@@ -155,7 +166,7 @@ function App() {
const doSearch = useCallback(
async (q: string, cat: Category = category, p: number = 1, size: ImageSize = imageSize, updateUrl = true) => {
async (q: string, cat: Category = category, p: number = 1, size: ImageSize = imageSize, updateUrl = true, sort: SortOrder = sortOrder) => {
if (!q.trim()) return;
setQuery(q);
setCategory(cat);
@@ -163,9 +174,9 @@ function App() {
setImageSize(size);
setLoading(true);
setHasSearched(true);
if (updateUrl) pushUrl(q, cat, p, cat === "images" ? size : "");
if (updateUrl) pushUrl(q, cat, p, cat === "images" ? size : "", "", sort);
try {
const res = await apiSearch(q, cat, p, cat === "images" ? size : "");
const res = await apiSearch(q, cat, p, cat === "images" ? size : "", sort);
setResponse(res);
} catch (err) {
setResponse({
@@ -185,13 +196,13 @@ function App() {
setLoading(false);
}
},
[category, imageSize]
[category, imageSize, sortOrder]
);
// Restore search from URL on initial load
useEffect(() => {
if (initial.q) {
doSearch(initial.q, initial.cat, initial.page, initial.imageSize, false);
doSearch(initial.q, initial.cat, initial.page, initial.imageSize, false, initial.sort);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
@@ -214,9 +225,10 @@ function App() {
setShowGallery(false);
setShowBookmarks(false);
setShowStats(false);
const { q, cat, page: p, imageSize: size } = parseUrlState();
const { q, cat, page: p, imageSize: size, sort } = parseUrlState();
if (q) {
doSearch(q, cat, p, size, false);
setSortOrder(sort);
doSearch(q, cat, p, size, false, sort);
} else {
setHasSearched(false);
setResponse(null);
@@ -232,7 +244,7 @@ function App() {
const handleCategoryChange = (cat: Category) => {
setCategory(cat);
if (query) doSearch(query, cat, 1, cat === "images" ? imageSize : "");
if (query) doSearch(query, cat, 1, cat === "images" ? imageSize : "", true, sortOrder);
};
const handlePageChange = (newPage: number) => {
@@ -243,7 +255,12 @@ function App() {
const handleImageSizeChange = (size: ImageSize) => {
setImageSize(size);
if (query) doSearch(query, category, 1, size);
if (query) doSearch(query, category, 1, size, true, sortOrder);
};
const handleSortChange = (sort: SortOrder) => {
setSortOrder(sort);
if (query) doSearch(query, category, page, imageSize, true, sort);
};
const handleGoHome = () => {
@@ -538,6 +555,27 @@ function App() {
))}
</>
)}
{/* Sort order — available for both web and images */}
<>
<div className="mx-2 h-5 w-px bg-border" aria-hidden="true" />
<span className="text-xs text-muted-foreground" aria-hidden="true">Sort:</span>
{SORT_OPTIONS.map(({ value, label }) => (
<button
key={value}
onClick={() => handleSortChange(value)}
aria-pressed={sortOrder === value}
className={cn(
"rounded-full px-3 py-1 text-xs font-medium transition-colors focus-visible:ring-2 focus-visible:ring-ring focus-visible:outline-none",
sortOrder === value
? "bg-secondary text-secondary-foreground"
: "text-muted-foreground hover:bg-accent"
)}
>
{label}
</button>
))}
</>
</nav>
</div>
+3
View File
@@ -202,6 +202,9 @@ export function ImageResults({ results, query = "", category = "images", bookmar
{getSize(selectedIndex) && (
<> {formatSize(getSize(selectedIndex)!.w, getSize(selectedIndex)!.h)}</>
)}
{selected.published_date && (
<> <span className="tabular-nums">{selected.published_date}</span></>
)}
<span className="ml-2 tabular-nums opacity-60">
{selectedIndex + 1} / {results.length}
</span>
+3
View File
@@ -28,6 +28,9 @@ export function WebResults({ results, query = "", category = "web", bookmarkedUr
/>
<span className="truncate">{new URL(result.url).hostname}</span>
<span className="rounded bg-muted px-1.5 py-0.5 text-xs">{result.engine}</span>
{result.published_date && (
<span className="text-xs text-muted-foreground/70">{result.published_date}</span>
)}
{onToggleBookmark && (
<button
onClick={() => onToggleBookmark(result)}
+5 -1
View File
@@ -7,6 +7,7 @@ export interface WebResult {
content: string;
engine: string;
rank: number;
published_date?: string;
}
export interface ImageResult {
@@ -20,6 +21,7 @@ export interface ImageResult {
rank: number;
width: number;
height: number;
published_date?: string;
}
export interface EngineError {
@@ -70,10 +72,12 @@ export async function search(
query: string,
category: "web" | "images" = "web",
page: number = 1,
imageSize: string = ""
imageSize: string = "",
sort: string = "default"
): Promise<SearchResponse> {
const params = new URLSearchParams({ q: query, category, page: String(page) });
if (imageSize) params.set("image_size", imageSize);
if (sort !== "default") params.set("sort", sort);
const resp = await fetch(`${API_BASE}/search?${params}`);
if (!resp.ok) throw new Error(`Search failed: ${resp.status}`);
return resp.json();