mirror of
https://github.com/wahyd4/hey-search.git
synced 2026-08-08 21:05:14 +10:00
update apis
This commit is contained in:
@@ -72,9 +72,10 @@ async def api_search(
|
||||
page: int = Query(1, ge=1, le=50, description="Page number"),
|
||||
engines: str | None = Query(None, description="Comma-separated engine names to use (e.g. 'google,bing')"),
|
||||
image_size: Literal["", "large", "medium", "small"] = Query("", description="Filter images by size (images category only)"),
|
||||
sort: Literal["default", "date_asc", "date_desc"] = Query("default", description="Sort results by publish date"),
|
||||
):
|
||||
engine_list = [e.strip() for e in engines.split(",")] if engines else None
|
||||
result = await search(q, category=category, page=page, engines=engine_list, image_size=image_size)
|
||||
result = await search(q, category=category, page=page, engines=engine_list, image_size=image_size, sort=sort)
|
||||
origin_ip = request.client.host if request.client else ""
|
||||
user_agent = request.headers.get("user-agent", "")
|
||||
_stats.record_search(
|
||||
|
||||
@@ -10,6 +10,7 @@ from lxml import html as lxml_html
|
||||
|
||||
from app.models import WebResult, ImageResult
|
||||
from app.engines.base import SearchEngine, get_http_client
|
||||
from app.engines.date_utils import parse_date_from_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -148,7 +149,13 @@ class BingEngine(SearchEngine):
|
||||
if len(txt) > len(content):
|
||||
content = txt
|
||||
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
|
||||
# Try news_dt span (present on news/recent results), fall back to content
|
||||
news_dt = el.xpath('.//span[contains(@class, "news_dt")]')
|
||||
published_date = (
|
||||
parse_date_from_text(news_dt[0].text_content().strip())
|
||||
if news_dt else parse_date_from_text(content)
|
||||
)
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=published_date))
|
||||
|
||||
# Rate limit detection: Bing silently resets to page 1
|
||||
if page > 1 and results:
|
||||
@@ -215,6 +222,7 @@ class BingEngine(SearchEngine):
|
||||
engine=self.name,
|
||||
width=int(data.get("mw", 0) or 0),
|
||||
height=int(data.get("mh", 0) or 0),
|
||||
published_date=parse_date_from_text(str(data.get("datePublished", "") or data.get("age", ""))),
|
||||
)
|
||||
)
|
||||
except _json.JSONDecodeError:
|
||||
@@ -260,7 +268,7 @@ class BingEngine(SearchEngine):
|
||||
|
||||
desc_els = el.xpath('.//div[contains(@class, "compText")]//p') or el.xpath(".//p")
|
||||
content = desc_els[0].text_content().strip() if desc_els else ""
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=parse_date_from_text(content)))
|
||||
|
||||
if not results:
|
||||
logger.warning("Yahoo fallback returned no results for '%s'", query)
|
||||
|
||||
@@ -9,6 +9,7 @@ from lxml import html as lxml_html
|
||||
|
||||
from app.models import WebResult, ImageResult
|
||||
from app.engines.base import SearchEngine, get_http_client
|
||||
from app.engines.date_utils import parse_date_from_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -48,7 +49,15 @@ class BraveEngine(SearchEngine):
|
||||
title = title_els[0].text_content().strip()
|
||||
content_els = el.xpath(".//div[contains(concat(' ', @class, ' '), ' content ')]")
|
||||
content = content_els[0].text_content().strip() if content_els else ""
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
|
||||
# Try to extract date from dedicated age element, fall back to content text
|
||||
date_els = el.xpath(".//div[contains(@class, 'snippet-description')]//time/@datetime") \
|
||||
or el.xpath(".//*[contains(@class, 'age')]")
|
||||
if date_els and isinstance(date_els[0], str):
|
||||
published_date = date_els[0][:10]
|
||||
else:
|
||||
age_text = date_els[0].text_content().strip() if date_els else ""
|
||||
published_date = parse_date_from_text(age_text) or parse_date_from_text(content)
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=published_date))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Date extraction utilities for search engine result snippets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
_MONTHS = {
|
||||
"jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
|
||||
"jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
|
||||
}
|
||||
|
||||
|
||||
def parse_date_from_text(text: str) -> str:
|
||||
"""
|
||||
Try to extract a publication date from the start of a snippet text.
|
||||
|
||||
Returns an ISO 8601 date string (YYYY-MM-DD) or empty string if not found.
|
||||
|
||||
Handles:
|
||||
- Relative: "3 hours ago", "2 days ago", "1 week ago", "5 months ago"
|
||||
- MDY: "Jan 15, 2024" or "January 15, 2024"
|
||||
- DMY: "15 Jan 2024" or "15 January 2024"
|
||||
- ISO: "2024-01-15"
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
text = text.strip()
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Relative: "X unit(s) ago"
|
||||
m = re.match(r"^(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago", text, re.IGNORECASE)
|
||||
if m:
|
||||
n = int(m.group(1))
|
||||
unit = m.group(2).lower()
|
||||
deltas: dict[str, timedelta] = {
|
||||
"second": timedelta(seconds=n),
|
||||
"minute": timedelta(minutes=n),
|
||||
"hour": timedelta(hours=n),
|
||||
"day": timedelta(days=n),
|
||||
"week": timedelta(weeks=n),
|
||||
"month": timedelta(days=n * 30),
|
||||
"year": timedelta(days=n * 365),
|
||||
}
|
||||
return (now - deltas[unit]).date().isoformat()
|
||||
|
||||
# MDY: "Jan 15, 2024" or "January 15, 2024"
|
||||
m = re.match(r"^([A-Za-z]{3,9})\s+(\d{1,2}),?\s+(\d{4})", text)
|
||||
if m:
|
||||
mon = m.group(1).lower()[:3]
|
||||
if mon in _MONTHS:
|
||||
try:
|
||||
return datetime(int(m.group(3)), _MONTHS[mon], int(m.group(2)), tzinfo=timezone.utc).date().isoformat()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# DMY: "15 Jan 2024" or "15 January 2024"
|
||||
m = re.match(r"^(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})", text)
|
||||
if m:
|
||||
mon = m.group(2).lower()[:3]
|
||||
if mon in _MONTHS:
|
||||
try:
|
||||
return datetime(int(m.group(3)), _MONTHS[mon], int(m.group(1)), tzinfo=timezone.utc).date().isoformat()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# ISO: "2024-01-15"
|
||||
m = re.match(r"^(\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01]))", text)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
return ""
|
||||
@@ -10,6 +10,7 @@ from lxml import html as lxml_html
|
||||
|
||||
from app.models import WebResult, ImageResult
|
||||
from app.engines.base import SearchEngine, get_http_client
|
||||
from app.engines.date_utils import parse_date_from_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,7 +51,7 @@ class DuckDuckGoEngine(SearchEngine):
|
||||
url = url_els[0]
|
||||
content_els = div.xpath('.//a[contains(@class, "result__snippet")]')
|
||||
content = content_els[0].text_content().strip() if content_els else ""
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=parse_date_from_text(content)))
|
||||
|
||||
return results
|
||||
|
||||
@@ -106,6 +107,7 @@ class DuckDuckGoEngine(SearchEngine):
|
||||
engine=self.name,
|
||||
width=int(item.get("width", 0) or 0),
|
||||
height=int(item.get("height", 0) or 0),
|
||||
published_date=parse_date_from_text(str(item.get("age", ""))),
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
@@ -14,6 +14,7 @@ from lxml import html as lxml_html
|
||||
|
||||
from app.models import WebResult, ImageResult
|
||||
from app.engines.base import SearchEngine, get_http_client
|
||||
from app.engines.date_utils import parse_date_from_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -133,7 +134,7 @@ class GoogleEngine(SearchEngine):
|
||||
content = txt
|
||||
break
|
||||
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name))
|
||||
results.append(WebResult(title=title, url=url, content=content, engine=self.name, published_date=parse_date_from_text(content)))
|
||||
|
||||
if not results:
|
||||
logger.warning("Google returned no parseable results for '%s'", query)
|
||||
|
||||
@@ -16,6 +16,7 @@ class WebResult(BaseModel):
|
||||
content: str = ""
|
||||
engine: str = ""
|
||||
rank: int = 0
|
||||
published_date: str = "" # ISO 8601 date (YYYY-MM-DD) or empty
|
||||
|
||||
|
||||
class ImageResult(BaseModel):
|
||||
@@ -30,6 +31,7 @@ class ImageResult(BaseModel):
|
||||
rank: int = 0
|
||||
width: int = 0
|
||||
height: int = 0
|
||||
published_date: str = "" # ISO 8601 date (YYYY-MM-DD) or empty
|
||||
|
||||
|
||||
class EngineError(BaseModel):
|
||||
|
||||
+19
-2
@@ -16,6 +16,18 @@ from app import cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SortOrder = str # "default" | "date_asc" | "date_desc"
|
||||
|
||||
|
||||
def _apply_sort(results: list, sort: SortOrder) -> None:
|
||||
"""Sort results in-place by published_date. Undated results go last."""
|
||||
if sort not in ("date_asc", "date_desc"):
|
||||
return
|
||||
with_date = [r for r in results if r.published_date]
|
||||
no_date = [r for r in results if not r.published_date]
|
||||
with_date.sort(key=lambda r: r.published_date, reverse=(sort == "date_desc"))
|
||||
results[:] = with_date + no_date
|
||||
|
||||
# Retry config: 2 retries with exponential backoff (0.5s, 1s)
|
||||
RETRY_DECORATOR = retry(
|
||||
stop=stop_after_attempt(2),
|
||||
@@ -60,15 +72,17 @@ async def search(
|
||||
page: int = 1,
|
||||
engines: list[str] | None = None,
|
||||
image_size: str = "",
|
||||
sort: SortOrder = "default",
|
||||
) -> SearchResponse:
|
||||
"""Search across all enabled engines concurrently, with optional Redis caching."""
|
||||
engines_key = ",".join(sorted(engines)) if engines else ""
|
||||
|
||||
# Check cache first
|
||||
# Check cache first (cache stores default-order results; sort applied after)
|
||||
cached_data = await cache.get_cached(query, category, page, image_size, engines_key)
|
||||
if cached_data is not None:
|
||||
resp = SearchResponse(**cached_data)
|
||||
resp.cached = True
|
||||
_apply_sort(resp.results, sort)
|
||||
return resp
|
||||
|
||||
enabled_engines = registry.get_enabled_engines()
|
||||
@@ -139,10 +153,13 @@ async def search(
|
||||
has_next=any(s.result_count > 0 for s in all_stats if s.status == "ok"),
|
||||
)
|
||||
|
||||
# Store in cache (only if we got results)
|
||||
# Store in cache before sorting (cache always holds default-order results)
|
||||
if unique_results:
|
||||
await cache.set_cached(query, category, page, image_size, engines_key, response.model_dump())
|
||||
|
||||
# Apply requested sort order
|
||||
_apply_sort(response.results, sort)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
|
||||
+50
-12
@@ -15,6 +15,7 @@ import { cn } from "@/lib/utils";
|
||||
|
||||
type Category = "web" | "images";
|
||||
type ImageSize = "" | "large" | "medium" | "small";
|
||||
type SortOrder = "default" | "date_desc" | "date_asc";
|
||||
|
||||
const IMAGE_SIZE_OPTIONS: { value: ImageSize; label: string }[] = [
|
||||
{ value: "", label: "All sizes" },
|
||||
@@ -23,7 +24,13 @@ const IMAGE_SIZE_OPTIONS: { value: ImageSize; label: string }[] = [
|
||||
{ value: "small", label: "Small" },
|
||||
];
|
||||
|
||||
function parseUrlState(): { q: string; cat: Category; page: number; imageSize: ImageSize; engines: string } {
|
||||
const SORT_OPTIONS: { value: SortOrder; label: string }[] = [
|
||||
{ value: "default", label: "Default" },
|
||||
{ value: "date_desc", label: "Newest" },
|
||||
{ value: "date_asc", label: "Oldest" },
|
||||
];
|
||||
|
||||
function parseUrlState(): { q: string; cat: Category; page: number; imageSize: ImageSize; engines: string; sort: SortOrder } {
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const q = params.get("q") ?? "";
|
||||
const cat = params.get("category") === "images" ? "images" : "web";
|
||||
@@ -31,16 +38,19 @@ function parseUrlState(): { q: string; cat: Category; page: number; imageSize: I
|
||||
const rawSize = params.get("image_size") ?? "";
|
||||
const imageSize: ImageSize = (["large", "medium", "small"].includes(rawSize) ? rawSize : "") as ImageSize;
|
||||
const engines = params.get("engines") ?? "";
|
||||
return { q, cat, page, imageSize, engines };
|
||||
const rawSort = params.get("sort") ?? "";
|
||||
const sort: SortOrder = (["date_desc", "date_asc"].includes(rawSort) ? rawSort : "default") as SortOrder;
|
||||
return { q, cat, page, imageSize, engines, sort };
|
||||
}
|
||||
|
||||
function pushUrl(q: string, cat: Category, page: number, imageSize: ImageSize = "", engines: string = "") {
|
||||
function pushUrl(q: string, cat: Category, page: number, imageSize: ImageSize = "", engines: string = "", sort: SortOrder = "default") {
|
||||
const params = new URLSearchParams();
|
||||
params.set("q", q);
|
||||
if (cat !== "web") params.set("category", cat);
|
||||
if (page > 1) params.set("page", String(page));
|
||||
if (imageSize) params.set("image_size", imageSize);
|
||||
if (engines) params.set("engines", engines);
|
||||
if (sort !== "default") params.set("sort", sort);
|
||||
const url = `/?${params.toString()}`;
|
||||
if (window.location.pathname + window.location.search !== url) {
|
||||
window.history.pushState(null, "", url);
|
||||
@@ -53,6 +63,7 @@ function App() {
|
||||
const [category, setCategory] = useState<Category>(initial.cat);
|
||||
const [page, setPage] = useState(initial.page);
|
||||
const [imageSize, setImageSize] = useState<ImageSize>(initial.imageSize);
|
||||
const [sortOrder, setSortOrder] = useState<SortOrder>(initial.sort);
|
||||
const [response, setResponse] = useState<SearchResponse | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [showSettings, setShowSettings] = useState(false);
|
||||
@@ -155,7 +166,7 @@ function App() {
|
||||
|
||||
|
||||
const doSearch = useCallback(
|
||||
async (q: string, cat: Category = category, p: number = 1, size: ImageSize = imageSize, updateUrl = true) => {
|
||||
async (q: string, cat: Category = category, p: number = 1, size: ImageSize = imageSize, updateUrl = true, sort: SortOrder = sortOrder) => {
|
||||
if (!q.trim()) return;
|
||||
setQuery(q);
|
||||
setCategory(cat);
|
||||
@@ -163,9 +174,9 @@ function App() {
|
||||
setImageSize(size);
|
||||
setLoading(true);
|
||||
setHasSearched(true);
|
||||
if (updateUrl) pushUrl(q, cat, p, cat === "images" ? size : "");
|
||||
if (updateUrl) pushUrl(q, cat, p, cat === "images" ? size : "", "", sort);
|
||||
try {
|
||||
const res = await apiSearch(q, cat, p, cat === "images" ? size : "");
|
||||
const res = await apiSearch(q, cat, p, cat === "images" ? size : "", sort);
|
||||
setResponse(res);
|
||||
} catch (err) {
|
||||
setResponse({
|
||||
@@ -185,13 +196,13 @@ function App() {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[category, imageSize]
|
||||
[category, imageSize, sortOrder]
|
||||
);
|
||||
|
||||
// Restore search from URL on initial load
|
||||
useEffect(() => {
|
||||
if (initial.q) {
|
||||
doSearch(initial.q, initial.cat, initial.page, initial.imageSize, false);
|
||||
doSearch(initial.q, initial.cat, initial.page, initial.imageSize, false, initial.sort);
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
@@ -214,9 +225,10 @@ function App() {
|
||||
setShowGallery(false);
|
||||
setShowBookmarks(false);
|
||||
setShowStats(false);
|
||||
const { q, cat, page: p, imageSize: size } = parseUrlState();
|
||||
const { q, cat, page: p, imageSize: size, sort } = parseUrlState();
|
||||
if (q) {
|
||||
doSearch(q, cat, p, size, false);
|
||||
setSortOrder(sort);
|
||||
doSearch(q, cat, p, size, false, sort);
|
||||
} else {
|
||||
setHasSearched(false);
|
||||
setResponse(null);
|
||||
@@ -232,7 +244,7 @@ function App() {
|
||||
|
||||
const handleCategoryChange = (cat: Category) => {
|
||||
setCategory(cat);
|
||||
if (query) doSearch(query, cat, 1, cat === "images" ? imageSize : "");
|
||||
if (query) doSearch(query, cat, 1, cat === "images" ? imageSize : "", true, sortOrder);
|
||||
};
|
||||
|
||||
const handlePageChange = (newPage: number) => {
|
||||
@@ -243,7 +255,12 @@ function App() {
|
||||
|
||||
const handleImageSizeChange = (size: ImageSize) => {
|
||||
setImageSize(size);
|
||||
if (query) doSearch(query, category, 1, size);
|
||||
if (query) doSearch(query, category, 1, size, true, sortOrder);
|
||||
};
|
||||
|
||||
const handleSortChange = (sort: SortOrder) => {
|
||||
setSortOrder(sort);
|
||||
if (query) doSearch(query, category, page, imageSize, true, sort);
|
||||
};
|
||||
|
||||
const handleGoHome = () => {
|
||||
@@ -538,6 +555,27 @@ function App() {
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Sort order — available for both web and images */}
|
||||
<>
|
||||
<div className="mx-2 h-5 w-px bg-border" aria-hidden="true" />
|
||||
<span className="text-xs text-muted-foreground" aria-hidden="true">Sort:</span>
|
||||
{SORT_OPTIONS.map(({ value, label }) => (
|
||||
<button
|
||||
key={value}
|
||||
onClick={() => handleSortChange(value)}
|
||||
aria-pressed={sortOrder === value}
|
||||
className={cn(
|
||||
"rounded-full px-3 py-1 text-xs font-medium transition-colors focus-visible:ring-2 focus-visible:ring-ring focus-visible:outline-none",
|
||||
sortOrder === value
|
||||
? "bg-secondary text-secondary-foreground"
|
||||
: "text-muted-foreground hover:bg-accent"
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -202,6 +202,9 @@ export function ImageResults({ results, query = "", category = "images", bookmar
|
||||
{getSize(selectedIndex) && (
|
||||
<> • {formatSize(getSize(selectedIndex)!.w, getSize(selectedIndex)!.h)}</>
|
||||
)}
|
||||
{selected.published_date && (
|
||||
<> • <span className="tabular-nums">{selected.published_date}</span></>
|
||||
)}
|
||||
<span className="ml-2 tabular-nums opacity-60">
|
||||
{selectedIndex + 1} / {results.length}
|
||||
</span>
|
||||
|
||||
@@ -28,6 +28,9 @@ export function WebResults({ results, query = "", category = "web", bookmarkedUr
|
||||
/>
|
||||
<span className="truncate">{new URL(result.url).hostname}</span>
|
||||
<span className="rounded bg-muted px-1.5 py-0.5 text-xs">{result.engine}</span>
|
||||
{result.published_date && (
|
||||
<span className="text-xs text-muted-foreground/70">{result.published_date}</span>
|
||||
)}
|
||||
{onToggleBookmark && (
|
||||
<button
|
||||
onClick={() => onToggleBookmark(result)}
|
||||
|
||||
@@ -7,6 +7,7 @@ export interface WebResult {
|
||||
content: string;
|
||||
engine: string;
|
||||
rank: number;
|
||||
published_date?: string;
|
||||
}
|
||||
|
||||
export interface ImageResult {
|
||||
@@ -20,6 +21,7 @@ export interface ImageResult {
|
||||
rank: number;
|
||||
width: number;
|
||||
height: number;
|
||||
published_date?: string;
|
||||
}
|
||||
|
||||
export interface EngineError {
|
||||
@@ -70,10 +72,12 @@ export async function search(
|
||||
query: string,
|
||||
category: "web" | "images" = "web",
|
||||
page: number = 1,
|
||||
imageSize: string = ""
|
||||
imageSize: string = "",
|
||||
sort: string = "default"
|
||||
): Promise<SearchResponse> {
|
||||
const params = new URLSearchParams({ q: query, category, page: String(page) });
|
||||
if (imageSize) params.set("image_size", imageSize);
|
||||
if (sort !== "default") params.set("sort", sort);
|
||||
const resp = await fetch(`${API_BASE}/search?${params}`);
|
||||
if (!resp.ok) throw new Error(`Search failed: ${resp.status}`);
|
||||
return resp.json();
|
||||
|
||||
Reference in New Issue
Block a user