mirror of
https://github.com/wahyd4/hey-search.git
synced 2026-08-08 21:05:14 +10:00
feat: Redis search cache with configurable TTL
- Add optional Redis caching for search results (REDIS_URL env var) - Cache TTL configurable via UI: 0h (disabled) to 168h (1 week), default 6h - New backend modules: cache.py (async Redis get/set/flush), settings.py (SQLite settings table) - New API endpoints: GET/PUT /api/settings, DELETE /api/cache - New 'Cache' tab in Settings modal with status indicator, preset buttons, slider, and flush - Cache keys are deterministic hashes of query+category+page+image_size+engines - Redis is fully optional: graceful no-op when REDIS_URL unset or Redis unreachable - Updated Dockerfile, README, CHANGELOG, FEATURES Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -5,12 +5,14 @@ from __future__ import annotations
|
||||
from typing import Literal
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.models import SearchResponse, EngineInfo, APIError
|
||||
from app.search import search, get_autocomplete
|
||||
from app.engines import registry
|
||||
from app.excluded import get_excluded_domains, add_excluded_domain, remove_excluded_domain
|
||||
from app.settings import get_all_settings, get_setting, set_setting
|
||||
from app.cache import is_cache_available, flush_cache
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -227,3 +229,83 @@ async def api_remove_excluded_domain(domain: str):
|
||||
content=APIError(code="not_found", message=f"Domain '{domain}' not in exclusion list").model_dump(),
|
||||
)
|
||||
return ExcludedDomainsResponse(domains=get_excluded_domains())
|
||||
|
||||
|
||||
# --- Settings ---
|
||||
|
||||
class SettingsResponse(BaseModel):
|
||||
cache_ttl_hours: float = Field(description="Cache TTL in hours (0 = disabled, max 168 = 1 week)")
|
||||
cache_available: bool = Field(description="Whether Redis is connected and available")
|
||||
|
||||
|
||||
class UpdateSettingsRequest(BaseModel):
|
||||
cache_ttl_hours: float = Field(ge=0, le=168, description="Cache TTL in hours (0 = disabled, max 168 = 1 week)")
|
||||
|
||||
|
||||
@router.get(
|
||||
"/settings",
|
||||
response_model=SettingsResponse,
|
||||
summary="Get application settings",
|
||||
description="""Returns current application settings including cache TTL.
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
curl '$BASE_URL/api/settings'
|
||||
```
|
||||
""",
|
||||
tags=["Settings"],
|
||||
)
|
||||
async def api_get_settings():
|
||||
settings = get_all_settings()
|
||||
return SettingsResponse(
|
||||
cache_ttl_hours=float(settings.get("cache_ttl_hours", "6")),
|
||||
cache_available=is_cache_available(),
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/settings",
|
||||
response_model=SettingsResponse,
|
||||
summary="Update application settings",
|
||||
description="""Update settings such as cache TTL. Set `cache_ttl_hours` to 0 to disable caching.
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
curl -X PUT '$BASE_URL/api/settings' \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{"cache_ttl_hours": 12}'
|
||||
```
|
||||
""",
|
||||
tags=["Settings"],
|
||||
)
|
||||
async def api_update_settings(body: UpdateSettingsRequest):
|
||||
set_setting("cache_ttl_hours", str(body.cache_ttl_hours))
|
||||
return SettingsResponse(
|
||||
cache_ttl_hours=body.cache_ttl_hours,
|
||||
cache_available=is_cache_available(),
|
||||
)
|
||||
|
||||
|
||||
# --- Cache Management ---
|
||||
|
||||
class CacheFlushResponse(BaseModel):
|
||||
keys_deleted: int
|
||||
message: str
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/cache",
|
||||
response_model=CacheFlushResponse,
|
||||
summary="Flush the search cache",
|
||||
description="""Delete all cached search results from Redis.
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
curl -X DELETE '$BASE_URL/api/cache'
|
||||
```
|
||||
""",
|
||||
tags=["Settings"],
|
||||
)
|
||||
async def api_flush_cache():
|
||||
count = await flush_cache()
|
||||
return CacheFlushResponse(keys_deleted=count, message=f"Deleted {count} cached entries")
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Optional Redis cache for search results.
|
||||
|
||||
When REDIS_URL is not set or Redis is unreachable, all operations
|
||||
gracefully degrade to no-ops (cache miss / skip).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
from app.settings import get_cache_ttl_seconds
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_redis: aioredis.Redis | None = None
|
||||
_available: bool = False
|
||||
|
||||
|
||||
async def init_redis() -> None:
|
||||
"""Try to connect to Redis. If it fails, caching is silently disabled."""
|
||||
global _redis, _available
|
||||
url = os.environ.get("REDIS_URL", "")
|
||||
if not url:
|
||||
logger.info("REDIS_URL not set — caching disabled")
|
||||
return
|
||||
try:
|
||||
_redis = aioredis.from_url(url, decode_responses=True, socket_connect_timeout=3)
|
||||
await _redis.ping()
|
||||
_available = True
|
||||
logger.info("Redis connected at %s — caching enabled", url)
|
||||
except Exception as e:
|
||||
logger.warning("Redis unavailable (%s) — caching disabled", e)
|
||||
_redis = None
|
||||
_available = False
|
||||
|
||||
|
||||
async def close_redis() -> None:
|
||||
global _redis, _available
|
||||
if _redis:
|
||||
await _redis.aclose()
|
||||
_redis = None
|
||||
_available = False
|
||||
|
||||
|
||||
def is_cache_available() -> bool:
|
||||
return _available and _redis is not None
|
||||
|
||||
|
||||
def _cache_key(query: str, category: str, page: int, image_size: str, engines: str) -> str:
|
||||
"""Build a deterministic cache key."""
|
||||
raw = f"hs:{category}:{page}:{image_size}:{engines}:{query}"
|
||||
h = hashlib.sha256(raw.encode()).hexdigest()[:16]
|
||||
return f"hs:search:{h}"
|
||||
|
||||
|
||||
async def get_cached(query: str, category: str, page: int, image_size: str, engines: str) -> dict | None:
|
||||
"""Return cached search response dict, or None on miss."""
|
||||
if not is_cache_available():
|
||||
return None
|
||||
ttl = get_cache_ttl_seconds()
|
||||
if ttl <= 0:
|
||||
return None
|
||||
key = _cache_key(query, category, page, image_size, engines)
|
||||
try:
|
||||
data = await _redis.get(key) # type: ignore[union-attr]
|
||||
if data:
|
||||
logger.debug("Cache HIT for key %s", key)
|
||||
return json.loads(data)
|
||||
except Exception as e:
|
||||
logger.warning("Cache get error: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
async def set_cached(query: str, category: str, page: int, image_size: str, engines: str, response_dict: dict) -> None:
|
||||
"""Store a search response in cache."""
|
||||
if not is_cache_available():
|
||||
return
|
||||
ttl = get_cache_ttl_seconds()
|
||||
if ttl <= 0:
|
||||
return
|
||||
key = _cache_key(query, category, page, image_size, engines)
|
||||
try:
|
||||
await _redis.set(key, json.dumps(response_dict, default=str), ex=ttl) # type: ignore[union-attr]
|
||||
logger.debug("Cache SET for key %s (ttl=%ds)", key, ttl)
|
||||
except Exception as e:
|
||||
logger.warning("Cache set error: %s", e)
|
||||
|
||||
|
||||
async def flush_cache() -> int:
|
||||
"""Delete all hey-search cache keys. Returns count deleted."""
|
||||
if not is_cache_available():
|
||||
return 0
|
||||
try:
|
||||
keys = []
|
||||
async for key in _redis.scan_iter("hs:search:*"): # type: ignore[union-attr]
|
||||
keys.append(key)
|
||||
if keys:
|
||||
await _redis.delete(*keys) # type: ignore[union-attr]
|
||||
return len(keys)
|
||||
except Exception as e:
|
||||
logger.warning("Cache flush error: %s", e)
|
||||
return 0
|
||||
@@ -15,14 +15,19 @@ from fastapi.staticfiles import StaticFiles
|
||||
from app.api.routes import router
|
||||
from app.engines import registry
|
||||
from app.excluded import init_db
|
||||
from app.settings import init_settings_table
|
||||
from app.cache import init_redis, close_redis
|
||||
from app.models import APIError
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(application: FastAPI):
|
||||
init_db()
|
||||
init_settings_table()
|
||||
registry.load_default_engines()
|
||||
await init_redis()
|
||||
yield
|
||||
await close_redis()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
|
||||
+17
-2
@@ -12,6 +12,7 @@ from app.models import WebResult, ImageResult, EngineError, EngineStat, SearchRe
|
||||
from app.engines.base import SearchEngine, SearchCategory
|
||||
from app.engines import registry
|
||||
from app.excluded import is_url_excluded
|
||||
from app import cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -60,7 +61,15 @@ async def search(
|
||||
engines: list[str] | None = None,
|
||||
image_size: str = "",
|
||||
) -> SearchResponse:
|
||||
"""Search across all enabled engines concurrently."""
|
||||
"""Search across all enabled engines concurrently, with optional Redis caching."""
|
||||
engines_key = ",".join(sorted(engines)) if engines else ""
|
||||
|
||||
# Check cache first
|
||||
cached = await cache.get_cached(query, category, page, image_size, engines_key)
|
||||
if cached is not None:
|
||||
resp = SearchResponse(**cached)
|
||||
return resp
|
||||
|
||||
enabled_engines = registry.get_enabled_engines()
|
||||
|
||||
if engines:
|
||||
@@ -118,7 +127,7 @@ async def search(
|
||||
for i, r in enumerate(unique_results):
|
||||
r.rank = i + 1
|
||||
|
||||
return SearchResponse(
|
||||
response = SearchResponse(
|
||||
query=query,
|
||||
category=category,
|
||||
page=page,
|
||||
@@ -129,6 +138,12 @@ async def search(
|
||||
has_next=any(s.result_count > 0 for s in all_stats if s.status == "ok"),
|
||||
)
|
||||
|
||||
# Store in cache (only if we got results)
|
||||
if unique_results:
|
||||
await cache.set_cached(query, category, page, image_size, engines_key, response.model_dump())
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def get_autocomplete(query: str) -> list[str]:
|
||||
"""Get autocomplete suggestions from enabled engines."""
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""SQLite-backed application settings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.excluded import _get_conn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default values
|
||||
DEFAULTS: dict[str, str] = {
|
||||
"cache_ttl_hours": "6",
|
||||
}
|
||||
|
||||
|
||||
def init_settings_table() -> None:
|
||||
"""Create the settings table and populate defaults."""
|
||||
conn = _get_conn()
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
# Insert defaults for any missing keys
|
||||
for key, default in DEFAULTS.items():
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)",
|
||||
(key, default),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.info("Settings table initialized")
|
||||
|
||||
|
||||
def get_setting(key: str) -> str:
|
||||
"""Get a single setting value. Returns default if not set."""
|
||||
conn = _get_conn()
|
||||
row = conn.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
|
||||
conn.close()
|
||||
if row:
|
||||
return row[0]
|
||||
return DEFAULTS.get(key, "")
|
||||
|
||||
|
||||
def set_setting(key: str, value: str) -> None:
|
||||
"""Set a single setting value."""
|
||||
conn = _get_conn()
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)",
|
||||
(key, value),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_all_settings() -> dict[str, str]:
|
||||
"""Get all settings as a dict."""
|
||||
conn = _get_conn()
|
||||
rows = conn.execute("SELECT key, value FROM settings").fetchall()
|
||||
conn.close()
|
||||
result = dict(DEFAULTS) # start with defaults
|
||||
result.update({k: v for k, v in rows})
|
||||
return result
|
||||
|
||||
|
||||
def get_cache_ttl_seconds() -> int:
|
||||
"""Get cache TTL in seconds. Returns 0 if caching is disabled."""
|
||||
try:
|
||||
hours = float(get_setting("cache_ttl_hours"))
|
||||
return max(0, int(hours * 3600))
|
||||
except (ValueError, TypeError):
|
||||
return 6 * 3600 # fallback: 6 hours
|
||||
@@ -11,4 +11,5 @@ dependencies = [
|
||||
"tenacity",
|
||||
"lxml",
|
||||
"beautifulsoup4",
|
||||
"redis>=5.0",
|
||||
]
|
||||
|
||||
Generated
+11
@@ -111,6 +111,7 @@ dependencies = [
|
||||
{ name = "httpx" },
|
||||
{ name = "lxml" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "redis" },
|
||||
{ name = "tenacity" },
|
||||
{ name = "uvicorn" },
|
||||
]
|
||||
@@ -122,6 +123,7 @@ requires-dist = [
|
||||
{ name = "httpx" },
|
||||
{ name = "lxml" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "redis", specifier = ">=5.0" },
|
||||
{ name = "tenacity" },
|
||||
{ name = "uvicorn" },
|
||||
]
|
||||
@@ -329,6 +331,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "7.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9f/32/6fac13a11e73e1bc67a2ae821a72bfe4c2d8c4c48f0267e4a952be0f1bae/redis-7.2.0.tar.gz", hash = "sha256:4dd5bf4bd4ae80510267f14185a15cba2a38666b941aff68cccf0256b51c1f26", size = 4901247, upload-time = "2026-02-16T17:16:22.797Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/86/cf/f6180b67f99688d83e15c84c5beda831d1d341e95872d224f87ccafafe61/redis-7.2.0-py3-none-any.whl", hash = "sha256:01f591f8598e483f1842d429e8ae3a820804566f1c73dca1b80e23af9fba0497", size = 394898, upload-time = "2026-02-16T17:16:20.693Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "2.8.3"
|
||||
|
||||
Reference in New Issue
Block a user