home/links
2
0
Fork 0
mirror of https://github.com/wahyd4/links.git synced 2026-08-09 05:06:16 +10:00
Code Issues Packages Projects Releases Wiki Activity

Update logic

Browse Source
This commit is contained in:
junv
2025-05-30 15:26:22 +10:00
parent 9c89b77006
commit 6f98ef653d
1 changed files with 57 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
links/image_api.py
+57 -130
View File
@@ -10,10 +10,7 @@ from django.http import HttpResponse, HttpResponseNotFound
from django.views.decorators.http import require_http_methods
from django.core.cache import cache
from django.conf import settings
from functools import lru_cache
from concurrent.futures import ThreadPoolExecutor
import asyncio
from asgiref.sync import sync_to_async
# Configure logging
logger = logging.getLogger(__name__)
@@ -22,11 +19,11 @@ FitMode = Literal['clip', 'crop', 'fill', 'scale']
# Configuration constants
IMAGES_FOLDER = os.getenv('IMAGES_FOLDER', '/Users/junv/Downloads')
CACHE_TIMEOUT = 600 # 10 minutes
CACHE_TIMEOUT = 600 # 10 minutes for HTTP caching
MAX_FILE_SIZE = 3 * 1024 * 1024 # 3MB in bytes
SUPPORTED_FORMATS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff'}
MAX_RANDOM_ATTEMPTS = 30 # Maximum number of attempts to find a valid random image
image_paths_cache_key = 'random_image_paths'
MINIMUM_IMAGE_SIZE = 400 # Minimum width and height for valid images
executor = ThreadPoolExecutor(max_workers=4)
# Initialize random seed for true randomness
@@ -81,7 +78,7 @@ def is_valid_image_fast(file_path: Path) -> bool:
try:
with Image.open(file_path) as img:
width, height = img.size
if width < 400 or height < 400:
if width < MINIMUM_IMAGE_SIZE or height < MINIMUM_IMAGE_SIZE:
logger.debug(f"Image too small: {file_path.name} ({width}x{height})")
return False
except Exception:
@@ -102,64 +99,30 @@ def is_valid_image_deep(file_path: Path) -> bool:
return False
def get_image_paths() -> List[Path]:
"""Get and cache the list of valid image paths with optimized scanning."""
"""Get a list of valid image paths from the root folder only.
Note: This function is only used for stats and is not part of the main API serving flow.
"""
if not IMAGES_FOLDER or not os.path.exists(IMAGES_FOLDER):
logger.warning(f"Images folder not found or not configured: {IMAGES_FOLDER}")
return []
image_paths = cache.get(image_paths_cache_key)
if image_paths is None:
logger.info(f"Scanning for images in: {IMAGES_FOLDER}")
logger.info(f"Scanning for images in root folder: {IMAGES_FOLDER}")
# First pass: collect all potential image files by extension only
potential_images = []
base_path = Path(IMAGES_FOLDER)
# Collect all image files from the root folder only
valid_images = []
base_path = Path(IMAGES_FOLDER)
for ext in SUPPORTED_FORMATS:
# Search case-insensitively
pattern_lower = f'**/*{ext.lower()}'
pattern_upper = f'**/*{ext.upper()}'
# Get files from the root folder
for item in base_path.iterdir():
if item.is_file() and item.suffix.lower() in SUPPORTED_FORMATS:
if is_valid_image_fast(item):
valid_images.append(item)
potential_images.extend(base_path.rglob(pattern_lower))
if ext.lower() != ext.upper(): # Avoid duplicates
potential_images.extend(base_path.rglob(pattern_upper))
logger.info(f"Found {len(valid_images)} valid images in the root folder")
return valid_images
# Second pass: fast validation (file size + magic number check)
valid_images = []
total_found = len(potential_images)
# Process in batches to avoid blocking
batch_size = 50
processed = 0
for i in range(0, len(potential_images), batch_size):
batch = potential_images[i:i + batch_size]
for img_path in batch:
if is_valid_image_fast(img_path):
valid_images.append(img_path)
processed += 1
# Yield control every batch to prevent blocking
if processed % batch_size == 0:
logger.debug(f"Processed {processed}/{total_found} files...")
# Small sleep to yield control to other threads/processes
time.sleep(0.001) # 1ms yield
logger.info(f"Found {total_found} potential image files, {len(valid_images)} passed fast validation and are under {MAX_FILE_SIZE/1024/1024:.1f}MB")
# Cache the valid image paths
cache.set(image_paths_cache_key, valid_images, CACHE_TIMEOUT)
return valid_images
return image_paths
def invalidate_image_cache():
"""Force invalidation of the image paths cache."""
cache.delete(image_paths_cache_key)
cache.delete('all_image_files')
logger.info("Image paths cache invalidated")
# Cache invalidation function removed as we no longer use caching in the new approach
def get_image_stats() -> dict:
"""Get statistics about available images."""
@@ -241,71 +204,57 @@ def resize_image(image: Image.Image, width: Optional[int], height: Optional[int]
ratio = height / orig_height
return image.resize((int(orig_width * ratio), height), Image.Resampling.LANCZOS)
def get_all_image_files() -> List[Path]:
"""Get all files with image extensions from the images folder."""
def get_supported_image_path() -> Optional[Path]:
"""Get a random valid image path from root folder only, without loading all files into memory."""
if not IMAGES_FOLDER or not os.path.exists(IMAGES_FOLDER):
logger.warning(f"Images folder not found or not configured: {IMAGES_FOLDER}")
return []
# Cache the file listing to avoid repeating directory traversal
all_image_files = cache.get('all_image_files')
if all_image_files is not None:
return all_image_files
# Get a list of all potential image files by extension only (no validation)
all_files = []
base_path = Path(IMAGES_FOLDER)
# Using os.walk instead of Path.rglob for better performance on large directories
for root, _, files in os.walk(str(base_path)):
for file in files:
# Check extension before adding to the list
ext = os.path.splitext(file)[1].lower()
if ext in SUPPORTED_FORMATS:
all_files.append(Path(os.path.join(root, file)))
logger.info(f"Found {len(all_files)} files with image extensions in {IMAGES_FOLDER}")
cache.set('all_image_files', all_files, CACHE_TIMEOUT)
return all_files
def get_random_image() -> Optional[Path]:
"""Get a random valid image path by sampling and validating on-the-fly."""
# Get all potential image files (only filtered by extension)
all_files = get_all_image_files()
if not all_files:
logger.warning("No image files found in the configured folder")
return None
# Try up to MAX_RANDOM_ATTEMPTS times to find a valid image
base_path = Path(IMAGES_FOLDER)
attempts = 0
while attempts < MAX_RANDOM_ATTEMPTS:
attempts += 1
# Ensure truly random selection by re-seeding with current time and system entropy
random.seed(time.time() + hash(str(os.urandom(8))))
try:
# Get all image files from the root folder only (no subdirectories)
files_in_dir = []
for item in base_path.iterdir():
if item.is_file() and item.suffix.lower() in SUPPORTED_FORMATS:
files_in_dir.append(item)
# Use cryptographically secure random choice
selected_file = random.SystemRandom().choice(all_files)
if not files_in_dir:
logger.warning("No image files found in root folder")
return None
# Check if file exists and passes fast validation
if selected_file.exists() and is_valid_image_fast(selected_file):
# Log the selected image with folder and filename details
relative_path = selected_file.relative_to(Path(IMAGES_FOLDER))
folder_name = relative_path.parent if relative_path.parent != Path('.') else 'root'
file_size_mb = selected_file.stat().st_size / (1024 * 1024)
# Ensure truly random selection
random.seed(time.time() + hash(str(os.urandom(8))))
logger.info(f"Selected random image: folder='{folder_name}', file='{selected_file.name}', "
f"size={file_size_mb:.2f}MB, attempt={attempts}/{MAX_RANDOM_ATTEMPTS}")
# Randomly pick a file
selected_file = random.SystemRandom().choice(files_in_dir)
return selected_file
# Validate the file
if selected_file.exists() and is_valid_image_fast(selected_file):
file_size_mb = selected_file.stat().st_size / (1024 * 1024)
if attempts % 5 == 0:
logger.debug(f"Still searching for valid image, attempt {attempts}/{MAX_RANDOM_ATTEMPTS}")
logger.info(f"Selected random image: file='{selected_file.name}', "
f"size={file_size_mb:.2f}MB, attempt={attempts}/{MAX_RANDOM_ATTEMPTS}")
return selected_file
if attempts % 5 == 0:
logger.debug(f"Still searching for valid image, attempt {attempts}/{MAX_RANDOM_ATTEMPTS}")
except (OSError, IOError, Exception) as e:
logger.warning(f"Error during file selection: {str(e)}, attempt {attempts}")
logger.warning(f"Failed to find valid image after {MAX_RANDOM_ATTEMPTS} attempts")
return None
def get_random_image() -> Optional[Path]:
"""Get a random valid image path by directly selecting files without building a complete list."""
return get_supported_image_path()
def process_image(image_path: Path, width: Optional[int] = None, height: Optional[int] = None,
fit: FitMode = 'scale') -> Tuple[bytes, str]:
"""Process image with the specified dimensions and fit mode."""
@@ -331,27 +280,7 @@ def process_image(image_path: Path, width: Optional[int] = None, height: Optiona
img.save(buffer, format='JPEG', quality=85, optimize=True)
return buffer.getvalue(), 'image/jpeg'
def warm_image_cache_background():
"""Warm up the image cache in a background thread to avoid blocking."""
def _warm_cache():
try:
logger.info("Starting background image cache warmup...")
# Only cache the file listing, not the full validation
get_all_image_files()
logger.info("Background image cache warmup completed")
except Exception as e:
logger.error(f"Error during background cache warmup: {e}")
# Run in background thread
import threading
thread = threading.Thread(target=_warm_cache, daemon=True)
thread.start()
@require_http_methods(["POST"])
def warm_cache(request):
"""Endpoint to manually trigger cache warming."""
warm_image_cache_background()
return HttpResponse("Cache warming started in background", content_type="text/plain")
# Cache warming functions removed as they are no longer needed with the on-demand approach
@require_http_methods(["GET"])
def random_image(request, width: Optional[int] = None, height: Optional[int] = None):
@@ -380,9 +309,9 @@ def random_image(request, width: Optional[int] = None, height: Optional[int] = N
return HttpResponseNotFound("Invalid fit mode. Supported modes: clip, crop, fill, scale")
# Validate dimensions
if width is not None and width <= 0:
if width is not None and width <= 400:
return HttpResponseNotFound("Width must be positive")
if height is not None and height <= 0:
if height is not None and height <= 400:
return HttpResponseNotFound("Height must be positive")
if width is not None and width > 4096:
return HttpResponseNotFound("Width too large (max 4096px)")
@@ -420,9 +349,7 @@ def random_image(request, width: Optional[int] = None, height: Optional[int] = N
except ValueError as e:
logger.error(f"Image validation error: {str(e)}")
# Invalidate cache and try once more
invalidate_image_cache()
return HttpResponseNotFound("Selected image is invalid, cache refreshed")
return HttpResponseNotFound("Selected image is invalid")
except Exception as e:
logger.error(f"Error processing image {image_path if 'image_path' in locals() else 'unknown'}: {str(e)}")
return HttpResponseNotFound(f"Error processing image: {str(e)}")