diff --git a/links/image_api.py b/links/image_api.py index cffce40..08cd332 100644 --- a/links/image_api.py +++ b/links/image_api.py @@ -10,10 +10,7 @@ from django.http import HttpResponse, HttpResponseNotFound from django.views.decorators.http import require_http_methods from django.core.cache import cache from django.conf import settings -from functools import lru_cache from concurrent.futures import ThreadPoolExecutor -import asyncio -from asgiref.sync import sync_to_async # Configure logging logger = logging.getLogger(__name__) @@ -22,11 +19,11 @@ FitMode = Literal['clip', 'crop', 'fill', 'scale'] # Configuration constants IMAGES_FOLDER = os.getenv('IMAGES_FOLDER', '/Users/junv/Downloads') -CACHE_TIMEOUT = 600 # 10 minutes +CACHE_TIMEOUT = 600 # 10 minutes for HTTP caching MAX_FILE_SIZE = 3 * 1024 * 1024 # 3MB in bytes SUPPORTED_FORMATS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff'} MAX_RANDOM_ATTEMPTS = 30 # Maximum number of attempts to find a valid random image -image_paths_cache_key = 'random_image_paths' +MINIMUM_IMAGE_SIZE = 400 # Minimum width and height for valid images executor = ThreadPoolExecutor(max_workers=4) # Initialize random seed for true randomness @@ -81,7 +78,7 @@ def is_valid_image_fast(file_path: Path) -> bool: try: with Image.open(file_path) as img: width, height = img.size - if width < 400 or height < 400: + if width < MINIMUM_IMAGE_SIZE or height < MINIMUM_IMAGE_SIZE: logger.debug(f"Image too small: {file_path.name} ({width}x{height})") return False except Exception: @@ -102,64 +99,30 @@ def is_valid_image_deep(file_path: Path) -> bool: return False def get_image_paths() -> List[Path]: - """Get and cache the list of valid image paths with optimized scanning.""" + """Get a list of valid image paths from the root folder only. + + Note: This function is only used for stats and is not part of the main API serving flow. + """ if not IMAGES_FOLDER or not os.path.exists(IMAGES_FOLDER): logger.warning(f"Images folder not found or not configured: {IMAGES_FOLDER}") return [] - image_paths = cache.get(image_paths_cache_key) - if image_paths is None: - logger.info(f"Scanning for images in: {IMAGES_FOLDER}") + logger.info(f"Scanning for images in root folder: {IMAGES_FOLDER}") - # First pass: collect all potential image files by extension only - potential_images = [] - base_path = Path(IMAGES_FOLDER) + # Collect all image files from the root folder only + valid_images = [] + base_path = Path(IMAGES_FOLDER) - for ext in SUPPORTED_FORMATS: - # Search case-insensitively - pattern_lower = f'**/*{ext.lower()}' - pattern_upper = f'**/*{ext.upper()}' + # Get files from the root folder + for item in base_path.iterdir(): + if item.is_file() and item.suffix.lower() in SUPPORTED_FORMATS: + if is_valid_image_fast(item): + valid_images.append(item) - potential_images.extend(base_path.rglob(pattern_lower)) - if ext.lower() != ext.upper(): # Avoid duplicates - potential_images.extend(base_path.rglob(pattern_upper)) + logger.info(f"Found {len(valid_images)} valid images in the root folder") + return valid_images - # Second pass: fast validation (file size + magic number check) - valid_images = [] - total_found = len(potential_images) - - # Process in batches to avoid blocking - batch_size = 50 - processed = 0 - - for i in range(0, len(potential_images), batch_size): - batch = potential_images[i:i + batch_size] - - for img_path in batch: - if is_valid_image_fast(img_path): - valid_images.append(img_path) - - processed += 1 - - # Yield control every batch to prevent blocking - if processed % batch_size == 0: - logger.debug(f"Processed {processed}/{total_found} files...") - # Small sleep to yield control to other threads/processes - time.sleep(0.001) # 1ms yield - - logger.info(f"Found {total_found} potential image files, {len(valid_images)} passed fast validation and are under {MAX_FILE_SIZE/1024/1024:.1f}MB") - - # Cache the valid image paths - cache.set(image_paths_cache_key, valid_images, CACHE_TIMEOUT) - return valid_images - - return image_paths - -def invalidate_image_cache(): - """Force invalidation of the image paths cache.""" - cache.delete(image_paths_cache_key) - cache.delete('all_image_files') - logger.info("Image paths cache invalidated") +# Cache invalidation function removed as we no longer use caching in the new approach def get_image_stats() -> dict: """Get statistics about available images.""" @@ -241,71 +204,57 @@ def resize_image(image: Image.Image, width: Optional[int], height: Optional[int] ratio = height / orig_height return image.resize((int(orig_width * ratio), height), Image.Resampling.LANCZOS) -def get_all_image_files() -> List[Path]: - """Get all files with image extensions from the images folder.""" +def get_supported_image_path() -> Optional[Path]: + """Get a random valid image path from root folder only, without loading all files into memory.""" if not IMAGES_FOLDER or not os.path.exists(IMAGES_FOLDER): logger.warning(f"Images folder not found or not configured: {IMAGES_FOLDER}") - return [] - - # Cache the file listing to avoid repeating directory traversal - all_image_files = cache.get('all_image_files') - if all_image_files is not None: - return all_image_files - - # Get a list of all potential image files by extension only (no validation) - all_files = [] - base_path = Path(IMAGES_FOLDER) - - # Using os.walk instead of Path.rglob for better performance on large directories - for root, _, files in os.walk(str(base_path)): - for file in files: - # Check extension before adding to the list - ext = os.path.splitext(file)[1].lower() - if ext in SUPPORTED_FORMATS: - all_files.append(Path(os.path.join(root, file))) - - logger.info(f"Found {len(all_files)} files with image extensions in {IMAGES_FOLDER}") - cache.set('all_image_files', all_files, CACHE_TIMEOUT) - return all_files - -def get_random_image() -> Optional[Path]: - """Get a random valid image path by sampling and validating on-the-fly.""" - # Get all potential image files (only filtered by extension) - all_files = get_all_image_files() - - if not all_files: - logger.warning("No image files found in the configured folder") return None - # Try up to MAX_RANDOM_ATTEMPTS times to find a valid image + base_path = Path(IMAGES_FOLDER) attempts = 0 + while attempts < MAX_RANDOM_ATTEMPTS: attempts += 1 - # Ensure truly random selection by re-seeding with current time and system entropy - random.seed(time.time() + hash(str(os.urandom(8)))) + try: + # Get all image files from the root folder only (no subdirectories) + files_in_dir = [] + for item in base_path.iterdir(): + if item.is_file() and item.suffix.lower() in SUPPORTED_FORMATS: + files_in_dir.append(item) - # Use cryptographically secure random choice - selected_file = random.SystemRandom().choice(all_files) + if not files_in_dir: + logger.warning("No image files found in root folder") + return None - # Check if file exists and passes fast validation - if selected_file.exists() and is_valid_image_fast(selected_file): - # Log the selected image with folder and filename details - relative_path = selected_file.relative_to(Path(IMAGES_FOLDER)) - folder_name = relative_path.parent if relative_path.parent != Path('.') else 'root' - file_size_mb = selected_file.stat().st_size / (1024 * 1024) + # Ensure truly random selection + random.seed(time.time() + hash(str(os.urandom(8)))) - logger.info(f"Selected random image: folder='{folder_name}', file='{selected_file.name}', " - f"size={file_size_mb:.2f}MB, attempt={attempts}/{MAX_RANDOM_ATTEMPTS}") + # Randomly pick a file + selected_file = random.SystemRandom().choice(files_in_dir) - return selected_file + # Validate the file + if selected_file.exists() and is_valid_image_fast(selected_file): + file_size_mb = selected_file.stat().st_size / (1024 * 1024) - if attempts % 5 == 0: - logger.debug(f"Still searching for valid image, attempt {attempts}/{MAX_RANDOM_ATTEMPTS}") + logger.info(f"Selected random image: file='{selected_file.name}', " + f"size={file_size_mb:.2f}MB, attempt={attempts}/{MAX_RANDOM_ATTEMPTS}") + + return selected_file + + if attempts % 5 == 0: + logger.debug(f"Still searching for valid image, attempt {attempts}/{MAX_RANDOM_ATTEMPTS}") + + except (OSError, IOError, Exception) as e: + logger.warning(f"Error during file selection: {str(e)}, attempt {attempts}") logger.warning(f"Failed to find valid image after {MAX_RANDOM_ATTEMPTS} attempts") return None +def get_random_image() -> Optional[Path]: + """Get a random valid image path by directly selecting files without building a complete list.""" + return get_supported_image_path() + def process_image(image_path: Path, width: Optional[int] = None, height: Optional[int] = None, fit: FitMode = 'scale') -> Tuple[bytes, str]: """Process image with the specified dimensions and fit mode.""" @@ -331,27 +280,7 @@ def process_image(image_path: Path, width: Optional[int] = None, height: Optiona img.save(buffer, format='JPEG', quality=85, optimize=True) return buffer.getvalue(), 'image/jpeg' -def warm_image_cache_background(): - """Warm up the image cache in a background thread to avoid blocking.""" - def _warm_cache(): - try: - logger.info("Starting background image cache warmup...") - # Only cache the file listing, not the full validation - get_all_image_files() - logger.info("Background image cache warmup completed") - except Exception as e: - logger.error(f"Error during background cache warmup: {e}") - - # Run in background thread - import threading - thread = threading.Thread(target=_warm_cache, daemon=True) - thread.start() - -@require_http_methods(["POST"]) -def warm_cache(request): - """Endpoint to manually trigger cache warming.""" - warm_image_cache_background() - return HttpResponse("Cache warming started in background", content_type="text/plain") +# Cache warming functions removed as they are no longer needed with the on-demand approach @require_http_methods(["GET"]) def random_image(request, width: Optional[int] = None, height: Optional[int] = None): @@ -380,9 +309,9 @@ def random_image(request, width: Optional[int] = None, height: Optional[int] = N return HttpResponseNotFound("Invalid fit mode. Supported modes: clip, crop, fill, scale") # Validate dimensions - if width is not None and width <= 0: + if width is not None and width <= 400: return HttpResponseNotFound("Width must be positive") - if height is not None and height <= 0: + if height is not None and height <= 400: return HttpResponseNotFound("Height must be positive") if width is not None and width > 4096: return HttpResponseNotFound("Width too large (max 4096px)") @@ -420,9 +349,7 @@ def random_image(request, width: Optional[int] = None, height: Optional[int] = N except ValueError as e: logger.error(f"Image validation error: {str(e)}") - # Invalidate cache and try once more - invalidate_image_cache() - return HttpResponseNotFound("Selected image is invalid, cache refreshed") + return HttpResponseNotFound("Selected image is invalid") except Exception as e: logger.error(f"Error processing image {image_path if 'image_path' in locals() else 'unknown'}: {str(e)}") return HttpResponseNotFound(f"Error processing image: {str(e)}")