from django.utils import timezone
from django.db.models import F
from datetime import timedelta
import requests
from bs4 import BeautifulSoup
import logging
import os
from django.conf import settings
from urllib.parse import quote
import asyncio
from threading import Thread
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError

logger = logging.getLogger(__name__)
SCREENSHOT_TIMEOUT = 60000  # 60 seconds in milliseconds for Playwright
FETCH_PAGE_TIMEOUT = 20
MAX_RETRIES = 3

def fibonacci(n):
    if n <= 0:
        return 0
    elif n == 1:
        return 1
    else:
        a, b = 0, 1
        for _ in range(2, n + 1):
            a, b = b, a + b
        return b

def extract_title_from_soup(soup):
    """Extract title from BeautifulSoup object using various methods"""
    if soup.title:
        return soup.title.string
    elif soup.find('h1'):
        return soup.find('h1').get_text(strip=True)
    elif soup.find('meta', property='og:title'):
        return soup.find('meta', property='og:title').get('content')
    return None

def extract_description_from_soup(soup):
    """Extract description from BeautifulSoup object"""
    # Try meta description first
    meta_desc = soup.find('meta', {'name': 'description'}) or soup.find('meta', {'property': 'og:description'})
    if meta_desc:
        return meta_desc.get('content')

    # If no meta description, try to find first meaningful paragraph
    for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
        tag.decompose()

    paragraphs = soup.find_all(['p', 'div'])
    for p in paragraphs:
        text = p.get_text(strip=True)
        if len(text) > 100:
            return text
    return None

async def _capture_screenshot_async(page_url, full_path, job_timeout_seconds=300):
    """Async function to capture screenshot using Playwright.

    Raises asyncio.TimeoutError if the entire operation exceeds job_timeout_seconds.
    """
    async def _run():
        async with async_playwright() as p:
            browser = await p.chromium.launch(
                headless=True,
                args=[
                    '--no-sandbox',
                    '--disable-setuid-sandbox',
                    '--disable-dev-shm-usage',
                ]
            )

            context = await browser.new_context(
                viewport={'width': 1366, 'height': 768},
                locale='zh-CN',
                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                extra_http_headers={
                    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
                },
                ignore_https_errors=True,
            )

            page = await context.new_page()

            try:
                await page.goto(
                    page_url,
                    wait_until='networkidle',
                    timeout=SCREENSHOT_TIMEOUT
                )

                await page.screenshot(
                    path=full_path,
                    full_page=True,
                    type='png'
                )

                logger.info(f"Successfully captured screenshot: {full_path}")

            finally:
                await context.close()
                await browser.close()

    await asyncio.wait_for(_run(), timeout=job_timeout_seconds)


def capture_screenshot(page_id, screenshot_id, retry_count=0):
    """Task to capture full-page screenshot with Playwright"""
    from .models import Page, Screenshot, SiteSettings

    screenshot = None
    try:
        page = Page.objects.get(id=page_id)
        screenshot = Screenshot.objects.get(id=screenshot_id)

        # Idempotency: skip if already completed
        if screenshot.status == Screenshot.Status.COMPLETED:
            logger.info(f"Screenshot {screenshot_id} already completed, skipping")
            return

        # Respect model-level retry limit (model tracks total attempts across restarts)
        if screenshot.retry_count >= MAX_RETRIES:
            logger.warning(f"Screenshot {screenshot_id} exceeded max retries ({MAX_RETRIES}), marking FAILED")
            screenshot.status = Screenshot.Status.FAILED
            screenshot.error = f"Exceeded maximum retries ({MAX_RETRIES})"
            screenshot.save()
            return

        job_timeout = SiteSettings.get().screenshot_job_timeout_seconds

        screenshot.status = Screenshot.Status.PROCESSING
        screenshot.error = None
        screenshot.save()

        # Create screenshots directory
        screenshots_dir = os.path.join(settings.MEDIA_ROOT, 'screenshots')
        os.makedirs(screenshots_dir, exist_ok=True)

        # Create filename with timestamp
        timestamp = timezone.now().strftime('%Y%m%d_%H%M%S')
        filename = f"page_{page_id}_{screenshot_id}_{timestamp}.png"
        filepath = os.path.join('screenshots', filename)
        full_path = os.path.join(settings.MEDIA_ROOT, filepath)

        # Run async screenshot capture in sync context
        try:
            asyncio.run(_capture_screenshot_async(page.url, full_path, job_timeout_seconds=job_timeout))

            # Update screenshot record
            screenshot.path = filepath
            screenshot.status = Screenshot.Status.COMPLETED
            screenshot.save()

            logger.info(f"Successfully captured screenshot for page {page_id}")

        except asyncio.TimeoutError:
            msg = f"Job timed out after {job_timeout}s"
            logger.error(f"Screenshot {screenshot_id}: {msg}")
            screenshot.status = Screenshot.Status.FAILED
            screenshot.error = msg
            screenshot.save()
            return
        except PlaywrightTimeoutError as e:
            logger.error(f"Timeout during screenshot capture: {str(e)}")
            screenshot.status = Screenshot.Status.FAILED
            screenshot.error = f"Timeout: {str(e)}"
            screenshot.save()
            raise
        except Exception as e:
            logger.error(f"Error during screenshot capture: {str(e)}")
            screenshot.status = Screenshot.Status.FAILED
            screenshot.error = str(e)
            screenshot.save()
            raise

    except Exception as exc:
        logger.error(f"Failed to capture screenshot for page {page_id}: {exc}", exc_info=True)
        if screenshot is None:
            return

        # Increment model-level retry count
        try:
            screenshot.retry_count = Screenshot.objects.filter(pk=screenshot.pk).values_list('retry_count', flat=True).first() or 0
            screenshot.retry_count += 1
            screenshot.save(update_fields=['retry_count', 'error', 'updated_at'])
        except Exception:
            pass

        if screenshot.retry_count >= MAX_RETRIES:
            logger.error(f"Max retries exceeded for screenshot {screenshot_id}")
            try:
                screenshot.status = Screenshot.Status.FAILED
                screenshot.error = f"Failed after {screenshot.retry_count} attempts: {str(exc)}"
                screenshot.save()
            except Exception:
                pass
        else:
            retry_delay = fibonacci(screenshot.retry_count)
            try:
                screenshot.error = f"Retrying in {retry_delay}s (attempt {screenshot.retry_count}/{MAX_RETRIES}): {str(exc)}"
                screenshot.save(update_fields=['error', 'updated_at'])
            except Exception:
                pass
            # Schedule retry using APScheduler
            from core.scheduler import scheduler
            from datetime import datetime
            run_date = datetime.now() + timedelta(seconds=retry_delay)
            try:
                scheduler.add_job(
                    capture_screenshot,
                    'date',
                    run_date=run_date,
                    args=[page_id, screenshot_id, screenshot.retry_count],
                    id=f'capture_screenshot_{screenshot_id}_retry_{screenshot.retry_count}',
                    replace_existing=True
                )
                logger.info(f"Scheduled retry {screenshot.retry_count} for screenshot {screenshot_id} in {retry_delay}s")
            except Exception as sched_exc:
                logger.error(f"Failed to schedule retry for screenshot {screenshot_id}: {sched_exc}")
                try:
                    screenshot.status = Screenshot.Status.FAILED
                    screenshot.error = f"Scheduler error, giving up: {str(sched_exc)}"
                    screenshot.save()
                except Exception:
                    pass

def process_page(page_id, retry_count=0):
    """Task to process a page and extract its metadata"""
    from .models import Page, Screenshot

    try:
        page = Page.objects.get(id=page_id)

        if not page.needs_processing():
            return

        page.process_status = Page.ProcessStatus.PROCESSING
        page.save()

        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
        }

        response = requests.get(page.url, headers=headers, timeout=FETCH_PAGE_TIMEOUT, verify=False)
        response.raise_for_status()

        if response.encoding == 'ISO-8859-1':
            response.encoding = response.apparent_encoding or 'utf-8'

        soup = BeautifulSoup(response.text, 'html.parser')

        # Extract title if not present
        if not page.title:
            title = extract_title_from_soup(soup)
            if title:
                page.title = title.strip()

        # Extract summary if not present
        if not page.summary:
            description = extract_description_from_soup(soup)
            if description:
                page.summary = description[:500]

        # Create a new screenshot record and trigger capture
        screenshot = Screenshot.objects.create(
            page=page,
            status=Screenshot.Status.PENDING
        )

        # Trigger screenshot capture asynchronously
        thread = Thread(target=capture_screenshot, args=(page.id, screenshot.id))
        thread.daemon = True
        thread.start()

        page.process_status = Page.ProcessStatus.COMPLETED
        page.save()

        logger.info(f"Successfully processed page {page_id}")

    except Exception as exc:
        logger.error(f"Failed to process page {page_id}: {exc}")
        try:
            page = Page.objects.get(id=page_id)
        except Page.DoesNotExist:
            return
        page.retry_count += 1
        page.last_retry_at = timezone.now()

        if page.retry_count >= MAX_RETRIES:
            page.process_status = Page.ProcessStatus.FAILED
            page.error_message = str(exc)
            page.save()
        else:
            page.process_status = Page.ProcessStatus.PENDING
            page.save()

            # Schedule retry using APScheduler
            retry_delay = fibonacci(retry_count)
            from core.scheduler import scheduler
            from datetime import datetime, timedelta
            run_date = datetime.now() + timedelta(seconds=retry_delay)
            scheduler.add_job(
                process_page,
                'date',
                run_date=run_date,
                args=[page_id, retry_count + 1],
                id=f'process_page_{page_id}_retry_{retry_count + 1}',
                replace_existing=True
            )
            logger.info(f"Scheduled retry {retry_count + 1} for page {page_id} in {retry_delay} seconds")

def schedule_pending_pages():
    """Periodic task to schedule processing of pending pages"""
    from .models import Page

    logger.info("Checking for pending pages...")
    pending_pages = Page.objects.filter(
        process_status=Page.ProcessStatus.PENDING,
        retry_count__lt=MAX_RETRIES
    ).exclude(
        last_retry_at__gte=timezone.now() - timedelta(seconds=fibonacci(MAX_RETRIES))
    )

    for page in pending_pages:
        logger.info(f"Scheduling processing for page {page.id}")
        # Run in background thread
        thread = Thread(target=process_page, args=(page.id,))
        thread.daemon = True
        thread.start()


def schedule_pending_screenshots():
    """Periodic task to retry screenshots stuck in PENDING or PROCESSING state.

    Screenshots can get stuck when their background thread is killed (e.g. server
    restart).  Any screenshot that hasn't been updated in more than 5 minutes and
    is not in a terminal state is considered stuck and will be re-queued — up to
    the configured concurrency limit to prevent OOM.
    """
    from .models import Screenshot, SiteSettings

    # First: mark any max-retried screenshots as FAILED so they stop being picked up
    over_limit = Screenshot.objects.filter(
        status__in=[Screenshot.Status.PENDING, Screenshot.Status.PROCESSING],
        retry_count__gte=MAX_RETRIES,
    )
    failed_count = over_limit.count()
    if failed_count:
        logger.warning(f"Marking {failed_count} screenshot(s) as FAILED (exceeded {MAX_RETRIES} retries)")
        over_limit.update(
            status=Screenshot.Status.FAILED,
            error=f"Exceeded maximum retries ({MAX_RETRIES})",
        )

    # Determine how many slots are available
    max_concurrent = SiteSettings.get().max_concurrent_screenshot_jobs
    currently_processing = Screenshot.objects.filter(
        status=Screenshot.Status.PROCESSING,
    ).count()
    available_slots = max(0, max_concurrent - currently_processing)

    if available_slots == 0:
        logger.info(f"All {max_concurrent} screenshot slots busy, skipping stuck check")
        return

    threshold = timezone.now() - timedelta(minutes=5)
    stuck = Screenshot.objects.filter(
        status__in=[Screenshot.Status.PENDING, Screenshot.Status.PROCESSING],
        updated_at__lt=threshold,
        retry_count__lt=MAX_RETRIES,
    )[:available_slots]

    count = stuck.count()
    if count:
        logger.info(f"Found {count} stuck screenshot(s) — retrying (slots available: {available_slots})")

    for screenshot in stuck:
        logger.info(f"Retrying screenshot {screenshot.id} (page {screenshot.page_id}, attempt {screenshot.retry_count + 1}/{MAX_RETRIES})")
        screenshot.status = Screenshot.Status.PENDING
        screenshot.retry_count += 1
        screenshot.save(update_fields=['status', 'retry_count', 'updated_at'])
        thread = Thread(target=capture_screenshot, args=(screenshot.page_id, screenshot.id))
        thread.daemon = True
        thread.start()

def fetch_webpage_content_from_crawl4ai(page_id, retry_count=0):
    """
    Fetch webpage content using Crawl4AI /md endpoint
    """
    from .models import Page

    if not settings.CRAWL4AI_ENABLED:
        logger.info("Crawl4AI is disabled, skipping webpage content extraction")
        return

    try:
        page = Page.objects.get(id=page_id)
        logger.info(f"Fetching webpage content for page {page_id} from Crawl4AI")

        # Call Crawl4AI /md endpoint
        api_url = f"{settings.CRAWL4AI_API_URL}/md"
        payload = {
            "url": page.url
        }

        response = requests.post(
            api_url,
            json=payload,
            timeout=60
        )
        response.raise_for_status()

        data = response.json()

        # Extract markdown content from response
        markdown_content = data.get('markdown', '')

        if markdown_content:
            page.content = markdown_content
            page.save()
            logger.info(f"Successfully fetched webpage content for page {page_id}")
        else:
            logger.warning(f"No markdown content returned for page {page_id}")

    except requests.RequestException as e:
        logger.error(f"Failed to fetch webpage content for page {page_id}: {e}")

        # Retry logic
        if retry_count < MAX_RETRIES:
            retry_delay = fibonacci(retry_count + 1)
            from core.scheduler import scheduler
            from datetime import datetime, timedelta
            run_date = datetime.now() + timedelta(seconds=retry_delay)
            scheduler.add_job(
                fetch_webpage_content_from_crawl4ai,
                'date',
                run_date=run_date,
                args=[page_id, retry_count + 1],
                id=f'fetch_crawl4ai_{page_id}_retry_{retry_count + 1}',
                replace_existing=True
            )
            logger.info(f"Scheduled Crawl4AI retry {retry_count + 1} for page {page_id} in {retry_delay} seconds")
    except Exception as e:
        logger.error(f"Unexpected error fetching webpage content for page {page_id}: {e}", exc_info=True)


def download_and_save_image(file_upload_id):
    """Background task: download an external image and save it to FILE_UPLOADS_FOLDER.

    The FileUpload record must already exist with source_url set. This task fills in
    the actual file content, size, and mime_type once the download completes.
    """
    from .models import FileUpload

    try:
        record = FileUpload.objects.get(pk=file_upload_id)
    except FileUpload.DoesNotExist:
        logger.error(f"download_and_save_image: FileUpload {file_upload_id} not found")
        return

    source_url = record.source_url
    if not source_url:
        logger.error(f"download_and_save_image: FileUpload {file_upload_id} has no source_url")
        return

    dest_path = record.file_path
    if os.path.exists(dest_path):
        logger.info(f"download_and_save_image: {dest_path} already exists, skipping download")
        return

    logger.info(f"download_and_save_image: downloading {source_url} → {dest_path}")
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                          '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        }
        response = requests.get(source_url, headers=headers, timeout=60, stream=True)
        response.raise_for_status()

        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
        content = response.content

        with open(dest_path, 'wb') as f:
            f.write(content)

        mime_type = response.headers.get('Content-Type', 'application/octet-stream').split(';')[0].strip()
        FileUpload.objects.filter(pk=file_upload_id).update(
            size=len(content),
            mime_type=mime_type,
        )
        logger.info(f"download_and_save_image: saved {len(content)} bytes for FileUpload {file_upload_id}")

    except Exception as exc:
        logger.error(f"download_and_save_image: failed for {source_url}: {exc}", exc_info=True)
        # Clean up partial file if it exists
        if os.path.exists(dest_path):
            try:
                os.remove(dest_path)
            except OSError:
                pass
        # Remove the stub record so a size=0 ghost doesn't appear in the file list.
        # The next import request for the same URL will create a fresh stub and retry.
        try:
            FileUpload.objects.filter(pk=file_upload_id, size=0).delete()
            logger.info(f"download_and_save_image: deleted stub FileUpload {file_upload_id} after failed download")
        except Exception as del_exc:
            logger.error(f"download_and_save_image: could not delete stub {file_upload_id}: {del_exc}")


def flush_click_buffer():
    """Periodic task: flush Redis-buffered click counts into the database.

    redirect_to_original increments Redis keys of the form 'clicks:{link_id}:{date}'
    instead of writing a ClickLog row per hit. This task drains those keys every ~60s
    and bulk-inserts the ClickLog records plus updates Link.click_count in one shot.
    """
    from .models import Link, ClickLog
    from django_redis import get_redis_connection

    try:
        redis_conn = get_redis_connection('default')
    except Exception as e:
        logger.error(f'flush_click_buffer: cannot get Redis connection: {e}')
        return

    cursor = 0
    pattern = 'clicks:*'
    # django-redis stores keys with a prefix like ':1:' — match it broadly
    prefix = redis_conn.connection_pool.connection_kwargs.get('db', 0)
    all_keys = []
    while True:
        cursor, keys = redis_conn.scan(cursor, match=f'*clicks:*', count=200)
        all_keys.extend(keys)
        if cursor == 0:
            break

    if not all_keys:
        return

    logs_to_create = []
    link_count_deltas = {}  # {link_id: total_clicks}

    pipeline = redis_conn.pipeline()
    for key in all_keys:
        pipeline.getdel(key)
    counts = pipeline.execute()

    for raw_key, count_bytes in zip(all_keys, counts):
        if not count_bytes:
            continue
        count = int(count_bytes)
        if count <= 0:
            continue
        try:
            # Key format: '<cache-prefix>clicks:{link_id}:{date}'
            key_str = raw_key.decode() if isinstance(raw_key, bytes) else raw_key
            # Strip any cache key prefix (':1:clicks:...' → 'clicks:...')
            parts = key_str.rsplit('clicks:', 1)
            if len(parts) != 2:
                continue
            remainder = parts[1]  # '{link_id}:{date}'
            link_id_str, date_str = remainder.split(':', 1)
            link_id = int(link_id_str)
        except (ValueError, IndexError):
            logger.warning(f'flush_click_buffer: unrecognised key format: {raw_key}')
            continue

        link_count_deltas[link_id] = link_count_deltas.get(link_id, 0) + count
        try:
            click_date = timezone.datetime.strptime(date_str, '%Y-%m-%d').replace(
                tzinfo=timezone.get_current_timezone()
            )
        except ValueError:
            click_date = timezone.now()

        for _ in range(count):
            logs_to_create.append(ClickLog(link_id=link_id, clicked_at=click_date))

    if logs_to_create:
        ClickLog.objects.bulk_create(logs_to_create, ignore_conflicts=True)
        logger.info(f'flush_click_buffer: inserted {len(logs_to_create)} ClickLog rows')

    for link_id, delta in link_count_deltas.items():
        Link.objects.filter(pk=link_id).update(click_count=F('click_count') + delta)

    if link_count_deltas:
        logger.info(f'flush_click_buffer: updated click_count for {len(link_count_deltas)} link(s)')


def retry_stuck_image_imports():
    """Periodic task: retry imported images that are stuck with size=0 and no file on disk.

    A stub FileUpload (size=0, source_url set) can be left behind when the background
    download thread is killed mid-flight (e.g. server restart). This task finds those
    orphaned stubs and re-kicks the download, provided the stub is old enough that we
    are confident it is not a currently in-progress download (>5 minutes since last update).
    """
    from .models import FileUpload

    threshold = timezone.now() - timedelta(minutes=5)
    stuck = FileUpload.objects.filter(
        size=0,
        source_url__isnull=False,
        updated_at__lt=threshold,
    ).exclude(source_url='')

    count = stuck.count()
    if count:
        logger.info(f"retry_stuck_image_imports: found {count} stuck import stub(s) — retrying")

    for record in stuck:
        if os.path.exists(record.file_path):
            # File landed on disk but DB wasn't updated — fix it now
            size = os.path.getsize(record.file_path)
            FileUpload.objects.filter(pk=record.pk, size=0).update(size=size)
            logger.info(f"retry_stuck_image_imports: fixed size for FileUpload {record.pk} ({size} bytes)")
            continue

        logger.info(f"retry_stuck_image_imports: re-queuing download for FileUpload {record.pk} ({record.source_url})")
        thread = Thread(target=download_and_save_image, args=(str(record.pk),), daemon=True)
        thread.start()
