<?php

namespace App\Jobs;

use App\Models\KbDocument;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;

class ScrapeWebsiteJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    /**
     * The number of times the job may be attempted.
     */
    public int $tries = 3;

    /**
     * Exponential backoff: 60s, 5min between retries.
     */
    public array $backoff = [60, 300];

    /**
     * The maximum number of seconds the job can run.
     */
    public int $timeout = 120;

    /**
     * The queue this job should be dispatched to.
     */

    /**
     * Maximum content length to process (5MB).
     */
    private const MAX_CONTENT_LENGTH = 5 * 1024 * 1024;

    public function __construct(
        private readonly string $url,
        private readonly int $workspaceId,
    ) {}

    public function handle(): void
    {
        $url = $this->url;
        $workspaceId = $this->workspaceId;

        Log::info('ScrapeWebsiteJob: starting', [
            'url' => $url,
            'workspace_id' => $workspaceId,
        ]);

        try {
            // Validate URL
            if (!filter_var($url, FILTER_VALIDATE_URL)) {
                throw new \RuntimeException("Invalid URL: {$url}");
            }

            // SSRF protection: block internal/private network URLs.
            // This is the last line of defense -- the Livewire component
            // should also validate before dispatching this job.
            if (!$this->isUrlSafe($url)) {
                throw new \RuntimeException('URL targets a private or reserved network and cannot be scraped.');
            }

            // Fetch page HTML
            $html = $this->fetchPage($url);

            if (empty($html)) {
                throw new \RuntimeException('Empty response from URL.');
            }

            // Extract main content
            $title = $this->extractTitle($html);
            $content = $this->extractContent($html);

            // Tolerance: only fail if we got literally nothing back.
            // Previously we threw on ANY empty result, but the new
            // bruteForceFallback() means we should almost always have
            // SOMETHING (title + meta tags at minimum). The only way to
            // hit this branch now is a truly empty document.
            if (trim($content) === '' && trim($title) === '') {
                throw new \RuntimeException('No extractable text content found on the page (even with fallback).');
            }
            // If we have at least a title and the body came back tiny,
            // backfill content with the title so the AI has SOMETHING.
            if (trim($content) === '') {
                $content = $title;
            }

            // Parse domain for metadata
            $parsedUrl = parse_url($url);
            $domain = $parsedUrl['host'] ?? $url;

            // Upsert the KbDocument. The Livewire component pre-creates a
            // `pending` row so the user sees instant feedback in their list;
            // we update that same row here with the scraped title + content
            // and bump status to `extracting`. If no pre-created row exists
            // (e.g. the job was dispatched from a CLI / API path), we create
            // it from scratch. Idempotent on retry either way.
            $document = KbDocument::updateOrCreate(
                ['workspace_id' => $workspaceId, 'source_url' => $url],
                [
                    'type' => 'website',
                    'title' => $title ?: $domain,
                    'content' => $content,
                    'status' => 'extracting', // ProcessKBDocumentJob bumps to 'ready'
                    'error_message' => null,
                ]
            );

            // Dispatch ProcessKBDocumentJob to chunk and embed the content
            ProcessKBDocumentJob::dispatch($document);

            Log::info('ScrapeWebsiteJob: completed, dispatched processing', [
                'document_id' => $document->id,
                'url' => $url,
                'title' => $title,
                'content_length' => mb_strlen($content),
            ]);
        } catch (\Exception $e) {
            Log::error('ScrapeWebsiteJob failed', [
                'url' => $url,
                'workspace_id' => $workspaceId,
                'error' => $e->getMessage(),
            ]);

            // Surface the failure on the user's list. updateOrCreate so we
            // update the `pending` row the Livewire component pre-created
            // instead of inserting a duplicate (which would either error on
            // a unique index or leave two visible rows for the same URL).
            KbDocument::updateOrCreate(
                ['workspace_id' => $workspaceId, 'source_url' => $url],
                [
                    'type' => 'website',
                    'title' => "Failed: {$url}",
                    'status' => 'failed',
                    'error_message' => mb_substr($e->getMessage(), 0, 255),
                ]
            );

            throw $e;
        }
    }

    /**
     * Fetch page HTML — tries multiple strategies in sequence so a single
     * misconfigured target site doesn't fail the whole scrape.
     *
     * Strategies, each used only if the previous threw / returned empty:
     *   1. Real Chrome User-Agent + TLS verify ON. Most sites take this.
     *      Identifying as a "browser" instead of a bot also gets us
     *      past Cloudflare's "Browser Integrity Check" on default settings.
     *   2. Same UA but TLS verify OFF. Catches sites with expired,
     *      self-signed, or misconfigured certs (very common on smaller
     *      business sites). We log it loudly so the operator knows the
     *      target had a bad cert, but we still scrape rather than refusing.
     *   3. Curl-style minimal headers as a last resort. Some old servers
     *      reject any request that doesn't look exactly like curl.
     *
     * Status / Content-Type are intentionally lenient: we accept anything
     * from 200-399, and any Content-Type whose body actually contains
     * `<html`. A 403 that still serves a real HTML body (typical of
     * "anti-bot" pages) is treated as a soft fail and gets one retry.
     */
    private function fetchPage(string $url): string
    {
        // Real Chrome 120 User-Agent. Picked because:
        //   - Includes all the headers a Chromium browser actually sends
        //   - Cloudflare's default bot challenge lets this through
        //   - Doesn't trigger "you're using an outdated browser" walls
        $chromeUA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
            . '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

        $strategies = [
            // 1. Browser-like, TLS on
            [
                'verify' => true,
                'headers' => $this->browserHeaders($chromeUA),
                'label' => 'browser-tls',
            ],
            // 2. Browser-like, TLS OFF. Last resort for sites with broken certs.
            [
                'verify' => false,
                'headers' => $this->browserHeaders($chromeUA),
                'label' => 'browser-notls',
            ],
            // 3. Bare curl-style. For sites that block anything that isn't curl.
            [
                'verify' => false,
                'headers' => [
                    'User-Agent' => 'curl/8.4.0',
                    'Accept' => '*/*',
                ],
                'label' => 'curl-bare',
            ],
        ];

        $lastError = null;
        foreach ($strategies as $strategy) {
            try {
                $body = $this->fetchWithStrategy($url, $strategy);
                if ($body !== '' && $this->looksLikeHtml($body)) {
                    if ($strategy['label'] !== 'browser-tls') {
                        Log::info('ScrapeWebsiteJob: succeeded with fallback strategy', [
                            'url' => $url, 'strategy' => $strategy['label'],
                        ]);
                    }
                    return $body;
                }
                $lastError = "Empty or non-HTML body via {$strategy['label']}";
            } catch (\Throwable $e) {
                $lastError = "{$strategy['label']}: " . $e->getMessage();
                Log::warning('ScrapeWebsiteJob: strategy failed, trying next', [
                    'url' => $url, 'strategy' => $strategy['label'], 'error' => $e->getMessage(),
                ]);
                continue;
            }
        }

        throw new \RuntimeException("All fetch strategies failed. Last error: {$lastError}");
    }

    /**
     * Per-strategy HTTP call. Lenient on status (anything < 400) and
     * Content-Type (returns body if it looks like HTML, even on weird
     * content-types).
     */
    private function fetchWithStrategy(string $url, array $strategy): string
    {
        $client = new Client([
            'timeout'         => 60,
            'connect_timeout' => 10,
            'allow_redirects' => [
                'max'       => 8,
                'strict'    => false,
                'referer'   => true,
                'protocols' => ['http', 'https'],
            ],
            'headers'     => $strategy['headers'],
            'verify'      => $strategy['verify'],
            'http_errors' => false, // we'll decide what counts as a failure
        ]);

        $response   = $client->get($url);
        $statusCode = $response->getStatusCode();

        // Accept 200-399. Anything 4xx/5xx is a hard fail UNLESS the body
        // still contains HTML — some sites return 403 with the real page
        // baked in (anti-bot scrape walls). We'll let the caller decide
        // based on looksLikeHtml().
        if ($statusCode >= 400) {
            $body = $response->getBody()->getContents();
            if (!$this->looksLikeHtml($body)) {
                throw new \RuntimeException("HTTP {$statusCode} response from {$url}");
            }
            // Fell through: 4xx but body is real HTML, use it.
            return strlen($body) > self::MAX_CONTENT_LENGTH ? '' : $body;
        }

        $body = $response->getBody()->getContents();
        if (strlen($body) > self::MAX_CONTENT_LENGTH) {
            throw new \RuntimeException('Page content exceeds 5MB limit.');
        }

        return $body;
    }

    /**
     * Headers that make us look like a real browser. Cloudflare /
     * Sucuri / WAFs check several of these, not just User-Agent.
     */
    private function browserHeaders(string $userAgent): array
    {
        return [
            'User-Agent'       => $userAgent,
            'Accept'           => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
            'Accept-Language'  => 'en-US,en;q=0.9',
            'Accept-Encoding'  => 'gzip, deflate, br',
            'Cache-Control'    => 'no-cache',
            'Pragma'           => 'no-cache',
            'Sec-Fetch-Dest'   => 'document',
            'Sec-Fetch-Mode'   => 'navigate',
            'Sec-Fetch-Site'   => 'none',
            'Sec-Fetch-User'   => '?1',
            'Upgrade-Insecure-Requests' => '1',
        ];
    }

    /**
     * Quick sniff: does the response body look like HTML at all?
     * Used to decide whether to accept a 4xx response or fall through
     * to the next strategy.
     */
    private function looksLikeHtml(string $body): bool
    {
        $head = mb_substr(ltrim($body), 0, 2000);
        return stripos($head, '<html') !== false
            || stripos($head, '<!doctype') !== false
            || stripos($head, '<body') !== false
            || stripos($head, '<title') !== false;
    }

    /**
     * Extract the page title from HTML.
     */
    private function extractTitle(string $html): string
    {
        // Try <title> tag first
        if (preg_match('/<title[^>]*>(.*?)<\/title>/is', $html, $matches)) {
            $title = html_entity_decode(trim($matches[1]), ENT_QUOTES, 'UTF-8');
            // Clean up common suffixes like " | Company Name"
            $title = preg_replace('/\s*[|\-]\s*$/', '', $title);
            return mb_substr($title, 0, 255);
        }

        // Try og:title meta tag
        if (preg_match('/<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']/', $html, $matches)) {
            return mb_substr(html_entity_decode(trim($matches[1]), ENT_QUOTES, 'UTF-8'), 0, 255);
        }

        // Try first h1
        if (preg_match('/<h1[^>]*>(.*?)<\/h1>/is', $html, $matches)) {
            return mb_substr(trim(strip_tags($matches[1])), 0, 255);
        }

        return '';
    }

    /**
     * Extract main content from HTML, stripping navigation, headers, footers, scripts, etc.
     */
    private function extractContent(string $html): string
    {
        // Suppress libxml errors for malformed HTML
        $previousUseErrors = libxml_use_internal_errors(true);

        $dom = new \DOMDocument();
        // Load HTML with UTF-8 encoding hint
        $dom->loadHTML('<?xml encoding="UTF-8">' . $html, LIBXML_NOWARNING | LIBXML_NOERROR);

        libxml_clear_errors();
        libxml_use_internal_errors($previousUseErrors);

        // Remove unwanted elements
        $tagsToRemove = ['script', 'style', 'nav', 'footer', 'header', 'aside', 'noscript', 'iframe', 'svg', 'form'];

        foreach ($tagsToRemove as $tag) {
            $elements = $dom->getElementsByTagName($tag);
            // Collect first since removing modifies the live NodeList
            $toRemove = [];
            for ($i = 0; $i < $elements->length; $i++) {
                $toRemove[] = $elements->item($i);
            }
            foreach ($toRemove as $element) {
                $element->parentNode?->removeChild($element);
            }
        }

        // Remove elements by common non-content class/id patterns
        $xpath = new \DOMXPath($dom);
        $nonContentSelectors = [
            '//*[contains(@class, "nav")]',
            '//*[contains(@class, "menu")]',
            '//*[contains(@class, "sidebar")]',
            '//*[contains(@class, "footer")]',
            '//*[contains(@class, "header")]',
            '//*[contains(@class, "cookie")]',
            '//*[contains(@class, "popup")]',
            '//*[contains(@class, "modal")]',
            '//*[contains(@class, "advertisement")]',
            '//*[contains(@class, "ad-")]',
            '//*[contains(@id, "nav")]',
            '//*[contains(@id, "menu")]',
            '//*[contains(@id, "sidebar")]',
            '//*[contains(@id, "footer")]',
            '//*[contains(@id, "header")]',
            '//*[contains(@id, "cookie")]',
        ];

        foreach ($nonContentSelectors as $selector) {
            try {
                $nodes = $xpath->query($selector);
                if ($nodes) {
                    $toRemove = [];
                    foreach ($nodes as $node) {
                        $toRemove[] = $node;
                    }
                    foreach ($toRemove as $node) {
                        $node->parentNode?->removeChild($node);
                    }
                }
            } catch (\Exception $e) {
                // Skip invalid selectors
                continue;
            }
        }

        // Try to find the main content area
        $contentNode = null;

        // Look for <main>, <article>, or common content containers
        $contentSelectors = [
            '//main',
            '//article',
            '//*[@id="content"]',
            '//*[@id="main-content"]',
            '//*[@class="content"]',
            '//*[@role="main"]',
        ];

        foreach ($contentSelectors as $selector) {
            try {
                $nodes = $xpath->query($selector);
                if ($nodes && $nodes->length > 0) {
                    $contentNode = $nodes->item(0);
                    break;
                }
            } catch (\Exception $e) {
                continue;
            }
        }

        // If no content container found, use body
        if (!$contentNode) {
            $bodies = $dom->getElementsByTagName('body');
            if ($bodies->length > 0) {
                $contentNode = $bodies->item(0);
            }
        }

        if (!$contentNode) {
            return $this->bruteForceFallback($html);
        }

        // Extract text with basic formatting preservation
        $text = $this->extractNodeText($contentNode);

        // Final cleanup
        $text = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/', '', $text);
        $text = preg_replace('/\n{3,}/', "\n\n", $text);
        $text = preg_replace('/[^\S\n]+/', ' ', $text);
        $text = trim($text);

        // If structured extraction came back with almost nothing — typical
        // of JS-rendered SPAs where the body is mostly <div id="root">
        // with one mounting tag — fall back to mining the raw HTML for
        // anything text-shaped. We'd rather give the AI 30 lines of
        // meta tags + footer copy than throw "no content".
        if (mb_strlen($text) < 80) {
            $fallback = $this->bruteForceFallback($html);
            if (mb_strlen($fallback) > mb_strlen($text)) {
                return $fallback;
            }
        }

        return $text;
    }

    /**
     * Last-resort content miner for sites where DOM-based extraction
     * yields nothing useful (JS-rendered SPAs, malformed HTML, exotic
     * markup). Strategy:
     *
     *   1. Pull every meta[name=description], meta[property=og:*] —
     *      SPAs usually leave at least these populated for SEO.
     *   2. Pull <title>.
     *   3. Strip tags from the whole document and normalize whitespace.
     *
     * Combined, these give the AI SOMETHING to learn from even on
     * fully-JS pages. Better than failing the whole row.
     */
    private function bruteForceFallback(string $html): string
    {
        $parts = [];

        if (preg_match('/<title[^>]*>(.*?)<\/title>/is', $html, $m)) {
            $parts[] = trim(html_entity_decode(strip_tags($m[1]), ENT_QUOTES, 'UTF-8'));
        }

        if (preg_match_all('/<meta[^>]+(?:name|property)=["\'](description|og:title|og:description|twitter:title|twitter:description)["\'][^>]*content=["\']([^"\']+)["\']/i', $html, $metaMatches)) {
            foreach ($metaMatches[2] as $val) {
                $parts[] = trim(html_entity_decode($val, ENT_QUOTES, 'UTF-8'));
            }
        }

        // Strip script + style fully before stripping all tags so their
        // contents don't leak into the output.
        $stripped = preg_replace('/<(script|style)\b[^>]*>.*?<\/\1>/is', '', $html);
        $stripped = strip_tags((string) $stripped);
        $stripped = html_entity_decode($stripped, ENT_QUOTES, 'UTF-8');
        $stripped = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/', '', $stripped);
        $stripped = preg_replace('/\n{3,}/', "\n\n", $stripped);
        $stripped = preg_replace('/[^\S\n]+/', ' ', $stripped);
        $parts[] = trim($stripped);

        return trim(implode("\n\n", array_filter($parts, fn ($p) => $p !== '')));
    }

    /**
     * Recursively extract text from a DOM node, preserving paragraph and heading structure.
     */
    private function extractNodeText(\DOMNode $node): string
    {
        if ($node->nodeType === XML_TEXT_NODE) {
            return $node->textContent;
        }

        if ($node->nodeType !== XML_ELEMENT_NODE) {
            return '';
        }

        $text = '';
        $tagName = strtolower($node->nodeName);

        // Block-level elements get newlines
        $blockTags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'tr', 'blockquote', 'pre', 'section'];

        if (in_array($tagName, $blockTags)) {
            $text .= "\n";
        }

        // Heading prefix
        if (preg_match('/^h([1-6])$/', $tagName)) {
            $text .= "\n";
        }

        foreach ($node->childNodes as $child) {
            $text .= $this->extractNodeText($child);
        }

        if (in_array($tagName, $blockTags)) {
            $text .= "\n";
        }

        // Line break elements
        if ($tagName === 'br') {
            $text .= "\n";
        }

        return $text;
    }

    /**
     * Validate that a URL does not target internal/private networks (SSRF protection).
     */
    private function isUrlSafe(string $url): bool
    {
        $parsed = parse_url($url);

        $scheme = strtolower($parsed['scheme'] ?? '');
        if (!in_array($scheme, ['http', 'https'])) {
            return false;
        }

        $host = $parsed['host'] ?? '';
        if ($host === '') {
            return false;
        }

        $hostLower = strtolower($host);

        $blockedHosts = [
            'localhost', '0.0.0.0',
            'metadata.google.internal', 'metadata.google',
            '169.254.169.254', 'metadata.azure.internal',
        ];
        if (in_array($hostLower, $blockedHosts)) {
            return false;
        }

        if (filter_var($host, FILTER_VALIDATE_IP)) {
            if (!filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
                return false;
            }
        }

        // Resolve hostname and verify resolved IPs are not private
        $resolvedIps = gethostbynamel($hostLower);
        if ($resolvedIps) {
            foreach ($resolvedIps as $ip) {
                if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
                    return false;
                }
            }
        }

        return true;
    }

    /**
     * Handle a job failure.
     */
    public function failed(?\Throwable $exception): void
    {
        Log::error('ScrapeWebsiteJob permanently failed', [
            'url' => $this->url,
            'workspace_id' => $this->workspaceId,
            'error' => $exception?->getMessage(),
        ]);
    }
}
