from __future__ import annotations

import json
import logging
import os
import random
import time
from typing import Iterable, List

import scrapy

from storage.config import load_config_from_env
from storage.s3_adapter import S3Storage
from extractor.extract import extract_article, detect_language
from features.rules import load_features, match_features

logger = logging.getLogger(__name__)




class PublicSpider(scrapy.Spider):
    name = "public_spider"
    custom_settings = {
        'DUPEFILTER_CLASS': 'scrapy.dupefilters.RFPDupeFilter',
        'DUPEFILTER_DEBUG': True,
        
        # Timeout and performance settings
        'DOWNLOAD_TIMEOUT': 30,  # Max 30 seconds per page (was 180+)
        'DOWNLOAD_DELAY': 0.5,  # 0.5 second delay between requests to same domain
        'CONCURRENT_REQUESTS': 16,  # Process 16 pages at once
        'CONCURRENT_REQUESTS_PER_DOMAIN': 4,  # Max 4 concurrent per domain
        'RETRY_TIMES': 2,  # Only retry twice (not 3+)
        'RETRY_HTTP_CODES': [500, 502, 503, 504, 522, 524, 408, 429],  # Retry only server errors
        
        # AutoThrottle for automatic speed adjustment
        'AUTOTHROTTLE_ENABLED': True,
        'AUTOTHROTTLE_START_DELAY': 0.5,
        'AUTOTHROTTLE_MAX_DELAY': 10,
        'AUTOTHROTTLE_TARGET_CONCURRENCY': 8.0,
        
        # Shutdown monitor extension - checks for shutdown signal every 30 seconds
        'EXTENSIONS': {
            'crawler.extensions.shutdown_monitor.ShutdownMonitor': 100,
        },
        
        # Enhanced headers to avoid bot detection
        'DEFAULT_REQUEST_HEADERS': {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
        }
    }

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.cfg = load_config_from_env()
        self.no_aws = os.getenv("NO_AWS", "false").lower() == "true"
        self.s3 = S3Storage(self.cfg, no_aws=self.no_aws)
        self.crawled_urls = set()  # Track crawled URLs to prevent duplicates
        self.pages_crawled = 0
        self.spider_closed = False  # Track if spider has been closed
        self.start_time = time.time()  # Track when crawler started
        self.last_shutdown_check = time.time()  # Track last time we checked for shutdown
        
        # Load features configuration for content analysis
        features_path = os.getenv("FEATURES_PATH", os.path.join("config", "features.yaml"))
        self.features_cfg = load_features(features_path)
        
        # Ensure S3 bucket exists
        try:
            self.s3.ensure_buckets_with_lifecycle()
            self.logger.info("S3 bucket ensured")
        except Exception as e:
            self.logger.warning(f"Could not ensure S3 bucket: {e}")
        
        # Check for shutdown signal
        self._check_shutdown_signal()
        
        # User agent rotation to avoid detection
        self.user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
        ]

    def _get_realistic_headers(self) -> Dict[str, str]:
        """Generate realistic browser headers to avoid detection"""
        user_agent = random.choice(self.user_agents)
        
        headers = {
            'User-Agent': user_agent,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'DNT': '1',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Cache-Control': 'max-age=0'
        }
        
        # Add browser-specific headers
        if 'Chrome' in user_agent:
            headers['sec-ch-ua'] = '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"'
            headers['sec-ch-ua-mobile'] = '?0'
            headers['sec-ch-ua-platform'] = '"Windows"' if 'Windows' in user_agent else '"macOS"' if 'Mac' in user_agent else '"Linux"'
        elif 'Firefox' in user_agent:
            headers['Sec-Fetch-Dest'] = 'document'
            headers['Sec-Fetch-Mode'] = 'navigate'
            headers['Sec-Fetch-Site'] = 'none'
            headers['Sec-Fetch-User'] = '?1'
        
        return headers

    def _clean_extracted_text(self, text: str) -> str:
        """Clean and validate extracted text to handle encoding issues"""
        if not text:
            return ""
        
        # Remove binary/control characters
        import re
        # Remove null bytes and other control characters
        text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
        
        # Remove excessive whitespace
        text = re.sub(r'\s+', ' ', text)
        
        # Check if text is mostly readable (not binary)
        readable_chars = sum(1 for c in text if c.isprintable() or c.isspace())
        total_chars = len(text)
        
        if total_chars > 0 and readable_chars / total_chars < 0.7:
            # Text is mostly non-readable, return empty
            self.logger.warning("Extracted text appears to be binary or corrupted")
            return ""
        
        return text.strip()

    def _check_shutdown_signal(self):
        """Check for shutdown signal in S3"""
        try:
            shutdown_data = self.s3.get_json(self.cfg.s3_bucket_processed, "shutdown_signal.json")
            if shutdown_data and shutdown_data.get('shutdown'):
                reason = shutdown_data.get('reason', 'Manual shutdown')
                self.logger.info(f"Shutdown signal detected: {reason}")
                self.logger.info("Initiating graceful shutdown...")
                self.spider_closed = True
                
                # Tell Scrapy to stop accepting new requests
                from scrapy import signals
                from scrapy.exceptions import CloseSpider
                raise CloseSpider(reason=f'Shutdown signal received: {reason}')
        except CloseSpider:
            # Re-raise CloseSpider to ensure Scrapy handles it
            raise
        except Exception as e:
            self.logger.debug(f"Could not check shutdown signal: {e}")
    
    def _calculate_relevance_score(self, text: str, url: str, features: dict) -> float:
        """
        Calculate a relevance score (0.0 to 1.0) for content based on multiple factors.
        Higher scores indicate more relevant cybersecurity/ransomware content.
        """
        try:
            score = 0.0
            text_lower = text.lower()
            url_lower = url.lower()
            
            # High-value keywords (worth more points)
            high_value_keywords = [
                'ransomware', 'lockbit', 'cl0p', 'royal', 'conti', 'maze', 'ryuk',
                'encryption', 'cryptocurrency', 'bitcoin', 'monero', 'ethereum',
                'phishing', 'social-engineering', 'lateral-movement', 'privilege-escalation',
                'command-control', 'c2', 'exfiltration', 'double-extortion'
            ]
            
            # Medium-value keywords
            medium_value_keywords = [
                'malware', 'virus', 'trojan', 'botnet', 'apt', 'advanced-persistent-threat',
                'zero-day', 'vulnerability', 'exploit', 'breach', 'incident', 'attack',
                'healthcare', 'hospital', 'finance', 'banking', 'government', 'education'
            ]
            
            # Count high-value keyword matches
            high_value_count = sum(1 for keyword in high_value_keywords if keyword in text_lower)
            medium_value_count = sum(1 for keyword in medium_value_keywords if keyword in text_lower)
            
            # Score based on keyword density
            score += (high_value_count * 0.1) + (medium_value_count * 0.05)
            
            # Bonus for URL containing security keywords
            url_security_keywords = ['security', 'cyber', 'threat', 'malware', 'ransomware', 'breach', 'attack']
            url_keyword_count = sum(1 for keyword in url_security_keywords if keyword in url_lower)
            score += url_keyword_count * 0.05
            
            # Bonus for substantial content (longer articles are often more detailed)
            if len(text) > 2000:
                score += 0.1
            elif len(text) > 1000:
                score += 0.05
            
            # Bonus for feature matches
            feature_count = sum(1 for v in features.values() if v)
            score += feature_count * 0.05
            
            # Cap the score at 1.0
            return min(score, 1.0)
            
        except Exception as e:
            self.logger.warning(f"Error calculating relevance score: {e}")
            return 0.0

    def start_requests(self) -> Iterable[scrapy.Request]:
        seeds = self._load_seeds()
        
        if seeds:
            self.logger.info(f"Loaded {len(seeds)} seed URLs: {seeds}")
            for i, url in enumerate(seeds):
                self.logger.info(f"Processing seed URL {i+1}/{len(seeds)}: {url}")
                if not self._allowed(url):
                    self.logger.warning(f"URL not allowed: {url}")
                    continue
                
                self.logger.info(f"Starting crawl for: {url}")
                # Scrapy handles delays automatically via DOWNLOAD_DELAY and AUTOTHROTTLE
                yield scrapy.Request(
                    url=url, 
                    callback=self.parse, 
                    errback=self.errback,
                    headers=self._get_realistic_headers(),
                    meta={'retry_times': 0, 'max_retries': 3}
                )
        else:
            # No seed URLs - start with keyword-based discovery
            self.logger.info("No seed URLs found - starting keyword-based discovery")
            discovery_urls = self._generate_discovery_urls()
            
            for i, url in enumerate(discovery_urls):
                self.logger.info(f"Starting discovery crawl {i+1}/{len(discovery_urls)}: {url}")
                # Scrapy handles delays automatically
                yield scrapy.Request(
                    url=url,
                    callback=self.parse,
                    errback=self.errback,
                    headers=self._get_realistic_headers(),
                    meta={'retry_times': 0, 'max_retries': 3}
                )

    def _load_seeds(self) -> List[str]:
        path = os.getenv("CRAWLER_CONFIG_PATH", os.path.join("config", "crawler-config.json"))
        if os.path.exists(path):
            with open(path, "r", encoding="utf-8") as f:
                try:
                    data = json.load(f)
                    return data.get("seeds", [])
                except Exception:
                    return []
        seeds_env = os.getenv("SEEDS_CSV", "")
        return [x.strip() for x in seeds_env.split(",") if x.strip()]

    def _generate_discovery_urls(self) -> List[str]:
        """Generate dynamic discovery URLs with time-based filtering to prevent cycling"""
        from datetime import datetime, timedelta
        import random
        
        # Get current time for date filtering
        now = datetime.now()
        week_ago = now - timedelta(days=7)
        month_ago = now - timedelta(days=30)
        
        # Format dates for search engines
        week_str = week_ago.strftime("%Y-%m-%d")
        month_str = month_ago.strftime("%Y-%m-%d")
        
        # Rotate through different ransomware families and keywords
        ransomware_families = [
            "lockbit", "cl0p", "royal", "conti", "maze", "ryuk", "sodinokibi", 
            "babuk", "avaddon", "egregor", "netwalker", "revil", "darkside"
        ]
        
        # Rotate through different time ranges
        time_ranges = [
            f"after:{week_str}",  # Last week
            f"after:{month_str}",  # Last month
            "2024",  # This year
            "2025"   # Current year
        ]
        
        # Get cycle state to avoid repeating searches
        cycle_state = self._get_cycle_state()
        used_queries = cycle_state.get('used_queries', set())
        
        # Reset cycle state if it gets too large (prevent infinite growth)
        if len(used_queries) > 100:  # Reset after 100 unique queries
            self.logger.info("Cycle state too large, resetting to prevent infinite growth")
            used_queries = set()
            self._reset_cycle_state()
        
        discovery_urls = []
        
        # Generate dynamic search queries
        search_queries = []
        for family in ransomware_families[:10]:  # Increased from 6 to 10 families
            for time_range in time_ranges[:3]:  # Increased from 2 to 3 time ranges
                query = f"{family}+ransomware+{time_range}"
                if query not in used_queries:
                    search_queries.append(query)
                    used_queries.add(query)
        
        # Add search engine URLs with dynamic queries
        for i, query in enumerate(search_queries[:20]):  # Increased from 8 to 20 search queries
            if i % 2 == 0:
                # Google News
                discovery_urls.append(f"https://www.google.com/search?q={query}&tbm=nws")
            else:
                # Bing News
                discovery_urls.append(f"https://www.bing.com/news/search?q={query}")
        
        # Add direct security news sources (these are stable and don't need cycling)
        stable_sources = [
            "https://www.bleepingcomputer.com/",
            "https://krebsonsecurity.com/",
            "https://www.securityweek.com/",
            "https://www.cyberscoop.com/",
            "https://feeds.feedburner.com/TheHackersNews",
            "https://krebsonsecurity.com/feed/",
            "https://www.bleepingcomputer.com/feed/",
            "https://blog.talosintelligence.com/",
            "https://unit42.paloaltonetworks.com/"
        ]
        
        # Add stable sources (these don't change much)
        discovery_urls.extend(stable_sources)
        
        # Update cycle state
        self._update_cycle_state(used_queries)
        
        self.logger.info(f"Generated {len(discovery_urls)} discovery URLs with dynamic queries")
        self.logger.info(f"Using {len(used_queries)} unique search queries to prevent cycling")
        return discovery_urls

    def _get_cycle_state(self) -> dict:
        """Get the current cycle state from S3 to track what's been searched"""
        try:
            state_data = self.s3.get_json(self.cfg.s3_bucket_processed, "crawler_cycle_state.json")
            if state_data:
                # Convert set back from list (JSON doesn't support sets)
                if 'used_queries' in state_data:
                    state_data['used_queries'] = set(state_data['used_queries'])
                return state_data
        except Exception as e:
            self.logger.debug(f"Could not load cycle state: {e}")
        
        return {'used_queries': set(), 'last_cycle': None, 'cycle_count': 0}

    def _update_cycle_state(self, used_queries: set):
        """Update the cycle state in S3 to track what's been searched"""
        try:
            from datetime import datetime
            
            cycle_state = {
                'used_queries': list(used_queries),  # Convert set to list for JSON
                'last_cycle': datetime.now().isoformat(),
                'cycle_count': self._get_cycle_state().get('cycle_count', 0) + 1
            }
            
            self.s3.put_json(self.cfg.s3_bucket_processed, "crawler_cycle_state.json", cycle_state)
            self.logger.info(f"Updated cycle state: {len(used_queries)} queries tracked")
            
        except Exception as e:
            self.logger.warning(f"Could not update cycle state: {e}")

    def _reset_cycle_state(self):
        """Reset the cycle state to start fresh"""
        try:
            from datetime import datetime
            
            cycle_state = {
                'used_queries': [],
                'last_cycle': datetime.now().isoformat(),
                'cycle_count': 0,
                'reset_reason': 'Query limit reached'
            }
            
            self.s3.put_json(self.cfg.s3_bucket_processed, "crawler_cycle_state.json", cycle_state)
            self.logger.info("Cycle state reset - starting fresh discovery cycle")
            
        except Exception as e:
            self.logger.warning(f"Could not reset cycle state: {e}")

    def _allowed(self, url: str) -> bool:
        """
        Minimal URL filtering - allow most URLs and let content analysis decide value.
        Block only obvious spam/malicious sites and non-relevant content.
        """
        from urllib.parse import urlparse
        
        try:
            url_lower = url.lower()
            parsed_url = urlparse(url)
            domain = parsed_url.netloc.lower()
            path = parsed_url.path.lower()
            
            # Block known problematic domains (social media, slow sites, paywalls)
            blocked_domains = [
                'facebook.com',
                'twitter.com',
                'instagram.com',
                'youtube.com',
                'tiktok.com',
                'pinterest.com',
                'reddit.com',
                'linkedin.com',
                'businesswire.com',  # Very slow to respond
                'prnewswire.com',    # Press release site - slow
                'globenewswire.com', # Press release site - slow
                'wsj.com',           # Paywall
                'nytimes.com',       # Paywall
                'ft.com'             # Paywall
            ]
            
            for blocked in blocked_domains:
                if blocked in domain:
                    self.logger.debug(f"URL {url} blocked - problematic domain: {blocked}")
                    return False
            
            # Block obvious non-security content paths
            blocked_patterns = [
                '/sports/', '/entertainment/', '/weather/', '/celebrity/',
                '/fashion/', '/cooking/', '/recipe/', '/travel/', '/video-games/',
                '/movies/', '/music/', '/lifestyle/', '/shopping/', '/deals/',
                '.jpg', '.png', '.gif', '.pdf', '.zip', '.exe', '.mp4', '.mp3'
            ]
            
            for pattern in blocked_patterns:
                if pattern in path:
                    self.logger.debug(f"URL {url} blocked by pattern: {pattern}")
                    return False
            
            # Allow everything else - content analysis will filter
            self.logger.debug(f"URL {url} allowed - will analyze content")
            return True
            
        except Exception as e:
            self.logger.warning(f"Error parsing URL {url}: {e}")
            return False

    def parse(self, response: scrapy.http.Response):
        # Check if spider has been closed
        if self.spider_closed:
            self.logger.info("Spider closed, stopping crawl")
            return
        
        # Periodically check for shutdown signal (every 30 seconds)
        current_time = time.time()
        if current_time - self.last_shutdown_check >= 30:
            try:
                self._check_shutdown_signal()
                self.last_shutdown_check = current_time
            except Exception as e:
                # Shutdown signal detected - this will stop the spider
                self.logger.info(f"Shutdown initiated: {e}")
                raise  # Re-raise to let Scrapy handle it
            
            if self.spider_closed:
                self.logger.info("Shutdown signal detected during crawl, stopping")
                return
            
        # Check for duplicates in memory only (not S3)
        url = response.url
        if url in self.crawled_urls:
            self.logger.debug(f"Skipping duplicate URL in this session: {url}")
            return
            
        # Only process successful responses
        if response.status != 200:
            self.logger.warning(f"Non-200 response for {url}: {response.status}")
            return
            
        self.crawled_urls.add(url)
        self.pages_crawled += 1
        
        html = response.text or ""
        
        # Extract article content
        try:
            self.logger.info(f"Extracting content from: {url}")
            extracted = extract_article(url, html)
            
            # Clean the extracted text
            cleaned_text = self._clean_extracted_text(extracted.text)
            if not cleaned_text:
                self.logger.warning(f"No readable text extracted from {url}")
                return
            
            # Update extracted object with cleaned text
            extracted.text = cleaned_text
            lang = detect_language(extracted.text)
            
            self.logger.info(f"Extracted - Title: {extracted.title[:100] if extracted.title else 'No title'}...")
            self.logger.info(f"Extracted - Text length: {len(extracted.text)} chars")
            self.logger.info(f"Extracted - Language: {lang}")
            
            # Check if content is relevant to ransomware/cybersecurity
            features = match_features(extracted.text, self.features_cfg)
            relevant_features = [k for k, v in features.items() if v]
            
            self.logger.info(f"Crawled page {self.pages_crawled}: {url}")
            self.logger.info(f"Found features: {relevant_features}")
            self.logger.info(f"All feature matches: {features}")
            
            # Enhanced content relevance scoring
            relevance_score = self._calculate_relevance_score(extracted.text, url, features)
            
            # Store in S3 if content is relevant OR substantial
            should_store = relevant_features or len(extracted.text) > 500 or relevance_score > 0.3
            self.logger.info(f"Should store document: {should_store} (relevant: {bool(relevant_features)}, substantial: {len(extracted.text) > 500}, relevance_score: {relevance_score:.2f})")
            
            if should_store:
                # Check if this specific URL already exists in S3 before saving
                if self.s3.check_website_exists(url):
                    self.logger.info(f"URL already exists in S3, skipping save: {url}")
                else:
                    # Create brief description from title and first 200 characters of content
                    brief_description = f"{extracted.title or 'No title'}"
                    if extracted.text:
                        # Get first 200 characters of meaningful text
                        clean_text = extracted.text.strip()[:200]
                        if clean_text:
                            brief_description += f" - {clean_text}..."
                    
                    payload = {
                        "doc_id": url,
                        "created_at": response.headers.get("Date", b"").decode("utf-8", errors="ignore"),
                        "url": url,
                        "title": extracted.title,
                        "brief_description": brief_description,
                        "author": extracted.author,
                        "date": extracted.date,
                        "canonical_url": extracted.canonical_url,
                        "language": lang,
                        "http_status": response.status,
                        "features": features,
                        "relevant_features": relevant_features,
                        "relevance_score": relevance_score,
                        "content_retrieval_required": True  # Flag to indicate LLM should retrieve full content
                    }
                    
                    try:
                        self.logger.info(f"Storing document in S3: {url}")
                        s3_path = self.s3.put_json_with_url(url, payload)
                        self.logger.info(f"Successfully stored document in S3: {s3_path}")
                        payload["s3_path"] = s3_path
                    except Exception as e:
                        self.logger.error(f"Failed to store document {url}: {e}")
                        import traceback
                        self.logger.error(f"Full error traceback: {traceback.format_exc()}")
                    
                    yield payload
            
        except Exception as e:
            self.logger.error(f"Failed to extract content from {url}: {e}")
            # Still yield basic info even if extraction fails
            yield {
                "url": url,
                "http_status": response.status,
                "fetched_at": response.headers.get("Date", b"").decode("utf-8", errors="ignore"),
                "page_number": self.pages_crawled,
                "error": str(e)
            }
        
        # Find and follow links to articles (continue indefinitely unless spider is closed)
        if not self.spider_closed:
            links_followed = 0
            
            for request in self._follow_links(response):
                links_followed += 1
                yield request
                    
            self.logger.info(f"Followed {links_followed} links from {url}")
        else:
            self.logger.info(f"Spider closed, not following links from {url}")
    
    def _follow_links(self, response: scrapy.http.Response) -> Iterable[scrapy.Request]:
        """Find and follow relevant links to articles"""
        url = response.url
        
        # Special handling for search engine results
        if 'google.com/search' in url or 'bing.com/search' in url or 'bing.com/news/search' in url:
            yield from self._extract_search_results(response)
            return
        
        # Special handling for RSS feeds
        if any(feed in url for feed in ['/feed/', '/rss/', 'feedburner.com', '.xml']):
            yield from self._extract_rss_links(response)
            return
        
        # Look for article links, news links, and blog post links
        article_selectors = [
            'a[href*="/news/"]',
            'a[href*="/article/"]', 
            'a[href*="/blog/"]',
            'a[href*="/post/"]',
            'a[href*="/story/"]',
            'a[href*="/alert/"]',
            'a[href*="/advisory/"]',
            'a[href*="/threat/"]',
            'a[href*="/ransomware/"]',
            'a[href*="/malware/"]',
            'a[href*="/security/"]',
            'a[href*="/cyber/"]',
            # Generic article patterns
            'article a',
            '.article a',
            '.post a',
            '.news-item a',
            '.story a',
            '.content a',
            '.main a',
            '.body a',
            # Any external links that might be relevant
            'a[href^="http"]',
            'a[href^="https"]',
            # Common news/security site patterns
            'a[href*="reuters.com"]',
            'a[href*="bbc.com"]',
            'a[href*="cnn.com"]',
            'a[href*="bleepingcomputer.com"]',
            'a[href*="krebsonsecurity.com"]',
            'a[href*="threatpost.com"]',
            'a[href*="securityweek.com"]'
        ]
        
        links_found = 0
        for selector in article_selectors:
            for link in response.css(selector):
                href = link.attrib.get('href', '')
                if not href:
                    continue
                    
                # Convert relative URLs to absolute
                absolute_url = response.urljoin(href)
                
                # Check if URL is allowed and not already crawled
                if absolute_url not in self.crawled_urls and links_found < 50:  # Increased from 3 to 50
                    
                    # Use the enhanced _allowed() method to check if link is relevant
                    if self._allowed(absolute_url):
                        links_found += 1
                        # Scrapy handles delays automatically
                        yield scrapy.Request(
                            url=absolute_url,
                            callback=self.parse,
                            errback=self.errback,
                            headers={'User-Agent': random.choice(self.user_agents)},
                            meta={'retry_times': 0, 'max_retries': 2}
                        )
                        self.logger.info(f"Following link {links_found}/50: {absolute_url}")
                        
                        if links_found >= 50:  # Increased limit
                            break
            
            if links_found >= 50:
                break

    def _extract_search_results(self, response: scrapy.http.Response) -> Iterable[scrapy.Request]:
        """Extract actual search result links from search engine pages"""
        url = response.url
        
        # Google search result selectors (updated for current structure)
        if 'google.com' in url:
            selectors = [
                'div.g a[href^="http"]',  # Google search result links
                'h3 a[href^="http"]',     # Google result titles
                '.yuRUbf a[href^="http"]', # Google result containers
                '.g a[href^="http"]',     # Generic Google results
                'a[href^="http"]:not([href*="google.com"])', # Any external links
                '.rc a[href^="http"]',    # Alternative Google result class
                '.r a[href^="http"]',     # Another Google result class
                'cite + a[href^="http"]'  # Links after citations
            ]
        
        # Bing search result selectors (updated for current structure)
        elif 'bing.com' in url:
            selectors = [
                '.b_algo a[href^="http"]', # Bing search result links
                'h2 a[href^="http"]',      # Bing result titles
                '.b_title a[href^="http"]', # Bing title links
                '.b_caption a[href^="http"]', # Bing caption links
                'a[href^="http"]:not([href*="bing.com"]):not([href*="microsoft.com"])', # Any external links
                '.b_attribution a[href^="http"]', # Attribution links
                '.b_factrow a[href^="http"]' # Fact row links
            ]
        
        else:
            return
        
        links_found = 0
        for selector in selectors:
            for link in response.css(selector):
                href = link.attrib.get('href', '')
                if not href or not href.startswith('http'):
                    continue
                    
                # Skip search engine internal links
                if any(domain in href for domain in ['google.com', 'bing.com', 'microsoft.com']):
                    continue
                    
                # Check if URL is relevant to ransomware
                if self._allowed(href):
                    absolute_url = response.urljoin(href)
                    
                    # Check for duplicates
                    if absolute_url not in self.crawled_urls:
                        links_found += 1
                        self.crawled_urls.add(absolute_url)
                        yield scrapy.Request(
                            url=absolute_url,
                            callback=self.parse,
                            meta={'referer': url}
                        )
                        
                        # Increased limit for more discovery
                        if links_found >= 30:
                            break
            if links_found >= 30:
                break
        
        self.logger.info(f"Extracted {links_found} search result links from {url}")

    def _extract_rss_links(self, response: scrapy.http.Response) -> Iterable[scrapy.Request]:
        """Extract links from RSS feeds"""
        url = response.url
        
        # RSS/XML selectors
        selectors = [
            'item link',           # Standard RSS item links
            'entry link',          # Atom feed links
            'link',                # Generic link elements
            'item guid',           # RSS item GUIDs (often URLs)
            'entry id',            # Atom entry IDs (often URLs)
            'a[href^="http"]'      # Any HTTP links in the feed
        ]
        
        links_found = 0
        for selector in selectors:
            for element in response.css(selector):
                # Get text content or href attribute
                href = element.attrib.get('href', '') or element.get()
                if not href or not href.startswith('http'):
                    continue
                
                # Skip feed URLs themselves
                if any(feed in href for feed in ['/feed/', '/rss/', 'feedburner.com', '.xml']):
                    continue
                
                # Check if URL is relevant to ransomware
                if self._allowed(href):
                    absolute_url = response.urljoin(href)
                    
                    # Check for duplicates
                    if absolute_url not in self.crawled_urls:
                        links_found += 1
                        self.crawled_urls.add(absolute_url)
                        yield scrapy.Request(
                            url=absolute_url,
                            callback=self.parse,
                            headers=self._get_realistic_headers(),
                            meta={'referer': url}
                        )
                        
                        # Increased limit for more content discovery
                        if links_found >= 50:
                            break
            if links_found >= 50:
                break
        
        self.logger.info(f"Extracted {links_found} RSS feed links from {url}")

    def errback(self, failure):
        request = failure.request
        self.logger.warning("Request failed: %s - %s", request.url, str(failure.value))
        
        # Log specific error types
        if hasattr(failure.value, 'response'):
            response = failure.value.response
            if response and hasattr(response, 'status'):
                self.logger.warning(f"HTTP {response.status} error for {request.url}")

                # Handle 403 errors with retry logic
                if response.status == 403:
                    retry_times = request.meta.get('retry_times', 0)
                    max_retries = request.meta.get('max_retries', 3)
                    
                    if retry_times < max_retries:
                        # Exponential backoff: wait longer between retries
                        wait_time = (2 ** retry_times) * random.uniform(10, 20)
                        self.logger.info(f"Retrying {request.url} in {wait_time:.1f} seconds (attempt {retry_times + 1}/{max_retries})")
                        
                        # Create a new request with updated retry count
                        new_request = request.replace(
                            dont_filter=True,
                            meta={
                                **request.meta,
                                'retry_times': retry_times + 1,
                                'max_retries': max_retries
                            },
                            headers={'User-Agent': random.choice(self.user_agents)}
                        )
                        
                        # Schedule retry with delay
                        return scrapy.utils.defer.Deferred.from_timer(wait_time).addCallback(
                            lambda _: new_request
                        )
                    else:
                        self.logger.error(f"Max retries exceeded for {request.url}, giving up")

    def closed(self, reason):
        """Called when spider is closed"""
        self.logger.info(f"Spider closed. Reason: {reason}")
        self.logger.info(f"Total pages crawled: {self.pages_crawled}")
        self.logger.info(f"Total unique URLs processed: {len(self.crawled_urls)}")
        
        # Log summary of what was accomplished
        if self.pages_crawled > 0:
            self.logger.info(f"Successfully crawled {self.pages_crawled} pages with keyword-based filtering")
            self.logger.info("Duplicate detection prevented re-crawling of existing websites")
        else:
            self.logger.warning("No pages were crawled")
