"""
Scrapy extension to monitor for shutdown signals.
Checks S3 periodically even when spider is idle.
"""
import logging
import time
from scrapy import signals
from scrapy.exceptions import CloseSpider
from twisted.internet import task

from storage.config import load_config_from_env
from storage.s3_adapter import S3Storage

logger = logging.getLogger(__name__)


class ShutdownMonitor:
    """Extension to monitor shutdown signal in S3"""
    
    def __init__(self, crawler):
        self.crawler = crawler
        self.check_interval = 30.0  # Check every 30 seconds
        self.loop_task = None
        
        # Initialize S3 connection
        import os
        self.cfg = load_config_from_env()
        no_aws = os.getenv("NO_AWS", "false").lower() == "true"
        self.s3 = S3Storage(self.cfg, no_aws=no_aws)
        
    @classmethod
    def from_crawler(cls, crawler):
        """Create extension from crawler"""
        ext = cls(crawler)
        
        # Connect to spider signals
        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
        
        return ext
    
    def spider_opened(self, spider):
        """Called when spider is opened"""
        logger.info("ShutdownMonitor: Starting periodic shutdown check")
        
        # Start periodic task to check for shutdown
        self.loop_task = task.LoopingCall(self.check_shutdown, spider)
        self.loop_task.start(self.check_interval, now=False)
        
    def spider_closed(self, spider, reason):
        """Called when spider is closed"""
        logger.info(f"ShutdownMonitor: Spider closed - {reason}")
        
        # Stop the periodic task
        if self.loop_task and self.loop_task.running:
            self.loop_task.stop()
    
    def check_shutdown(self, spider):
        """Check for shutdown signal in S3"""
        try:
            shutdown_data = self.s3.get_json(self.cfg.s3_bucket_processed, "shutdown_signal.json")
            
            if shutdown_data and shutdown_data.get('shutdown'):
                reason = shutdown_data.get('reason', 'Manual shutdown')
                logger.info(f"ShutdownMonitor: Shutdown signal detected - {reason}")
                logger.info("ShutdownMonitor: Closing spider gracefully...")
                
                # Close the spider
                self.crawler.engine.close_spider(spider, f'Shutdown signal: {reason}')
                
        except Exception as e:
            logger.debug(f"ShutdownMonitor: Error checking shutdown signal: {e}")

