import os

BOT_NAME = "defensive_ransomware_crawler"

SPIDER_MODULES = ["crawler.spiders"]
NEWSPIDER_MODULE = "crawler.spiders"

ROBOTSTXT_OBEY = True

# Request settings - more conservative to avoid blocking
CONCURRENT_REQUESTS = int(os.getenv("CRAWLER_CONCURRENCY", "1"))  # Very conservative
DOWNLOAD_DELAY = float(os.getenv("CRAWLER_PER_DOMAIN_DELAY", "10.0"))  # Even longer delay
CONCURRENT_REQUESTS_PER_DOMAIN = int(os.getenv("CRAWLER_CONCURRENCY_PER_DOMAIN", "1"))  # One at a time
RANDOMIZE_DOWNLOAD_DELAY = True
DOWNLOAD_DELAY_RANDOMIZE_FACTOR = 0.8  # Random delay between 2-18 seconds

# Timeout settings
DOWNLOAD_TIMEOUT = int(os.getenv("DOWNLOAD_TIMEOUT", "30"))  # 30 second timeout
DOWNLOAD_FAIL_ON_DATALOSS = False

# More realistic User-Agent to avoid blocking
DEFAULT_REQUEST_HEADERS = {
    "User-Agent": os.getenv("CRAWLER_USER_AGENT", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Accept-Encoding": "gzip, deflate",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
}

# Duplicate prevention
DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"
DUPEFILTER_DEBUG = True

# Retry settings - include 403 errors
RETRY_TIMES = int(os.getenv("RETRY_TIMES", "3"))
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429, 403]  # Added 403 to retry codes

LOG_LEVEL = os.getenv("CRAWLER_LOG_LEVEL", "INFO")

# Logging configuration
LOG_FILE = os.getenv("CRAWLER_LOG_FILE", "scrapy.log")
LOG_ENABLED = True
LOG_STDOUT = True
LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'


