#!/usr/bin/env python3

import os
import sys
import json
import time
import threading
import hashlib
import platform
import uuid
import subprocess
import requests
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlparse, urljoin
from concurrent.futures import ThreadPoolExecutor
from bs4 import BeautifulSoup
import re

class Colors:
    HEADER = '\033[95m'
    BLUE = '\033[94m'
    CYAN = '\033[96m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    GREY = '\033[90m'
    BOLD = '\033[1m'
    DIM = '\033[2m'
    RESET = '\033[0m'

class Config:
    def __init__(self):
        self.config_file = Path(__file__).parent / "config.json"
        self.api_url = "https://www.crawlio.online/dash_api.php"
        self.load_config()
    
    def load_config(self):
        if self.config_file.exists():
            with open(self.config_file, 'r') as f:
                data = json.load(f)
                self.api_url = data.get('api_url', self.api_url)
    
    def save_config(self):
        with open(self.config_file, 'w') as f:
            json.dump({'api_url': self.api_url}, f, indent=2)

class SecurityManager:
    @staticmethod
    def get_hardware_id():
        try:
            mac = hex(uuid.getnode()).replace('0x', '').upper()
            hostname = platform.node()
            if sys.platform == "win32":
                try:
                    cpu = subprocess.check_output("wmic cpu get processorid", shell=True).decode().strip().split('\n')[1].strip()
                except:
                    cpu = "unknown"
                try:
                    disk = subprocess.check_output("wmic diskdrive get serialnumber", shell=True).decode().strip().split('\n')[1].strip()
                except:
                    disk = "unknown"
            else:
                try:
                    cpu = subprocess.check_output("cat /proc/cpuinfo | grep serial | tail -1", shell=True).decode().strip().split(':')[-1].strip()
                except:
                    cpu = "unknown"
                try:
                    disk = subprocess.check_output("lsblk -o NAME,SERIAL | grep -v NAME | head -1", shell=True).decode().strip().split(' ')[-1].strip()
                except:
                    disk = "unknown"
            combined = f"{mac}_{hostname}_{cpu}_{disk}"
            return hashlib.sha256(combined.encode()).hexdigest()[:64]
        except:
            return hashlib.sha256(platform.node().encode()).hexdigest()[:64]

class LicenseManager:
    def __init__(self):
        self.config = Config()
        self.hwid = SecurityManager.get_hardware_id()
        self.license_file = Path(__file__).parent / "license.dat"
        self.license_data = None
        self.load_license()
    
    def load_license(self):
        if self.license_file.exists():
            try:
                with open(self.license_file, 'r') as f:
                    self.license_data = json.load(f)
            except:
                self.license_data = None
    
    def save_license(self, data):
        with open(self.license_file, 'w') as f:
            json.dump(data, f, indent=2)
        self.license_data = data
    
    def activate(self, key):
        try:
            response = requests.post(
                self.config.api_url,
                json={'action': 'activate', 'key': key, 'hwid': self.hwid},
                timeout=30
            )
            data = response.json()
            
            if data.get('success'):
                self.save_license({
                    'key': key,
                    'hwid': self.hwid,
                    'type': data.get('type'),
                    'expires': data.get('expires'),
                    'activated': datetime.now().isoformat()
                })
                return True, data
            else:
                return False, data.get('message', 'Activation failed')
        except Exception as e:
            return False, str(e)
    
    def check(self):
        if not self.license_data:
            return False, "No license found"
        
        try:
            response = requests.post(
                self.config.api_url,
                json={'action': 'check', 'key': self.license_data.get('key'), 'hwid': self.hwid},
                timeout=30
            )
            data = response.json()
            
            if data.get('success'):
                self.license_data['expires'] = data.get('expires')
                self.save_license(self.license_data)
                return True, data
            else:
                if self.license_file.exists():
                    self.license_file.unlink()
                return False, data.get('message', 'License invalid')
        except Exception as e:
            return False, str(e)
    
    def get_info(self):
        if not self.license_data:
            return None
        return self.license_data

class WebsiteCrawler:
    def __init__(self, base_url, log_cb=None, stats_cb=None, stop_event=None):
        self.base_url = base_url.rstrip('/')
        self.parsed_base = urlparse(self.base_url)
        self.domain = self.parsed_base.netloc
        self.log_cb = log_cb or (lambda msg: None)
        self.stats_cb = stats_cb or (lambda **kwargs: None)
        self.stop_event = stop_event or threading.Event()
        domain = self.domain.replace('.', '_')
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        downloads_path = Path.home() / "Downloads"
        self.output_dir = downloads_path / f"{domain}_{timestamp}"
        self.max_workers = 20
        self.delay = 0.3
        self.session = requests.Session()
        self.session.headers.update({
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36"
        })
        self.processed_urls = set()
        self.pending_urls = set([self.base_url])
        self.all_urls = set()
        self.broken_links = []
        self.lock = threading.Lock()
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.downloaded = 0
        self.failed = 0
        self.images = 0
        self.videos = 0
        self.pages = 0
        self.broken_count = 0
        self.total_checked = 0
        self.css_files = 0
        self.js_files = 0
        self.fonts = 0
    
    def should_download(self, url):
        parsed = urlparse(url)
        skip_patterns = ['mailto:', 'tel:', 'javascript:', '#', '?utm_', '?ref=']
        for pattern in skip_patterns:
            if pattern in url.lower():
                return False
        if parsed.netloc == self.domain:
            return True
        cdn_domains = ['cdn.', 'cdnjs.', 'cloudflare.', 'googleapis.', 'gstatic.', 
                      'fontawesome.', 'bootstrap.', 'jquery.', 'unpkg.', 'jsdelivr.']
        for cdn in cdn_domains:
            if cdn in parsed.netloc.lower():
                return True
        return False
    
    def get_filepath(self, url):
        parsed = urlparse(url)
        path = parsed.path or 'index.html'
        if path.endswith('/'):
            path += 'index.html'
        if '?' in path:
            path = path.split('?')[0]
        if path.startswith('/'):
            path = path[1:]
        if parsed.netloc and parsed.netloc != self.domain:
            domain_folder = parsed.netloc.replace('.', '_')
            path = f"external/{domain_folder}/{path}"
        return path
    
    def save_resource(self, url, content, content_type=None):
        filepath = self.get_filepath(url)
        if content_type:
            content_type = content_type.split(';')[0].strip()
            ext_map = {
                'css': '.css', 'javascript': '.js', 'ecmascript': '.js',
                'woff2': '.woff2', 'woff': '.woff', 'ttf': '.ttf',
                'svg': '.svg', 'png': '.png', 'jpg': '.jpg', 'jpeg': '.jpeg',
                'gif': '.gif', 'webp': '.webp', 'mp4': '.mp4', 'webm': '.webm'
            }
            for key, ext in ext_map.items():
                if key in content_type:
                    if not filepath.endswith(ext):
                        filepath = filepath.rsplit('.', 1)[0] if '.' in filepath.split('/')[-1] else filepath
                        filepath += ext
                    break
        full_path = self.output_dir / filepath
        full_path.parent.mkdir(parents=True, exist_ok=True)
        with open(full_path, 'wb') as f:
            f.write(content)
        return full_path
    
    def check_broken_link(self, url):
        try:
            response = self.session.head(url, timeout=10, allow_redirects=True)
            return response.status_code >= 400, response.status_code
        except:
            return True, 0
    
    def extract_all_resources(self, html, base_url):
        soup = BeautifulSoup(html, 'html.parser')
        resources = set()
        links = set()
        for tag in soup.find_all('a'):
            href = tag.get('href')
            if href:
                full_url = urljoin(base_url, href)
                if full_url.startswith(('http://', 'https://')):
                    links.add(full_url)
                    self.all_urls.add(full_url)
        resource_tags = ['img', 'script', 'link', 'video', 'audio', 'source', 
                        'embed', 'iframe', 'object', 'meta', 'track']
        for tag in soup.find_all(resource_tags):
            for attr in ['src', 'href', 'data', 'poster', 'content', 'data-src', 'srcset']:
                value = tag.get(attr)
                if value:
                    if attr == 'srcset':
                        urls = re.findall(r'(https?://[^\s]+)', value)
                        for url in urls:
                            full_url = urljoin(base_url, url)
                            if full_url.startswith(('http://', 'https://')):
                                resources.add(full_url)
                                self.all_urls.add(full_url)
                    else:
                        full_url = urljoin(base_url, value)
                        if full_url.startswith(('http://', 'https://')):
                            resources.add(full_url)
                            self.all_urls.add(full_url)
        return links, resources
    
    def process_page(self, url):
        if self.stop_event.is_set():
            return
        try:
            time.sleep(self.delay)
            response = self.session.get(url, timeout=30, allow_redirects=True)
            response.raise_for_status()
            content_type = response.headers.get('content-type', '').lower()
            if 'text/html' in content_type:
                self.save_resource(url, response.content, content_type)
                links, resources = self.extract_all_resources(response.text, url)
                for link in list(links)[:100]:
                    if self.stop_event.is_set():
                        break
                    is_broken, status = self.check_broken_link(link)
                    if is_broken:
                        with self.lock:
                            self.broken_links.append((url, link, status))
                            self.broken_count += 1
                    with self.lock:
                        self.total_checked += 1
                with self.lock:
                    self.downloaded += 1
                    self.pages += 1
                    for item in links | resources:
                        if self.should_download(item):
                            if item not in self.processed_urls and item not in self.pending_urls:
                                self.pending_urls.add(item)
                self.log_cb(f"OK: {url}")
                self.stats_cb(
                    pages=self.pages,
                    images=self.images,
                    videos=self.videos,
                    downloaded=self.downloaded,
                    failed=self.failed,
                    broken=self.broken_count,
                    pending=len(self.pending_urls),
                    total_links=len(self.all_urls),
                    total_checked=self.total_checked
                )
            else:
                self.save_resource(url, response.content, content_type)
                with self.lock:
                    self.downloaded += 1
                    if 'image' in content_type:
                        self.images += 1
                    elif 'video' in content_type:
                        self.videos += 1
                    elif 'css' in content_type:
                        self.css_files += 1
                    elif 'javascript' in content_type:
                        self.js_files += 1
                    elif 'font' in content_type:
                        self.fonts += 1
                self.log_cb(f"OK: {url}")
        except Exception:
            with self.lock:
                self.failed += 1
            self.log_cb(f"FAIL: {url}")
    
    def crawl(self):
        self.log_cb(f"Crawling: {self.base_url}")
        self.log_cb(f"Output: {self.output_dir}")
        self.log_cb("")
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            futures = {}
            while self.pending_urls and not self.stop_event.is_set():
                with self.lock:
                    current_pending = list(self.pending_urls)
                    self.pending_urls.clear()
                for url in current_pending:
                    if self.stop_event.is_set():
                        break
                    if url not in self.processed_urls:
                        self.processed_urls.add(url)
                        future = executor.submit(self.process_page, url)
                        futures[future] = url
                for future in list(futures.keys()):
                    if self.stop_event.is_set():
                        for f in futures:
                            f.cancel()
                        break
                    try:
                        future.result(timeout=60)
                    except:
                        pass
                    del futures[future]
        self.save_report()
        self.log_cb("")
        self.log_cb("=" * 50)
        self.log_cb("CRAWL COMPLETE")
        self.log_cb("=" * 50)
        self.log_cb(f"Pages: {self.pages}")
        self.log_cb(f"Images: {self.images}")
        self.log_cb(f"Videos: {self.videos}")
        self.log_cb(f"CSS Files: {self.css_files}")
        self.log_cb(f"JS Files: {self.js_files}")
        self.log_cb(f"Fonts: {self.fonts}")
        self.log_cb(f"Downloaded: {self.downloaded}")
        self.log_cb(f"Failed: {self.failed}")
        self.log_cb(f"Broken Links: {self.broken_count}")
        self.log_cb(f"Total URLs: {len(self.all_urls)}")
        self.log_cb(f"Saved to: {self.output_dir}")
    
    def save_report(self):
        report = {
            "timestamp": datetime.now().isoformat(),
            "base_url": self.base_url,
            "summary": {
                "pages": self.pages,
                "images": self.images,
                "videos": self.videos,
                "css_files": self.css_files,
                "js_files": self.js_files,
                "fonts": self.fonts,
                "downloaded": self.downloaded,
                "failed": self.failed,
                "broken_links": self.broken_count,
                "total_urls_found": len(self.all_urls)
            },
            "all_urls": list(self.all_urls)[:500],
            "broken_links": self.broken_links[:100]
        }
        try:
            with open(self.output_dir / "crawl_report.json", 'w') as f:
                json.dump(report, f, indent=2)
        except:
            pass

class CrawlioCLI:
    def __init__(self):
        self.license_manager = LicenseManager()
        self.is_licensed = False
        self.license_info = None
        self.current_crawler = None
        self.stop_event = threading.Event()
        self.is_crawling = False
        self.crawl_stats = {
            "pages": 0, "images": 0, "videos": 0,
            "downloaded": 0, "failed": 0, "broken": 0,
            "pending": 0, "total_links": 0
        }
        self.check_license()
    
    def check_license(self):
        valid, data = self.license_manager.check()
        if valid:
            self.is_licensed = True
            self.license_info = self.license_manager.get_info()
        else:
            self.is_licensed = False
            self.license_info = None
    
    def clear_screen(self):
        os.system('cls' if os.name == 'nt' else 'clear')
    
    def print_header(self):
        print(f"{Colors.BOLD}{Colors.CYAN}Crawlio{Colors.RESET}")
        print(f"{Colors.GREY}crawl everything discover anything{Colors.RESET}")
        print()
    
    def get_time_left(self):
        if not self.license_info:
            return "No license"
        expiry = datetime.fromisoformat(self.license_info["expires"])
        days_left = (expiry - datetime.now()).days
        hours_left = (expiry - datetime.now()).seconds // 3600
        if days_left > 0:
            return f"{days_left}d {hours_left}h"
        return f"{hours_left}h"
    
    def print_dashboard(self):
        self.clear_screen()
        self.print_header()
        
        status = f"{Colors.GREEN}ACTIVE{Colors.RESET}" if self.is_licensed else f"{Colors.RED}INACTIVE{Colors.RESET}"
        print(f"Status: {status}")
        if self.is_licensed:
            print(f"License: {self.license_info.get('type', 'Unknown')} · {self.get_time_left()} remaining")
            print(f"HWID: {self.license_info.get('hwid', '')[:16]}...")
        print()
        print(f"{Colors.GREY}{'-' * 50}{Colors.RESET}")
        print()
        
        if self.is_crawling:
            print(f"{Colors.GREEN}▶ Crawling in progress...{Colors.RESET}")
            print(f"  Pages: {self.crawl_stats['pages']} | Images: {self.crawl_stats['images']} | Videos: {self.crawl_stats['videos']}")
            print(f"  Downloaded: {self.crawl_stats['downloaded']} | Failed: {self.crawl_stats['failed']} | Broken: {self.crawl_stats['broken']}")
        else:
            print(f"{Colors.DIM}No active crawl{Colors.RESET}")
        
        print()
        print(f"{Colors.GREY}{'-' * 50}{Colors.RESET}")
        print()
        print(f"{Colors.BOLD}Commands:{Colors.RESET}")
        if not self.is_licensed:
            print(f"  {Colors.CYAN}activate <key>{Colors.RESET}  - Activate license")
        else:
            print(f"  {Colors.CYAN}crawl <url>{Colors.RESET}   - Start crawling")
            print(f"  {Colors.CYAN}stop{Colors.RESET}          - Stop crawling")
            print(f"  {Colors.CYAN}info{Colors.RESET}          - Show license info")
        print(f"  {Colors.CYAN}exit{Colors.RESET}          - Exit program")
    
    def activate_license(self, key):
        print(f"\n{Colors.YELLOW}Activating license...{Colors.RESET}")
        success, result = self.license_manager.activate(key)
        if success:
            self.is_licensed = True
            self.license_info = self.license_manager.get_info()
            print(f"{Colors.GREEN}✓ License activated successfully!{Colors.RESET}")
            print(f"Type: {result.get('type')}")
            print(f"Expires: {result.get('expires')}")
            print(f"{Colors.GREY}New key generated: {result.get('new_key', 'N/A')}{Colors.RESET}")
        else:
            print(f"{Colors.RED}✗ {result}{Colors.RESET}")
        time.sleep(2)
    
    def start_crawl(self, url):
        if not self.is_licensed:
            print(f"{Colors.RED}✗ No active license{Colors.RESET}")
            return
        
        if self.is_crawling:
            print(f"{Colors.YELLOW}⚠ Crawl already running{Colors.RESET}")
            return
        
        if not url.startswith(('http://', 'https://')):
            url = 'https://' + url
        
        self.stop_event.clear()
        self.is_crawling = True
        self.crawl_stats = {"pages": 0, "images": 0, "videos": 0, "downloaded": 0, "failed": 0, "broken": 0, "pending": 0, "total_links": 0}
        
        self.current_crawler = WebsiteCrawler(
            base_url=url,
            log_cb=self.crawl_log,
            stats_cb=self.crawl_stats_update,
            stop_event=self.stop_event
        )
        
        def run_crawl():
            try:
                self.current_crawler.crawl()
            except Exception as e:
                print(f"{Colors.RED}Error: {e}{Colors.RESET}")
            finally:
                self.is_crawling = False
        
        self.crawl_thread = threading.Thread(target=run_crawl, daemon=True)
        self.crawl_thread.start()
        print(f"{Colors.GREEN}✓ Crawl started!{Colors.RESET}")
    
    def crawl_log(self, msg):
        print(f"{Colors.GREY}{msg}{Colors.RESET}")
    
    def crawl_stats_update(self, **kwargs):
        for key, value in kwargs.items():
            if key in self.crawl_stats:
                self.crawl_stats[key] = value
    
    def stop_crawl(self):
        if self.is_crawling:
            self.stop_event.set()
            print(f"{Colors.YELLOW}Stopping crawl...{Colors.RESET}")
            time.sleep(1)
        else:
            print(f"{Colors.DIM}No crawl running{Colors.RESET}")
    
    def show_info(self):
        if not self.license_info:
            print(f"{Colors.RED}✗ No license active{Colors.RESET}")
            return
        
        print(f"\n{Colors.BOLD}License Information{Colors.RESET}")
        print(f"{Colors.GREY}{'-' * 40}{Colors.RESET}")
        print(f"Type: {self.license_info.get('type', 'Unknown')}")
        print(f"Hardware ID: {self.license_info.get('hwid', '')[:16]}...")
        print(f"Activated: {self.license_info.get('activated', 'Unknown')}")
        print(f"Expires: {self.license_info.get('expires', 'Unknown')}")
        print(f"Time Left: {self.get_time_left()}")
        input(f"\n{Colors.DIM}Press Enter to continue...{Colors.RESET}")
    
    def run(self):
        while True:
            try:
                self.print_dashboard()
                cmd = input(f"{Colors.BOLD}{Colors.CYAN}> {Colors.RESET}").strip()
                
                if cmd in ["exit", "quit", "q"]:
                    if self.is_crawling:
                        self.stop_event.set()
                        print(f"\n{Colors.YELLOW}Stopping crawl...{Colors.RESET}")
                        time.sleep(2)
                    print(f"\n{Colors.GREEN}Goodbye!{Colors.RESET}")
                    break
                
                elif cmd.startswith("activate"):
                    parts = cmd.split()
                    if len(parts) > 1:
                        self.activate_license(parts[1])
                    else:
                        print(f"{Colors.RED}✗ Usage: activate <key>{Colors.RESET}")
                        time.sleep(1)
                
                elif cmd.startswith("crawl"):
                    parts = cmd.split()
                    if len(parts) > 1:
                        self.start_crawl(parts[1])
                    else:
                        print(f"{Colors.RED}✗ Usage: crawl <url>{Colors.RESET}")
                        time.sleep(1)
                
                elif cmd == "stop":
                    self.stop_crawl()
                
                elif cmd == "info":
                    self.show_info()
                
                elif cmd == "":
                    continue
                
                else:
                    print(f"{Colors.RED}✗ Unknown command: {cmd}{Colors.RESET}")
                    time.sleep(1)
            
            except KeyboardInterrupt:
                if self.is_crawling:
                    print(f"\n{Colors.YELLOW}Stopping crawl...{Colors.RESET}")
                    self.stop_event.set()
                    self.is_crawling = False
                    time.sleep(2)
                else:
                    print(f"\n{Colors.GREEN}Goodbye!{Colors.RESET}")
                    break

def main():
    try:
        app = CrawlioCLI()
        app.run()
    except KeyboardInterrupt:
        print(f"\n{Colors.GREEN}Goodbye!{Colors.RESET}")
        sys.exit(0)
    except Exception as e:
        print(f"{Colors.RED}Fatal error: {e}{Colors.RESET}")
        sys.exit(1)

if __name__ == "__main__":
    main()