#!/usr/bin/env python3
"""
ULTIMATE AI-ML HYBRID LOTTO GENERATOR V2.2 - ERWEITERT
Erweiterte Version mit verbessertem Feature Engineering

NEUE FEATURES V2.2:
- Erweiterte Feature Engineering (4 → 18+ Features)
- Gap-Analyse (Ziehungen seit letztem Auftreten)
- Momentum & Trend-Features
- Beziehungs-Features (Summen, Parität, Dekaden)
- Nachbarschafts-Features (Cluster-Erkennung)
- Zyklus-Features (Monat, Woche, Saison)

OPTIMIERUNGEN (V2.1):
- Progress Bar für Training
- Reduzierte Iterationen (150 -> 50)
- Model Caching mit intelligenter Update-Erkennung
- Paralleles Training (wenn möglich)
"""

import pandas as pd
import numpy as np
import random
from collections import Counter, defaultdict, deque
import datetime
import pickle
import os
import json
import sys

# ML Imports
try:
    from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
    from sklearn.neural_network import MLPRegressor
    from sklearn.preprocessing import StandardScaler
    from sklearn.model_selection import train_test_split, cross_val_score
    from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error
    import joblib
    ML_AVAILABLE = True
except ImportError:
    ML_AVAILABLE = False

# Deep Learning (optional) - PyTorch or TensorFlow
DEEP_LEARNING_AVAILABLE = False
try:
    import torch
    DEEP_LEARNING_AVAILABLE = True
except ImportError:
    try:
        import tensorflow as tf
        from tensorflow.keras.models import Sequential
        from tensorflow.keras.layers import LSTM, Dense, Dropout
        from tensorflow.keras.optimizers import Adam
        DEEP_LEARNING_AVAILABLE = True
    except ImportError:
        DEEP_LEARNING_AVAILABLE = False

# Import Deep Learning Engine (PyTorch-based for Python 3.14+ compatibility)
try:
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'utils'))
    from deep_learning_engine_pytorch import DeepLearningEngine, HybridDeepLearningPredictor
    DL_ENGINE_AVAILABLE = True
except ImportError:
    try:
        from deep_learning_engine import DeepLearningEngine, HybridDeepLearningPredictor
        DL_ENGINE_AVAILABLE = True
    except ImportError:
        DL_ENGINE_AVAILABLE = False
        DeepLearningEngine = None
        HybridDeepLearningPredictor = None


class UltimateAIMLHybridGenerator:
    """
    Ultimate Generator - OPTIMIZED VERSION
    """

    # Empirisch belegte "griffige" Einzelzahlen, die Spieler überproportional oft frei
    # wählen (Quelle: Analysen realer Tippscheine, z.B. frz. 6aus49 über 25 Jahre:
    # populärste Zahlen 5,7,9,11,12,13). Unabhängig vom Geburtstags-Bias (<=31).
    POPULAR_PLAYER_PICKS = {5, 7, 9, 11, 12, 13}

    def __init__(self, data_path, fast_mode=True):
        self.data_path = data_path
        self.df = None
        self.fast_mode = fast_mode  # Schnellerer Modus
        
        # Configuration
        self.model_cache_path = os.path.join(os.path.dirname(data_path), "ultimate_ml_models")

        # Core Systems
        self.ai_ml_engine = AIMLEngine(fast_mode=fast_mode)
        self.pattern_engine = PatternEngine()
        self.hybrid_optimizer = HybridOptimizer(fast_mode=fast_mode)
        self.feature_engineer = FeatureEngineer()
        self.real_time_learner = RealTimeLearner(cache_path=self.model_cache_path)  # Mit Persistenz
        self.performance_tracker = PerformanceTracker()
        
        # Strategy Management: 4 echte Strategien mit echter Differenzierung
        self.strategy_weights = {
            'hybrid_optimized': 0.40,   # AI + Pattern Optimierung (bester Baseline)
            'balanced_spread': 0.30,    # Erzwingt N≥1 M≥1 H≥1 aus Top-Mustern
            'high_ev': 0.20,            # EV-Optimierung: 2-3 Zahlen >31, wenig Lucky
            'soft_contrarian': 0.10,    # Bevorzugt unterrepräsentierte Zahlen (letzte 30)
        }

        self.is_trained = False
        
        print("🚀 ULTIMATE AI-ML HYBRID GENERATOR V2.2 - ERWEITERT")
        print("=" * 70)
        print("🧠 AI/ML + 🎨 Pattern + ⚡ Hybrid + 📊 Real-Time Learning")
        print("✨ Neue Features: Gap-Analyse, Momentum, Trends, Beziehungen")
        if fast_mode:
            print("⚡ FAST MODE: Aktiviert (reduzierte Iterationen)")
        print("=" * 70)
        
        # Initialize
        self.initialize_system()
    
    def initialize_system(self):
        """Initialisiert alle Systeme."""
        print("\n🔧 SYSTEM INITIALIZATION...")
        print("-" * 70)
        
        # 1. Load Data
        print("📊 Loading data...", end=" ", flush=True)
        if not self._load_data():
            return
        print("✅")
        
        # 2. Feature Engineering
        print("🔍 Feature Engineering...", end=" ", flush=True)
        self.features_df = self.feature_engineer.create_features(self.df)
        print(f"✅ ({len(self.features_df.columns)} features)")
        
        # 3. Initialize Pattern Engine
        print("🎨 Analyzing patterns...", end=" ", flush=True)
        self.pattern_engine.initialize(self.df)
        print(f"✅ ({len(self.pattern_engine.pattern_frequencies)} patterns)")
        
        # 4. Initialize AI/ML Engine
        print("🧠 Initializing AI/ML Engine...")
        self.ai_ml_engine.data_file = self.data_path  # Setze data_file für Retraining-Check
        self.ai_ml_engine.initialize(self.df, self.features_df, self.model_cache_path)
        
        # 5. Initialize Hybrid Optimizer
        print("⚡ Initializing Hybrid Optimizer...", end=" ", flush=True)
        self.hybrid_optimizer.initialize(self.ai_ml_engine, self.pattern_engine)
        print("✅")
        
        # 6. Initialize Real-Time Learning
        print("📚 Activating Real-Time Learning...", end=" ", flush=True)
        self.real_time_learner.initialize(self.features_df)
        _counts = self.df[['Z1','Z2','Z3','Z4','Z5','Z6']].stack().value_counts()
        freq = {n: int(_counts.get(n, 0)) for n in range(1, 50)}
        self.real_time_learner.set_frequency_prior(freq)
        print("✅")
        
        self.is_trained = self.ai_ml_engine.is_trained
        
        print("\n✅ ALL SYSTEMS READY!")
        self._print_system_status()
    
    def _load_data(self):
        """Lädt und validiert Daten."""
        try:
            self.df = pd.read_csv(self.data_path, sep=';')
            
            if 'datum' in self.df.columns:
                self.df['datum'] = pd.to_datetime(self.df['datum'], format='%Y-%m-%d', errors='coerce')
                self.df = self.df.sort_values('datum')
            
            # Validate
            required_cols = ['Z1', 'Z2', 'Z3', 'Z4', 'Z5', 'Z6']
            if not all(col in self.df.columns for col in required_cols):
                print("❌ Fehlende Spalten")
                return False
            
            return True
            
        except Exception as e:
            print(f"❌ Error: {e}")
            return False
    
    def _print_system_status(self):
        """Zeigt System-Status."""
        print("\n📊 SYSTEM STATUS:")
        print(f"   🧠 ML Available: {'✅' if ML_AVAILABLE else '❌ (pip install scikit-learn)'}")
        print(f"   🚀 Deep Learning: {'✅ ACTIVE' if self.ai_ml_engine.use_deep_learning else '⚠️ Optional (pip install tensorflow)'}")
        print(f"   🎯 Models Trained: {'✅' if self.is_trained else '⚠️ Using frequency-based fallback'}")
        print(f"   📁 Data Size: {len(self.df):,} drawings")
        print(f"   🎨 Patterns: {len(self.pattern_engine.pattern_frequencies)}")
        print(f"   ⚡ Fast Mode: {'ON' if self.fast_mode else 'OFF'}")
        if self.ai_ml_engine.use_deep_learning:
            print(f"   🔮 LSTM: Hybrid Mode (RF 40% + DL 60%)")
    
    def generate_ultimate_tips(self, num_tips=10):
        """
        Generiert Ultimate Tips mit allen 4 Strategien.
        """
        print(f"\n🎯 GENERATING {num_tips} ULTIMATE TIPS")
        print("=" * 95)
        
        if len(self.df) == 0:
            print("❌ Keine Daten verfügbar")
            return []
        
        # Get AI Predictions
        print("🧠 Computing AI predictions...", end=" ", flush=True)
        ai_predictions = self.ai_ml_engine.predict_all_numbers(self.features_df)
        ai_predictions = self.real_time_learner.adjust_predictions(ai_predictions)
        print("✅")

        # Recency-Counts für Soft-Contrarian vorberechnen
        self._recency_counts = self._compute_recency_counts(30)

        # Determine Strategy Distribution
        distribution = self._calculate_strategy_distribution(num_tips)
        
        print(f"\n📊 STRATEGY DISTRIBUTION:")
        for strategy, count in distribution.items():
            weight = self.strategy_weights[strategy]
            print(f"   {strategy.upper()}: {count} tips (Weight: {weight:.1%})")
        
        # Show Top AI Predictions
        self._show_top_ai_predictions(ai_predictions)
        
        # Generate Tips
        print(f"\n🎲 GENERATING TIPS...")
        all_tips = []
        tip_counter = 1
        
        # Generate by strategy with progress
        for strategy, count in distribution.items():
            strategy_name = strategy.upper()
            print(f"   {strategy_name}...", end=" ", flush=True)
            tips = self._generate_tips_by_strategy(
                strategy, count, tip_counter, ai_predictions
            )
            all_tips.extend(tips)
            tip_counter += len(tips)
            print(f"✅ ({count} tips)")
        
        # Strukturfilter: Summe + Parität korrigieren
        fixed_count = 0
        for tip in all_tips:
            if not self._passes_structural_constraints(tip['numbers']):
                tip['numbers'] = self._apply_structural_fix(tip['numbers'], ai_predictions)
                tip['ai_score'] = np.mean([ai_predictions.get(n, 0.1) for n in tip['numbers']])
                tip['pattern_weight'] = self.pattern_engine.calculate_pattern_weight(tip['numbers'])
                tip['confidence'] = (tip['ai_score'] + tip['pattern_weight']) / 2
                tip['quality'] = self._calculate_quality_score(tip['numbers'], ai_predictions, tip['pattern_weight'])
                fixed_count += 1
        if fixed_count:
            print(f"   🔧 {fixed_count} Tip(s) via Strukturfilter korrigiert (Summe/Parität)")

        # Output all tips
        print(f"\n📋 RESULTS:")
        print("=" * 95)
        print("Nr  6 Numbers              SZ  Strategy           AI-Score  Pattern-W  Confidence  Quality")
        print("-" * 95)
        
        for tip in all_tips:
            self._print_tip_line(tip)
        
        # Analysis
        self._analyze_portfolio(all_tips)
        
        # Update adaptive weights
        self._update_strategy_weights(all_tips)
        
        # Track performance
        self.performance_tracker.log_generated_tips(all_tips)
        
        return all_tips
    
    def _calculate_strategy_distribution(self, num_tips):
        """Berechnet Tip-Verteilung über Strategien."""
        distribution = {}
        remaining = num_tips
        
        # Calculate based on weights
        for strategy, weight in sorted(self.strategy_weights.items(), key=lambda x: x[1], reverse=True):
            count = max(1, int(num_tips * weight))
            count = min(count, remaining)
            distribution[strategy] = count
            remaining -= count
        
        # Distribute remaining
        if remaining > 0:
            best_strategy = max(self.strategy_weights.items(), key=lambda x: x[1])[0]
            distribution[best_strategy] += remaining
        
        return distribution
    
    def _show_top_ai_predictions(self, ai_predictions):
        """Zeigt Top AI Predictions."""
        print(f"\n🧠 TOP 10 AI PREDICTIONS:")
        sorted_preds = sorted(ai_predictions.items(), key=lambda x: x[1], reverse=True)[:10]
        
        for i, (num, score) in enumerate(sorted_preds, 1):
            status = "🔥" if score > 0.7 else "🌡️" if score > 0.5 else "💧"
            print(f"   {i:2}. Zahl {num:2}: {score:.4f} {status}")
    
    def _generate_tips_by_strategy(self, strategy, count, start_number, ai_predictions):
        """Generiert Tips für spezifische Strategie."""
        tips = []
        
        for i in range(count):
            tip_number = start_number + i
            
            if strategy == 'hybrid_optimized':
                tip = self._generate_hybrid_tip(tip_number, ai_predictions)
            elif strategy == 'balanced_spread':
                tip = self._generate_balanced_spread_tip(tip_number, ai_predictions)
            elif strategy == 'high_ev':
                tip = self._generate_high_ev_tip(tip_number, ai_predictions)
            else:  # soft_contrarian
                tip = self._generate_soft_contrarian_tip(tip_number, ai_predictions)
            
            tips.append(tip)
        
        return tips
    
    def _generate_balanced_spread_tip(self, tip_number, ai_predictions):
        """Balanced Spread: erzwingt N≥1 M≥1 H≥1, AI-gewichtet innerhalb Zonen, max. 1 Consecutive Pair."""
        random.seed(42 + tip_number * 37)

        zones = {
            'N': list(range(1, 17)),
            'M': list(range(17, 33)),
            'H': list(range(33, 50))
        }

        # Nur Muster mit allen 3 Zonen (historisch top)
        top_patterns = self.pattern_engine.get_top_patterns(10)
        valid_patterns = [p for p in top_patterns if 'N' in p and 'M' in p and 'H' in p]
        if not valid_patterns:
            valid_patterns = ['NNMMHH', 'NNMHHH', 'NMMHHH', 'NNMMMH', 'NNNMMH']

        target_pattern = valid_patterns[tip_number % len(valid_patterns)]
        pattern_counts = Counter(target_pattern)

        selected = []
        for zone_char, count in pattern_counts.items():
            available = [n for n in zones[zone_char] if n not in selected]
            weights = [ai_predictions.get(n, 0.1) for n in available]
            for _ in range(count):
                if not available:
                    break
                choice = random.choices(available, weights=weights)[0]
                selected.append(choice)
                idx = available.index(choice)
                available.pop(idx)
                weights.pop(idx)

        selected = sorted(selected)

        # Soft consecutive reduction: max 1 Paar erlaubt
        for _ in range(5):
            consec = sum(1 for i in range(len(selected) - 1) if selected[i + 1] - selected[i] == 1)
            if consec <= 1:
                break
            for i in range(len(selected) - 1):
                if selected[i + 1] - selected[i] == 1:
                    n = selected[i + 1]
                    zone_char = 'N' if n <= 16 else 'M' if n <= 32 else 'H'
                    alts = [x for x in zones[zone_char] if x not in selected
                            and abs(x - selected[i]) > 1
                            and (i + 2 >= len(selected) or abs(x - selected[i + 2]) > 1)]
                    if alts:
                        alt_w = [ai_predictions.get(x, 0.1) for x in alts]
                        selected[i + 1] = random.choices(alts, weights=alt_w)[0]
                        selected = sorted(selected)
                    break

        # Soft sum range (Q1–Q3: 127–171)
        for _ in range(5):
            s = sum(selected)
            if 127 <= s <= 171:
                break
            if s < 127:
                alts = [x for x in range(selected[0] + 1, 50) if x not in selected]
                if alts:
                    selected[0] = random.choices(alts, weights=[ai_predictions.get(x, 0.1) for x in alts])[0]
                    selected = sorted(selected)
            else:
                alts = [x for x in range(1, selected[-1]) if x not in selected]
                if alts:
                    selected[-1] = random.choices(alts, weights=[ai_predictions.get(x, 0.1) for x in alts])[0]
                    selected = sorted(selected)

        superzahl = self._get_smart_superzahl(tip_number)
        ai_score = np.mean([ai_predictions.get(n, 0.1) for n in selected])
        pattern_weight = self.pattern_engine.calculate_pattern_weight(selected)
        confidence = ai_score * 0.7 + pattern_weight * 0.3
        quality = self._calculate_quality_score(selected, ai_predictions, pattern_weight)

        return {
            'tip_number': tip_number,
            'numbers': selected,
            'superzahl': superzahl,
            'strategy': 'BALANCED-SPREAD',
            'ai_score': ai_score,
            'pattern_weight': pattern_weight,
            'confidence': confidence,
            'quality': quality
        }

    def _generate_high_ev_tip(self, tip_number, ai_predictions):
        """High-EV: 2–3 Zahlen >31, max. 1 Lucky Number, soft Consecutive-Vermeidung."""
        random.seed(42 + tip_number * 41)

        lucky_numbers = UltimateAIMLHybridGenerator.POPULAR_PLAYER_PICKS
        above_31_target = 2 + (tip_number % 2)  # wechselt zwischen 2 und 3

        selected = []

        # Zahlen >31 wählen (ohne Consecutives)
        above_pool = list(range(32, 50))
        above_weights = [ai_predictions.get(n, 0.1) for n in above_pool]
        for _ in range(above_31_target):
            if not above_pool:
                break
            choice = random.choices(above_pool, weights=above_weights)[0]
            selected.append(choice)
            # Benachbarte aus Pool entfernen → keine Consecutives innerhalb above-31
            new_pool, new_weights = [], []
            for n, w in zip(above_pool, above_weights):
                if n != choice and abs(n - choice) > 1:
                    new_pool.append(n)
                    new_weights.append(w)
            above_pool, above_weights = new_pool, new_weights

        # Zahlen ≤31 wählen (max. 1 Lucky, soft Consecutive-Penalty)
        below_pool = list(range(1, 32))
        lucky_picked = 0
        for _ in range(6 - above_31_target):
            if not below_pool:
                break
            weights = []
            for n in below_pool:
                w = ai_predictions.get(n, 0.1)
                if n in lucky_numbers:
                    w *= (0.2 if lucky_picked >= 1 else 0.6)
                if any(abs(n - s) == 1 for s in selected):
                    w *= 0.25  # soft Penalty für Consecutive
                weights.append(max(0.001, w))

            choice = random.choices(below_pool, weights=weights)[0]
            if choice in lucky_numbers:
                lucky_picked += 1
            selected.append(choice)
            below_pool = [n for n in below_pool if n != choice]

        selected = sorted(selected)
        superzahl = self._get_smart_superzahl(tip_number)
        ai_score = np.mean([ai_predictions.get(n, 0.1) for n in selected])
        pattern_weight = self.pattern_engine.calculate_pattern_weight(selected)
        pop_score = self._calculate_popularity_score(selected)
        confidence = ai_score * 0.5 + pattern_weight * 0.2 + pop_score * 0.3
        quality = self._calculate_quality_score(selected, ai_predictions, pattern_weight)

        return {
            'tip_number': tip_number,
            'numbers': selected,
            'superzahl': superzahl,
            'strategy': 'HIGH-EV',
            'ai_score': ai_score,
            'pattern_weight': pattern_weight,
            'confidence': confidence,
            'quality': quality
        }

    def _generate_soft_contrarian_tip(self, tip_number, ai_predictions):
        """Soft Contrarian: bevorzugt Zahlen die in letzten 30 Ziehungen unterrepräsentiert waren."""
        random.seed(42 + tip_number * 43)

        expected_freq = 30 * 6 / 49  # ~3.67 Vorkommen pro Zahl erwartet

        selected = []
        for _ in range(6):
            candidates = [n for n in range(1, 50) if n not in selected]

            weights = []
            selected_zones = {'N' if s <= 16 else 'M' if s <= 32 else 'H' for s in selected}
            for c in candidates:
                ai_s = ai_predictions.get(c, 0.1)
                actual = self._recency_counts.get(c, 0)
                recency_s = max(0.0, (expected_freq - actual) / expected_freq)
                zone_char = 'N' if c <= 16 else 'M' if c <= 32 else 'H'
                zone_s = 0.8 if zone_char not in selected_zones else 0.4
                weights.append(max(0.001, ai_s * 0.5 + recency_s * 0.3 + zone_s * 0.2))

            choice = random.choices(candidates, weights=weights)[0]
            selected.append(choice)

        selected = sorted(selected)
        superzahl = self._get_smart_superzahl(tip_number)
        ai_score = np.mean([ai_predictions.get(n, 0.1) for n in selected])
        pattern_weight = self.pattern_engine.calculate_pattern_weight(selected)
        recency_avg = np.mean([
            max(0.0, (expected_freq - self._recency_counts.get(n, 0)) / expected_freq)
            for n in selected
        ])
        confidence = ai_score * 0.5 + pattern_weight * 0.3 + recency_avg * 0.2
        quality = self._calculate_quality_score(selected, ai_predictions, pattern_weight)

        return {
            'tip_number': tip_number,
            'numbers': selected,
            'superzahl': superzahl,
            'strategy': 'SOFT-CONTRARIAN',
            'ai_score': ai_score,
            'pattern_weight': pattern_weight,
            'confidence': confidence,
            'quality': quality
        }

    def _compute_recency_counts(self, lookback=30):
        """Zählt Vorkommen jeder Zahl in den letzten N Ziehungen."""
        recent = self.df.tail(lookback)
        counts = Counter()
        for _, row in recent.iterrows():
            for col in ['Z1', 'Z2', 'Z3', 'Z4', 'Z5', 'Z6']:
                counts[int(row[col])] += 1
        return counts

    def _generate_hybrid_tip(self, tip_number, ai_predictions):
        """Hybrid Strategie."""
        result = self.hybrid_optimizer.optimize(tip_number, ai_predictions)
        
        superzahl = self._get_smart_superzahl(tip_number)
        quality = self._calculate_quality_score(
            result['numbers'], ai_predictions, result['pattern_weight']
        )
        
        return {
            'tip_number': tip_number,
            'numbers': result['numbers'],
            'superzahl': superzahl,
            'strategy': 'HYBRID-OPT',
            'ai_score': result['ai_score'],
            'pattern_weight': result['pattern_weight'],
            'confidence': result['confidence'],
            'quality': quality
        }
    
    def _passes_structural_constraints(self, numbers):
        """Prüft Summenbereich (122-176) und Parität (mind. 1G + 1U)."""
        s = sum(numbers)
        if s < 100 or s > 200:
            return False
        even = sum(1 for n in numbers if n % 2 == 0)
        if even == 0 or even == 6:
            return False
        return True

    def _apply_structural_fix(self, numbers, ai_preds):
        """Korrigiert Summe/Parität durch minimalen Tausch."""
        nums = list(numbers)
        even = [n for n in nums if n % 2 == 0]
        odd  = [n for n in nums if n % 2 != 0]
        if len(even) == 0:
            worst = min(odd, key=lambda n: ai_preds.get(n, 0))
            cands = sorted([n for n in range(2, 50, 2) if n not in nums],
                           key=lambda n: -ai_preds.get(n, 0))
            if cands:
                nums.remove(worst)
                nums.append(cands[0])
        elif len(odd) == 0:
            worst = min(even, key=lambda n: ai_preds.get(n, 0))
            cands = sorted([n for n in range(1, 50, 2) if n not in nums],
                           key=lambda n: -ai_preds.get(n, 0))
            if cands:
                nums.remove(worst)
                nums.append(cands[0])
        for _ in range(10):
            s = sum(nums)
            if 100 <= s <= 200:
                break
            if s > 200:
                highest = max(nums)
                cands = sorted([n for n in range(1, highest) if n not in nums],
                               key=lambda n: -ai_preds.get(n, 0))
                if cands:
                    nums.remove(highest)
                    nums.append(cands[0])
            else:
                lowest = min(nums)
                cands = sorted([n for n in range(lowest + 1, 50) if n not in nums],
                               key=lambda n: -ai_preds.get(n, 0))
                if cands:
                    nums.remove(lowest)
                    nums.append(cands[0])
        return sorted(nums)

    def _calculate_diversity_score(self, candidate, selected):
        """Berechnet Diversitäts-Score."""
        if not selected:
            return 0.5
        
        distances = [abs(candidate - s) for s in selected]
        avg_distance = np.mean(distances)
        distance_score = min(avg_distance / 10.0, 1.0)
        
        def get_range(n):
            if n <= 16: return 0
            elif n <= 32: return 1
            else: return 2
        
        candidate_range = get_range(candidate)
        selected_ranges = [get_range(s) for s in selected]
        range_counts = Counter(selected_ranges)
        
        if range_counts[candidate_range] < 2:
            range_score = 0.8
        elif range_counts[candidate_range] == 2:
            range_score = 0.4
        else:
            range_score = 0.1
        
        even_count = sum(1 for s in selected if s % 2 == 0)
        candidate_even = candidate % 2 == 0
        
        if len(selected) < 3:
            balance_score = 0.5
        elif (even_count < 3 and candidate_even) or (even_count >= 3 and not candidate_even):
            balance_score = 0.7
        else:
            balance_score = 0.3
        
        return (distance_score * 0.4 + range_score * 0.4 + balance_score * 0.2)
    
    def _get_smart_superzahl(self, tip_number):
        """
        EV-optimierte Superzahl: bevorzugt Ziffern, die Spieler erfahrungsgemäß
        seltener frei wählen (0 wirkt nicht wie eine "eigene" Zahl; 7/3 gelten als
        klassische Glückszahlen und werden überproportional gewählt). Historische
        Ziehungshäufigkeit fließt bewusst NICHT mehr ein - die SZ wird pro Ziehung
        unabhängig und gleichverteilt (0-9) gezogen, "häufig gezogen in letzten 30
        Ziehungen" ist reines Rauschen ohne Vorhersagewert.
        """
        ev_order = [0, 9, 8, 4, 2, 6, 1, 5, 3, 7]  # unpopulärste zuerst
        return ev_order[tip_number % len(ev_order)]
    
    @staticmethod
    def _calculate_popularity_score(numbers):
        """
        Schätzt den Erwartungswert-Vorteil durch Vermeidung populärer Zahlenkombinationen.
        Höher = unpopulärer bei Mitspielern = höherer Gewinnanteil bei einem Treffer
        (Hit-Wahrscheinlichkeit selbst ist bei i.i.d. Ziehungen nicht beeinflussbar).

        Basis: dokumentierte Spielerverhalten-Biases (Henze/Riedwyl-artige Analysen
        realer Tippscheine): Geburtstagszahlen (<=31), einzelne "griffige" Zahlen,
        Zahlenfolgen/arithmetische Muster (z.B. 5-10-15-20-25-30) und "zufällig
        aussehende" ausgeglichene Odd/Even-Splits werden von Menschen
        überproportional oft gewählt - jede einzelne Kombination ist aber exakt
        gleich wahrscheinlich, unabhängig von diesen Eigenschaften.

        Hinweis: eine "Summe nahe am Mittel"-Komponente wurde bewusst NICHT
        aufgenommen - sie widerspricht sich mit der Muster-Erkennung, da gerade
        die meistgespielten sequenziellen Kombinationen (z.B. 1-2-3-4-5-6) durch
        ihre enge Zahlen-Clusterung zugleich eine extreme Summe erzeugen. Ohne
        Trennung von "Summe extrem durch Streuung" vs. "Summe extrem durch engen
        Cluster" hätte dieses Merkmal genau die falschesten Kombinationen
        aufgewertet.
        """
        n = len(numbers)
        sorted_nums = sorted(numbers)

        # 1) Geburtstags-Range (1-31) meiden
        above_31_ratio = sum(1 for x in numbers if x > 31) / n

        # 2) Empirisch belegte populäre Einzelzahlen meiden
        popular_ratio = sum(
            1 for x in numbers
            if x in UltimateAIMLHybridGenerator.POPULAR_PLAYER_PICKS
        ) / n

        # 3) Muster/Zahlenfolgen meiden: direkte Nachbarn UND allgemeine
        #    arithmetische Folgen (konstante Schrittweite, z.B. 5-10-15-20-25-30
        #    zählt zu den meistgespielten Kombinationen überhaupt)
        diffs = [sorted_nums[i + 1] - sorted_nums[i] for i in range(n - 1)]
        small_step_ratio = sum(1 for d in diffs if d <= 5) / (n - 1) if diffs else 0
        is_perfect_progression = len(diffs) > 1 and len(set(diffs)) == 1
        pattern_penalty = small_step_ratio * (0.6 if not is_perfect_progression else 1.0)

        # 4) Odd/Even-Split: Menschen bevorzugen "ausgeglichen aussehende" 3-3
        #    Splits, obwohl jede einzelne Kombination gleich wahrscheinlich ist
        even_count = sum(1 for x in numbers if x % 2 == 0)
        split_extremity = abs(even_count - 3) / 3  # 0.0 bei 3-3, 1.0 bei 6-0/0-6

        score = (
            above_31_ratio * 0.30
            + (1 - popular_ratio) * 0.20
            + (1 - pattern_penalty) * 0.30
            + split_extremity * 0.20
        )
        return min(max(score, 0.0), 1.0)

    def _calculate_quality_score(self, numbers, ai_predictions, pattern_weight):
        """Berechnet Qualitäts-Score aus allen 5 Strategie-Perspektiven."""
        # HYBRID-OPT Perspektive: AI-Score gewichtet mit Streuung
        ai_scores = [ai_predictions.get(n, 0.1) for n in numbers]
        ai_quality = np.mean(ai_scores) * (1 + np.std(ai_scores))

        # BALANCED-SPREAD Perspektive: historisches Mustergewicht
        pattern_quality = pattern_weight

        # Zonenspreizung: mittlere paarweise Distanz
        distances = []
        for i, n1 in enumerate(numbers):
            for n2 in numbers[i+1:]:
                distances.append(abs(n1 - n2))
        diversity_quality = min(np.mean(distances) / 8.0, 1.0) if distances else 0.5

        # HIGH-EV Perspektive: Popularitäts-/EV-Score
        popularity_quality = self._calculate_popularity_score(numbers)

        # SOFT-CONTRARIAN Perspektive: Recency-Score (wie stark unterrepräsentiert?)
        if hasattr(self, '_recency_counts'):
            expected_freq = 30 * 6 / 49
            recency_quality = np.mean([
                max(0.0, (expected_freq - self._recency_counts.get(n, 0)) / expected_freq)
                for n in numbers
            ])
        else:
            recency_quality = 0.5  # neutral wenn noch nicht berechnet

        # Popularity/EV dominiert bewusst: AI-Score, Pattern und Recency sagen nichts
        # über die (unbeeinflussbare) Trefferwahrscheinlichkeit aus - Recency/Pattern
        # folgen zudem Heuristiken, die auch andere Systemspieler nutzen und damit die
        # Popularity eher untergraben statt sie zu unterstützen.
        quality = (
            ai_quality        * 0.15
            + pattern_quality * 0.10
            + diversity_quality * 0.15
            + popularity_quality * 0.50
            + recency_quality * 0.10
        )
        return min(quality, 1.0)
    
    def _print_tip_line(self, tip):
        """Druckt Tip-Zeile."""
        zahlen_str = '-'.join([f"{n:2d}" for n in tip['numbers']])
        
        quality_emoji = "⭐" if tip['quality'] > 0.7 else "🌟" if tip['quality'] > 0.5 else "💫"
        
        print(f"{tip['tip_number']:2d}  {zahlen_str}  {tip['superzahl']:2d}  "
              f"{tip['strategy']:<14}  {tip['ai_score']:.4f}    {tip['pattern_weight']:.4f}     "
              f"{tip['confidence']:.4f}    {quality_emoji} {tip['quality']:.3f}")
    
    def _analyze_portfolio(self, tips):
        """Analysiert Tip-Portfolio."""
        print(f"\n📊 PORTFOLIO ANALYSIS:")
        print("=" * 70)
        
        strategy_stats = defaultdict(lambda: {'count': 0, 'avg_conf': [], 'avg_quality': []})
        
        for tip in tips:
            strategy = tip['strategy']
            strategy_stats[strategy]['count'] += 1
            strategy_stats[strategy]['avg_conf'].append(tip['confidence'])
            strategy_stats[strategy]['avg_quality'].append(tip['quality'])
        
        for strategy, stats in strategy_stats.items():
            avg_conf = np.mean(stats['avg_conf'])
            avg_qual = np.mean(stats['avg_quality'])
            print(f"{strategy}:")
            print(f"   Count: {stats['count']}, Confidence: {avg_conf:.4f}, Quality: {avg_qual:.4f}")
        
        best_by_confidence = max(tips, key=lambda x: x['confidence'])
        best_by_quality = max(tips, key=lambda x: x['quality'])
        
        print(f"\n⭐ TOP RECOMMENDATIONS:")
        print(f"\n🎯 Highest Confidence:")
        zahlen = '-'.join([f"{n:2d}" for n in best_by_confidence['numbers']])
        print(f"   Tip #{best_by_confidence['tip_number']}: {zahlen} + SZ {best_by_confidence['superzahl']}")
        print(f"   Strategy: {best_by_confidence['strategy']}, Confidence: {best_by_confidence['confidence']:.4f}")
        
        print(f"\n💎 Highest Quality:")
        zahlen = '-'.join([f"{n:2d}" for n in best_by_quality['numbers']])
        print(f"   Tip #{best_by_quality['tip_number']}: {zahlen} + SZ {best_by_quality['superzahl']}")
        print(f"   Strategy: {best_by_quality['strategy']}, Quality: {best_by_quality['quality']:.4f}")
    
    def _update_strategy_weights(self, tips):
        """Updated Strategy-Gewichte."""
        strategy_performance = defaultdict(list)
        
        for tip in tips:
            perf_score = (tip['confidence'] * 0.6 + tip['quality'] * 0.4)
            strategy_performance[tip['strategy']].append(perf_score)
        
        strategy_avg = {}
        total = 0
        
        for strategy in ['hybrid_optimized', 'balanced_spread', 'high_ev', 'soft_contrarian']:
            strategy_key = {
                'hybrid_optimized': 'HYBRID-OPT',
                'balanced_spread': 'BALANCED-SPREAD',
                'high_ev': 'HIGH-EV',
                'soft_contrarian': 'SOFT-CONTRARIAN'
            }[strategy]
            
            if strategy_key in strategy_performance:
                avg = np.mean(strategy_performance[strategy_key])
                strategy_avg[strategy] = avg
                total += avg
        
        if total > 0:
            for strategy in self.strategy_weights:
                if strategy in strategy_avg:
                    new_weight = strategy_avg[strategy] / total
                    self.strategy_weights[strategy] = (
                        self.strategy_weights[strategy] * 0.7 + new_weight * 0.3
                    )
        
        print(f"\n🔄 UPDATED STRATEGY WEIGHTS:")
        for strategy, weight in sorted(self.strategy_weights.items(), key=lambda x: x[1], reverse=True):
            print(f"   {strategy.upper()}: {weight:.1%}")


# ============================================================================
# OPTIMIZED SUPPORT CLASSES
# ============================================================================

class AIMLEngine:
    """AI/ML Engine - OPTIMIZED mit Deep Learning Support."""

    def __init__(self, fast_mode=True):
        self.models = {}
        self.trained_models = {}
        self.scaler = StandardScaler()
        self.is_trained = False
        self.fast_mode = fast_mode

        # Deep Learning
        self.dl_engine = None
        self.hybrid_predictor = None
        self.use_deep_learning = DL_ENGINE_AVAILABLE and DEEP_LEARNING_AVAILABLE
    
    def initialize(self, df, features_df, cache_path):
        """Initialisiert AI/ML System mit intelligentem Retraining."""
        self.cache_path = cache_path
        self.data_file = None  # Wird vom Generator gesetzt
        self.df = df  # Store for DL
        self.features_df = features_df  # Store for DL

        if not ML_AVAILABLE:
            print("   ⚠️ ML not available - using frequency-based fallback")
            return

        # Setup models
        n_estimators = 100 if self.fast_mode else 200
        self.models = {
            'random_forest': RandomForestRegressor(
                n_estimators=n_estimators, max_depth=8, random_state=42, n_jobs=-1
            ),
            'gradient_boost': GradientBoostingRegressor(
                n_estimators=n_estimators, learning_rate=0.1, max_depth=5, random_state=42
            )
        }

        # Intelligentes Retraining
        cache_file = os.path.join(cache_path, "trained_models.pkl")
        needs_retrain = self._needs_retraining(cache_file)

        if os.path.exists(cache_file) and not needs_retrain:
            print("   📂 Loading cached models...", end=" ", flush=True)
            self._load_models()
            print("✅")
        elif len(df) >= 50:
            if needs_retrain:
                print("   🔄 Data updated - retraining models...")
            self._train_models(df, features_df)
        else:
            print("   ⚠️ Insufficient data - using fallback")

        # Initialize Deep Learning
        if self.use_deep_learning:
            print("   🧠 Initializing Deep Learning (LSTM)...", end=" ", flush=True)
            try:
                self.dl_engine = DeepLearningEngine(
                    num_numbers=49,
                    sequence_length=20,
                    cache_dir=os.path.join(cache_path, 'deep_learning'),
                    fast_mode=self.fast_mode
                )

                # Train or load LSTM
                self.dl_engine.train(df, features_df, force_retrain=needs_retrain)

                # Create hybrid predictor
                self.hybrid_predictor = HybridDeepLearningPredictor(
                    dl_engine=self.dl_engine,
                    rf_weight=0.4,
                    dl_weight=0.6
                )
                print("✅")
            except Exception as e:
                print(f"⚠️  DL init failed: {e}")
                self.use_deep_learning = False
        else:
            if not DEEP_LEARNING_AVAILABLE:
                print("   ℹ️  Deep Learning disabled (TensorFlow not installed)")
            elif not DL_ENGINE_AVAILABLE:
                print("   ℹ️  Deep Learning Engine not available")

    def _needs_retraining(self, cache_file):
        """Prüft ob Retraining nötig ist."""
        if not os.path.exists(cache_file):
            return True

        # Prüfe ob CSV neuer als Cache
        data_file = self.data_file or os.path.join(
            os.path.dirname(self.cache_path),
            "AlleLottozahlen.csv"
        )

        if os.path.exists(data_file):
            cache_mtime = os.path.getmtime(cache_file)
            data_mtime = os.path.getmtime(data_file)

            if data_mtime > cache_mtime:
                print("   🔍 Detected updated data (CSV newer than cache)")
                return True

        return False
    
    def _train_models(self, df, features_df):
        """Trainiert ML-Modelle mit Progress."""
        print("   🎯 Training AI models...")
        
        training_results = {}
        
        # Train only for most frequently drawn numbers (optimization)
        number_freq = Counter()
        for _, row in df.iterrows():
            for col in ['Z1', 'Z2', 'Z3', 'Z4', 'Z5', 'Z6']:
                number_freq[int(row[col])] += 1
        
        # Train for top 40 numbers (saves time)
        top_numbers = [num for num, _ in number_freq.most_common(40)]
        
        total = len(top_numbers)
        for idx, number in enumerate(top_numbers, 1):
            # Progress indicator
            progress = idx / total
            bar_length = 30
            filled = int(bar_length * progress)
            bar = '█' * filled + '░' * (bar_length - filled)
            print(f"\r   [{bar}] {progress:.0%} - Training number {number}", end='', flush=True)
            
            X, y = self._prepare_training_data(df, features_df, number)
            
            if len(X) < 20:
                continue
            
            X_train, X_test, y_train, y_test = train_test_split(
                X, y, test_size=0.2, random_state=42
            )
            
            scaler = StandardScaler()
            X_train_scaled = scaler.fit_transform(X_train)
            X_test_scaled = scaler.transform(X_test)
            
            number_models = {}
            
            for model_name, model in self.models.items():
                try:
                    model.fit(X_train_scaled, y_train)
                    y_pred = model.predict(X_test_scaled)
                    score = r2_score(y_test, y_pred)
                    
                    number_models[model_name] = {
                        'model': model,
                        'scaler': scaler,
                        'score': max(score, 0.1)
                    }
                except:
                    pass
            
            if number_models:
                training_results[number] = number_models
        
        print()  # New line after progress bar
        
        self.trained_models = training_results
        self._save_models()
        self.is_trained = True
        print(f"   ✅ AI models trained ({len(training_results)} numbers)")
    
    def _prepare_training_data(self, df, features_df, number):
        """Bereitet Trainingsdaten vor - SIMPLIFIED."""
        X = []
        y = []
        
        window_size = 3  # Reduced from 5
        
        for i in range(window_size, min(len(features_df), 100)):  # Max 100 samples
            features = []
            for j in range(i - window_size, i):
                row_features = features_df.iloc[j].values.tolist()
                features.extend(row_features[:4])  # Only first 4 features
            
            X.append(features)
            
            current_numbers = [df.iloc[i][f'Z{k}'] for k in range(1, 7)]
            y.append(1 if number in current_numbers else 0)
        
        return np.array(X), np.array(y)
    
    def predict_all_numbers(self, features_df):
        """Vorhersage für alle Zahlen - mit Deep Learning Hybrid."""
        predictions = {}

        # Get RandomForest predictions
        if not self.is_trained or not self.trained_models:
            # Fallback
            for num in range(1, 50):
                predictions[num] = 0.2 + random.random() * 0.3
        else:
            current_features = self._get_current_features(features_df)

            for number in range(1, 50):
                if number not in self.trained_models:
                    predictions[number] = 0.15 + (number % 10) * 0.03
                    continue

                number_models = self.trained_models[number]
                ensemble_pred = 0
                total_weight = 0

                for model_name, model_data in number_models.items():
                    try:
                        model = model_data['model']
                        scaler = model_data['scaler']
                        score = model_data['score']

                        features_scaled = scaler.transform([current_features])
                        pred = model.predict(features_scaled)[0]

                        weight = score
                        ensemble_pred += pred * weight
                        total_weight += weight
                    except:
                        pass

                if total_weight > 0:
                    predictions[number] = np.clip(ensemble_pred / total_weight, 0, 1)
                else:
                    predictions[number] = 0.2 + random.random() * 0.2

        # Use Deep Learning Hybrid if available
        if self.use_deep_learning and self.hybrid_predictor:
            try:
                predictions = self.hybrid_predictor.predict(
                    predictions,
                    self.df,
                    self.features_df
                )
            except Exception as e:
                print(f"⚠️  DL prediction failed: {e}, using RF only")

        return predictions
    
    def _get_current_features(self, features_df):
        """Holt aktuelle Features - SIMPLIFIED."""
        if len(features_df) == 0:
            return [0.0] * 12
        
        features = []
        for i in range(max(0, len(features_df)-3), len(features_df)):
            row_features = features_df.iloc[i].values.tolist()
            features.extend(row_features[:4])
        
        while len(features) < 12:
            features.append(0.0)
        
        return features[:12]
    
    def _save_models(self):
        """Speichert Modelle."""
        os.makedirs(self.cache_path, exist_ok=True)
        filepath = os.path.join(self.cache_path, "trained_models.pkl")
        
        with open(filepath, 'wb') as f:
            pickle.dump(self.trained_models, f)
    
    def _load_models(self):
        """Lädt Modelle."""
        filepath = os.path.join(self.cache_path, "trained_models.pkl")
        
        try:
            with open(filepath, 'rb') as f:
                self.trained_models = pickle.load(f)
            self.is_trained = True
        except:
            pass


class PatternEngine:
    """Pattern Analysis Engine."""
    
    def __init__(self):
        self.pattern_frequencies = Counter()
        self.pattern_weights = {}
        self.number_patterns = defaultdict(list)
    
    def initialize(self, df):
        """Initialisiert Pattern Engine."""
        self._analyze_patterns(df)
        self._analyze_number_patterns(df)
    
    def _analyze_patterns(self, df):
        """Analysiert Patterns."""
        for _, row in df.iterrows():
            numbers = sorted([row['Z1'], row['Z2'], row['Z3'], 
                            row['Z4'], row['Z5'], row['Z6']])
            pattern = self._get_pattern(numbers)
            self.pattern_frequencies[pattern] += 1
        
        total = sum(self.pattern_frequencies.values())
        for pattern, count in self.pattern_frequencies.items():
            self.pattern_weights[pattern] = count / total if total > 0 else 0
    
    def _analyze_number_patterns(self, df):
        """Analysiert Zahlen-Muster."""
        for _, row in df.tail(100).iterrows():  # Only last 100 for speed
            numbers = [row['Z1'], row['Z2'], row['Z3'], 
                      row['Z4'], row['Z5'], row['Z6']]
            for num in numbers:
                self.number_patterns[num].append(numbers)
    
    def _get_pattern(self, numbers):
        """Bestimmt Pattern."""
        pattern = ""
        for num in numbers:
            if num <= 16:
                pattern += "N"
            elif num <= 32:
                pattern += "M"
            else:
                pattern += "H"
        return pattern
    
    def calculate_pattern_weight(self, numbers):
        """Berechnet Pattern-Gewicht."""
        pattern = self._get_pattern(sorted(numbers))
        return self.pattern_weights.get(pattern, 0.01)
    
    def get_top_patterns(self, count):
        """Liefert Top Patterns."""
        return [p for p, _ in self.pattern_frequencies.most_common(count)]
    
    def generate_for_pattern(self, target_pattern, seed):
        """Generiert für Pattern."""
        random.seed(42 + seed * 19)
        
        ranges = {
            'N': list(range(1, 17)),
            'M': list(range(17, 33)),
            'H': list(range(33, 50))
        }
        
        pattern_counts = Counter(target_pattern)
        selected = []
        
        for char, count in pattern_counts.items():
            if char in ranges:
                available = [n for n in ranges[char] if n not in selected]
                if len(available) >= count:
                    selected.extend(random.sample(available, count))
        
        while len(selected) < 6:
            available = [n for n in range(1, 50) if n not in selected]
            if available:
                selected.append(random.choice(available))
        
        return sorted(selected[:6])
    
    def get_number_pattern_score(self, number):
        """Pattern-Score für Zahl."""
        if number not in self.number_patterns:
            return 0.1
        
        frequency = len(self.number_patterns[number])
        max_freq = max(len(patterns) for patterns in self.number_patterns.values()) if self.number_patterns else 1
        
        return min(frequency / max_freq, 1.0) if max_freq > 0 else 0.1
    
    def update_with_drawing(self, drawing):
        """Update mit Ziehung."""
        numbers = sorted([drawing['Z1'], drawing['Z2'], drawing['Z3'],
                         drawing['Z4'], drawing['Z5'], drawing['Z6']])
        pattern = self._get_pattern(numbers)
        self.pattern_frequencies[pattern] += 1
        
        total = sum(self.pattern_frequencies.values())
        for p, count in self.pattern_frequencies.items():
            self.pattern_weights[p] = count / total


class HybridOptimizer:
    """Hybrid Optimizer - OPTIMIZED."""
    
    def __init__(self, fast_mode=True):
        self.ai_engine = None
        self.pattern_engine = None
        self.fast_mode = fast_mode
        self.iterations = 50 if fast_mode else 150
    
    def initialize(self, ai_engine, pattern_engine):
        """Initialisiert Optimizer."""
        self.ai_engine = ai_engine
        self.pattern_engine = pattern_engine
    
    def optimize(self, seed, ai_predictions):
        """Multi-Objective Optimization - FASTER."""
        random.seed(42 + seed * 29)
        
        best_score = -1
        best_combination = None
        best_ai_score = 0
        best_pattern_weight = 0
        
        for attempt in range(self.iterations):
            candidate = self._generate_candidate(ai_predictions, attempt, seed)
            
            ai_score = np.mean([ai_predictions.get(n, 0.1) for n in candidate])
            pattern_weight = self.pattern_engine.calculate_pattern_weight(candidate)
            diversity = self._calculate_diversity(candidate)
            
            combined_score = (
                ai_score * 0.45 +
                pattern_weight * 0.35 +
                diversity * 0.20
            )
            
            if combined_score > best_score:
                best_score = combined_score
                best_combination = candidate
                best_ai_score = ai_score
                best_pattern_weight = pattern_weight
        
        return {
            'numbers': best_combination if best_combination else sorted(random.sample(range(1, 50), 6)),
            'ai_score': best_ai_score,
            'pattern_weight': best_pattern_weight,
            'confidence': best_score
        }
    
    def _generate_candidate(self, ai_predictions, attempt, seed):
        """Generiert Kandidaten."""
        if attempt < self.iterations // 3:
            sorted_preds = sorted(ai_predictions.items(), key=lambda x: x[1], reverse=True)
            candidates = [num for num, _ in sorted_preds[:25]]
            return sorted(random.sample(candidates, min(6, len(candidates))))
        
        elif attempt < 2 * self.iterations // 3:
            top_patterns = self.pattern_engine.get_top_patterns(5)
            if top_patterns:
                pattern = random.choice(top_patterns)
                return self.pattern_engine.generate_for_pattern(pattern, seed + attempt)
        
        sorted_preds = sorted(ai_predictions.items(), key=lambda x: x[1], reverse=True)
        ai_nums = [num for num, _ in sorted_preds[:15]]
        
        selected = random.sample(ai_nums, 3)
        remaining = [n for n in range(1, 50) if n not in selected]
        selected.extend(random.sample(remaining, 3))
        
        return sorted(selected)
    
    def _calculate_diversity(self, numbers):
        """Berechnet Diversität."""
        if len(numbers) < 2:
            return 0.5
        
        distances = []
        for i, n1 in enumerate(numbers):
            for n2 in numbers[i+1:]:
                distances.append(abs(n1 - n2))
        
        avg_distance = np.mean(distances)
        return min(avg_distance / 10.0, 1.0)


class FeatureEngineer:
    """Feature Engineering - ERWEITERT V2.2."""

    def create_features(self, df):
        """Erstellt Features."""
        features_list = []

        # Calculate number-specific features once
        self.number_gaps = self._calculate_all_gaps(df)
        self.number_frequencies = self._calculate_all_frequencies(df)

        for i in range(len(df)):
            features = self._extract_features(df, i)
            features_list.append(features)

        return pd.DataFrame(features_list)

    def _calculate_all_gaps(self, df):
        """Berechnet Gap-Statistiken für alle Zahlen."""
        gaps = {num: [] for num in range(1, 50)}
        last_seen = {num: -1 for num in range(1, 50)}

        for idx, row in df.iterrows():
            numbers = [row['Z1'], row['Z2'], row['Z3'], row['Z4'], row['Z5'], row['Z6']]
            for num in range(1, 50):
                if num in numbers:
                    if last_seen[num] >= 0:
                        gaps[num].append(idx - last_seen[num])
                    last_seen[num] = idx

        return gaps

    def _calculate_all_frequencies(self, df):
        """Berechnet Häufigkeiten für alle Zahlen."""
        freq = {num: 0 for num in range(1, 50)}

        for _, row in df.iterrows():
            numbers = [row['Z1'], row['Z2'], row['Z3'], row['Z4'], row['Z5'], row['Z6']]
            for num in numbers:
                freq[num] += 1

        return freq

    def _extract_features(self, df, idx):
        """Extrahiert Features - ERWEITERT."""
        features = {}

        # 1. BASIC FREQUENCY FEATURES (original)
        for window in [5, 10]:
            start = max(0, idx - window)
            hist = df.iloc[start:idx]

            if len(hist) > 0:
                all_nums = []
                for _, row in hist.iterrows():
                    all_nums.extend([row['Z1'], row['Z2'], row['Z3'],
                                   row['Z4'], row['Z5'], row['Z6']])
                features[f'freq_{window}'] = len(set(all_nums)) / (window * 6) if window > 0 else 0
            else:
                features[f'freq_{window}'] = 0

        # 2. GAP ANALYSIS FEATURES (neu)
        for window in [20, 50]:
            start = max(0, idx - window)
            hist = df.iloc[start:idx]

            if len(hist) > 0:
                gap_features = self._extract_gap_features(hist)
                features[f'avg_gap_{window}'] = gap_features['avg_gap']
                features[f'gap_variance_{window}'] = gap_features['gap_variance']
            else:
                features[f'avg_gap_{window}'] = 0
                features[f'gap_variance_{window}'] = 0

        # 3. MOMENTUM FEATURES (neu)
        features.update(self._extract_momentum_features(df, idx))

        # 4. RELATIONSHIP FEATURES (neu)
        if idx > 0:
            prev_row = df.iloc[idx-1]
            prev_nums = [prev_row['Z1'], prev_row['Z2'], prev_row['Z3'],
                        prev_row['Z4'], prev_row['Z5'], prev_row['Z6']]

            # Sum tendency - normalized from min (6) to max (264)
            prev_sum_raw = sum(prev_nums)
            features['prev_sum'] = (prev_sum_raw - 6) / (264 - 6)  # Normalize to [0,1]
            features['prev_sum_norm'] = prev_sum_raw / (49 * 6)  # Alternative normalization

            # Parity balance
            even_count = sum(1 for n in prev_nums if n % 2 == 0)
            features['prev_parity'] = even_count / 6

            # Decade distribution
            decades = [0] * 5  # 1-10, 11-20, 21-30, 31-40, 41-49
            for num in prev_nums:
                decade_idx = min((num - 1) // 10, 4)
                decades[decade_idx] += 1
            features['prev_decade_spread'] = len([d for d in decades if d > 0]) / 5
        else:
            features['prev_sum'] = 0
            features['prev_sum_norm'] = 0
            features['prev_parity'] = 0.5
            features['prev_decade_spread'] = 0

        # 5. TEMPORAL FEATURES (original + erweitert)
        if 'datum' in df.columns and pd.notna(df.iloc[idx].get('datum')):
            date = df.iloc[idx]['datum']
            features['day_of_week'] = date.dayofweek / 6  # Normalize 0-6 to [0,1]
            # Normalize sin/cos from [-1,1] to [0,1]
            features['seasonal'] = (np.sin(2 * np.pi * date.dayofyear / 365) + 1) / 2

            # Cycle features (neu) - normalized
            features['month'] = (date.month - 1) / 11  # Normalize 1-12 to [0,1]
            features['week_of_year'] = (date.isocalendar()[1] - 1) / 52  # Normalize 1-53 to [0,1]
            features['seasonal_cos'] = (np.cos(2 * np.pi * date.dayofyear / 365) + 1) / 2  # [-1,1] to [0,1]
        else:
            features['day_of_week'] = 0
            features['seasonal'] = 0.5
            features['month'] = 0
            features['week_of_year'] = 0
            features['seasonal_cos'] = 0.5

        # 6. NEIGHBORHOOD FEATURES (neu)
        if idx > 0:
            prev_row = df.iloc[idx-1]
            prev_nums = set([prev_row['Z1'], prev_row['Z2'], prev_row['Z3'],
                           prev_row['Z4'], prev_row['Z5'], prev_row['Z6']])

            # Count numbers that are ±1 or ±2 from previous
            neighbor_1 = sum(1 for n in prev_nums if any(abs(n-m) == 1 for m in prev_nums if m != n))
            neighbor_2 = sum(1 for n in prev_nums if any(abs(n-m) == 2 for m in prev_nums if m != n))

            features['prev_neighbor_1'] = neighbor_1 / 6
            features['prev_neighbor_2'] = neighbor_2 / 6
        else:
            features['prev_neighbor_1'] = 0
            features['prev_neighbor_2'] = 0

        return features

    def _extract_gap_features(self, hist):
        """Extrahiert Gap-Features aus Historie."""
        if len(hist) == 0:
            return {'avg_gap': 0, 'gap_variance': 0}

        # Track when each number was last seen
        last_seen = {num: -1 for num in range(1, 50)}
        gaps = {num: [] for num in range(1, 50)}

        for local_idx, (_, row) in enumerate(hist.iterrows()):
            numbers = [row['Z1'], row['Z2'], row['Z3'], row['Z4'], row['Z5'], row['Z6']]

            for num in range(1, 50):
                if num in numbers:
                    if last_seen[num] >= 0:
                        gaps[num].append(local_idx - last_seen[num])
                    last_seen[num] = local_idx

        # Calculate average gap across all numbers
        all_gaps = [g for gap_list in gaps.values() for g in gap_list if gap_list]

        if all_gaps:
            avg_gap = np.mean(all_gaps)
            gap_variance = np.var(all_gaps) if len(all_gaps) > 1 else 0
        else:
            avg_gap = 0
            gap_variance = 0

        return {
            'avg_gap': min(avg_gap / 10.0, 1.0),  # Normalized
            'gap_variance': min(gap_variance / 50.0, 1.0)  # Normalized
        }

    def _extract_momentum_features(self, df, idx):
        """Extrahiert Momentum-Features."""
        features = {}

        if idx < 10:
            # Not enough data for momentum
            features['trend_5'] = 0
            features['trend_10'] = 0
            features['acceleration'] = 0
            return features

        # Count unique numbers in windows
        recent_5 = set()
        recent_10 = set()
        old_10 = set()

        for i in range(max(0, idx-5), idx):
            row = df.iloc[i]
            nums = [row['Z1'], row['Z2'], row['Z3'], row['Z4'], row['Z5'], row['Z6']]
            recent_5.update(nums)

        for i in range(max(0, idx-10), idx):
            row = df.iloc[i]
            nums = [row['Z1'], row['Z2'], row['Z3'], row['Z4'], row['Z5'], row['Z6']]
            recent_10.update(nums)

        for i in range(max(0, idx-20), max(0, idx-10)):
            row = df.iloc[i]
            nums = [row['Z1'], row['Z2'], row['Z3'], row['Z4'], row['Z5'], row['Z6']]
            old_10.update(nums)

        # Trend: Increasing or decreasing diversity
        div_5 = len(recent_5) / 30.0  # Max 30 numbers in 5 draws
        div_10 = len(recent_10) / 49.0  # Normalized
        div_old = len(old_10) / 49.0 if len(old_10) > 0 else div_10

        features['trend_5'] = div_5
        features['trend_10'] = div_10
        # Normalize acceleration from [-1, 1] to [0, 1]
        accel_raw = max(min((div_10 - div_old), 1.0), -1.0)
        features['acceleration'] = (accel_raw + 1.0) / 2.0  # Map [-1,1] to [0,1]

        return features


class RealTimeLearner:
    """Real-Time Learning mit Persistenz - ERWEITERT V2.2."""

    def __init__(self, cache_path=None):
        self.learning_rate = 0.1
        self.adjustments = {}
        self.stats = defaultdict(int)
        self.cache_path = cache_path
        self.learning_state_file = os.path.join(cache_path, 'learning_state.json') if cache_path else None
        self.freq_prior = {}
        self.freq_alpha = 0.2

        # Lade gespeicherten State
        if self.learning_state_file and os.path.exists(self.learning_state_file):
            self._load_learning_state()

    def initialize(self, features_df):
        """Initialisiert Learning."""
        self.features_df = features_df

    def set_frequency_prior(self, freq):
        """Setzt Langzeit-Frequenz als Dämpfer (normalisiert auf [0,1])."""
        max_val = max(freq.values(), default=0)
        norm = max_val if max_val > 0 else 1
        self.freq_prior = {n: v / norm for n, v in freq.items()}

    def adjust_predictions(self, predictions):
        """Passt Predictions an, gedämpft durch Langzeit-Frequenz."""
        adjusted = {}
        for number, pred in predictions.items():
            adjustment = self.adjustments.get(number, 0)
            learner_val = np.clip(pred + adjustment * self.learning_rate, 0, 1)
            if self.freq_prior:
                alpha = max(0.0, min(1.0, self.freq_alpha))
                freq_val = self.freq_prior.get(number, 0.5)
                adjusted[number] = np.clip((1 - alpha) * learner_val + alpha * freq_val, 0, 1)
            else:
                adjusted[number] = learner_val
        return adjusted

    def learn_from_result(self, drawing):
        """Lernt aus Ziehung."""
        actual = [drawing[f'Z{i}'] for i in range(1, 7)]

        for number in range(1, 50):
            if number not in self.adjustments:
                self.adjustments[number] = 0

            if number in actual:
                self.adjustments[number] += 0.01
                self.stats['correct'] += 1
            else:
                self.adjustments[number] -= 0.005
                self.stats['incorrect'] += 1

            # Langfristiger Decay (wöchentlich statt pro Zyklus)
            self.adjustments[number] *= 0.995  # Langsamer Decay

        self.stats['cycles'] += 1
        self.stats['last_update'] = datetime.datetime.now().isoformat()

        # Speichere Learning State
        self._save_learning_state()

    def _save_learning_state(self):
        """Speichert Learning State persistent."""
        if not self.learning_state_file:
            return

        try:
            # Erstelle Verzeichnis falls nötig
            os.makedirs(os.path.dirname(self.learning_state_file), exist_ok=True)

            state = {
                'adjustments': {str(k): float(v) for k, v in self.adjustments.items()},
                'stats': dict(self.stats),
                'learning_rate': self.learning_rate,
                'last_saved': datetime.datetime.now().isoformat()
            }

            with open(self.learning_state_file, 'w') as f:
                json.dump(state, f, indent=2)

        except Exception as e:
            print(f"⚠️  Warning: Konnte Learning State nicht speichern: {e}")

    def _load_learning_state(self):
        """Lädt gespeicherten Learning State."""
        try:
            with open(self.learning_state_file, 'r') as f:
                state = json.load(f)

            self.adjustments = {int(k): float(v) for k, v in state.get('adjustments', {}).items()}
            self.stats = defaultdict(int, state.get('stats', {}))
            self.learning_rate = state.get('learning_rate', 0.1)

            last_saved = state.get('last_saved', 'Unbekannt')
            cycles = self.stats.get('cycles', 0)
            print(f"📚 Learning State geladen: {cycles} Zyklen (Stand: {last_saved[:10]})")

        except Exception as e:
            print(f"⚠️  Warning: Konnte Learning State nicht laden: {e}")
            self.adjustments = {}
            self.stats = defaultdict(int)

    def get_learning_summary(self):
        """Gibt Learning-Zusammenfassung zurück."""
        if not self.adjustments:
            return "Noch kein Learning durchgeführt"

        # Top Adjustments
        top_positive = sorted(self.adjustments.items(), key=lambda x: x[1], reverse=True)[:5]
        top_negative = sorted(self.adjustments.items(), key=lambda x: x[1])[:5]

        avg_adjustment = np.mean(list(self.adjustments.values()))

        summary = {
            'total_cycles': self.stats.get('cycles', 0),
            'correct_predictions': self.stats.get('correct', 0),
            'incorrect_predictions': self.stats.get('incorrect', 0),
            'avg_adjustment': avg_adjustment,
            'top_positive': top_positive,
            'top_negative': top_negative,
            'last_update': self.stats.get('last_update', 'Nie')
        }

        return summary
    
    def get_stats(self):
        """Stats."""
        return dict(self.stats)


class PerformanceTracker:
    """Performance Tracking."""
    
    def __init__(self):
        self.generated_tips = []
        self.evaluations = []
    
    def log_generated_tips(self, tips):
        """Loggt Tips."""
        self.generated_tips.extend(tips)
    
    def evaluate_predictions(self, drawing):
        """Evaluiert."""
        actual = [drawing[f'Z{i}'] for i in range(1, 7)]
        
        if self.generated_tips:
            recent = self.generated_tips[-10:]
            
            for tip in recent:
                matches = len(set(tip['numbers']) & set(actual))
                self.evaluations.append({
                    'matches': matches,
                    'strategy': tip['strategy'],
                    'timestamp': datetime.datetime.now()
                })
    
    def get_statistics(self):
        """Stats."""
        return {
            'total_tips': len(self.generated_tips),
            'total_evaluations': len(self.evaluations)
        }


# ============================================================================
# MAIN
# ============================================================================

def main():
    """Main Function - OPTIMIZED."""
    print("🚀 ULTIMATE AI-ML HYBRID LOTTO GENERATOR V2.1")
    print("⚡ OPTIMIZED VERSION with Progress Indicators")
    print("=" * 70)
    
    data_path = "/Users/sebastianfrohlich/Library/Mobile Documents/com~apple~CloudDocs/Jupyter Notebooks/AlleLottozahlen.csv"
    
    try:
        # Initialize with fast mode
        generator = UltimateAIMLHybridGenerator(data_path, fast_mode=True)
        
        # Generate Tips
        tips = generator.generate_ultimate_tips(10)
        
        if tips:
            print("\n🏆 GENERATION COMPLETED!")
            print("=" * 70)
            print(f"✅ {len(tips)} Ultimate Tips generated")
            print(f"🧠 AI/ML: {'✅' if ML_AVAILABLE else '⚠️'}")
            print(f"🎨 Patterns: ✅")
            print(f"⚡ Optimization: ✅")
            print(f"📚 Learning: ✅")
            
            print("\n💡 ADVANTAGES:")
            print("   🔬 4 Strategien: Hybrid-OPT, Balanced-Spread, High-EV, Soft-Contrarian")
            print("   🧠 AI/ML Ensemble: RandomForest + GradientBoosting")
            print("   🎨 Pattern Analysis: Historische Verteilungen")
            print("   ⚡ Multi-Objective: AI + Pattern + Diversity")
            print("   📚 Real-Time Learning: Kontinuierliche Anpassung")
            print("   ⚡ Fast Mode: Optimierte Performance")
    
    except Exception as e:
        print(f"❌ Error: {e}")
        import traceback
        traceback.print_exc()


if __name__ == "__main__":
    random.seed(42)
    np.random.seed(42)
    main()
