#!/usr/bin/env python3
"""
Deep Learning Engine mit LSTM für Lotto-Vorhersagen
====================================================

Implementiert LSTM-basierte Modelle zur Vorhersage von Lotto-Zahlen
basierend auf historischen Sequenzen und Features.

Features:
- LSTM-Netzwerk für zeitliche Sequenzen
- Sequence-to-Probability Mapping
- Feature Engineering Integration
- Model Persistence & Caching
- Hybrid mit RandomForest
"""

import numpy as np
import pandas as pd
import os
import json
import pickle
from datetime import datetime
from typing import Dict, List, Tuple, Optional
import warnings
warnings.filterwarnings('ignore')

# TensorFlow/Keras Imports
try:
    import tensorflow as tf
    from tensorflow import keras
    from tensorflow.keras.models import Sequential, load_model
    from tensorflow.keras.layers import LSTM, Dense, Dropout, Bidirectional, BatchNormalization
    from tensorflow.keras.optimizers import Adam
    from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
    from tensorflow.keras.regularizers import l2
    TENSORFLOW_AVAILABLE = True
except ImportError:
    TENSORFLOW_AVAILABLE = False
    print("⚠️  TensorFlow not available. Install with: pip install tensorflow")


class DeepLearningEngine:
    """
    LSTM-basierter Deep Learning Engine für Lotto-Vorhersagen.

    Architecture:
    - Input: Sequence of historical draws + features
    - LSTM layers: Learn temporal patterns
    - Dense layers: Map to probability distribution
    - Output: Probability for each number
    """

    def __init__(
        self,
        num_numbers: int = 49,
        sequence_length: int = 20,
        cache_dir: str = None,
        fast_mode: bool = True
    ):
        """
        Args:
            num_numbers: Maximum number (49 for Lotto, 50 for Eurojackpot)
            sequence_length: How many past draws to consider
            cache_dir: Directory for model persistence
            fast_mode: Use faster training (fewer epochs)
        """
        self.num_numbers = num_numbers
        self.sequence_length = sequence_length
        self.cache_dir = cache_dir
        self.fast_mode = fast_mode

        # Model components
        self.model = None
        self.is_trained = False
        self.training_history = {}

        # Configuration
        self.config = {
            'lstm_units_1': 128,
            'lstm_units_2': 64,
            'dense_units': 128,
            'dropout_rate': 0.3,
            'learning_rate': 0.001,
            'batch_size': 32,
            'epochs': 30 if fast_mode else 100,
            'validation_split': 0.2
        }

        if cache_dir:
            os.makedirs(cache_dir, exist_ok=True)
            self.model_path = os.path.join(cache_dir, f'lstm_model_{num_numbers}.h5')
            self.config_path = os.path.join(cache_dir, f'lstm_config_{num_numbers}.json')
        else:
            self.model_path = None
            self.config_path = None

        print(f"🧠 Deep Learning Engine initialized")
        print(f"   Numbers: 1-{num_numbers}")
        print(f"   Sequence Length: {sequence_length}")
        print(f"   Fast Mode: {fast_mode}")

    def _build_model(self, num_features: int) -> Sequential:
        """
        Builds LSTM architecture.

        Architecture:
        Input (sequence_length, num_features)
          ↓
        Bidirectional LSTM(128) + Dropout(0.3)
          ↓
        Bidirectional LSTM(64) + Dropout(0.3)
          ↓
        Dense(128, relu) + BatchNorm + Dropout(0.3)
          ↓
        Dense(num_numbers, sigmoid)
        """
        model = Sequential([
            # First Bidirectional LSTM layer
            Bidirectional(
                LSTM(
                    self.config['lstm_units_1'],
                    return_sequences=True,
                    kernel_regularizer=l2(0.01)
                ),
                input_shape=(self.sequence_length, num_features)
            ),
            Dropout(self.config['dropout_rate']),
            BatchNormalization(),

            # Second Bidirectional LSTM layer
            Bidirectional(
                LSTM(
                    self.config['lstm_units_2'],
                    return_sequences=False,
                    kernel_regularizer=l2(0.01)
                )
            ),
            Dropout(self.config['dropout_rate']),
            BatchNormalization(),

            # Dense layers
            Dense(
                self.config['dense_units'],
                activation='relu',
                kernel_regularizer=l2(0.01)
            ),
            BatchNormalization(),
            Dropout(self.config['dropout_rate']),

            # Output layer - probability for each number
            Dense(self.num_numbers, activation='sigmoid')
        ])

        # Compile
        model.compile(
            optimizer=Adam(learning_rate=self.config['learning_rate']),
            loss='binary_crossentropy',
            metrics=['accuracy', 'AUC']
        )

        return model

    def _prepare_sequences(
        self,
        df: pd.DataFrame,
        features_df: pd.DataFrame
    ) -> Tuple[np.ndarray, np.ndarray]:
        """
        Prepares sequences for LSTM training.

        Args:
            df: Historical draws (with columns Z1-Z6)
            features_df: Engineered features

        Returns:
            X: (num_samples, sequence_length, num_features)
            y: (num_samples, num_numbers) - binary matrix
        """
        print(f"   Preparing sequences (length={self.sequence_length})...")

        # Ensure data is sorted by date
        if 'datum' in df.columns:
            df = df.sort_values('datum').reset_index(drop=True)

        # Extract number columns
        num_cols = [col for col in df.columns if col.startswith('Z')]

        # Normalize features to [0, 1]
        features_normalized = features_df.copy()
        for col in features_normalized.columns:
            min_val = features_normalized[col].min()
            max_val = features_normalized[col].max()
            if max_val > min_val:
                features_normalized[col] = (features_normalized[col] - min_val) / (max_val - min_val)
            else:
                features_normalized[col] = 0.5

        X_sequences = []
        y_targets = []

        # Create sequences
        for i in range(self.sequence_length, len(df)):
            # Get sequence of features
            sequence = features_normalized.iloc[i - self.sequence_length:i].values
            X_sequences.append(sequence)

            # Target: next draw as binary vector
            target = np.zeros(self.num_numbers)
            next_draw = df.iloc[i][num_cols].values
            for num in next_draw:
                if 1 <= num <= self.num_numbers:
                    target[int(num) - 1] = 1
            y_targets.append(target)

        X = np.array(X_sequences)
        y = np.array(y_targets)

        print(f"   ✅ Created {len(X)} sequences")
        print(f"   Shape: X={X.shape}, y={y.shape}")

        return X, y

    def train(
        self,
        df: pd.DataFrame,
        features_df: pd.DataFrame,
        force_retrain: bool = False
    ) -> bool:
        """
        Trains LSTM model on historical data.

        Args:
            df: Historical draws
            features_df: Engineered features
            force_retrain: Retrain even if cached model exists

        Returns:
            Success status
        """
        if not TENSORFLOW_AVAILABLE:
            print("❌ TensorFlow not available")
            return False

        # Check for cached model
        if not force_retrain and self.model_path and os.path.exists(self.model_path):
            print("   📦 Loading cached LSTM model...")
            try:
                self.model = load_model(self.model_path)
                self.is_trained = True

                # Load config
                if os.path.exists(self.config_path):
                    with open(self.config_path, 'r') as f:
                        self.training_history = json.load(f)

                print(f"   ✅ Loaded cached model")
                return True
            except Exception as e:
                print(f"   ⚠️  Failed to load cached model: {e}")
                print("   🔄 Training new model...")

        print(f"\n🧠 TRAINING DEEP LEARNING MODEL (LSTM)")
        print("=" * 70)

        # Prepare data
        X, y = self._prepare_sequences(df, features_df)

        if len(X) < 100:
            print("   ⚠️  Not enough data for training (need >100 sequences)")
            return False

        # Build model
        print(f"   Building LSTM architecture...")
        num_features = X.shape[2]
        self.model = self._build_model(num_features)

        # Show summary
        print(f"\n   📊 Model Summary:")
        total_params = self.model.count_params()
        print(f"   Total parameters: {total_params:,}")

        # Callbacks
        callbacks = [
            EarlyStopping(
                monitor='val_loss',
                patience=10,
                restore_best_weights=True,
                verbose=0
            ),
            ReduceLROnPlateau(
                monitor='val_loss',
                factor=0.5,
                patience=5,
                verbose=0
            )
        ]

        # Train
        print(f"\n   🚀 Training for {self.config['epochs']} epochs...")
        print(f"   Batch size: {self.config['batch_size']}")
        print(f"   Validation split: {self.config['validation_split']:.1%}")

        try:
            history = self.model.fit(
                X, y,
                batch_size=self.config['batch_size'],
                epochs=self.config['epochs'],
                validation_split=self.config['validation_split'],
                callbacks=callbacks,
                verbose=1
            )

            # Store training history
            self.training_history = {
                'trained_at': datetime.now().isoformat(),
                'num_samples': len(X),
                'num_features': num_features,
                'final_loss': float(history.history['loss'][-1]),
                'final_val_loss': float(history.history['val_loss'][-1]),
                'final_accuracy': float(history.history['accuracy'][-1]),
                'final_val_accuracy': float(history.history['val_accuracy'][-1]),
                'epochs_trained': len(history.history['loss'])
            }

            self.is_trained = True

            # Save model
            if self.model_path:
                print(f"\n   💾 Saving model to cache...")
                self.model.save(self.model_path)

                with open(self.config_path, 'w') as f:
                    json.dump(self.training_history, f, indent=2)

                print(f"   ✅ Model saved")

            # Print results
            print(f"\n   ✅ TRAINING COMPLETED")
            print(f"   Final Loss: {self.training_history['final_loss']:.4f}")
            print(f"   Final Val Loss: {self.training_history['final_val_loss']:.4f}")
            print(f"   Final Accuracy: {self.training_history['final_accuracy']:.4f}")
            print(f"   Epochs: {self.training_history['epochs_trained']}")

            return True

        except Exception as e:
            print(f"\n   ❌ Training failed: {e}")
            import traceback
            traceback.print_exc()
            return False

    def predict(
        self,
        recent_df: pd.DataFrame,
        recent_features: pd.DataFrame
    ) -> Dict[int, float]:
        """
        Predicts probabilities for each number.

        Args:
            recent_df: Recent draws (at least sequence_length)
            recent_features: Recent features

        Returns:
            {number: probability} for numbers 1-num_numbers
        """
        if not self.is_trained or self.model is None:
            print("⚠️  Model not trained, returning uniform distribution")
            return {i: 0.5 for i in range(1, self.num_numbers + 1)}

        # Prepare last sequence
        if len(recent_df) < self.sequence_length:
            print(f"⚠️  Not enough recent data (need {self.sequence_length}, got {len(recent_df)})")
            return {i: 0.5 for i in range(1, self.num_numbers + 1)}

        # Get last sequence
        recent_features_normalized = recent_features.copy()
        for col in recent_features_normalized.columns:
            min_val = recent_features_normalized[col].min()
            max_val = recent_features_normalized[col].max()
            if max_val > min_val:
                recent_features_normalized[col] = (recent_features_normalized[col] - min_val) / (max_val - min_val)
            else:
                recent_features_normalized[col] = 0.5

        sequence = recent_features_normalized.iloc[-self.sequence_length:].values
        X = np.array([sequence])  # Shape: (1, sequence_length, num_features)

        # Predict
        predictions = self.model.predict(X, verbose=0)[0]  # Shape: (num_numbers,)

        # Convert to dictionary
        result = {i + 1: float(predictions[i]) for i in range(self.num_numbers)}

        return result

    def get_model_info(self) -> Dict:
        """Returns model information."""
        return {
            'is_trained': self.is_trained,
            'tensorflow_available': TENSORFLOW_AVAILABLE,
            'num_numbers': self.num_numbers,
            'sequence_length': self.sequence_length,
            'config': self.config,
            'training_history': self.training_history,
            'model_exists': self.model is not None
        }


class HybridDeepLearningPredictor:
    """
    Kombiniert RandomForest + LSTM für robustere Vorhersagen.
    """

    def __init__(
        self,
        dl_engine: DeepLearningEngine,
        rf_weight: float = 0.4,
        dl_weight: float = 0.6
    ):
        """
        Args:
            dl_engine: Deep Learning Engine
            rf_weight: Weight for RandomForest predictions
            dl_weight: Weight for Deep Learning predictions
        """
        self.dl_engine = dl_engine
        self.rf_weight = rf_weight
        self.dl_weight = dl_weight

        print(f"🔀 Hybrid Predictor: RF={rf_weight:.1%} + DL={dl_weight:.1%}")

    def predict(
        self,
        rf_predictions: Dict[int, float],
        recent_df: pd.DataFrame,
        recent_features: pd.DataFrame
    ) -> Dict[int, float]:
        """
        Combines RandomForest and Deep Learning predictions.

        Args:
            rf_predictions: Predictions from RandomForest
            recent_df: Recent draws for DL
            recent_features: Recent features for DL

        Returns:
            Combined predictions
        """
        # Get DL predictions
        dl_predictions = self.dl_engine.predict(recent_df, recent_features)

        # Combine
        combined = {}
        for num in range(1, self.dl_engine.num_numbers + 1):
            rf_score = rf_predictions.get(num, 0.5)
            dl_score = dl_predictions.get(num, 0.5)

            combined[num] = (
                self.rf_weight * rf_score +
                self.dl_weight * dl_score
            )

        # Normalize to [0, 1]
        min_score = min(combined.values())
        max_score = max(combined.values())
        if max_score > min_score:
            combined = {
                num: (score - min_score) / (max_score - min_score)
                for num, score in combined.items()
            }

        return combined