Place-Order-Trading-Bot/mt5_pnl_tracker.py

#!/usr/bin/env python3
"""
📊 MT5 P&L Tracker with Automatic History Import
Automatically imports MT5 trading history and tracks real P&L performance
"""

import MetaTrader5 as mt5
import sqlite3
import pandas as pd
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
import json
from pathlib import Path


class MT5PnLTracker:
    """
    Automatic MT5 History Import and P&L Tracking

    Features:
    - Automatic deal import from MT5 history
    - Position matching (Entry + Exit deals)
    - Real P&L calculation from closed trades
    - Win Rate, Profit Factor, Max Drawdown
    - Session/Confidence/Time analysis
    - Daily/Weekly/Monthly reports
    """

    def __init__(self, db_path: str = "trading_bot.db", magic_number: int = None):
        """
        Initialize MT5 P&L Tracker

        Args:
            db_path: Path to SQLite database
            magic_number: EA magic number (None = all trades)
        """
        self.db_path = db_path
        self.magic_number = magic_number
        self.conn = None

    def connect_db(self):
        """Connect to database"""
        self.conn = sqlite3.connect(self.db_path)
        self.conn.row_factory = sqlite3.Row
        self._ensure_tables()

    def _ensure_tables(self):
        """Ensure required tables exist"""
        cursor = self.conn.cursor()

        # MT5 Deals table
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS mt5_deals (
                deal_id INTEGER PRIMARY KEY,
                ticket INTEGER,
                order_ticket INTEGER,
                time TEXT,
                time_msc INTEGER,
                type INTEGER,
                entry INTEGER,
                magic INTEGER,
                position_id INTEGER,
                reason INTEGER,
                volume REAL,
                price REAL,
                commission REAL,
                swap REAL,
                profit REAL,
                fee REAL,
                symbol TEXT,
                comment TEXT,
                external_id TEXT,
                imported_at TEXT,
                UNIQUE(deal_id)
            )
        """)

        # Matched Positions table (Entry + Exit pairs)
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS matched_positions (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                position_id INTEGER UNIQUE,
                symbol TEXT,
                entry_deal_id INTEGER,
                exit_deal_id INTEGER,
                type TEXT,
                volume REAL,
                entry_price REAL,
                exit_price REAL,
                entry_time TEXT,
                exit_time TEXT,
                duration_hours REAL,
                profit REAL,
                commission REAL,
                swap REAL,
                net_profit REAL,
                pips REAL,
                is_win BOOLEAN,
                magic INTEGER,
                matched_at TEXT
            )
        """)

        # P&L Summary table
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS pnl_summary (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                period_type TEXT,
                period_start TEXT,
                period_end TEXT,
                total_trades INTEGER,
                winning_trades INTEGER,
                losing_trades INTEGER,
                win_rate REAL,
                total_profit REAL,
                total_loss REAL,
                net_profit REAL,
                profit_factor REAL,
                avg_win REAL,
                avg_loss REAL,
                max_drawdown REAL,
                largest_win REAL,
                largest_loss REAL,
                calculated_at TEXT
            )
        """)

        self.conn.commit()

    def import_mt5_history(self, days_back: int = 30) -> Dict:
        """
        Import trading history from MT5

        Args:
            days_back: Number of days to import

        Returns:
            Dict with import statistics
        """
        if not mt5.initialize():
            return {
                'success': False,
                'error': 'MT5 initialization failed',
                'new_deals': 0
            }

        try:
            # Get deals from MT5
            from_date = datetime.now() - timedelta(days=days_back)
            to_date = datetime.now()

            deals = mt5.history_deals_get(from_date, to_date)

            if deals is None or len(deals) == 0:
                return {
                    'success': True,
                    'message': 'No deals found',
                    'new_deals': 0,
                    'total_deals': 0
                }

            # Filter by magic number if specified
            if self.magic_number is not None:
                deals = [d for d in deals if d.magic == self.magic_number]

            # Import deals to database
            new_deals = 0
            cursor = self.conn.cursor()

            for deal in deals:
                try:
                    cursor.execute("""
                        INSERT OR IGNORE INTO mt5_deals (
                            deal_id, ticket, order_ticket, time, time_msc, type, entry,
                            magic, position_id, reason, volume, price, commission,
                            swap, profit, fee, symbol, comment, external_id, imported_at
                        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """, (
                        deal.ticket,  # deal_id
                        deal.ticket,
                        deal.order,
                        datetime.fromtimestamp(deal.time).strftime('%Y-%m-%d %H:%M:%S'),
                        deal.time_msc,
                        deal.type,
                        deal.entry,
                        deal.magic,
                        deal.position_id,
                        deal.reason,
                        deal.volume,
                        deal.price,
                        deal.commission,
                        deal.swap,
                        deal.profit,
                        deal.fee,
                        deal.symbol,
                        deal.comment,
                        deal.external_id,
                        datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                    ))

                    if cursor.rowcount > 0:
                        new_deals += 1

                except sqlite3.IntegrityError:
                    # Deal already exists
                    continue

            self.conn.commit()

            return {
                'success': True,
                'new_deals': new_deals,
                'total_deals': len(deals),
                'period': f'{from_date.strftime("%Y-%m-%d")} to {to_date.strftime("%Y-%m-%d")}'
            }

        except Exception as e:
            return {
                'success': False,
                'error': str(e),
                'new_deals': 0
            }
        finally:
            mt5.shutdown()

    def match_positions(self) -> Dict:
        """
        Match Entry and Exit deals to create complete positions

        Returns:
            Dict with matching statistics
        """
        cursor = self.conn.cursor()

        # Get all deals ordered by position_id and time
        cursor.execute("""
            SELECT * FROM mt5_deals
            WHERE position_id > 0
            ORDER BY position_id, time
        """)

        deals = cursor.fetchall()

        if not deals:
            return {
                'success': True,
                'message': 'No deals to match',
                'matched': 0
            }

        # Group by position_id
        positions = {}
        for deal in deals:
            pos_id = deal['position_id']
            if pos_id not in positions:
                positions[pos_id] = []
            positions[pos_id].append(dict(deal))

        # Match positions
        matched = 0

        for pos_id, pos_deals in positions.items():
            if len(pos_deals) < 2:
                # Incomplete position (still open or only one deal)
                continue

            # Find entry and exit deals
            entry_deal = None
            exit_deal = None

            for deal in pos_deals:
                # Entry: type 0 (buy) or 1 (sell), entry flag 0 (in)
                if deal['entry'] == 0:  # IN
                    entry_deal = deal
                # Exit: entry flag 1 (out)
                elif deal['entry'] == 1:  # OUT
                    exit_deal = deal

            if not entry_deal or not exit_deal:
                continue

            # Calculate metrics
            entry_time = datetime.strptime(entry_deal['time'], '%Y-%m-%d %H:%M:%S')
            exit_time = datetime.strptime(exit_deal['time'], '%Y-%m-%d %H:%M:%S')
            duration_hours = (exit_time - entry_time).total_seconds() / 3600

            # Calculate total P&L
            total_profit = exit_deal['profit']
            total_commission = entry_deal['commission'] + exit_deal['commission']
            total_swap = entry_deal['swap'] + exit_deal['swap']
            net_profit = total_profit + total_commission + total_swap

            # Calculate pips
            pip_value = 0.0001 if 'JPY' not in entry_deal['symbol'] else 0.01
            if entry_deal['type'] == 0:  # BUY
                pips = (exit_deal['price'] - entry_deal['price']) / pip_value
            else:  # SELL
                pips = (entry_deal['price'] - exit_deal['price']) / pip_value

            # Determine trade type
            trade_type = 'LONG' if entry_deal['type'] == 0 else 'SHORT'

            # Insert matched position
            try:
                cursor.execute("""
                    INSERT OR REPLACE INTO matched_positions (
                        position_id, symbol, entry_deal_id, exit_deal_id,
                        type, volume, entry_price, exit_price,
                        entry_time, exit_time, duration_hours,
                        profit, commission, swap, net_profit, pips, is_win,
                        magic, matched_at
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """, (
                    pos_id,
                    entry_deal['symbol'],
                    entry_deal['deal_id'],
                    exit_deal['deal_id'],
                    trade_type,
                    entry_deal['volume'],
                    entry_deal['price'],
                    exit_deal['price'],
                    entry_deal['time'],
                    exit_deal['time'],
                    duration_hours,
                    total_profit,
                    total_commission,
                    total_swap,
                    net_profit,
                    pips,
                    1 if net_profit > 0 else 0,
                    entry_deal['magic'],
                    datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                ))

                if cursor.rowcount > 0:
                    matched += 1

            except Exception as e:
                print(f"Error matching position {pos_id}: {e}")
                continue

        self.conn.commit()

        return {
            'success': True,
            'matched': matched,
            'total_positions': len(positions)
        }

    def calculate_pnl_metrics(self, period: str = 'all') -> Dict:
        """
        Calculate P&L metrics for specified period

        Args:
            period: 'all', 'today', 'week', 'month'

        Returns:
            Dict with P&L metrics
        """
        cursor = self.conn.cursor()

        # Build date filter
        where_clause = ""
        if period == 'today':
            where_clause = f"WHERE DATE(exit_time) = DATE('now')"
        elif period == 'week':
            where_clause = f"WHERE exit_time >= DATE('now', '-7 days')"
        elif period == 'month':
            where_clause = f"WHERE exit_time >= DATE('now', '-30 days')"

        # Get positions
        cursor.execute(f"""
            SELECT * FROM matched_positions
            {where_clause}
            ORDER BY exit_time DESC
        """)

        positions = cursor.fetchall()

        if not positions:
            return {
                'period': period,
                'total_trades': 0,
                'message': 'No trades found for period'
            }

        # Convert to DataFrame for analysis
        df = pd.DataFrame([dict(pos) for pos in positions])

        # Calculate metrics
        total_trades = len(df)
        winning_trades = len(df[df['is_win'] == 1])
        losing_trades = total_trades - winning_trades

        win_rate = (winning_trades / total_trades * 100) if total_trades > 0 else 0

        total_profit = df[df['net_profit'] > 0]['net_profit'].sum()
        total_loss = abs(df[df['net_profit'] <= 0]['net_profit'].sum())
        net_profit = df['net_profit'].sum()

        avg_win = df[df['is_win'] == 1]['net_profit'].mean() if winning_trades > 0 else 0
        avg_loss = df[df['is_win'] == 0]['net_profit'].mean() if losing_trades > 0 else 0

        profit_factor = abs(total_profit / total_loss) if total_loss > 0 else 0

        # Drawdown calculation
        df_sorted = df.sort_values('exit_time')
        df_sorted['cumulative'] = df_sorted['net_profit'].cumsum()
        df_sorted['running_max'] = df_sorted['cumulative'].cummax()
        df_sorted['drawdown'] = df_sorted['cumulative'] - df_sorted['running_max']
        max_drawdown = df_sorted['drawdown'].min()

        largest_win = df['net_profit'].max()
        largest_loss = df['net_profit'].min()

        metrics = {
            'period': period,
            'total_trades': int(total_trades),
            'winning_trades': int(winning_trades),
            'losing_trades': int(losing_trades),
            'win_rate': float(win_rate),
            'total_profit': float(total_profit),
            'total_loss': float(total_loss),
            'net_profit': float(net_profit),
            'profit_factor': float(profit_factor),
            'avg_win': float(avg_win),
            'avg_loss': float(avg_loss),
            'max_drawdown': float(max_drawdown),
            'largest_win': float(largest_win),
            'largest_loss': float(largest_loss),
            'avg_duration_hours': float(df['duration_hours'].mean()),
            'total_pips': float(df['pips'].sum())
        }

        return metrics

    def generate_dashboard(self) -> str:
        """
        Generate P&L dashboard text

        Returns:
            Formatted dashboard string
        """
        # Get metrics for different periods
        all_time = self.calculate_pnl_metrics('all')
        today = self.calculate_pnl_metrics('today')
        week = self.calculate_pnl_metrics('week')
        month = self.calculate_pnl_metrics('month')

        dashboard = []
        dashboard.append("=" * 80)
        dashboard.append("💰 MT5 P&L TRACKER - LIVE PERFORMANCE DASHBOARD")
        dashboard.append("=" * 80)
        dashboard.append(f"\nGenerated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")

        # All Time
        dashboard.append("\n" + "=" * 80)
        dashboard.append("📊 ALL TIME PERFORMANCE")
        dashboard.append("=" * 80)

        if all_time['total_trades'] > 0:
            dashboard.append(f"\nTotal Trades:        {all_time['total_trades']}")
            dashboard.append(f"Winning Trades:      {all_time['winning_trades']} ({all_time['win_rate']:.1f}%)")
            dashboard.append(f"Losing Trades:       {all_time['losing_trades']}")
            dashboard.append(f"\nNet Profit:          ${all_time['net_profit']:.2f}")
            dashboard.append(f"Total Profit:        ${all_time['total_profit']:.2f}")
            dashboard.append(f"Total Loss:          ${all_time['total_loss']:.2f}")
            dashboard.append(f"Profit Factor:       {all_time['profit_factor']:.2f}")
            dashboard.append(f"\nAverage Win:         ${all_time['avg_win']:.2f}")
            dashboard.append(f"Average Loss:        ${all_time['avg_loss']:.2f}")
            dashboard.append(f"Largest Win:         ${all_time['largest_win']:.2f}")
            dashboard.append(f"Largest Loss:        ${all_time['largest_loss']:.2f}")
            dashboard.append(f"\nMax Drawdown:        ${all_time['max_drawdown']:.2f}")
            dashboard.append(f"Avg Duration:        {all_time['avg_duration_hours']:.1f} hours")
            dashboard.append(f"Total Pips:          {all_time['total_pips']:.1f}")
        else:
            dashboard.append("\n❌ No trades found")

        # Month
        dashboard.append("\n" + "=" * 80)
        dashboard.append("📅 THIS MONTH")
        dashboard.append("=" * 80)

        if month['total_trades'] > 0:
            dashboard.append(f"\nTrades:      {month['total_trades']} ({month['win_rate']:.1f}% WR)")
            dashboard.append(f"Net Profit:  ${month['net_profit']:.2f}")
            dashboard.append(f"Profit/Loss: +${month['total_profit']:.2f} / -${month['total_loss']:.2f}")
        else:
            dashboard.append("\n❌ No trades this month")

        # Week
        dashboard.append("\n" + "=" * 80)
        dashboard.append("📅 THIS WEEK")
        dashboard.append("=" * 80)

        if week['total_trades'] > 0:
            dashboard.append(f"\nTrades:      {week['total_trades']} ({week['win_rate']:.1f}% WR)")
            dashboard.append(f"Net Profit:  ${week['net_profit']:.2f}")
            dashboard.append(f"Profit/Loss: +${week['total_profit']:.2f} / -${week['total_loss']:.2f}")
        else:
            dashboard.append("\n❌ No trades this week")

        # Today
        dashboard.append("\n" + "=" * 80)
        dashboard.append("📅 TODAY")
        dashboard.append("=" * 80)

        if today['total_trades'] > 0:
            dashboard.append(f"\nTrades:      {today['total_trades']} ({today['win_rate']:.1f}% WR)")
            dashboard.append(f"Net Profit:  ${today['net_profit']:.2f}")
            dashboard.append(f"Profit/Loss: +${today['total_profit']:.2f} / -${today['total_loss']:.2f}")
        else:
            dashboard.append("\n❌ No trades today")

        dashboard.append("\n" + "=" * 80)

        return "\n".join(dashboard)

    def sync_and_update(self, days_back: int = 30) -> Dict:
        """
        Complete sync: Import MT5 history → Match positions → Calculate metrics

        Args:
            days_back: Number of days to import

        Returns:
            Dict with sync results
        """
        results = {
            'timestamp': datetime.now().isoformat(),
            'steps': {}
        }

        # Step 1: Import MT5 history
        import_result = self.import_mt5_history(days_back)
        results['steps']['import'] = import_result

        if not import_result['success']:
            results['success'] = False
            results['error'] = import_result.get('error', 'Import failed')
            return results

        # Step 2: Match positions
        match_result = self.match_positions()
        results['steps']['match'] = match_result

        if not match_result['success']:
            results['success'] = False
            results['error'] = 'Position matching failed'
            return results

        # Step 3: Calculate current metrics
        metrics = self.calculate_pnl_metrics('all')
        results['steps']['metrics'] = metrics

        results['success'] = True
        results['summary'] = {
            'new_deals': import_result['new_deals'],
            'matched_positions': match_result['matched'],
            'total_trades': metrics.get('total_trades', 0),
            'win_rate': metrics.get('win_rate', 0),
            'net_profit': metrics.get('net_profit', 0)
        }

        return results

    def get_recent_trades(self, limit: int = 10) -> pd.DataFrame:
        """
        Get recent closed positions

        Args:
            limit: Number of trades to return

        Returns:
            DataFrame with recent trades
        """
        query = f"""
            SELECT
                position_id, symbol, type, volume,
                entry_price, exit_price, entry_time, exit_time,
                duration_hours, net_profit, pips, is_win
            FROM matched_positions
            ORDER BY exit_time DESC
            LIMIT {limit}
        """

        df = pd.read_sql_query(query, self.conn)
        return df

    def close(self):
        """Close database connection"""
        if self.conn:
            self.conn.close()


# ==========================================
# SCHEDULER INTEGRATION
# ==========================================

def scheduled_pnl_sync(tracker: MT5PnLTracker, days_back: int = 7):
    """
    Scheduled job for automatic P&L sync

    Args:
        tracker: MT5PnLTracker instance
        days_back: Days to sync
    """
    try:
        print(f"\n[{datetime.now().strftime('%H:%M:%S')}] 🔄 Running scheduled P&L sync...")

        results = tracker.sync_and_update(days_back)

        if results['success']:
            summary = results['summary']
            print(f"✅ Sync complete: {summary['new_deals']} new deals, "
                  f"{summary['matched_positions']} matched positions")
            print(f"📊 Total: {summary['total_trades']} trades, "
                  f"{summary['win_rate']:.1f}% WR, ${summary['net_profit']:.2f} P&L")
        else:
            print(f"❌ Sync failed: {results.get('error', 'Unknown error')}")

    except Exception as e:
        print(f"❌ P&L sync error: {e}")


# ==========================================
# MAIN EXECUTION
# ==========================================

if __name__ == "__main__":
    # Create tracker
    tracker = MT5PnLTracker(db_path="trading_bot.db")
    tracker.connect_db()

    print("=" * 80)
    print("🚀 MT5 P&L TRACKER - INITIAL SYNC")
    print("=" * 80)

    # Sync history
    print("\n📥 Importing MT5 history...")
    results = tracker.sync_and_update(days_back=30)

    if results['success']:
        print(f"\n✅ Sync successful!")
        print(f"   New deals: {results['summary']['new_deals']}")
        print(f"   Matched positions: {results['summary']['matched_positions']}")

        # Show dashboard
        print("\n" + tracker.generate_dashboard())

        # Show recent trades
        print("\n" + "=" * 80)
        print("📜 RECENT TRADES (Last 10)")
        print("=" * 80)
        recent = tracker.get_recent_trades(10)
        if not recent.empty:
            print("\n" + recent.to_string(index=False))
        else:
            print("\n❌ No recent trades")

    else:
        print(f"\n❌ Sync failed: {results.get('error', 'Unknown error')}")

    tracker.close()
    print("\n" + "=" * 80)
    print("✅ Complete!")
    print("=" * 80)