Place-Order-Trading-Bot/import_json_to_db.py

#!/usr/bin/env python3
"""
📥 JSON Trade History → SQLite Database Importer
Importiert alte Trade-Daten aus JSON-Dateien in die trading_bot.db
"""

import json
import sqlite3
import os
from datetime import datetime
import glob

# ==========================================
# CONFIGURATION
# ==========================================

DB_PATH = "trading_bot.db"
JSON_PATTERN = "trade_performance_*.json"  # Alle JSON-Dateien mit diesem Muster

# ==========================================
# DATABASE CONNECTION
# ==========================================

def get_db_connection():
    """Verbindung zur Datenbank herstellen"""
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    return conn

# ==========================================
# IMPORT FUNCTIONS
# ==========================================

def parse_trade_data(trade, index=0):
    """
    Konvertiert JSON Trade-Daten ins Datenbank-Format
    """
    # Generate unique ticket from timestamp + index (für historische Daten ohne echtes Ticket)
    timestamp = trade.get('timestamp', '')
    ticket = hash(timestamp + str(index)) % 1000000000  # Unique ticket generieren

    # Basis-Daten
    data = {
        'ticket': ticket,
        'position_id': 0,  # Nicht in JSON vorhanden
        'symbol': trade.get('symbol', 'XAUUSD'),
        'strategy_name': 'TradingBot_' + trade.get('version', 'V1.6'),
        'type': 'BUY' if trade.get('entry_signal') == 1 else 'SELL' if trade.get('entry_signal') == -1 else 'UNKNOWN',
        'volume': 0.01,  # Default, nicht in JSON
        'entry_price': 0.0,  # Nicht in JSON
        'exit_price': None,
        'sl_price': None,
        'tp_price': None,
        'entry_time': trade.get('timestamp'),
        'exit_time': None,
        'duration_hours': None,
        'session': trade.get('session', 'unknown'),
        'regime': trade.get('market_regime', 'unknown'),
        'quality': trade.get('signal_quality', 'unknown'),
        'confidence': trade.get('confidence', 0.0),
        'timeframe_alignment': trade.get('timeframe_alignment', 2),
        'profit': None,  # Nicht in JSON
        'commission': 0.0,
        'swap': 0.0,
        'net_profit': None,
        'profit_pct': None,
        'risk_amount': trade.get('risk_amount'),
        'risk_pct': trade.get('risk_pct', 0.01),
        'rr_ratio': None,
        'status': 'historical',  # Markiere als historische Daten
        'exit_reason': 'imported_from_json'
    }

    return data

def import_json_file(filepath, conn):
    """
    Importiert eine JSON-Datei in die Datenbank
    """
    print(f"\n📄 Processing: {os.path.basename(filepath)}")

    try:
        with open(filepath, 'r') as f:
            trades = json.load(f)

        if not isinstance(trades, list):
            print(f"  ⚠️ Skipped: Not a list of trades")
            return 0

        imported_count = 0
        skipped_count = 0

        cursor = conn.cursor()

        for i, trade in enumerate(trades):
            try:
                # Parse trade data (mit Index für unique ticket)
                data = parse_trade_data(trade, index=i)

                # Check if already exists (by timestamp + symbol)
                cursor.execute("""
                    SELECT id FROM trades
                    WHERE entry_time = ? AND symbol = ?
                """, (data['entry_time'], data['symbol']))

                if cursor.fetchone():
                    skipped_count += 1
                    continue

                # Insert into database
                cursor.execute("""
                    INSERT INTO trades (
                        ticket, position_id, symbol, strategy_name, type, volume,
                        entry_price, exit_price, sl_price, tp_price,
                        entry_time, exit_time, duration_hours,
                        session, regime, quality, confidence, timeframe_alignment,
                        profit, commission, swap, net_profit, profit_pct,
                        risk_amount, risk_pct, rr_ratio,
                        status, exit_reason
                    ) VALUES (
                        ?, ?, ?, ?, ?, ?,
                        ?, ?, ?, ?,
                        ?, ?, ?,
                        ?, ?, ?, ?, ?,
                        ?, ?, ?, ?, ?,
                        ?, ?, ?,
                        ?, ?
                    )
                """, (
                    data['ticket'], data['position_id'], data['symbol'],
                    data['strategy_name'], data['type'], data['volume'],
                    data['entry_price'], data['exit_price'], data['sl_price'], data['tp_price'],
                    data['entry_time'], data['exit_time'], data['duration_hours'],
                    data['session'], data['regime'], data['quality'],
                    data['confidence'], data['timeframe_alignment'],
                    data['profit'], data['commission'], data['swap'],
                    data['net_profit'], data['profit_pct'],
                    data['risk_amount'], data['risk_pct'], data['rr_ratio'],
                    data['status'], data['exit_reason']
                ))

                imported_count += 1

            except Exception as e:
                print(f"  ⚠️ Error importing trade: {e}")
                continue

        conn.commit()

        print(f"  ✅ Imported: {imported_count} trades")
        print(f"  ⏭️  Skipped: {skipped_count} trades (duplicates)")

        return imported_count

    except Exception as e:
        print(f"  ❌ Error reading file: {e}")
        return 0

def import_all_json_files():
    """
    Importiert alle JSON-Dateien im aktuellen Verzeichnis
    """
    print("=" * 80)
    print("📥 JSON TO DATABASE IMPORTER")
    print("=" * 80)

    # Finde alle JSON-Dateien
    json_files = glob.glob(JSON_PATTERN)

    if not json_files:
        print(f"\n⚠️ No JSON files found matching pattern: {JSON_PATTERN}")
        print("\nSearching for any trade_performance*.json files...")
        json_files = glob.glob("trade_performance*.json")

    if not json_files:
        print("\n❌ No JSON files found!")
        print("\nMake sure you have JSON files in this directory:")
        print(f"   {os.getcwd()}")
        return

    print(f"\n📊 Found {len(json_files)} JSON file(s):")
    for f in json_files:
        print(f"   • {os.path.basename(f)}")

    # Verbinde zur Datenbank
    print(f"\n🔌 Connecting to database: {DB_PATH}")

    if not os.path.exists(DB_PATH):
        print(f"\n❌ Database not found: {DB_PATH}")
        print("\n💡 Please run the Trading Bot first to create the database!")
        return

    conn = get_db_connection()

    # Importiere jede Datei
    total_imported = 0

    for json_file in json_files:
        imported = import_json_file(json_file, conn)
        total_imported += imported

    conn.close()

    # Summary
    print("\n" + "=" * 80)
    print("📊 IMPORT SUMMARY")
    print("=" * 80)
    print(f"Files processed: {len(json_files)}")
    print(f"Total trades imported: {total_imported}")
    print(f"\n✅ Import complete!")

    # Zeige Statistik
    show_database_stats()

def show_database_stats():
    """
    Zeigt Statistiken der importierten Daten
    """
    conn = get_db_connection()
    cursor = conn.cursor()

    print("\n" + "=" * 80)
    print("📈 DATABASE STATISTICS")
    print("=" * 80)

    # Total trades
    cursor.execute("SELECT COUNT(*) FROM trades")
    total = cursor.fetchone()[0]
    print(f"\nTotal trades in database: {total}")

    # By session
    cursor.execute("""
        SELECT session, COUNT(*) as count
        FROM trades
        GROUP BY session
        ORDER BY count DESC
    """)

    print("\nTrades by Session:")
    for row in cursor.fetchall():
        session = row[0] or 'unknown'
        count = row[1]
        pct = (count / total * 100) if total > 0 else 0
        print(f"   {session:10s}: {count:4d} ({pct:5.1f}%)")

    # By version
    cursor.execute("""
        SELECT strategy_name, COUNT(*) as count
        FROM trades
        GROUP BY strategy_name
        ORDER BY count DESC
    """)

    print("\nTrades by Version:")
    for row in cursor.fetchall():
        version = row[0]
        count = row[1]
        print(f"   {version:20s}: {count:4d}")

    # Date range
    cursor.execute("""
        SELECT
            MIN(entry_time) as first_trade,
            MAX(entry_time) as last_trade
        FROM trades
    """)

    row = cursor.fetchone()
    if row[0]:
        print(f"\nDate Range:")
        print(f"   First trade: {row[0]}")
        print(f"   Last trade:  {row[1]}")

    conn.close()

    print("\n" + "=" * 80)

# ==========================================
# MANUAL IMPORT FUNCTION
# ==========================================

def import_specific_file(filepath):
    """
    Importiert eine spezifische JSON-Datei

    Usage:
        python import_json_to_db.py <filepath>
    """
    print("=" * 80)
    print("📥 MANUAL JSON IMPORT")
    print("=" * 80)

    if not os.path.exists(filepath):
        print(f"\n❌ File not found: {filepath}")
        return

    if not os.path.exists(DB_PATH):
        print(f"\n❌ Database not found: {DB_PATH}")
        print("\n💡 Please run the Trading Bot first to create the database!")
        return

    conn = get_db_connection()
    imported = import_json_file(filepath, conn)
    conn.close()

    print(f"\n✅ Import complete! Imported {imported} trades.")
    show_database_stats()

# ==========================================
# MAIN
# ==========================================

if __name__ == "__main__":
    import sys

    if len(sys.argv) > 1:
        # Manual import of specific file
        filepath = sys.argv[1]
        import_specific_file(filepath)
    else:
        # Auto-import all JSON files
        import_all_json_files()