Lotto-Tip-Generator/scripts/utils/update_from_api.py

#!/usr/bin/env python3
"""
Lotto 6aus49 API Data Updater

Lädt Lotto-Ziehungen von verschiedenen APIs:
1. lottoAPI (https://lottoapi.herokuapp.com)
2. Lotto.de API
3. Fallback Quellen

Features:
- Automatisches Laden von APIs
- Fallback auf andere APIs wenn primäre fehlschlägt
- Duplikate-Vermeidung
- Automatisches Backup
"""

import requests
import pandas as pd
from datetime import datetime
import shutil
import os
from typing import List, Dict, Optional
import time


class LottoAPIUpdater:
    """Updater für Lotto 6aus49 über APIs."""

    def __init__(self, data_file: str):
        self.data_file = data_file
        self.backup_file = None
        self.df_existing = None

        # API-Endpoints (Lottoland zuerst, da am aktuellsten)
        self.apis = {
            'lottoland': {
                'url': 'https://media.lottoland.com/api/drawings/german6aus49',
                'name': 'Lottoland API',
                'parser': self._parse_lottoland
            },
            'github': {
                'url': 'https://johannesfriedrich.github.io/LottoNumberArchive/Lottonumbers_tidy_complete.json',
                'name': 'GitHub Lotto Archive',
                'parser': self._parse_github_archive
            },
            'lottoapi': {
                'url': 'https://lottoapi.herokuapp.com/lotto/6aus49/100',
                'name': 'lottoAPI (Backup)',
                'parser': self._parse_lottoapi
            }
        }

        print("🔄 LOTTO 6AUS49 API UPDATER")
        print("=" * 70)
        print(f"📁 Datei: {os.path.basename(data_file)}")
        print("=" * 70)

    def load_existing_data(self) -> bool:
        """Lädt existierende Daten."""
        print("\n📂 Lade existierende Daten...")

        try:
            if not os.path.exists(self.data_file):
                print(f"   ⚠️  Datei existiert nicht - erstelle neue")
                self.df_existing = pd.DataFrame(
                    columns=['datum', 'Z1', 'Z2', 'Z3', 'Z4', 'Z5', 'Z6', 'SZ']
                )
                return True

            self.df_existing = pd.read_csv(self.data_file, sep=';')

            # Konvertiere Datum
            if 'datum' in self.df_existing.columns:
                self.df_existing['datum'] = pd.to_datetime(
                    self.df_existing['datum'],
                    format='%Y-%m-%d',
                    errors='coerce'
                )

            print(f"   ✅ {len(self.df_existing)} Ziehungen geladen")

            if len(self.df_existing) > 0:
                latest = self.df_existing['datum'].max()
                oldest = self.df_existing['datum'].min()
                print(f"   📅 Zeitraum: {oldest.strftime('%Y-%m-%d')} bis {latest.strftime('%Y-%m-%d')}")

            return True

        except Exception as e:
            print(f"   ❌ Fehler: {e}")
            return False

    def create_backup(self) -> bool:
        """Erstellt Backup."""
        if not os.path.exists(self.data_file):
            return True

        try:
            print("\n💾 Erstelle Backup...")

            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            backup_dir = os.path.join(os.path.dirname(self.data_file), "data", "backups")
            os.makedirs(backup_dir, exist_ok=True)

            filename = os.path.basename(self.data_file)
            self.backup_file = os.path.join(backup_dir, f"{filename}.backup_{timestamp}")

            shutil.copy2(self.data_file, self.backup_file)
            print(f"   ✅ Backup: {os.path.basename(self.backup_file)}")

            return True

        except Exception as e:
            print(f"   ⚠️  Backup-Fehler: {e}")
            return False

    def fetch_from_api(self, api_name: str) -> List[Dict]:
        """
        Holt Daten von spezifischer API.

        Args:
            api_name: Name der API ('lottoapi')

        Returns:
            Liste von Ziehungen
        """
        if api_name not in self.apis:
            print(f"   ❌ Unbekannte API: {api_name}")
            return []

        api = self.apis[api_name]
        print(f"\n🌐 Versuche {api['name']}...")
        print(f"   URL: {api['url']}")

        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
            }

            response = requests.get(api['url'], headers=headers, timeout=15)
            response.raise_for_status()

            print(f"   ✅ Antwort erhalten ({len(response.content)} bytes)")

            # Parse mit spezifischem Parser
            draws = api['parser'](response.json())

            if draws:
                print(f"   ✅ {len(draws)} Ziehungen extrahiert")
            else:
                print(f"   ⚠️  Keine Ziehungen extrahiert")

            return draws

        except requests.RequestException as e:
            print(f"   ❌ Netzwerkfehler: {e}")
            return []
        except Exception as e:
            print(f"   ❌ Fehler: {e}")
            return []

    def _parse_github_archive(self, data: List[Dict]) -> List[Dict]:
        """
        Parst GitHub Lotto Archive Response.

        Expected format:
        [
          {
            "id": 4963,
            "date": "26.11.2025",
            "variable": "Lottozahl",
            "value": 2
          },
          ...
        ]
        """
        draws = []

        try:
            # Gruppiere nach ID (jede ID = eine Ziehung)
            draws_by_id = {}

            for entry in data:
                draw_id = entry.get('id')
                if draw_id not in draws_by_id:
                    draws_by_id[draw_id] = {
                        'date': entry.get('date'),
                        'numbers': [],
                        'superzahl': None
                    }

                variable = entry.get('variable')
                value = entry.get('value')

                if variable == 'Lottozahl':
                    draws_by_id[draw_id]['numbers'].append(value)
                elif variable == 'Superzahl':
                    draws_by_id[draw_id]['superzahl'] = value

            # Konvertiere zu unserem Format
            for draw_id, draw_data in draws_by_id.items():
                try:
                    # Parse Datum (Format: DD.MM.YYYY)
                    date_str = draw_data['date']
                    day, month, year = date_str.split('.')
                    date_obj = datetime(int(year), int(month), int(day))

                    # Sortiere Zahlen
                    numbers = sorted(draw_data['numbers'])

                    # Validierung
                    if len(numbers) != 6:
                        continue

                    superzahl = draw_data['superzahl']
                    if superzahl is None:
                        continue

                    draw = {
                        'datum': date_obj,
                        'Z1': numbers[0],
                        'Z2': numbers[1],
                        'Z3': numbers[2],
                        'Z4': numbers[3],
                        'Z5': numbers[4],
                        'Z6': numbers[5],
                        'SZ': superzahl
                    }

                    if self._validate_draw(draw):
                        draws.append(draw)

                except Exception as e:
                    continue

        except Exception as e:
            print(f"   ⚠️  Parse-Fehler: {e}")

        return draws

    def _parse_lottoapi(self, data: List[Dict]) -> List[Dict]:
        """
        Parst lottoAPI Response.

        Expected format:
        [
          {
            "date": "2025-01-22",
            "numbers": [3, 9, 12, 24, 39, 45],
            "superzahl": 7
          },
          ...
        ]
        """
        draws = []

        try:
            for item in data:
                # Datum
                date_str = item.get('date')
                if not date_str:
                    continue

                date_obj = datetime.strptime(date_str, '%Y-%m-%d')

                # Hauptzahlen (6 Zahlen)
                numbers = sorted(item.get('numbers', []))
                if len(numbers) != 6:
                    continue

                # Superzahl
                superzahl = item.get('superzahl')
                if superzahl is None:
                    continue

                draw = {
                    'datum': date_obj,
                    'Z1': numbers[0],
                    'Z2': numbers[1],
                    'Z3': numbers[2],
                    'Z4': numbers[3],
                    'Z5': numbers[4],
                    'Z6': numbers[5],
                    'SZ': superzahl
                }

                if self._validate_draw(draw):
                    draws.append(draw)

        except Exception as e:
            print(f"   ⚠️  Parse-Fehler: {e}")

        return draws

    def _parse_lottoland(self, data: Dict) -> List[Dict]:
        """
        Parst Lottoland API Response.

        Expected format:
        {
          "last": {
            "date": {"day": 31, "month": 1, "year": 2026, ...},
            "numbers": [8, 19, 30, 37, 38, 49],
            "superzahl": 3
          }
        }
        """
        draws = []

        try:
            # Lottoland liefert nur die letzte Ziehung unter "last"
            last_draw = data.get('last')
            if not last_draw:
                return draws

            # Datum extrahieren
            date_info = last_draw.get('date', {})
            day = date_info.get('day')
            month = date_info.get('month')
            year = date_info.get('year')

            if not all([day, month, year]):
                return draws

            date_obj = datetime(year, month, day)
            print(f"   ✅ Ziehung vom {date_obj.strftime('%Y-%m-%d')} extrahiert")

            # Hauptzahlen (6 Zahlen)
            numbers = sorted(last_draw.get('numbers', []))
            if len(numbers) != 6:
                return draws

            # Superzahl
            superzahl = last_draw.get('superzahl')
            if superzahl is None:
                return draws

            draw = {
                'datum': date_obj,
                'Z1': numbers[0],
                'Z2': numbers[1],
                'Z3': numbers[2],
                'Z4': numbers[3],
                'Z5': numbers[4],
                'Z6': numbers[5],
                'SZ': int(superzahl)
            }

            if self._validate_draw(draw):
                draws.append(draw)

        except Exception as e:
            print(f"   ⚠️  Parse-Fehler: {e}")

        return draws

    def _validate_draw(self, draw: Dict) -> bool:
        """Validiert eine Ziehung."""
        try:
            required_fields = ['datum', 'Z1', 'Z2', 'Z3', 'Z4', 'Z5', 'Z6', 'SZ']
            if not all(field in draw for field in required_fields):
                return False

            # Hauptzahlen (1-49)
            main_numbers = [draw[f'Z{i}'] for i in range(1, 7)]
            if not all(isinstance(n, int) and 1 <= n <= 49 for n in main_numbers):
                return False

            if len(set(main_numbers)) != 6:
                return False

            # Superzahl (0-9)
            if not isinstance(draw['SZ'], int) or not 0 <= draw['SZ'] <= 9:
                return False

            if not isinstance(draw['datum'], datetime):
                return False

            return True

        except Exception:
            return False

    def fetch_from_all_apis(self) -> List[Dict]:
        """
        Versucht alle APIs nacheinander.

        Returns:
            Liste von Ziehungen
        """
        print("\n🔍 SUCHE NACH DATEN VON APIs")
        print("=" * 70)

        all_draws = []
        success_count = 0

        # Strategie: Lottoland für aktuelle Ziehung, GitHub für Historie
        # Lottoland liefert nur die neueste Ziehung, GitHub hat die komplette Historie

        # 1. Versuche Lottoland für die aktuellste Ziehung
        lottoland_draws = self.fetch_from_api('lottoland')
        if lottoland_draws:
            all_draws.extend(lottoland_draws)
            success_count += 1

        time.sleep(1)

        # 2. Versuche GitHub für historische Daten
        github_draws = self.fetch_from_api('github')
        if github_draws:
            all_draws.extend(github_draws)
            success_count += 1

        # 3. Falls beide fehlschlagen, versuche lottoapi als Fallback
        if success_count == 0:
            time.sleep(1)
            lottoapi_draws = self.fetch_from_api('lottoapi')
            if lottoapi_draws:
                all_draws.extend(lottoapi_draws)

        if not all_draws:
            print("\n   ❌ Keine Daten von APIs erhalten")
            print("   💡 Tipp: Verwende simple_update.py für manuelle Eingabe")

        return all_draws

    def merge_with_existing(self, new_draws: List[Dict]) -> pd.DataFrame:
        """Merged neue Ziehungen mit existierenden Daten."""
        print(f"\n🔀 Merge mit existierenden Daten...")

        if not new_draws:
            print("   ⚠️  Keine neuen Ziehungen zum Mergen")
            return self.df_existing

        df_new = pd.DataFrame(new_draws)

        if self.df_existing is None or len(self.df_existing) == 0:
            df_combined = df_new
            print(f"   ✅ Neue Datei erstellt mit {len(df_combined)} Ziehungen")
        else:
            df_combined = pd.concat([self.df_existing, df_new], ignore_index=True)

            before_dedup = len(df_combined)
            df_combined = df_combined.drop_duplicates(subset=['datum'], keep='first')
            after_dedup = len(df_combined)

            duplicates = before_dedup - after_dedup
            if duplicates > 0:
                print(f"   🗑️  {duplicates} Duplikat(e) entfernt")

        df_combined = df_combined.sort_values('datum', ascending=True)
        df_combined = df_combined.reset_index(drop=True)

        new_entries = len(df_combined) - (len(self.df_existing) if self.df_existing is not None else 0)

        print(f"   ✅ Merge abgeschlossen:")
        print(f"      Vorher: {len(self.df_existing) if self.df_existing is not None else 0} Ziehungen")
        print(f"      Neue: {new_entries} Ziehungen")
        print(f"      Gesamt: {len(df_combined)} Ziehungen")

        return df_combined

    def save_data(self, df: pd.DataFrame) -> bool:
        """Speichert Daten."""
        try:
            print(f"\n💾 Speichere Daten...")

            df_export = df.copy()
            df_export['datum'] = df_export['datum'].dt.strftime('%Y-%m-%d')

            column_order = ['datum', 'Z1', 'Z2', 'Z3', 'Z4', 'Z5', 'Z6', 'SZ']
            df_export = df_export[column_order]

            df_export.to_csv(self.data_file, sep=';', index=False)

            print(f"   ✅ Gespeichert: {self.data_file}")
            print(f"   📊 {len(df)} Ziehungen total")

            if len(df) > 0:
                latest = df['datum'].max()
                oldest = df['datum'].min()
                print(f"   📅 Zeitraum: {oldest.strftime('%Y-%m-%d')} bis {latest.strftime('%Y-%m-%d')}")

            return True

        except Exception as e:
            print(f"   ❌ Speicherfehler: {e}")

            if self.backup_file and os.path.exists(self.backup_file):
                print("   🔄 Stelle Backup wieder her...")
                shutil.copy2(self.backup_file, self.data_file)
                print("   ✅ Backup wiederhergestellt")

            return False

    def update(self, api_name: str = 'all', create_backup: bool = True) -> bool:
        """
        Führt komplettes Update durch.

        Args:
            api_name: 'all' oder 'lottoapi'
            create_backup: Backup erstellen

        Returns:
            True bei Erfolg
        """
        print("\n🚀 STARTE UPDATE")
        print("=" * 70)

        if not self.load_existing_data():
            return False

        if create_backup:
            self.create_backup()

        if api_name == 'all':
            new_draws = self.fetch_from_all_apis()
        else:
            new_draws = self.fetch_from_api(api_name)

        if not new_draws:
            print("\n❌ Keine Daten von APIs geladen")
            print("💡 Tipp: Verwende simple_update.py für manuelle Eingabe")
            return False

        df_updated = self.merge_with_existing(new_draws)
        success = self.save_data(df_updated)

        print("\n" + "=" * 70)
        if success:
            print("✅ UPDATE ERFOLGREICH ABGESCHLOSSEN")
        else:
            print("❌ UPDATE FEHLGESCHLAGEN")
        print("=" * 70)

        return success


def main():
    """Hauptfunktion."""
    import sys

    print()
    print("=" * 70)
    print("  LOTTO 6AUS49 API UPDATER")
    print("  Unterstützt: GitHub Lotto Archive, lottoAPI")
    print("=" * 70)
    print()

    default_file = "/Users/sebastianfrohlich/Projekte/Lotto/data/AlleLottozahlen.csv"

    if len(sys.argv) > 1:
        data_file = sys.argv[1]
        api_name = sys.argv[2] if len(sys.argv) > 2 else 'all'
    else:
        print(f"Standard-Datei: {default_file}")
        use_default = input("\nStandard verwenden? (j/n): ").strip().lower()

        if use_default in ['j', 'ja', 'y', 'yes', '']:
            data_file = default_file
        else:
            data_file = input("Pfad zur Datei: ").strip()

        api_name = 'all'

    print()

    updater = LottoAPIUpdater(data_file)
    success = updater.update(api_name=api_name)

    sys.exit(0 if success else 1)


if __name__ == "__main__":
    main()