feat: extractor, normalizer, ingest v2 – OCR pipeline funktioniert

extractor.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""
+extractor.py — Text-Extraktion aus verschiedenen Dateiformaten.
+
+Unterstützte Formate:
+    .txt .md            → direkt lesen
+    .docx               → python-docx
+    .pdf                → pypdf (mit OCR-Fallback bei Scans)
+    .png .jpg .jpeg     → OCR via Tesseract
+    .tif .tiff .bmp     → OCR via Tesseract
+    .webp               → OCR via Tesseract
+
+Tesseract muss systemweit installiert sein:
+    brew install tesseract tesseract-lang
+"""
+
+import re
+from pathlib import Path
+
+try:
+    from pypdf import PdfReader
+    _PDF_SUPPORT = True
+except ImportError:
+    _PDF_SUPPORT = False
+
+try:
+    from docx import Document as DocxDocument
+    _DOCX_SUPPORT = True
+except ImportError:
+    _DOCX_SUPPORT = False
+
+try:
+    from PIL import Image
+    import pytesseract
+    _OCR_SUPPORT = True
+except ImportError:
+    _OCR_SUPPORT = False
+
+try:
+    from pdf2image import convert_from_path
+    _PDF2IMAGE_SUPPORT = True
+except ImportError:
+    _PDF2IMAGE_SUPPORT = False
+
+
+SUPPORTED_EXTENSIONS = {
+    ".txt", ".md",
+    ".pdf",
+    ".docx",
+    ".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp",
+}
+
+_PDF_MIN_TEXT_LENGTH = 150
+_OCR_LANG = "deu+eng"
+
+
+def extract(path: Path) -> str:
+    if not path.exists():
+        raise FileNotFoundError(f"Datei nicht gefunden: {path}")
+
+    suffix = path.suffix.lower()
+
+    if suffix not in SUPPORTED_EXTENSIONS:
+        raise ValueError(
+            f"Nicht unterstütztes Format '{suffix}'. "
+            f"Unterstützt: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
+        )
+
+    if suffix in (".txt", ".md"):
+        return _read_text(path)
+    elif suffix == ".docx":
+        return _read_docx(path)
+    elif suffix == ".pdf":
+        return _read_pdf(path)
+    else:
+        return _ocr_image(path)
+
+
+def is_supported(path: Path) -> bool:
+    return path.suffix.lower() in SUPPORTED_EXTENSIONS
+
+
+def _read_text(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+def _read_docx(path: Path) -> str:
+    if not _DOCX_SUPPORT:
+        raise ImportError("python-docx nicht installiert: pip install python-docx")
+
+    doc = DocxDocument(str(path))
+    parts = []
+
+    for element in doc.element.body:
+        tag = element.tag.split("}")[-1]
+        if tag == "p":
+            runs = "".join(
+                r.text for r in element.iter()
+                if r.tag.endswith("}t") and r.text
+            )
+            text = runs.strip()
+            if text:
+                parts.append(text)
+        elif tag == "tbl":
+            for row in element.iter():
+                if row.tag.endswith("}tr"):
+                    cells = [
+                        "".join(
+                            t.text or "" for t in cell.iter()
+                            if t.tag.endswith("}t")
+                        ).strip()
+                        for cell in row
+                        if cell.tag.endswith("}tc")
+                    ]
+                    if any(cells):
+                        parts.append(" | ".join(cells))
+
+    return "\n\n".join(parts)
+
+
+def _read_pdf(path: Path) -> str:
+    if not _PDF_SUPPORT:
+        raise ImportError("pypdf nicht installiert: pip install pypdf")
+
+    reader = PdfReader(str(path))
+    pages = [page.extract_text() or "" for page in reader.pages]
+    text = "\n\n".join(pages).strip()
+
+    if len(text) < _PDF_MIN_TEXT_LENGTH:
+        if not _PDF2IMAGE_SUPPORT or not _OCR_SUPPORT:
+            raise ImportError(
+                "PDF scheint ein Scan zu sein, aber pdf2image oder pytesseract fehlen. "
+                "Installieren: pip install pdf2image pytesseract && brew install tesseract"
+            )
+        print("    ℹ️  PDF scheint ein Scan – verwende OCR-Fallback...")
+        text = _ocr_pdf(path)
+
+    return text
+
+
+def _ocr_pdf(path: Path) -> str:
+    images = convert_from_path(str(path), dpi=300)
+    pages = []
+    for i, img in enumerate(images, 1):
+        page_text = pytesseract.image_to_string(img, lang=_OCR_LANG)
+        pages.append(page_text)
+        print(f"    📄 OCR Seite {i}/{len(images)} abgeschlossen")
+    return _clean_ocr("\n\n".join(pages))
+
+
+def _ocr_image(path: Path) -> str:
+    if not _OCR_SUPPORT:
+        raise ImportError(
+            "pytesseract oder Pillow nicht installiert. "
+            "Installieren: pip install pytesseract Pillow && brew install tesseract tesseract-lang"
+        )
+    img = Image.open(str(path))
+    raw = pytesseract.image_to_string(img, lang=_OCR_LANG)
+    return _clean_ocr(raw)
+
+
+def _clean_ocr(text: str) -> str:
+    text = re.sub(r"[^\S\n]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    text = re.sub(r"[|}{~`\\^]", "", text)
+    text = re.sub(r"(\w)-\n(\w)", r"\1\2", text)
+    return text.strip()

ingest_V2.py

@@ -0,0 +1,489 @@
+#!/usr/bin/env python3
+"""
+ingest.py — RAG Ingestion Pipeline (v2)
+
+Ablauf pro Datei:
+    1.  Extraktion     → extractor.py  (Text, OCR, DOCX, PDF, Bilder)
+    2.  Normalisierung → normalizer.py (KI: Rohtext → Template-Struktur)
+    3.  Chunking       → token-basiert mit Overlap
+    4.  Embedding      → multilingual-e5-small (lokal)
+    5.  Speichern      → pgvector via SSH-Tunnel (Hetzner / anythingllm)
+
+CLI-Befehle:
+    python ingest.py file  <pfad>          Einzelne Datei
+    python ingest.py dir   <pfad>          Ganzes Verzeichnis
+    python ingest.py watch <pfad>          Ordner live beobachten
+    python ingest.py list                  DB-Inhalt anzeigen
+    python ingest.py delete <titel>        Dokument aus DB löschen
+
+Flags:
+    --force             Bestehende Chunks überschreiben
+    --no-normalize      KI-Normalisierung überspringen
+    --dry-run           Extrahieren + normalisieren, aber nicht in DB speichern
+    --quality-min 0.4   Mindest-Qualitäts-Score (Standard: 0.0 = alles speichern)
+"""
+
+import os
+import sys
+import uuid
+import json
+import argparse
+from datetime import datetime
+from pathlib import Path
+
+import tiktoken
+import psycopg2
+from psycopg2.extras import execute_values
+from sshtunnel import SSHTunnelForwarder
+from dotenv import load_dotenv
+
+from extractor import extract, is_supported, SUPPORTED_EXTENSIONS
+from normalizer import normalize
+
+load_dotenv(".env.ingest")
+
+# ---------------------------------------------------------------------------
+# Konfiguration (aus .env.ingest)
+# ---------------------------------------------------------------------------
+SSH_HOST           = os.getenv("SSH_HOST")
+SSH_USER           = os.getenv("SSH_USER", "root")
+SSH_KEY_PATH       = os.path.expanduser(os.getenv("SSH_KEY_PATH", "~/.ssh/id_ed25519"))
+SSH_KEY_PASSPHRASE = os.getenv("SSH_KEY_PASSPHRASE")
+DB_HOST            = os.getenv("DB_HOST", "127.0.0.1")
+DB_PORT            = int(os.getenv("DB_PORT", 5432))
+DB_NAME            = os.getenv("DB_NAME", "anythingllm")
+DB_USER            = os.getenv("DB_USER", "anythingllm")
+DB_PASSWORD        = os.getenv("DB_PASSWORD")
+NAMESPACE          = os.getenv("NAMESPACE", "mein-workspace")
+CHUNK_SIZE         = int(os.getenv("CHUNK_SIZE", 500))
+CHUNK_OVERLAP      = int(os.getenv("CHUNK_OVERLAP", 50))
+EMBED_MODEL        = "intfloat/multilingual-e5-small"
+
+_embed_model = None  # lazy load
+
+
+# ---------------------------------------------------------------------------
+# Chunking
+# ---------------------------------------------------------------------------
+
+def chunk_text(text: str) -> list[str]:
+    """Teilt Text in überlappende Token-Chunks auf."""
+    enc    = tiktoken.get_encoding("cl100k_base")
+    tokens = enc.encode(text)
+    chunks = []
+    start  = 0
+    while start < len(tokens):
+        end = min(start + CHUNK_SIZE, len(tokens))
+        chunks.append(enc.decode(tokens[start:end]))
+        start += CHUNK_SIZE - CHUNK_OVERLAP
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# Embedding
+# ---------------------------------------------------------------------------
+
+def embed(texts: list[str]) -> list[list[float]]:
+    """Erstellt Embeddings via multilingual-e5-small (lokal, einmaliger Download ~120 MB)."""
+    global _embed_model
+    if _embed_model is None:
+        from sentence_transformers import SentenceTransformer
+        print("  ⏳ Lade Embedding-Modell (einmalig)...")
+        _embed_model = SentenceTransformer(EMBED_MODEL)
+    prefixed = [f"passage: {t}" for t in texts]
+    return _embed_model.encode(prefixed).tolist()
+
+
+# ---------------------------------------------------------------------------
+# Datenbank
+# ---------------------------------------------------------------------------
+
+def _db_connect(tunnel_port: int):
+    return psycopg2.connect(
+        host="127.0.0.1",
+        port=tunnel_port,
+        dbname=DB_NAME,
+        user=DB_USER,
+        password=DB_PASSWORD,
+    )
+
+
+def _tunnel():
+    return SSHTunnelForwarder(
+        (SSH_HOST, 22),
+        ssh_username=SSH_USER,
+        ssh_pkey=SSH_KEY_PATH,
+        ssh_private_key_password=SSH_KEY_PASSPHRASE,
+        remote_bind_address=("127.0.0.1", DB_PORT),
+    )
+
+
+def _count_existing(cur, title: str) -> int:
+    cur.execute(
+        "SELECT COUNT(*) FROM anythingllm_vectors "
+        "WHERE metadata->>'title' = %s AND namespace = %s",
+        (title, NAMESPACE),
+    )
+    return cur.fetchone()[0]
+
+
+def _delete_existing(cur, title: str):
+    cur.execute(
+        "DELETE FROM anythingllm_vectors "
+        "WHERE metadata->>'title' = %s AND namespace = %s",
+        (title, NAMESPACE),
+    )
+
+
+def _insert_chunks(cur, chunks, embeddings, source_path: Path, extra_meta: dict):
+    now     = datetime.now().isoformat()
+    records = []
+    for chunk, embedding in zip(chunks, embeddings):
+        metadata = {
+            "id":          str(uuid.uuid4()),
+            "url":         f"file://{source_path.resolve()}",
+            "text":        chunk,
+            "title":       source_path.name,
+            "docSource":   "rag-ingestion-v2",
+            "published":   now,
+            "wordCount":   len(chunk.split()),
+            "chunkSource": str(source_path.resolve()),
+            **extra_meta,
+        }
+        records.append((
+            str(uuid.uuid4()),
+            json.dumps(metadata),
+            NAMESPACE,
+            embedding,
+        ))
+    execute_values(
+        cur,
+        "INSERT INTO anythingllm_vectors (id, metadata, namespace, embedding) VALUES %s",
+        records,
+        template="(%s::uuid, %s::jsonb, %s, %s::vector)",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Kern-Pipeline
+# ---------------------------------------------------------------------------
+
+def ingest_file(
+    file_path: str,
+    force: bool = False,
+    normalize_doc: bool = True,
+    dry_run: bool = False,
+    quality_min: float = 0.0,
+) -> bool:
+    path = Path(file_path).resolve()
+
+    if not path.exists():
+        print(f"❌ Datei nicht gefunden: {path}")
+        return False
+
+    if not is_supported(path):
+        print(f"⏭️  Übersprungen (Format nicht unterstützt): {path.suffix}")
+        return False
+
+    print(f"\n📄 {path.name}")
+
+    # ── 1. Extraktion ────────────────────────────────────────────────────────
+    try:
+        raw_text = extract(path)
+        print(f"  ✅ Text extrahiert ({len(raw_text):,} Zeichen)")
+    except Exception as e:
+        print(f"  ❌ Extraktion fehlgeschlagen: {e}")
+        return False
+
+    if not raw_text.strip():
+        print("  ⚠️  Kein Text extrahiert — Datei übersprungen")
+        return False
+
+    # ── 2. KI-Normalisierung ─────────────────────────────────────────────────
+    try:
+        norm = normalize(
+            raw_text,
+            filename=path.name,
+            skip_normalization=not normalize_doc,
+        )
+        text_to_index = norm["normalized_text"]
+    except Exception as e:
+        print(f"  ❌ Normalisierung fehlgeschlagen: {e}")
+        print("  ↩️  Verwende Rohtext als Fallback")
+        norm = {
+            "normalized_text": raw_text,
+            "doc_type":        "unbekannt",
+            "template_file":   "",
+            "quality_score":   0.5,
+            "missing_fields":  0,
+            "was_normalized":  False,
+        }
+        text_to_index = raw_text
+
+    # Qualitäts-Filter
+    if norm["quality_score"] < quality_min:
+        print(
+            f"  ⏭️  Qualitäts-Score {norm['quality_score']:.0%} "
+            f"< Mindest-Score {quality_min:.0%} — übersprungen"
+        )
+        return False
+
+    extra_meta = {
+        "doc_type":       norm["doc_type"],
+        "quality_score":  norm["quality_score"],
+        "missing_fields": norm["missing_fields"],
+        "was_normalized": norm["was_normalized"],
+        "template_file":  norm["template_file"],
+        "source_format":  path.suffix.lower(),
+    }
+
+    # ── 3. Chunking ──────────────────────────────────────────────────────────
+    chunks = chunk_text(text_to_index)
+    print(f"  ✅ {len(chunks)} Chunks erstellt")
+
+    # ── 4. Embedding ─────────────────────────────────────────────────────────
+    try:
+        embeddings = embed(chunks)
+        print(f"  ✅ {len(embeddings)} Embeddings erstellt")
+    except Exception as e:
+        print(f"  ❌ Embedding fehlgeschlagen: {e}")
+        return False
+
+    # ── Dry-Run Ende ─────────────────────────────────────────────────────────
+    if dry_run:
+        print(f"  🔍 Dry-Run — Typ: {norm['doc_type']} | "
+              f"Score: {norm['quality_score']:.0%} | "
+              f"Fehlend: {norm['missing_fields']}")
+        if norm["was_normalized"]:
+            print("\n" + "─" * 60)
+            print(text_to_index[:600] + ("..." if len(text_to_index) > 600 else ""))
+            print("─" * 60)
+        return True
+
+    # ── 5. Datenbank ─────────────────────────────────────────────────────────
+    print("  ⏳ Verbinde mit Datenbank (SSH-Tunnel)...")
+    try:
+        with _tunnel() as tunnel:
+            conn = _db_connect(tunnel.local_bind_port)
+            cur  = conn.cursor()
+
+            existing = _count_existing(cur, path.name)
+            if existing > 0:
+                if force:
+                    _delete_existing(cur, path.name)
+                    print(f"  🗑️  {existing} bestehende Chunks gelöscht")
+                else:
+                    print(
+                        f"  ⚠️  '{path.name}' bereits in DB ({existing} Chunks). "
+                        "Nutze --force zum Überschreiben."
+                    )
+                    cur.close(); conn.close()
+                    return False
+
+            _insert_chunks(cur, chunks, embeddings, path, extra_meta)
+            conn.commit()
+            cur.close(); conn.close()
+            print(f"  ✅ {len(chunks)} Chunks gespeichert "
+                  f"(Typ: {norm['doc_type']}, Score: {norm['quality_score']:.0%})")
+    except Exception as e:
+        print(f"  ❌ Datenbankfehler: {e}")
+        return False
+
+    return True
+
+
+def ingest_directory(
+    dir_path: str,
+    force: bool = False,
+    normalize_doc: bool = True,
+    dry_run: bool = False,
+    quality_min: float = 0.0,
+):
+    path = Path(dir_path).resolve()
+    if not path.is_dir():
+        print(f"❌ Verzeichnis nicht gefunden: {path}")
+        return
+
+    files = [f for f in sorted(path.iterdir()) if f.is_file() and is_supported(f)]
+    if not files:
+        print(f"❌ Keine unterstützten Dateien in: {path}")
+        print(f"   Unterstützte Formate: {', '.join(sorted(SUPPORTED_EXTENSIONS))}")
+        return
+
+    print(f"\n📁 {len(files)} Datei(en) in: {path}")
+    success = 0
+    for f in files:
+        if ingest_file(str(f), force=force, normalize_doc=normalize_doc,
+                       dry_run=dry_run, quality_min=quality_min):
+            success += 1
+
+    print(f"\n{'─' * 50}")
+    print(f"✅ Fertig: {success}/{len(files)} Dateien verarbeitet")
+
+
+def watch_directory(
+    dir_path: str,
+    force: bool = False,
+    normalize_doc: bool = True,
+    quality_min: float = 0.0,
+):
+    try:
+        from watchdog.observers import Observer
+        from watchdog.events import FileSystemEventHandler
+    except ImportError:
+        print("❌ watchdog nicht installiert: pip install watchdog")
+        sys.exit(1)
+
+    class _Handler(FileSystemEventHandler):
+        def on_created(self, event):
+            if event.is_directory:
+                return
+            p = Path(event.src_path)
+            if is_supported(p):
+                print(f"\n🆕 Neue Datei erkannt: {p.name}")
+                ingest_file(str(p), force=force, normalize_doc=normalize_doc,
+                            quality_min=quality_min)
+
+    observer = Observer()
+    observer.schedule(_Handler(), dir_path, recursive=False)
+    observer.start()
+    print(f"👁️  Beobachte: {dir_path}")
+    print("   Drücke Ctrl+C zum Beenden\n")
+    try:
+        import time
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+    observer.join()
+
+
+def list_documents():
+    print(f"\n📋 Dokumente in Namespace '{NAMESPACE}':\n")
+    try:
+        with _tunnel() as tunnel:
+            conn = _db_connect(tunnel.local_bind_port)
+            cur  = conn.cursor()
+            cur.execute("""
+                SELECT
+                    metadata->>'title'         AS titel,
+                    metadata->>'doc_type'      AS typ,
+                    metadata->>'quality_score' AS score,
+                    metadata->>'source_format' AS format,
+                    COUNT(*)                   AS chunks,
+                    MAX(metadata->>'published') AS datum
+                FROM anythingllm_vectors
+                WHERE namespace = %s
+                GROUP BY titel, typ, score, format
+                ORDER BY datum DESC
+            """, (NAMESPACE,))
+            rows = cur.fetchall()
+            cur.close(); conn.close()
+
+        if not rows:
+            print("  Keine Dokumente gefunden.")
+            return
+
+        header = f"  {'Titel':<45} {'Typ':<20} {'Score':>6}  {'Fmt':>5}  {'Chunks':>6}  Datum"
+        print(header)
+        print("  " + "─" * (len(header) - 2))
+        for titel, typ, score, fmt, chunks, datum in rows:
+            score_str = f"{float(score):.0%}" if score else "–"
+            print(
+                f"  {(titel or '?'):<45} {(typ or '?'):<20} "
+                f"{score_str:>6}  {(fmt or '?'):>5}  {chunks:>6}  "
+                f"{(datum or '?')[:19]}"
+            )
+        print()
+    except Exception as e:
+        print(f"❌ Datenbankfehler: {e}")
+
+
+def delete_document(title: str):
+    print(f"\n🗑️  Lösche '{title}' aus Namespace '{NAMESPACE}'...")
+    try:
+        with _tunnel() as tunnel:
+            conn     = _db_connect(tunnel.local_bind_port)
+            cur      = conn.cursor()
+            existing = _count_existing(cur, title)
+            if existing == 0:
+                print(f"  ⚠️  Kein Dokument mit Titel '{title}' gefunden")
+                cur.close(); conn.close()
+                return
+            _delete_existing(cur, title)
+            conn.commit()
+            cur.close(); conn.close()
+            print(f"  ✅ {existing} Chunks gelöscht")
+    except Exception as e:
+        print(f"  ❌ Datenbankfehler: {e}")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        description="RAG Ingestion v2 — Dokumente in pgvector importieren",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Beispiele:
+  python ingest.py file  ~/Downloads/fremdes_cv.pdf
+  python ingest.py file  ~/Desktop/scan.png --dry-run
+  python ingest.py dir   ~/Dokumente/bewerbung/ --force
+  python ingest.py watch ~/Desktop/scan-eingang/
+  python ingest.py list
+  python ingest.py delete "fremdes_cv.pdf"
+        """
+    )
+    sub = p.add_subparsers(dest="command", required=True)
+
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--force",        action="store_true", help="Bestehende Chunks überschreiben")
+    common.add_argument("--no-normalize", action="store_true", help="KI-Normalisierung überspringen")
+    common.add_argument("--quality-min",  type=float, default=0.0, metavar="0.0-1.0",
+                        help="Mindest-Qualitäts-Score (Standard: 0.0)")
+    common.add_argument("--dry-run",      action="store_true",
+                        help="Nur extrahieren + normalisieren, nicht speichern")
+
+    pf = sub.add_parser("file",  parents=[common], help="Einzelne Datei importieren")
+    pf.add_argument("path", help="Pfad zur Datei")
+
+    pd = sub.add_parser("dir",   parents=[common], help="Verzeichnis importieren")
+    pd.add_argument("path", help="Pfad zum Verzeichnis")
+
+    pw = sub.add_parser("watch", parents=[common], help="Ordner live beobachten")
+    pw.add_argument("path", help="Pfad zum Verzeichnis")
+
+    sub.add_parser("list", help="Alle Dokumente in der DB anzeigen")
+
+    pdel = sub.add_parser("delete", help="Dokument aus DB löschen")
+    pdel.add_argument("title", help="Titel (Dateiname) des Dokuments")
+
+    return p
+
+
+def main():
+    args         = _build_parser().parse_args()
+    do_normalize = not getattr(args, "no_normalize", False)
+    force        = getattr(args, "force", False)
+    dry_run      = getattr(args, "dry_run", False)
+    quality_min  = getattr(args, "quality_min", 0.0)
+
+    if args.command == "file":
+        ingest_file(args.path, force=force, normalize_doc=do_normalize,
+                    dry_run=dry_run, quality_min=quality_min)
+    elif args.command == "dir":
+        ingest_directory(args.path, force=force, normalize_doc=do_normalize,
+                         dry_run=dry_run, quality_min=quality_min)
+    elif args.command == "watch":
+        watch_directory(args.path, force=force, normalize_doc=do_normalize,
+                        quality_min=quality_min)
+    elif args.command == "list":
+        list_documents()
+    elif args.command == "delete":
+        delete_document(args.title)
+
+
+if __name__ == "__main__":
+    main()

normalizer.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+normalizer.py — KI-gestützte Dokumenten-Normalisierung.
+
+Ablauf:
+    1. Dokumenttyp via KI erkennen (Claude Haiku — günstig + schnell)
+    2. Passendes Template aus templates/ laden
+    3. Rohtext → Template-Struktur transformieren (Claude Sonnet)
+    4. Qualitäts-Score berechnen (Anteil befüllter Felder)
+
+API-Key: ANTHROPIC_API_KEY in .env.ingest
+"""
+
+import os
+import re
+from pathlib import Path
+
+import anthropic
+from dotenv import load_dotenv
+
+load_dotenv(".env.ingest")
+
+_client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
+
+TEMPLATES_DIR   = Path(__file__).parent / "templates"
+AUSGEFUELLT_DIR = TEMPLATES_DIR / "ausgefuellt"
+
+DOC_TYPE_MAP: dict[str, str] = {
+    "projektbeschreibung":  "Projektbeschreibung_TEMPLATE.md",
+    "anschreiben":          "Muster-Anschreiben_TEMPLATE.md",
+    "karriereziele":        "Karriereziele_TEMPLATE.md",
+    "rahmenbedingungen":    "Rahmenbedingungen_TEMPLATE.md",
+    "referenzen":           "Referenzen_TEMPLATE.md",
+    "technologie":          "Technologie-Bewertung_TEMPLATE.md",
+    "elevatorpitch":        "Ueber-mich_Elevator-Pitch_TEMPLATE.md",
+    "zertifikate":          "Zertifikate_Weiterbildungen_TEMPLATE.md",
+    "zielstellen":          "Zielstellen-Profil_TEMPLATE.md",
+    "cv":                   "Projektbeschreibung_TEMPLATE.md",
+    "lebenslauf":           "Projektbeschreibung_TEMPLATE.md",
+    "motivation":           "Muster-Anschreiben_TEMPLATE.md",
+    "skills":               "Technologie-Bewertung_TEMPLATE.md",
+    "skillmatrix":          "Technologie-Bewertung_TEMPLATE.md",
+    "pitch":                "Ueber-mich_Elevator-Pitch_TEMPLATE.md",
+    "profil":               "Ueber-mich_Elevator-Pitch_TEMPLATE.md",
+    "weiterbildung":        "Zertifikate_Weiterbildungen_TEMPLATE.md",
+    "arbeitszeugnis":       "Projektbeschreibung_TEMPLATE.md",
+    "zeugnis":              "Projektbeschreibung_TEMPLATE.md",
+}
+
+_MISSING_MARKER         = "[FEHLT]"
+_QUALITY_WARN_THRESHOLD = 0.5
+
+
+def normalize(raw_text: str, filename: str = "", skip_normalization: bool = False) -> dict:
+    if skip_normalization or not raw_text.strip():
+        return _passthrough(raw_text)
+
+    print("    🤖 Erkenne Dokumenttyp...")
+    doc_type = _detect_doc_type(raw_text, filename)
+    print(f"    ✅ Erkannter Typ: '{doc_type}'")
+
+    template_filename = DOC_TYPE_MAP.get(doc_type, "")
+    template_text     = _load_template(template_filename)
+
+    if not template_text:
+        print(f"    ⚠️  Kein Template für Typ '{doc_type}' — Rohtext wird unverändert verwendet")
+        return _passthrough(raw_text, doc_type=doc_type)
+
+    print(f"    🤖 Normalisiere in Template '{template_filename}'...")
+    example_text = _load_best_example(doc_type)
+    normalized   = _transform(raw_text, template_text, example_text)
+
+    missing_count     = len(re.findall(re.escape(_MISSING_MARKER), normalized))
+    placeholder_count = len(re.findall(r"\[.+?\]", template_text))
+    quality_score     = round(
+        max(0.0, 1.0 - (missing_count / max(placeholder_count, 1))),
+        2,
+    )
+
+    if quality_score < _QUALITY_WARN_THRESHOLD:
+        print(
+            f"    ⚠️  Niedriger Qualitäts-Score ({quality_score:.0%}) — "
+            f"{missing_count} Felder konnten nicht befüllt werden"
+        )
+    else:
+        print(f"    ✅ Qualitäts-Score: {quality_score:.0%} ({missing_count} fehlende Felder)")
+
+    return {
+        "normalized_text": normalized,
+        "doc_type":        doc_type,
+        "template_file":   template_filename,
+        "quality_score":   quality_score,
+        "missing_fields":  missing_count,
+        "was_normalized":  True,
+    }
+
+
+def list_doc_types() -> list[str]:
+    return sorted(set(DOC_TYPE_MAP.keys()))
+
+
+def _detect_doc_type(raw_text: str, filename: str = "") -> str:
+    known_types   = ", ".join(sorted(set(DOC_TYPE_MAP.keys())))
+    filename_hint = f"\nDateiname: {filename}" if filename else ""
+
+    response = _client.messages.create(
+        model="claude-haiku-4-5-20251001",
+        max_tokens=30,
+        temperature=0,
+        messages=[{
+            "role": "user",
+            "content": (
+                f"Bestimme den Dokumenttyp dieses Textes.\n"
+                f"Erlaubte Typen: {known_types}\n"
+                f"{filename_hint}\n\n"
+                f"Antworte NUR mit einem der erlaubten Typen, nichts anderes.\n\n"
+                f"Text (erste 1200 Zeichen):\n{raw_text[:1200]}"
+            ),
+        }],
+    )
+
+    detected = response.content[0].text.strip().lower()
+    detected = re.sub(r"[^a-zäöüß]", "", detected)
+
+    if detected in DOC_TYPE_MAP:
+        return detected
+    for key in DOC_TYPE_MAP:
+        if key in detected or detected in key:
+            return key
+
+    return "projektbeschreibung"
+
+
+def _load_template(filename: str) -> str:
+    if not filename:
+        return ""
+    path = TEMPLATES_DIR / filename
+    return path.read_text(encoding="utf-8") if path.exists() else ""
+
+
+def _load_best_example(doc_type: str) -> str:
+    if not AUSGEFUELLT_DIR.exists():
+        return ""
+
+    template_name = DOC_TYPE_MAP.get(doc_type, "")
+    keyword       = template_name.replace("_TEMPLATE.md", "").split("_")[0].lower()
+
+    for f in AUSGEFUELLT_DIR.glob("*.md"):
+        if keyword in f.name.lower():
+            return f.read_text(encoding="utf-8")[:800]
+
+    return ""
+
+
+def _transform(raw_text: str, template: str, example: str = "") -> str:
+    example_block = (
+        f"\n\nReferenzbeispiel (so soll das Ergebnis aussehen):\n{example}\n"
+        if example else ""
+    )
+
+    system_prompt = (
+        "Du bist ein Dokumenten-Strukturierungs-Assistent für Bewerbungsunterlagen.\n"
+        "Deine Aufgabe: Extrahiere Informationen aus einem Rohtext und fülle sie "
+        "exakt in die vorgegebene Markdown-Template-Struktur ein.\n\n"
+        "REGELN:\n"
+        "1. Übernimm AUSSCHLIESSLICH Informationen die tatsächlich im Rohtext stehen\n"
+        "2. Erfinde KEINE Daten, halluziniere NICHT\n"
+        f"3. Felder die nicht befüllt werden können → mit '{_MISSING_MARKER}' markieren\n"
+        "4. Behalte die EXAKTE Markdown-Struktur (Überschriften, Tabellen, Listen) bei\n"
+        "5. Antworte NUR mit dem ausgefüllten Markdown, KEIN Kommentar davor/danach\n"
+        "6. Sprache: Deutsch (wie im Original-Template)\n"
+        "7. Platzhalterwörter wie [z.B. ...] immer durch echte Daten oder [FEHLT] ersetzen"
+    )
+
+    user_prompt = (
+        f"Template-Struktur:\n{template}"
+        f"{example_block}"
+        f"\n\n---\nRohtext des Dokuments:\n{raw_text}\n\n---\n"
+        "Fülle das Template mit den Informationen aus dem Rohtext aus."
+    )
+
+    response = _client.messages.create(
+        model="claude-sonnet-4-6",
+        max_tokens=3000,
+        temperature=0.1,
+        system=system_prompt,
+        messages=[{"role": "user", "content": user_prompt}],
+    )
+
+    return response.content[0].text.strip()
+
+
+def _passthrough(raw_text: str, doc_type: str = "unbekannt") -> dict:
+    return {
+        "normalized_text": raw_text,
+        "doc_type":        doc_type,
+        "template_file":   "",
+        "quality_score":   1.0,
+        "missing_fields":  0,
+        "was_normalized":  False,
+    }