diff --git a/extractor.py b/extractor.py new file mode 100644 index 0000000..83c27f4 --- /dev/null +++ b/extractor.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +extractor.py — Text-Extraktion aus verschiedenen Dateiformaten. + +Unterstützte Formate: + .txt .md → direkt lesen + .docx → python-docx + .pdf → pypdf (mit OCR-Fallback bei Scans) + .png .jpg .jpeg → OCR via Tesseract + .tif .tiff .bmp → OCR via Tesseract + .webp → OCR via Tesseract + +Tesseract muss systemweit installiert sein: + brew install tesseract tesseract-lang +""" + +import re +from pathlib import Path + +try: + from pypdf import PdfReader + _PDF_SUPPORT = True +except ImportError: + _PDF_SUPPORT = False + +try: + from docx import Document as DocxDocument + _DOCX_SUPPORT = True +except ImportError: + _DOCX_SUPPORT = False + +try: + from PIL import Image + import pytesseract + _OCR_SUPPORT = True +except ImportError: + _OCR_SUPPORT = False + +try: + from pdf2image import convert_from_path + _PDF2IMAGE_SUPPORT = True +except ImportError: + _PDF2IMAGE_SUPPORT = False + + +SUPPORTED_EXTENSIONS = { + ".txt", ".md", + ".pdf", + ".docx", + ".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp", +} + +_PDF_MIN_TEXT_LENGTH = 150 +_OCR_LANG = "deu+eng" + + +def extract(path: Path) -> str: + if not path.exists(): + raise FileNotFoundError(f"Datei nicht gefunden: {path}") + + suffix = path.suffix.lower() + + if suffix not in SUPPORTED_EXTENSIONS: + raise ValueError( + f"Nicht unterstütztes Format '{suffix}'. " + f"Unterstützt: {', '.join(sorted(SUPPORTED_EXTENSIONS))}" + ) + + if suffix in (".txt", ".md"): + return _read_text(path) + elif suffix == ".docx": + return _read_docx(path) + elif suffix == ".pdf": + return _read_pdf(path) + else: + return _ocr_image(path) + + +def is_supported(path: Path) -> bool: + return path.suffix.lower() in SUPPORTED_EXTENSIONS + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _read_docx(path: Path) -> str: + if not _DOCX_SUPPORT: + raise ImportError("python-docx nicht installiert: pip install python-docx") + + doc = DocxDocument(str(path)) + parts = [] + + for element in doc.element.body: + tag = element.tag.split("}")[-1] + if tag == "p": + runs = "".join( + r.text for r in element.iter() + if r.tag.endswith("}t") and r.text + ) + text = runs.strip() + if text: + parts.append(text) + elif tag == "tbl": + for row in element.iter(): + if row.tag.endswith("}tr"): + cells = [ + "".join( + t.text or "" for t in cell.iter() + if t.tag.endswith("}t") + ).strip() + for cell in row + if cell.tag.endswith("}tc") + ] + if any(cells): + parts.append(" | ".join(cells)) + + return "\n\n".join(parts) + + +def _read_pdf(path: Path) -> str: + if not _PDF_SUPPORT: + raise ImportError("pypdf nicht installiert: pip install pypdf") + + reader = PdfReader(str(path)) + pages = [page.extract_text() or "" for page in reader.pages] + text = "\n\n".join(pages).strip() + + if len(text) < _PDF_MIN_TEXT_LENGTH: + if not _PDF2IMAGE_SUPPORT or not _OCR_SUPPORT: + raise ImportError( + "PDF scheint ein Scan zu sein, aber pdf2image oder pytesseract fehlen. " + "Installieren: pip install pdf2image pytesseract && brew install tesseract" + ) + print(" ℹ️ PDF scheint ein Scan – verwende OCR-Fallback...") + text = _ocr_pdf(path) + + return text + + +def _ocr_pdf(path: Path) -> str: + images = convert_from_path(str(path), dpi=300) + pages = [] + for i, img in enumerate(images, 1): + page_text = pytesseract.image_to_string(img, lang=_OCR_LANG) + pages.append(page_text) + print(f" 📄 OCR Seite {i}/{len(images)} abgeschlossen") + return _clean_ocr("\n\n".join(pages)) + + +def _ocr_image(path: Path) -> str: + if not _OCR_SUPPORT: + raise ImportError( + "pytesseract oder Pillow nicht installiert. " + "Installieren: pip install pytesseract Pillow && brew install tesseract tesseract-lang" + ) + img = Image.open(str(path)) + raw = pytesseract.image_to_string(img, lang=_OCR_LANG) + return _clean_ocr(raw) + + +def _clean_ocr(text: str) -> str: + text = re.sub(r"[^\S\n]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + text = re.sub(r"[|}{~`\\^]", "", text) + text = re.sub(r"(\w)-\n(\w)", r"\1\2", text) + return text.strip() \ No newline at end of file diff --git a/ingest_V2.py b/ingest_V2.py new file mode 100644 index 0000000..8363752 --- /dev/null +++ b/ingest_V2.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +""" +ingest.py — RAG Ingestion Pipeline (v2) + +Ablauf pro Datei: + 1. Extraktion → extractor.py (Text, OCR, DOCX, PDF, Bilder) + 2. Normalisierung → normalizer.py (KI: Rohtext → Template-Struktur) + 3. Chunking → token-basiert mit Overlap + 4. Embedding → multilingual-e5-small (lokal) + 5. Speichern → pgvector via SSH-Tunnel (Hetzner / anythingllm) + +CLI-Befehle: + python ingest.py file Einzelne Datei + python ingest.py dir Ganzes Verzeichnis + python ingest.py watch Ordner live beobachten + python ingest.py list DB-Inhalt anzeigen + python ingest.py delete Dokument aus DB löschen + +Flags: + --force Bestehende Chunks überschreiben + --no-normalize KI-Normalisierung überspringen + --dry-run Extrahieren + normalisieren, aber nicht in DB speichern + --quality-min 0.4 Mindest-Qualitäts-Score (Standard: 0.0 = alles speichern) +""" + +import os +import sys +import uuid +import json +import argparse +from datetime import datetime +from pathlib import Path + +import tiktoken +import psycopg2 +from psycopg2.extras import execute_values +from sshtunnel import SSHTunnelForwarder +from dotenv import load_dotenv + +from extractor import extract, is_supported, SUPPORTED_EXTENSIONS +from normalizer import normalize + +load_dotenv(".env.ingest") + +# --------------------------------------------------------------------------- +# Konfiguration (aus .env.ingest) +# --------------------------------------------------------------------------- +SSH_HOST = os.getenv("SSH_HOST") +SSH_USER = os.getenv("SSH_USER", "root") +SSH_KEY_PATH = os.path.expanduser(os.getenv("SSH_KEY_PATH", "~/.ssh/id_ed25519")) +SSH_KEY_PASSPHRASE = os.getenv("SSH_KEY_PASSPHRASE") +DB_HOST = os.getenv("DB_HOST", "127.0.0.1") +DB_PORT = int(os.getenv("DB_PORT", 5432)) +DB_NAME = os.getenv("DB_NAME", "anythingllm") +DB_USER = os.getenv("DB_USER", "anythingllm") +DB_PASSWORD = os.getenv("DB_PASSWORD") +NAMESPACE = os.getenv("NAMESPACE", "mein-workspace") +CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", 500)) +CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", 50)) +EMBED_MODEL = "intfloat/multilingual-e5-small" + +_embed_model = None # lazy load + + +# --------------------------------------------------------------------------- +# Chunking +# --------------------------------------------------------------------------- + +def chunk_text(text: str) -> list[str]: + """Teilt Text in überlappende Token-Chunks auf.""" + enc = tiktoken.get_encoding("cl100k_base") + tokens = enc.encode(text) + chunks = [] + start = 0 + while start < len(tokens): + end = min(start + CHUNK_SIZE, len(tokens)) + chunks.append(enc.decode(tokens[start:end])) + start += CHUNK_SIZE - CHUNK_OVERLAP + return chunks + + +# --------------------------------------------------------------------------- +# Embedding +# --------------------------------------------------------------------------- + +def embed(texts: list[str]) -> list[list[float]]: + """Erstellt Embeddings via multilingual-e5-small (lokal, einmaliger Download ~120 MB).""" + global _embed_model + if _embed_model is None: + from sentence_transformers import SentenceTransformer + print(" ⏳ Lade Embedding-Modell (einmalig)...") + _embed_model = SentenceTransformer(EMBED_MODEL) + prefixed = [f"passage: {t}" for t in texts] + return _embed_model.encode(prefixed).tolist() + + +# --------------------------------------------------------------------------- +# Datenbank +# --------------------------------------------------------------------------- + +def _db_connect(tunnel_port: int): + return psycopg2.connect( + host="127.0.0.1", + port=tunnel_port, + dbname=DB_NAME, + user=DB_USER, + password=DB_PASSWORD, + ) + + +def _tunnel(): + return SSHTunnelForwarder( + (SSH_HOST, 22), + ssh_username=SSH_USER, + ssh_pkey=SSH_KEY_PATH, + ssh_private_key_password=SSH_KEY_PASSPHRASE, + remote_bind_address=("127.0.0.1", DB_PORT), + ) + + +def _count_existing(cur, title: str) -> int: + cur.execute( + "SELECT COUNT(*) FROM anythingllm_vectors " + "WHERE metadata->>'title' = %s AND namespace = %s", + (title, NAMESPACE), + ) + return cur.fetchone()[0] + + +def _delete_existing(cur, title: str): + cur.execute( + "DELETE FROM anythingllm_vectors " + "WHERE metadata->>'title' = %s AND namespace = %s", + (title, NAMESPACE), + ) + + +def _insert_chunks(cur, chunks, embeddings, source_path: Path, extra_meta: dict): + now = datetime.now().isoformat() + records = [] + for chunk, embedding in zip(chunks, embeddings): + metadata = { + "id": str(uuid.uuid4()), + "url": f"file://{source_path.resolve()}", + "text": chunk, + "title": source_path.name, + "docSource": "rag-ingestion-v2", + "published": now, + "wordCount": len(chunk.split()), + "chunkSource": str(source_path.resolve()), + **extra_meta, + } + records.append(( + str(uuid.uuid4()), + json.dumps(metadata), + NAMESPACE, + embedding, + )) + execute_values( + cur, + "INSERT INTO anythingllm_vectors (id, metadata, namespace, embedding) VALUES %s", + records, + template="(%s::uuid, %s::jsonb, %s, %s::vector)", + ) + + +# --------------------------------------------------------------------------- +# Kern-Pipeline +# --------------------------------------------------------------------------- + +def ingest_file( + file_path: str, + force: bool = False, + normalize_doc: bool = True, + dry_run: bool = False, + quality_min: float = 0.0, +) -> bool: + path = Path(file_path).resolve() + + if not path.exists(): + print(f"❌ Datei nicht gefunden: {path}") + return False + + if not is_supported(path): + print(f"⏭️ Übersprungen (Format nicht unterstützt): {path.suffix}") + return False + + print(f"\n📄 {path.name}") + + # ── 1. Extraktion ──────────────────────────────────────────────────────── + try: + raw_text = extract(path) + print(f" ✅ Text extrahiert ({len(raw_text):,} Zeichen)") + except Exception as e: + print(f" ❌ Extraktion fehlgeschlagen: {e}") + return False + + if not raw_text.strip(): + print(" ⚠️ Kein Text extrahiert — Datei übersprungen") + return False + + # ── 2. KI-Normalisierung ───────────────────────────────────────────────── + try: + norm = normalize( + raw_text, + filename=path.name, + skip_normalization=not normalize_doc, + ) + text_to_index = norm["normalized_text"] + except Exception as e: + print(f" ❌ Normalisierung fehlgeschlagen: {e}") + print(" ↩️ Verwende Rohtext als Fallback") + norm = { + "normalized_text": raw_text, + "doc_type": "unbekannt", + "template_file": "", + "quality_score": 0.5, + "missing_fields": 0, + "was_normalized": False, + } + text_to_index = raw_text + + # Qualitäts-Filter + if norm["quality_score"] < quality_min: + print( + f" ⏭️ Qualitäts-Score {norm['quality_score']:.0%} " + f"< Mindest-Score {quality_min:.0%} — übersprungen" + ) + return False + + extra_meta = { + "doc_type": norm["doc_type"], + "quality_score": norm["quality_score"], + "missing_fields": norm["missing_fields"], + "was_normalized": norm["was_normalized"], + "template_file": norm["template_file"], + "source_format": path.suffix.lower(), + } + + # ── 3. Chunking ────────────────────────────────────────────────────────── + chunks = chunk_text(text_to_index) + print(f" ✅ {len(chunks)} Chunks erstellt") + + # ── 4. Embedding ───────────────────────────────────────────────────────── + try: + embeddings = embed(chunks) + print(f" ✅ {len(embeddings)} Embeddings erstellt") + except Exception as e: + print(f" ❌ Embedding fehlgeschlagen: {e}") + return False + + # ── Dry-Run Ende ───────────────────────────────────────────────────────── + if dry_run: + print(f" 🔍 Dry-Run — Typ: {norm['doc_type']} | " + f"Score: {norm['quality_score']:.0%} | " + f"Fehlend: {norm['missing_fields']}") + if norm["was_normalized"]: + print("\n" + "─" * 60) + print(text_to_index[:600] + ("..." if len(text_to_index) > 600 else "")) + print("─" * 60) + return True + + # ── 5. Datenbank ───────────────────────────────────────────────────────── + print(" ⏳ Verbinde mit Datenbank (SSH-Tunnel)...") + try: + with _tunnel() as tunnel: + conn = _db_connect(tunnel.local_bind_port) + cur = conn.cursor() + + existing = _count_existing(cur, path.name) + if existing > 0: + if force: + _delete_existing(cur, path.name) + print(f" 🗑️ {existing} bestehende Chunks gelöscht") + else: + print( + f" ⚠️ '{path.name}' bereits in DB ({existing} Chunks). " + "Nutze --force zum Überschreiben." + ) + cur.close(); conn.close() + return False + + _insert_chunks(cur, chunks, embeddings, path, extra_meta) + conn.commit() + cur.close(); conn.close() + print(f" ✅ {len(chunks)} Chunks gespeichert " + f"(Typ: {norm['doc_type']}, Score: {norm['quality_score']:.0%})") + except Exception as e: + print(f" ❌ Datenbankfehler: {e}") + return False + + return True + + +def ingest_directory( + dir_path: str, + force: bool = False, + normalize_doc: bool = True, + dry_run: bool = False, + quality_min: float = 0.0, +): + path = Path(dir_path).resolve() + if not path.is_dir(): + print(f"❌ Verzeichnis nicht gefunden: {path}") + return + + files = [f for f in sorted(path.iterdir()) if f.is_file() and is_supported(f)] + if not files: + print(f"❌ Keine unterstützten Dateien in: {path}") + print(f" Unterstützte Formate: {', '.join(sorted(SUPPORTED_EXTENSIONS))}") + return + + print(f"\n📁 {len(files)} Datei(en) in: {path}") + success = 0 + for f in files: + if ingest_file(str(f), force=force, normalize_doc=normalize_doc, + dry_run=dry_run, quality_min=quality_min): + success += 1 + + print(f"\n{'─' * 50}") + print(f"✅ Fertig: {success}/{len(files)} Dateien verarbeitet") + + +def watch_directory( + dir_path: str, + force: bool = False, + normalize_doc: bool = True, + quality_min: float = 0.0, +): + try: + from watchdog.observers import Observer + from watchdog.events import FileSystemEventHandler + except ImportError: + print("❌ watchdog nicht installiert: pip install watchdog") + sys.exit(1) + + class _Handler(FileSystemEventHandler): + def on_created(self, event): + if event.is_directory: + return + p = Path(event.src_path) + if is_supported(p): + print(f"\n🆕 Neue Datei erkannt: {p.name}") + ingest_file(str(p), force=force, normalize_doc=normalize_doc, + quality_min=quality_min) + + observer = Observer() + observer.schedule(_Handler(), dir_path, recursive=False) + observer.start() + print(f"👁️ Beobachte: {dir_path}") + print(" Drücke Ctrl+C zum Beenden\n") + try: + import time + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + observer.join() + + +def list_documents(): + print(f"\n📋 Dokumente in Namespace '{NAMESPACE}':\n") + try: + with _tunnel() as tunnel: + conn = _db_connect(tunnel.local_bind_port) + cur = conn.cursor() + cur.execute(""" + SELECT + metadata->>'title' AS titel, + metadata->>'doc_type' AS typ, + metadata->>'quality_score' AS score, + metadata->>'source_format' AS format, + COUNT(*) AS chunks, + MAX(metadata->>'published') AS datum + FROM anythingllm_vectors + WHERE namespace = %s + GROUP BY titel, typ, score, format + ORDER BY datum DESC + """, (NAMESPACE,)) + rows = cur.fetchall() + cur.close(); conn.close() + + if not rows: + print(" Keine Dokumente gefunden.") + return + + header = f" {'Titel':<45} {'Typ':<20} {'Score':>6} {'Fmt':>5} {'Chunks':>6} Datum" + print(header) + print(" " + "─" * (len(header) - 2)) + for titel, typ, score, fmt, chunks, datum in rows: + score_str = f"{float(score):.0%}" if score else "–" + print( + f" {(titel or '?'):<45} {(typ or '?'):<20} " + f"{score_str:>6} {(fmt or '?'):>5} {chunks:>6} " + f"{(datum or '?')[:19]}" + ) + print() + except Exception as e: + print(f"❌ Datenbankfehler: {e}") + + +def delete_document(title: str): + print(f"\n🗑️ Lösche '{title}' aus Namespace '{NAMESPACE}'...") + try: + with _tunnel() as tunnel: + conn = _db_connect(tunnel.local_bind_port) + cur = conn.cursor() + existing = _count_existing(cur, title) + if existing == 0: + print(f" ⚠️ Kein Dokument mit Titel '{title}' gefunden") + cur.close(); conn.close() + return + _delete_existing(cur, title) + conn.commit() + cur.close(); conn.close() + print(f" ✅ {existing} Chunks gelöscht") + except Exception as e: + print(f" ❌ Datenbankfehler: {e}") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="RAG Ingestion v2 — Dokumente in pgvector importieren", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Beispiele: + python ingest.py file ~/Downloads/fremdes_cv.pdf + python ingest.py file ~/Desktop/scan.png --dry-run + python ingest.py dir ~/Dokumente/bewerbung/ --force + python ingest.py watch ~/Desktop/scan-eingang/ + python ingest.py list + python ingest.py delete "fremdes_cv.pdf" + """ + ) + sub = p.add_subparsers(dest="command", required=True) + + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--force", action="store_true", help="Bestehende Chunks überschreiben") + common.add_argument("--no-normalize", action="store_true", help="KI-Normalisierung überspringen") + common.add_argument("--quality-min", type=float, default=0.0, metavar="0.0-1.0", + help="Mindest-Qualitäts-Score (Standard: 0.0)") + common.add_argument("--dry-run", action="store_true", + help="Nur extrahieren + normalisieren, nicht speichern") + + pf = sub.add_parser("file", parents=[common], help="Einzelne Datei importieren") + pf.add_argument("path", help="Pfad zur Datei") + + pd = sub.add_parser("dir", parents=[common], help="Verzeichnis importieren") + pd.add_argument("path", help="Pfad zum Verzeichnis") + + pw = sub.add_parser("watch", parents=[common], help="Ordner live beobachten") + pw.add_argument("path", help="Pfad zum Verzeichnis") + + sub.add_parser("list", help="Alle Dokumente in der DB anzeigen") + + pdel = sub.add_parser("delete", help="Dokument aus DB löschen") + pdel.add_argument("title", help="Titel (Dateiname) des Dokuments") + + return p + + +def main(): + args = _build_parser().parse_args() + do_normalize = not getattr(args, "no_normalize", False) + force = getattr(args, "force", False) + dry_run = getattr(args, "dry_run", False) + quality_min = getattr(args, "quality_min", 0.0) + + if args.command == "file": + ingest_file(args.path, force=force, normalize_doc=do_normalize, + dry_run=dry_run, quality_min=quality_min) + elif args.command == "dir": + ingest_directory(args.path, force=force, normalize_doc=do_normalize, + dry_run=dry_run, quality_min=quality_min) + elif args.command == "watch": + watch_directory(args.path, force=force, normalize_doc=do_normalize, + quality_min=quality_min) + elif args.command == "list": + list_documents() + elif args.command == "delete": + delete_document(args.title) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/normalizer.py b/normalizer.py new file mode 100644 index 0000000..fbb8841 --- /dev/null +++ b/normalizer.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +normalizer.py — KI-gestützte Dokumenten-Normalisierung. + +Ablauf: + 1. Dokumenttyp via KI erkennen (Claude Haiku — günstig + schnell) + 2. Passendes Template aus templates/ laden + 3. Rohtext → Template-Struktur transformieren (Claude Sonnet) + 4. Qualitäts-Score berechnen (Anteil befüllter Felder) + +API-Key: ANTHROPIC_API_KEY in .env.ingest +""" + +import os +import re +from pathlib import Path + +import anthropic +from dotenv import load_dotenv + +load_dotenv(".env.ingest") + +_client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) + +TEMPLATES_DIR = Path(__file__).parent / "templates" +AUSGEFUELLT_DIR = TEMPLATES_DIR / "ausgefuellt" + +DOC_TYPE_MAP: dict[str, str] = { + "projektbeschreibung": "Projektbeschreibung_TEMPLATE.md", + "anschreiben": "Muster-Anschreiben_TEMPLATE.md", + "karriereziele": "Karriereziele_TEMPLATE.md", + "rahmenbedingungen": "Rahmenbedingungen_TEMPLATE.md", + "referenzen": "Referenzen_TEMPLATE.md", + "technologie": "Technologie-Bewertung_TEMPLATE.md", + "elevatorpitch": "Ueber-mich_Elevator-Pitch_TEMPLATE.md", + "zertifikate": "Zertifikate_Weiterbildungen_TEMPLATE.md", + "zielstellen": "Zielstellen-Profil_TEMPLATE.md", + "cv": "Projektbeschreibung_TEMPLATE.md", + "lebenslauf": "Projektbeschreibung_TEMPLATE.md", + "motivation": "Muster-Anschreiben_TEMPLATE.md", + "skills": "Technologie-Bewertung_TEMPLATE.md", + "skillmatrix": "Technologie-Bewertung_TEMPLATE.md", + "pitch": "Ueber-mich_Elevator-Pitch_TEMPLATE.md", + "profil": "Ueber-mich_Elevator-Pitch_TEMPLATE.md", + "weiterbildung": "Zertifikate_Weiterbildungen_TEMPLATE.md", + "arbeitszeugnis": "Projektbeschreibung_TEMPLATE.md", + "zeugnis": "Projektbeschreibung_TEMPLATE.md", +} + +_MISSING_MARKER = "[FEHLT]" +_QUALITY_WARN_THRESHOLD = 0.5 + + +def normalize(raw_text: str, filename: str = "", skip_normalization: bool = False) -> dict: + if skip_normalization or not raw_text.strip(): + return _passthrough(raw_text) + + print(" 🤖 Erkenne Dokumenttyp...") + doc_type = _detect_doc_type(raw_text, filename) + print(f" ✅ Erkannter Typ: '{doc_type}'") + + template_filename = DOC_TYPE_MAP.get(doc_type, "") + template_text = _load_template(template_filename) + + if not template_text: + print(f" ⚠️ Kein Template für Typ '{doc_type}' — Rohtext wird unverändert verwendet") + return _passthrough(raw_text, doc_type=doc_type) + + print(f" 🤖 Normalisiere in Template '{template_filename}'...") + example_text = _load_best_example(doc_type) + normalized = _transform(raw_text, template_text, example_text) + + missing_count = len(re.findall(re.escape(_MISSING_MARKER), normalized)) + placeholder_count = len(re.findall(r"\[.+?\]", template_text)) + quality_score = round( + max(0.0, 1.0 - (missing_count / max(placeholder_count, 1))), + 2, + ) + + if quality_score < _QUALITY_WARN_THRESHOLD: + print( + f" ⚠️ Niedriger Qualitäts-Score ({quality_score:.0%}) — " + f"{missing_count} Felder konnten nicht befüllt werden" + ) + else: + print(f" ✅ Qualitäts-Score: {quality_score:.0%} ({missing_count} fehlende Felder)") + + return { + "normalized_text": normalized, + "doc_type": doc_type, + "template_file": template_filename, + "quality_score": quality_score, + "missing_fields": missing_count, + "was_normalized": True, + } + + +def list_doc_types() -> list[str]: + return sorted(set(DOC_TYPE_MAP.keys())) + + +def _detect_doc_type(raw_text: str, filename: str = "") -> str: + known_types = ", ".join(sorted(set(DOC_TYPE_MAP.keys()))) + filename_hint = f"\nDateiname: {filename}" if filename else "" + + response = _client.messages.create( + model="claude-haiku-4-5-20251001", + max_tokens=30, + temperature=0, + messages=[{ + "role": "user", + "content": ( + f"Bestimme den Dokumenttyp dieses Textes.\n" + f"Erlaubte Typen: {known_types}\n" + f"{filename_hint}\n\n" + f"Antworte NUR mit einem der erlaubten Typen, nichts anderes.\n\n" + f"Text (erste 1200 Zeichen):\n{raw_text[:1200]}" + ), + }], + ) + + detected = response.content[0].text.strip().lower() + detected = re.sub(r"[^a-zäöüß]", "", detected) + + if detected in DOC_TYPE_MAP: + return detected + for key in DOC_TYPE_MAP: + if key in detected or detected in key: + return key + + return "projektbeschreibung" + + +def _load_template(filename: str) -> str: + if not filename: + return "" + path = TEMPLATES_DIR / filename + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def _load_best_example(doc_type: str) -> str: + if not AUSGEFUELLT_DIR.exists(): + return "" + + template_name = DOC_TYPE_MAP.get(doc_type, "") + keyword = template_name.replace("_TEMPLATE.md", "").split("_")[0].lower() + + for f in AUSGEFUELLT_DIR.glob("*.md"): + if keyword in f.name.lower(): + return f.read_text(encoding="utf-8")[:800] + + return "" + + +def _transform(raw_text: str, template: str, example: str = "") -> str: + example_block = ( + f"\n\nReferenzbeispiel (so soll das Ergebnis aussehen):\n{example}\n" + if example else "" + ) + + system_prompt = ( + "Du bist ein Dokumenten-Strukturierungs-Assistent für Bewerbungsunterlagen.\n" + "Deine Aufgabe: Extrahiere Informationen aus einem Rohtext und fülle sie " + "exakt in die vorgegebene Markdown-Template-Struktur ein.\n\n" + "REGELN:\n" + "1. Übernimm AUSSCHLIESSLICH Informationen die tatsächlich im Rohtext stehen\n" + "2. Erfinde KEINE Daten, halluziniere NICHT\n" + f"3. Felder die nicht befüllt werden können → mit '{_MISSING_MARKER}' markieren\n" + "4. Behalte die EXAKTE Markdown-Struktur (Überschriften, Tabellen, Listen) bei\n" + "5. Antworte NUR mit dem ausgefüllten Markdown, KEIN Kommentar davor/danach\n" + "6. Sprache: Deutsch (wie im Original-Template)\n" + "7. Platzhalterwörter wie [z.B. ...] immer durch echte Daten oder [FEHLT] ersetzen" + ) + + user_prompt = ( + f"Template-Struktur:\n{template}" + f"{example_block}" + f"\n\n---\nRohtext des Dokuments:\n{raw_text}\n\n---\n" + "Fülle das Template mit den Informationen aus dem Rohtext aus." + ) + + response = _client.messages.create( + model="claude-sonnet-4-6", + max_tokens=3000, + temperature=0.1, + system=system_prompt, + messages=[{"role": "user", "content": user_prompt}], + ) + + return response.content[0].text.strip() + + +def _passthrough(raw_text: str, doc_type: str = "unbekannt") -> dict: + return { + "normalized_text": raw_text, + "doc_type": doc_type, + "template_file": "", + "quality_score": 1.0, + "missing_fields": 0, + "was_normalized": False, + } \ No newline at end of file