feat: extractor, normalizer, ingest v2 – OCR pipeline funktioniert
This commit is contained in:
+167
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
extractor.py — Text-Extraktion aus verschiedenen Dateiformaten.
|
||||
|
||||
Unterstützte Formate:
|
||||
.txt .md → direkt lesen
|
||||
.docx → python-docx
|
||||
.pdf → pypdf (mit OCR-Fallback bei Scans)
|
||||
.png .jpg .jpeg → OCR via Tesseract
|
||||
.tif .tiff .bmp → OCR via Tesseract
|
||||
.webp → OCR via Tesseract
|
||||
|
||||
Tesseract muss systemweit installiert sein:
|
||||
brew install tesseract tesseract-lang
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
_PDF_SUPPORT = True
|
||||
except ImportError:
|
||||
_PDF_SUPPORT = False
|
||||
|
||||
try:
|
||||
from docx import Document as DocxDocument
|
||||
_DOCX_SUPPORT = True
|
||||
except ImportError:
|
||||
_DOCX_SUPPORT = False
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
import pytesseract
|
||||
_OCR_SUPPORT = True
|
||||
except ImportError:
|
||||
_OCR_SUPPORT = False
|
||||
|
||||
try:
|
||||
from pdf2image import convert_from_path
|
||||
_PDF2IMAGE_SUPPORT = True
|
||||
except ImportError:
|
||||
_PDF2IMAGE_SUPPORT = False
|
||||
|
||||
|
||||
SUPPORTED_EXTENSIONS = {
|
||||
".txt", ".md",
|
||||
".pdf",
|
||||
".docx",
|
||||
".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp",
|
||||
}
|
||||
|
||||
_PDF_MIN_TEXT_LENGTH = 150
|
||||
_OCR_LANG = "deu+eng"
|
||||
|
||||
|
||||
def extract(path: Path) -> str:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Datei nicht gefunden: {path}")
|
||||
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
if suffix not in SUPPORTED_EXTENSIONS:
|
||||
raise ValueError(
|
||||
f"Nicht unterstütztes Format '{suffix}'. "
|
||||
f"Unterstützt: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
|
||||
)
|
||||
|
||||
if suffix in (".txt", ".md"):
|
||||
return _read_text(path)
|
||||
elif suffix == ".docx":
|
||||
return _read_docx(path)
|
||||
elif suffix == ".pdf":
|
||||
return _read_pdf(path)
|
||||
else:
|
||||
return _ocr_image(path)
|
||||
|
||||
|
||||
def is_supported(path: Path) -> bool:
|
||||
return path.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _read_docx(path: Path) -> str:
|
||||
if not _DOCX_SUPPORT:
|
||||
raise ImportError("python-docx nicht installiert: pip install python-docx")
|
||||
|
||||
doc = DocxDocument(str(path))
|
||||
parts = []
|
||||
|
||||
for element in doc.element.body:
|
||||
tag = element.tag.split("}")[-1]
|
||||
if tag == "p":
|
||||
runs = "".join(
|
||||
r.text for r in element.iter()
|
||||
if r.tag.endswith("}t") and r.text
|
||||
)
|
||||
text = runs.strip()
|
||||
if text:
|
||||
parts.append(text)
|
||||
elif tag == "tbl":
|
||||
for row in element.iter():
|
||||
if row.tag.endswith("}tr"):
|
||||
cells = [
|
||||
"".join(
|
||||
t.text or "" for t in cell.iter()
|
||||
if t.tag.endswith("}t")
|
||||
).strip()
|
||||
for cell in row
|
||||
if cell.tag.endswith("}tc")
|
||||
]
|
||||
if any(cells):
|
||||
parts.append(" | ".join(cells))
|
||||
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _read_pdf(path: Path) -> str:
|
||||
if not _PDF_SUPPORT:
|
||||
raise ImportError("pypdf nicht installiert: pip install pypdf")
|
||||
|
||||
reader = PdfReader(str(path))
|
||||
pages = [page.extract_text() or "" for page in reader.pages]
|
||||
text = "\n\n".join(pages).strip()
|
||||
|
||||
if len(text) < _PDF_MIN_TEXT_LENGTH:
|
||||
if not _PDF2IMAGE_SUPPORT or not _OCR_SUPPORT:
|
||||
raise ImportError(
|
||||
"PDF scheint ein Scan zu sein, aber pdf2image oder pytesseract fehlen. "
|
||||
"Installieren: pip install pdf2image pytesseract && brew install tesseract"
|
||||
)
|
||||
print(" ℹ️ PDF scheint ein Scan – verwende OCR-Fallback...")
|
||||
text = _ocr_pdf(path)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _ocr_pdf(path: Path) -> str:
|
||||
images = convert_from_path(str(path), dpi=300)
|
||||
pages = []
|
||||
for i, img in enumerate(images, 1):
|
||||
page_text = pytesseract.image_to_string(img, lang=_OCR_LANG)
|
||||
pages.append(page_text)
|
||||
print(f" 📄 OCR Seite {i}/{len(images)} abgeschlossen")
|
||||
return _clean_ocr("\n\n".join(pages))
|
||||
|
||||
|
||||
def _ocr_image(path: Path) -> str:
|
||||
if not _OCR_SUPPORT:
|
||||
raise ImportError(
|
||||
"pytesseract oder Pillow nicht installiert. "
|
||||
"Installieren: pip install pytesseract Pillow && brew install tesseract tesseract-lang"
|
||||
)
|
||||
img = Image.open(str(path))
|
||||
raw = pytesseract.image_to_string(img, lang=_OCR_LANG)
|
||||
return _clean_ocr(raw)
|
||||
|
||||
|
||||
def _clean_ocr(text: str) -> str:
|
||||
text = re.sub(r"[^\S\n]+", " ", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
text = re.sub(r"[|}{~`\\^]", "", text)
|
||||
text = re.sub(r"(\w)-\n(\w)", r"\1\2", text)
|
||||
return text.strip()
|
||||
+489
@@ -0,0 +1,489 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ingest.py — RAG Ingestion Pipeline (v2)
|
||||
|
||||
Ablauf pro Datei:
|
||||
1. Extraktion → extractor.py (Text, OCR, DOCX, PDF, Bilder)
|
||||
2. Normalisierung → normalizer.py (KI: Rohtext → Template-Struktur)
|
||||
3. Chunking → token-basiert mit Overlap
|
||||
4. Embedding → multilingual-e5-small (lokal)
|
||||
5. Speichern → pgvector via SSH-Tunnel (Hetzner / anythingllm)
|
||||
|
||||
CLI-Befehle:
|
||||
python ingest.py file <pfad> Einzelne Datei
|
||||
python ingest.py dir <pfad> Ganzes Verzeichnis
|
||||
python ingest.py watch <pfad> Ordner live beobachten
|
||||
python ingest.py list DB-Inhalt anzeigen
|
||||
python ingest.py delete <titel> Dokument aus DB löschen
|
||||
|
||||
Flags:
|
||||
--force Bestehende Chunks überschreiben
|
||||
--no-normalize KI-Normalisierung überspringen
|
||||
--dry-run Extrahieren + normalisieren, aber nicht in DB speichern
|
||||
--quality-min 0.4 Mindest-Qualitäts-Score (Standard: 0.0 = alles speichern)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
import json
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import tiktoken
|
||||
import psycopg2
|
||||
from psycopg2.extras import execute_values
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from extractor import extract, is_supported, SUPPORTED_EXTENSIONS
|
||||
from normalizer import normalize
|
||||
|
||||
load_dotenv(".env.ingest")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Konfiguration (aus .env.ingest)
|
||||
# ---------------------------------------------------------------------------
|
||||
SSH_HOST = os.getenv("SSH_HOST")
|
||||
SSH_USER = os.getenv("SSH_USER", "root")
|
||||
SSH_KEY_PATH = os.path.expanduser(os.getenv("SSH_KEY_PATH", "~/.ssh/id_ed25519"))
|
||||
SSH_KEY_PASSPHRASE = os.getenv("SSH_KEY_PASSPHRASE")
|
||||
DB_HOST = os.getenv("DB_HOST", "127.0.0.1")
|
||||
DB_PORT = int(os.getenv("DB_PORT", 5432))
|
||||
DB_NAME = os.getenv("DB_NAME", "anythingllm")
|
||||
DB_USER = os.getenv("DB_USER", "anythingllm")
|
||||
DB_PASSWORD = os.getenv("DB_PASSWORD")
|
||||
NAMESPACE = os.getenv("NAMESPACE", "mein-workspace")
|
||||
CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", 500))
|
||||
CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", 50))
|
||||
EMBED_MODEL = "intfloat/multilingual-e5-small"
|
||||
|
||||
_embed_model = None # lazy load
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chunking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def chunk_text(text: str) -> list[str]:
|
||||
"""Teilt Text in überlappende Token-Chunks auf."""
|
||||
enc = tiktoken.get_encoding("cl100k_base")
|
||||
tokens = enc.encode(text)
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(tokens):
|
||||
end = min(start + CHUNK_SIZE, len(tokens))
|
||||
chunks.append(enc.decode(tokens[start:end]))
|
||||
start += CHUNK_SIZE - CHUNK_OVERLAP
|
||||
return chunks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def embed(texts: list[str]) -> list[list[float]]:
|
||||
"""Erstellt Embeddings via multilingual-e5-small (lokal, einmaliger Download ~120 MB)."""
|
||||
global _embed_model
|
||||
if _embed_model is None:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
print(" ⏳ Lade Embedding-Modell (einmalig)...")
|
||||
_embed_model = SentenceTransformer(EMBED_MODEL)
|
||||
prefixed = [f"passage: {t}" for t in texts]
|
||||
return _embed_model.encode(prefixed).tolist()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datenbank
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _db_connect(tunnel_port: int):
|
||||
return psycopg2.connect(
|
||||
host="127.0.0.1",
|
||||
port=tunnel_port,
|
||||
dbname=DB_NAME,
|
||||
user=DB_USER,
|
||||
password=DB_PASSWORD,
|
||||
)
|
||||
|
||||
|
||||
def _tunnel():
|
||||
return SSHTunnelForwarder(
|
||||
(SSH_HOST, 22),
|
||||
ssh_username=SSH_USER,
|
||||
ssh_pkey=SSH_KEY_PATH,
|
||||
ssh_private_key_password=SSH_KEY_PASSPHRASE,
|
||||
remote_bind_address=("127.0.0.1", DB_PORT),
|
||||
)
|
||||
|
||||
|
||||
def _count_existing(cur, title: str) -> int:
|
||||
cur.execute(
|
||||
"SELECT COUNT(*) FROM anythingllm_vectors "
|
||||
"WHERE metadata->>'title' = %s AND namespace = %s",
|
||||
(title, NAMESPACE),
|
||||
)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def _delete_existing(cur, title: str):
|
||||
cur.execute(
|
||||
"DELETE FROM anythingllm_vectors "
|
||||
"WHERE metadata->>'title' = %s AND namespace = %s",
|
||||
(title, NAMESPACE),
|
||||
)
|
||||
|
||||
|
||||
def _insert_chunks(cur, chunks, embeddings, source_path: Path, extra_meta: dict):
|
||||
now = datetime.now().isoformat()
|
||||
records = []
|
||||
for chunk, embedding in zip(chunks, embeddings):
|
||||
metadata = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"url": f"file://{source_path.resolve()}",
|
||||
"text": chunk,
|
||||
"title": source_path.name,
|
||||
"docSource": "rag-ingestion-v2",
|
||||
"published": now,
|
||||
"wordCount": len(chunk.split()),
|
||||
"chunkSource": str(source_path.resolve()),
|
||||
**extra_meta,
|
||||
}
|
||||
records.append((
|
||||
str(uuid.uuid4()),
|
||||
json.dumps(metadata),
|
||||
NAMESPACE,
|
||||
embedding,
|
||||
))
|
||||
execute_values(
|
||||
cur,
|
||||
"INSERT INTO anythingllm_vectors (id, metadata, namespace, embedding) VALUES %s",
|
||||
records,
|
||||
template="(%s::uuid, %s::jsonb, %s, %s::vector)",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kern-Pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ingest_file(
|
||||
file_path: str,
|
||||
force: bool = False,
|
||||
normalize_doc: bool = True,
|
||||
dry_run: bool = False,
|
||||
quality_min: float = 0.0,
|
||||
) -> bool:
|
||||
path = Path(file_path).resolve()
|
||||
|
||||
if not path.exists():
|
||||
print(f"❌ Datei nicht gefunden: {path}")
|
||||
return False
|
||||
|
||||
if not is_supported(path):
|
||||
print(f"⏭️ Übersprungen (Format nicht unterstützt): {path.suffix}")
|
||||
return False
|
||||
|
||||
print(f"\n📄 {path.name}")
|
||||
|
||||
# ── 1. Extraktion ────────────────────────────────────────────────────────
|
||||
try:
|
||||
raw_text = extract(path)
|
||||
print(f" ✅ Text extrahiert ({len(raw_text):,} Zeichen)")
|
||||
except Exception as e:
|
||||
print(f" ❌ Extraktion fehlgeschlagen: {e}")
|
||||
return False
|
||||
|
||||
if not raw_text.strip():
|
||||
print(" ⚠️ Kein Text extrahiert — Datei übersprungen")
|
||||
return False
|
||||
|
||||
# ── 2. KI-Normalisierung ─────────────────────────────────────────────────
|
||||
try:
|
||||
norm = normalize(
|
||||
raw_text,
|
||||
filename=path.name,
|
||||
skip_normalization=not normalize_doc,
|
||||
)
|
||||
text_to_index = norm["normalized_text"]
|
||||
except Exception as e:
|
||||
print(f" ❌ Normalisierung fehlgeschlagen: {e}")
|
||||
print(" ↩️ Verwende Rohtext als Fallback")
|
||||
norm = {
|
||||
"normalized_text": raw_text,
|
||||
"doc_type": "unbekannt",
|
||||
"template_file": "",
|
||||
"quality_score": 0.5,
|
||||
"missing_fields": 0,
|
||||
"was_normalized": False,
|
||||
}
|
||||
text_to_index = raw_text
|
||||
|
||||
# Qualitäts-Filter
|
||||
if norm["quality_score"] < quality_min:
|
||||
print(
|
||||
f" ⏭️ Qualitäts-Score {norm['quality_score']:.0%} "
|
||||
f"< Mindest-Score {quality_min:.0%} — übersprungen"
|
||||
)
|
||||
return False
|
||||
|
||||
extra_meta = {
|
||||
"doc_type": norm["doc_type"],
|
||||
"quality_score": norm["quality_score"],
|
||||
"missing_fields": norm["missing_fields"],
|
||||
"was_normalized": norm["was_normalized"],
|
||||
"template_file": norm["template_file"],
|
||||
"source_format": path.suffix.lower(),
|
||||
}
|
||||
|
||||
# ── 3. Chunking ──────────────────────────────────────────────────────────
|
||||
chunks = chunk_text(text_to_index)
|
||||
print(f" ✅ {len(chunks)} Chunks erstellt")
|
||||
|
||||
# ── 4. Embedding ─────────────────────────────────────────────────────────
|
||||
try:
|
||||
embeddings = embed(chunks)
|
||||
print(f" ✅ {len(embeddings)} Embeddings erstellt")
|
||||
except Exception as e:
|
||||
print(f" ❌ Embedding fehlgeschlagen: {e}")
|
||||
return False
|
||||
|
||||
# ── Dry-Run Ende ─────────────────────────────────────────────────────────
|
||||
if dry_run:
|
||||
print(f" 🔍 Dry-Run — Typ: {norm['doc_type']} | "
|
||||
f"Score: {norm['quality_score']:.0%} | "
|
||||
f"Fehlend: {norm['missing_fields']}")
|
||||
if norm["was_normalized"]:
|
||||
print("\n" + "─" * 60)
|
||||
print(text_to_index[:600] + ("..." if len(text_to_index) > 600 else ""))
|
||||
print("─" * 60)
|
||||
return True
|
||||
|
||||
# ── 5. Datenbank ─────────────────────────────────────────────────────────
|
||||
print(" ⏳ Verbinde mit Datenbank (SSH-Tunnel)...")
|
||||
try:
|
||||
with _tunnel() as tunnel:
|
||||
conn = _db_connect(tunnel.local_bind_port)
|
||||
cur = conn.cursor()
|
||||
|
||||
existing = _count_existing(cur, path.name)
|
||||
if existing > 0:
|
||||
if force:
|
||||
_delete_existing(cur, path.name)
|
||||
print(f" 🗑️ {existing} bestehende Chunks gelöscht")
|
||||
else:
|
||||
print(
|
||||
f" ⚠️ '{path.name}' bereits in DB ({existing} Chunks). "
|
||||
"Nutze --force zum Überschreiben."
|
||||
)
|
||||
cur.close(); conn.close()
|
||||
return False
|
||||
|
||||
_insert_chunks(cur, chunks, embeddings, path, extra_meta)
|
||||
conn.commit()
|
||||
cur.close(); conn.close()
|
||||
print(f" ✅ {len(chunks)} Chunks gespeichert "
|
||||
f"(Typ: {norm['doc_type']}, Score: {norm['quality_score']:.0%})")
|
||||
except Exception as e:
|
||||
print(f" ❌ Datenbankfehler: {e}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def ingest_directory(
|
||||
dir_path: str,
|
||||
force: bool = False,
|
||||
normalize_doc: bool = True,
|
||||
dry_run: bool = False,
|
||||
quality_min: float = 0.0,
|
||||
):
|
||||
path = Path(dir_path).resolve()
|
||||
if not path.is_dir():
|
||||
print(f"❌ Verzeichnis nicht gefunden: {path}")
|
||||
return
|
||||
|
||||
files = [f for f in sorted(path.iterdir()) if f.is_file() and is_supported(f)]
|
||||
if not files:
|
||||
print(f"❌ Keine unterstützten Dateien in: {path}")
|
||||
print(f" Unterstützte Formate: {', '.join(sorted(SUPPORTED_EXTENSIONS))}")
|
||||
return
|
||||
|
||||
print(f"\n📁 {len(files)} Datei(en) in: {path}")
|
||||
success = 0
|
||||
for f in files:
|
||||
if ingest_file(str(f), force=force, normalize_doc=normalize_doc,
|
||||
dry_run=dry_run, quality_min=quality_min):
|
||||
success += 1
|
||||
|
||||
print(f"\n{'─' * 50}")
|
||||
print(f"✅ Fertig: {success}/{len(files)} Dateien verarbeitet")
|
||||
|
||||
|
||||
def watch_directory(
|
||||
dir_path: str,
|
||||
force: bool = False,
|
||||
normalize_doc: bool = True,
|
||||
quality_min: float = 0.0,
|
||||
):
|
||||
try:
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
except ImportError:
|
||||
print("❌ watchdog nicht installiert: pip install watchdog")
|
||||
sys.exit(1)
|
||||
|
||||
class _Handler(FileSystemEventHandler):
|
||||
def on_created(self, event):
|
||||
if event.is_directory:
|
||||
return
|
||||
p = Path(event.src_path)
|
||||
if is_supported(p):
|
||||
print(f"\n🆕 Neue Datei erkannt: {p.name}")
|
||||
ingest_file(str(p), force=force, normalize_doc=normalize_doc,
|
||||
quality_min=quality_min)
|
||||
|
||||
observer = Observer()
|
||||
observer.schedule(_Handler(), dir_path, recursive=False)
|
||||
observer.start()
|
||||
print(f"👁️ Beobachte: {dir_path}")
|
||||
print(" Drücke Ctrl+C zum Beenden\n")
|
||||
try:
|
||||
import time
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
observer.stop()
|
||||
observer.join()
|
||||
|
||||
|
||||
def list_documents():
|
||||
print(f"\n📋 Dokumente in Namespace '{NAMESPACE}':\n")
|
||||
try:
|
||||
with _tunnel() as tunnel:
|
||||
conn = _db_connect(tunnel.local_bind_port)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT
|
||||
metadata->>'title' AS titel,
|
||||
metadata->>'doc_type' AS typ,
|
||||
metadata->>'quality_score' AS score,
|
||||
metadata->>'source_format' AS format,
|
||||
COUNT(*) AS chunks,
|
||||
MAX(metadata->>'published') AS datum
|
||||
FROM anythingllm_vectors
|
||||
WHERE namespace = %s
|
||||
GROUP BY titel, typ, score, format
|
||||
ORDER BY datum DESC
|
||||
""", (NAMESPACE,))
|
||||
rows = cur.fetchall()
|
||||
cur.close(); conn.close()
|
||||
|
||||
if not rows:
|
||||
print(" Keine Dokumente gefunden.")
|
||||
return
|
||||
|
||||
header = f" {'Titel':<45} {'Typ':<20} {'Score':>6} {'Fmt':>5} {'Chunks':>6} Datum"
|
||||
print(header)
|
||||
print(" " + "─" * (len(header) - 2))
|
||||
for titel, typ, score, fmt, chunks, datum in rows:
|
||||
score_str = f"{float(score):.0%}" if score else "–"
|
||||
print(
|
||||
f" {(titel or '?'):<45} {(typ or '?'):<20} "
|
||||
f"{score_str:>6} {(fmt or '?'):>5} {chunks:>6} "
|
||||
f"{(datum or '?')[:19]}"
|
||||
)
|
||||
print()
|
||||
except Exception as e:
|
||||
print(f"❌ Datenbankfehler: {e}")
|
||||
|
||||
|
||||
def delete_document(title: str):
|
||||
print(f"\n🗑️ Lösche '{title}' aus Namespace '{NAMESPACE}'...")
|
||||
try:
|
||||
with _tunnel() as tunnel:
|
||||
conn = _db_connect(tunnel.local_bind_port)
|
||||
cur = conn.cursor()
|
||||
existing = _count_existing(cur, title)
|
||||
if existing == 0:
|
||||
print(f" ⚠️ Kein Dokument mit Titel '{title}' gefunden")
|
||||
cur.close(); conn.close()
|
||||
return
|
||||
_delete_existing(cur, title)
|
||||
conn.commit()
|
||||
cur.close(); conn.close()
|
||||
print(f" ✅ {existing} Chunks gelöscht")
|
||||
except Exception as e:
|
||||
print(f" ❌ Datenbankfehler: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
description="RAG Ingestion v2 — Dokumente in pgvector importieren",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Beispiele:
|
||||
python ingest.py file ~/Downloads/fremdes_cv.pdf
|
||||
python ingest.py file ~/Desktop/scan.png --dry-run
|
||||
python ingest.py dir ~/Dokumente/bewerbung/ --force
|
||||
python ingest.py watch ~/Desktop/scan-eingang/
|
||||
python ingest.py list
|
||||
python ingest.py delete "fremdes_cv.pdf"
|
||||
"""
|
||||
)
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument("--force", action="store_true", help="Bestehende Chunks überschreiben")
|
||||
common.add_argument("--no-normalize", action="store_true", help="KI-Normalisierung überspringen")
|
||||
common.add_argument("--quality-min", type=float, default=0.0, metavar="0.0-1.0",
|
||||
help="Mindest-Qualitäts-Score (Standard: 0.0)")
|
||||
common.add_argument("--dry-run", action="store_true",
|
||||
help="Nur extrahieren + normalisieren, nicht speichern")
|
||||
|
||||
pf = sub.add_parser("file", parents=[common], help="Einzelne Datei importieren")
|
||||
pf.add_argument("path", help="Pfad zur Datei")
|
||||
|
||||
pd = sub.add_parser("dir", parents=[common], help="Verzeichnis importieren")
|
||||
pd.add_argument("path", help="Pfad zum Verzeichnis")
|
||||
|
||||
pw = sub.add_parser("watch", parents=[common], help="Ordner live beobachten")
|
||||
pw.add_argument("path", help="Pfad zum Verzeichnis")
|
||||
|
||||
sub.add_parser("list", help="Alle Dokumente in der DB anzeigen")
|
||||
|
||||
pdel = sub.add_parser("delete", help="Dokument aus DB löschen")
|
||||
pdel.add_argument("title", help="Titel (Dateiname) des Dokuments")
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def main():
|
||||
args = _build_parser().parse_args()
|
||||
do_normalize = not getattr(args, "no_normalize", False)
|
||||
force = getattr(args, "force", False)
|
||||
dry_run = getattr(args, "dry_run", False)
|
||||
quality_min = getattr(args, "quality_min", 0.0)
|
||||
|
||||
if args.command == "file":
|
||||
ingest_file(args.path, force=force, normalize_doc=do_normalize,
|
||||
dry_run=dry_run, quality_min=quality_min)
|
||||
elif args.command == "dir":
|
||||
ingest_directory(args.path, force=force, normalize_doc=do_normalize,
|
||||
dry_run=dry_run, quality_min=quality_min)
|
||||
elif args.command == "watch":
|
||||
watch_directory(args.path, force=force, normalize_doc=do_normalize,
|
||||
quality_min=quality_min)
|
||||
elif args.command == "list":
|
||||
list_documents()
|
||||
elif args.command == "delete":
|
||||
delete_document(args.title)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
normalizer.py — KI-gestützte Dokumenten-Normalisierung.
|
||||
|
||||
Ablauf:
|
||||
1. Dokumenttyp via KI erkennen (Claude Haiku — günstig + schnell)
|
||||
2. Passendes Template aus templates/ laden
|
||||
3. Rohtext → Template-Struktur transformieren (Claude Sonnet)
|
||||
4. Qualitäts-Score berechnen (Anteil befüllter Felder)
|
||||
|
||||
API-Key: ANTHROPIC_API_KEY in .env.ingest
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import anthropic
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(".env.ingest")
|
||||
|
||||
_client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
|
||||
|
||||
TEMPLATES_DIR = Path(__file__).parent / "templates"
|
||||
AUSGEFUELLT_DIR = TEMPLATES_DIR / "ausgefuellt"
|
||||
|
||||
DOC_TYPE_MAP: dict[str, str] = {
|
||||
"projektbeschreibung": "Projektbeschreibung_TEMPLATE.md",
|
||||
"anschreiben": "Muster-Anschreiben_TEMPLATE.md",
|
||||
"karriereziele": "Karriereziele_TEMPLATE.md",
|
||||
"rahmenbedingungen": "Rahmenbedingungen_TEMPLATE.md",
|
||||
"referenzen": "Referenzen_TEMPLATE.md",
|
||||
"technologie": "Technologie-Bewertung_TEMPLATE.md",
|
||||
"elevatorpitch": "Ueber-mich_Elevator-Pitch_TEMPLATE.md",
|
||||
"zertifikate": "Zertifikate_Weiterbildungen_TEMPLATE.md",
|
||||
"zielstellen": "Zielstellen-Profil_TEMPLATE.md",
|
||||
"cv": "Projektbeschreibung_TEMPLATE.md",
|
||||
"lebenslauf": "Projektbeschreibung_TEMPLATE.md",
|
||||
"motivation": "Muster-Anschreiben_TEMPLATE.md",
|
||||
"skills": "Technologie-Bewertung_TEMPLATE.md",
|
||||
"skillmatrix": "Technologie-Bewertung_TEMPLATE.md",
|
||||
"pitch": "Ueber-mich_Elevator-Pitch_TEMPLATE.md",
|
||||
"profil": "Ueber-mich_Elevator-Pitch_TEMPLATE.md",
|
||||
"weiterbildung": "Zertifikate_Weiterbildungen_TEMPLATE.md",
|
||||
"arbeitszeugnis": "Projektbeschreibung_TEMPLATE.md",
|
||||
"zeugnis": "Projektbeschreibung_TEMPLATE.md",
|
||||
}
|
||||
|
||||
_MISSING_MARKER = "[FEHLT]"
|
||||
_QUALITY_WARN_THRESHOLD = 0.5
|
||||
|
||||
|
||||
def normalize(raw_text: str, filename: str = "", skip_normalization: bool = False) -> dict:
|
||||
if skip_normalization or not raw_text.strip():
|
||||
return _passthrough(raw_text)
|
||||
|
||||
print(" 🤖 Erkenne Dokumenttyp...")
|
||||
doc_type = _detect_doc_type(raw_text, filename)
|
||||
print(f" ✅ Erkannter Typ: '{doc_type}'")
|
||||
|
||||
template_filename = DOC_TYPE_MAP.get(doc_type, "")
|
||||
template_text = _load_template(template_filename)
|
||||
|
||||
if not template_text:
|
||||
print(f" ⚠️ Kein Template für Typ '{doc_type}' — Rohtext wird unverändert verwendet")
|
||||
return _passthrough(raw_text, doc_type=doc_type)
|
||||
|
||||
print(f" 🤖 Normalisiere in Template '{template_filename}'...")
|
||||
example_text = _load_best_example(doc_type)
|
||||
normalized = _transform(raw_text, template_text, example_text)
|
||||
|
||||
missing_count = len(re.findall(re.escape(_MISSING_MARKER), normalized))
|
||||
placeholder_count = len(re.findall(r"\[.+?\]", template_text))
|
||||
quality_score = round(
|
||||
max(0.0, 1.0 - (missing_count / max(placeholder_count, 1))),
|
||||
2,
|
||||
)
|
||||
|
||||
if quality_score < _QUALITY_WARN_THRESHOLD:
|
||||
print(
|
||||
f" ⚠️ Niedriger Qualitäts-Score ({quality_score:.0%}) — "
|
||||
f"{missing_count} Felder konnten nicht befüllt werden"
|
||||
)
|
||||
else:
|
||||
print(f" ✅ Qualitäts-Score: {quality_score:.0%} ({missing_count} fehlende Felder)")
|
||||
|
||||
return {
|
||||
"normalized_text": normalized,
|
||||
"doc_type": doc_type,
|
||||
"template_file": template_filename,
|
||||
"quality_score": quality_score,
|
||||
"missing_fields": missing_count,
|
||||
"was_normalized": True,
|
||||
}
|
||||
|
||||
|
||||
def list_doc_types() -> list[str]:
|
||||
return sorted(set(DOC_TYPE_MAP.keys()))
|
||||
|
||||
|
||||
def _detect_doc_type(raw_text: str, filename: str = "") -> str:
|
||||
known_types = ", ".join(sorted(set(DOC_TYPE_MAP.keys())))
|
||||
filename_hint = f"\nDateiname: {filename}" if filename else ""
|
||||
|
||||
response = _client.messages.create(
|
||||
model="claude-haiku-4-5-20251001",
|
||||
max_tokens=30,
|
||||
temperature=0,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Bestimme den Dokumenttyp dieses Textes.\n"
|
||||
f"Erlaubte Typen: {known_types}\n"
|
||||
f"{filename_hint}\n\n"
|
||||
f"Antworte NUR mit einem der erlaubten Typen, nichts anderes.\n\n"
|
||||
f"Text (erste 1200 Zeichen):\n{raw_text[:1200]}"
|
||||
),
|
||||
}],
|
||||
)
|
||||
|
||||
detected = response.content[0].text.strip().lower()
|
||||
detected = re.sub(r"[^a-zäöüß]", "", detected)
|
||||
|
||||
if detected in DOC_TYPE_MAP:
|
||||
return detected
|
||||
for key in DOC_TYPE_MAP:
|
||||
if key in detected or detected in key:
|
||||
return key
|
||||
|
||||
return "projektbeschreibung"
|
||||
|
||||
|
||||
def _load_template(filename: str) -> str:
|
||||
if not filename:
|
||||
return ""
|
||||
path = TEMPLATES_DIR / filename
|
||||
return path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
|
||||
|
||||
def _load_best_example(doc_type: str) -> str:
|
||||
if not AUSGEFUELLT_DIR.exists():
|
||||
return ""
|
||||
|
||||
template_name = DOC_TYPE_MAP.get(doc_type, "")
|
||||
keyword = template_name.replace("_TEMPLATE.md", "").split("_")[0].lower()
|
||||
|
||||
for f in AUSGEFUELLT_DIR.glob("*.md"):
|
||||
if keyword in f.name.lower():
|
||||
return f.read_text(encoding="utf-8")[:800]
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _transform(raw_text: str, template: str, example: str = "") -> str:
|
||||
example_block = (
|
||||
f"\n\nReferenzbeispiel (so soll das Ergebnis aussehen):\n{example}\n"
|
||||
if example else ""
|
||||
)
|
||||
|
||||
system_prompt = (
|
||||
"Du bist ein Dokumenten-Strukturierungs-Assistent für Bewerbungsunterlagen.\n"
|
||||
"Deine Aufgabe: Extrahiere Informationen aus einem Rohtext und fülle sie "
|
||||
"exakt in die vorgegebene Markdown-Template-Struktur ein.\n\n"
|
||||
"REGELN:\n"
|
||||
"1. Übernimm AUSSCHLIESSLICH Informationen die tatsächlich im Rohtext stehen\n"
|
||||
"2. Erfinde KEINE Daten, halluziniere NICHT\n"
|
||||
f"3. Felder die nicht befüllt werden können → mit '{_MISSING_MARKER}' markieren\n"
|
||||
"4. Behalte die EXAKTE Markdown-Struktur (Überschriften, Tabellen, Listen) bei\n"
|
||||
"5. Antworte NUR mit dem ausgefüllten Markdown, KEIN Kommentar davor/danach\n"
|
||||
"6. Sprache: Deutsch (wie im Original-Template)\n"
|
||||
"7. Platzhalterwörter wie [z.B. ...] immer durch echte Daten oder [FEHLT] ersetzen"
|
||||
)
|
||||
|
||||
user_prompt = (
|
||||
f"Template-Struktur:\n{template}"
|
||||
f"{example_block}"
|
||||
f"\n\n---\nRohtext des Dokuments:\n{raw_text}\n\n---\n"
|
||||
"Fülle das Template mit den Informationen aus dem Rohtext aus."
|
||||
)
|
||||
|
||||
response = _client.messages.create(
|
||||
model="claude-sonnet-4-6",
|
||||
max_tokens=3000,
|
||||
temperature=0.1,
|
||||
system=system_prompt,
|
||||
messages=[{"role": "user", "content": user_prompt}],
|
||||
)
|
||||
|
||||
return response.content[0].text.strip()
|
||||
|
||||
|
||||
def _passthrough(raw_text: str, doc_type: str = "unbekannt") -> dict:
|
||||
return {
|
||||
"normalized_text": raw_text,
|
||||
"doc_type": doc_type,
|
||||
"template_file": "",
|
||||
"quality_score": 1.0,
|
||||
"missing_fields": 0,
|
||||
"was_normalized": False,
|
||||
}
|
||||
Reference in New Issue
Block a user