diff --git a/--help b/--help new file mode 100644 index 0000000..e69de29 diff --git a/.env b/.env index 3d29774..85901cb 100644 --- a/.env +++ b/.env @@ -20,13 +20,21 @@ LOG_STACK=single LOG_DEPRECATIONS_CHANNEL=null LOG_LEVEL=debug -DB_CONNECTION=sqlite +# DB_CONNECTION=pgsql # DB_HOST=127.0.0.1 # DB_PORT=3306 # DB_DATABASE=laravel # DB_USERNAME=root # DB_PASSWORD= +# Secondary PostgreSQL Connection (for backend schema) +DB_CONNECTION2=pgsql +DB_HOST2=127.0.0.1 +DB_PORT2=5433 +DB_DATABASE2=risk_ingest_db +DB_USERNAME2=risk_ingest_user +DB_PASSWORD2=S0prast3r1a + SESSION_DRIVER=database SESSION_LIFETIME=120 SESSION_ENCRYPT=false diff --git a/app/Models/Backend/Transaction.php b/app/Models/Backend/Transaction.php new file mode 100644 index 0000000..fffd473 --- /dev/null +++ b/app/Models/Backend/Transaction.php @@ -0,0 +1,166 @@ + 'decimal:2', + ]; + + /** + * KI-generierte Outputs für diese Transaktion + */ + public function outputs(): HasMany + { + return $this->hasMany(TransactionOutput::class, 'transaction_id'); + } + + /** + * Hole spezifischen Output-Typ + */ + public function getOutput(string $key): ?array + { + return $this->outputs() + ->where('output_key', $key) + ->first() + ?->content; + } + + /** + * Hole alle Outputs als Key-Value Array + */ + public function getOutputsArray(): array + { + return $this->outputs() + ->get() + ->pluck('content', 'output_key') + ->toArray(); + } + + /** + * Company Info aus Outputs + */ + public function getCompanyInfo(): ?array + { + return $this->getOutput('company_info'); + } + + /** + * Risk Assessment aus Outputs + */ + public function getRiskAssessment(): ?array + { + return $this->getOutput('risk_assessment'); + } + + /** + * Sanctions Check aus Outputs + */ + public function getSanctionsCheck(): ?array + { + return $this->getOutput('sanctions'); + } + + /** + * PEP Check aus Outputs + */ + public function getPepCheck(): ?array + { + return $this->getOutput('pep'); + } + + /** + * Helper: Hat diese Transaction einen bestimmten Output? + */ + public function hasOutput(string $key): bool + { + return $this->outputs()->where('output_key', $key)->exists(); + } + + /** + * Ist KI-Verarbeitung abgeschlossen? + */ + public function isProcessed(): bool + { + return $this->status === self::STATUS_COMPLETED; + } + + /** + * Benötigt Review? + */ + public function requiresReview(): bool + { + $risk = $this->getRiskAssessment(); + + if (!$risk) { + return true; // Keine Risk-Assessment → Review + } + + return ($risk['score'] ?? 0) >= 70 || ($risk['requires_review'] ?? false); + } + + /** + * Get display-friendly company name + */ + public function getCompanyName(): string + { + $companyInfo = $this->getCompanyInfo(); + + return $companyInfo['name'] ?? $this->corporate_entity; + } + + /** + * Get risk score (0-100) + */ + public function getRiskScore(): int + { + $risk = $this->getRiskAssessment(); + + return $risk['score'] ?? 0; + } + + /** + * Get risk level (low, medium, high) + */ + public function getRiskLevel(): string + { + $risk = $this->getRiskAssessment(); + + return $risk['level'] ?? 'unknown'; + } +} diff --git a/app/Models/Backend/TransactionOutput.php b/app/Models/Backend/TransactionOutput.php new file mode 100644 index 0000000..618f55b --- /dev/null +++ b/app/Models/Backend/TransactionOutput.php @@ -0,0 +1,109 @@ + 'array', + ]; + + /** + * Transaction zu der dieser Output gehört + */ + public function transaction(): BelongsTo + { + return $this->belongsTo(Transaction::class, 'transaction_id'); + } + + /** + * Output-Key Konstanten für Type Safety + */ + public const KEY_COMPANY_INFO = 'company_info'; + + public const KEY_RISK_ASSESSMENT = 'risk_assessment'; + + public const KEY_SANCTIONS = 'sanctions'; + + public const KEY_PEP = 'pep'; + + public const KEY_REGISTRY = 'registry'; + + public const KEY_GLEIF = 'gleif'; + + public const KEY_INSOLVENCY = 'insolvency'; + + public const KEY_BUNDESANZEIGER = 'bundesanzeiger'; + + public const KEY_RSS = 'rss'; + + public const KEY_EU_SANCTIONS = 'eu_sanctions'; + + public const KEY_HANDELSREGISTER = 'handelsregister'; + + public const KEY_GENESIS = 'genesis'; + + public const KEY_GOVDATA = 'govdata'; + + /** + * Alle verfügbaren Output-Keys + */ + public static function availableKeys(): array + { + return [ + self::KEY_COMPANY_INFO, + self::KEY_RISK_ASSESSMENT, + self::KEY_SANCTIONS, + self::KEY_PEP, + self::KEY_REGISTRY, + self::KEY_GLEIF, + self::KEY_INSOLVENCY, + self::KEY_BUNDESANZEIGER, + self::KEY_RSS, + self::KEY_EU_SANCTIONS, + self::KEY_HANDELSREGISTER, + self::KEY_GENESIS, + self::KEY_GOVDATA, + ]; + } + + /** + * Get human-readable label for output key + */ + public static function getKeyLabel(string $key): string + { + return match ($key) { + self::KEY_COMPANY_INFO => 'Company Information', + self::KEY_RISK_ASSESSMENT => 'Risk Assessment', + self::KEY_SANCTIONS => 'Sanctions Check', + self::KEY_PEP => 'PEP Check', + self::KEY_REGISTRY => 'Registry Data', + self::KEY_GLEIF => 'GLEIF Data', + self::KEY_INSOLVENCY => 'Insolvency Check', + self::KEY_BUNDESANZEIGER => 'Bundesanzeiger', + self::KEY_RSS => 'RSS Alerts', + self::KEY_EU_SANCTIONS => 'EU Sanctions', + self::KEY_HANDELSREGISTER => 'Handelsregister', + self::KEY_GENESIS => 'Genesis Data', + self::KEY_GOVDATA => 'GovData', + default => ucfirst(str_replace('_', ' ', $key)), + }; + } +} diff --git a/app/Repositories/TransactionRepository.php b/app/Repositories/TransactionRepository.php new file mode 100644 index 0000000..a6bd226 --- /dev/null +++ b/app/Repositories/TransactionRepository.php @@ -0,0 +1,229 @@ +orderBy('created_at', 'desc') + ->get(); + } + + /** + * Finde Transaction by ID mit Outputs + */ + public function find(int $id): ?Transaction + { + return Transaction::with('outputs')->find($id); + } + + /** + * Transaktionen die Review benötigen + */ + public function requiresReview(): Collection + { + return Transaction::with('outputs') + ->where('status', Transaction::STATUS_COMPLETED) + ->get() + ->filter(fn ($t) => $t->requiresReview()); + } + + /** + * High-Risk Transaktionen + */ + public function highRisk(int $threshold = 70): Collection + { + return $this->all() + ->filter(function ($transaction) use ($threshold) { + $risk = $transaction->getRiskAssessment(); + + return ($risk['score'] ?? 0) >= $threshold; + }); + } + + /** + * Transaktionen nach Firma + */ + public function byCompany(string $companyName): Collection + { + return Transaction::with('outputs') + ->where('corporate_entity', 'ILIKE', "%{$companyName}%") + ->orderBy('tx_date', 'desc') + ->get(); + } + + /** + * Transaktionen nach Status + */ + public function byStatus(string $status): Collection + { + return Transaction::with('outputs') + ->where('status', $status) + ->orderBy('created_at', 'desc') + ->get(); + } + + /** + * Pending Transaktionen (warten auf KI-Verarbeitung) + */ + public function pending(): Collection + { + return $this->byStatus(Transaction::STATUS_PENDING); + } + + /** + * Processing Transaktionen (werden gerade verarbeitet) + */ + public function processing(): Collection + { + return $this->byStatus(Transaction::STATUS_PROCESSING); + } + + /** + * Completed Transaktionen + */ + public function completed(): Collection + { + return $this->byStatus(Transaction::STATUS_COMPLETED); + } + + /** + * Failed Transaktionen + */ + public function failed(): Collection + { + return $this->byStatus(Transaction::STATUS_FAILED); + } + + /** + * Statistiken + */ + public function stats(): array + { + $total = Transaction::count(); + $pending = Transaction::where('status', Transaction::STATUS_PENDING)->count(); + $processing = Transaction::where('status', Transaction::STATUS_PROCESSING)->count(); + $completed = Transaction::where('status', Transaction::STATUS_COMPLETED)->count(); + $failed = Transaction::where('status', Transaction::STATUS_FAILED)->count(); + + // Review Statistics + $needsReview = $this->completed() + ->filter(fn ($t) => $t->requiresReview()) + ->count(); + + // Risk Statistics + $highRisk = $this->completed() + ->filter(fn ($t) => $t->getRiskScore() >= 70) + ->count(); + + $mediumRisk = $this->completed() + ->filter(fn ($t) => $t->getRiskScore() >= 40 && $t->getRiskScore() < 70) + ->count(); + + $lowRisk = $this->completed() + ->filter(fn ($t) => $t->getRiskScore() < 40) + ->count(); + + return compact( + 'total', + 'pending', + 'processing', + 'completed', + 'failed', + 'needsReview', + 'highRisk', + 'mediumRisk', + 'lowRisk' + ); + } + + /** + * Paginierte Transaktionen + */ + public function paginated(int $perPage = 20) + { + return Transaction::with('outputs') + ->orderBy('created_at', 'desc') + ->paginate($perPage); + } + + /** + * Suche in Transaktionen + */ + public function search(string $query): Collection + { + return Transaction::with('outputs') + ->where(function ($q) use ($query) { + $q->where('corporate_entity', 'ILIKE', "%{$query}%") + ->orWhere('corporate_counterparty', 'ILIKE', "%{$query}%") + ->orWhere('tx_purpose', 'ILIKE', "%{$query}%"); + }) + ->orderBy('created_at', 'desc') + ->get(); + } + + /** + * Transaktionen nach Datum-Range + */ + public function byDateRange(string $startDate, string $endDate): Collection + { + return Transaction::with('outputs') + ->whereBetween('tx_date', [$startDate, $endDate]) + ->orderBy('tx_date', 'desc') + ->get(); + } + + /** + * Alle unique Firmen + */ + public function getAllCompanies(): Collection + { + return DB::connection('backend') + ->table('transactions') + ->select('corporate_entity') + ->distinct() + ->orderBy('corporate_entity') + ->get(); + } + + /** + * Dashboard-Daten + */ + public function dashboardData(): array + { + $stats = $this->stats(); + + $recentTransactions = Transaction::with('outputs') + ->orderBy('created_at', 'desc') + ->limit(10) + ->get(); + + $recentHighRisk = $this->completed() + ->filter(fn ($t) => $t->getRiskScore() >= 70) + ->take(5); + + $topCompanies = DB::connection('backend') + ->table('transactions') + ->select('corporate_entity', DB::raw('COUNT(*) as count')) + ->groupBy('corporate_entity') + ->orderByDesc('count') + ->limit(10) + ->get(); + + return [ + 'stats' => $stats, + 'recent_transactions' => $recentTransactions, + 'recent_high_risk' => $recentHighRisk, + 'top_companies' => $topCompanies, + ]; + } +} diff --git a/config/database.php b/config/database.php index de11492..493c08a 100644 --- a/config/database.php +++ b/config/database.php @@ -16,7 +16,7 @@ return [ | */ - 'default' => env('DB_CONNECTION', 'sqlite'), + 'default' => env('DB_CONNECTION', 'pgsql_second'), /* |-------------------------------------------------------------------------- @@ -82,6 +82,35 @@ return [ ]) : [], ], + 'pgsql' => [ + 'driver' => 'pgsql', + 'url' => env('DB_URL'), + 'host' => env('DB_HOST2', '127.0.0.1'), + 'port' => env('DB_PORT2', '5433'), + 'database' => env('DB_DATABASE2', 'risk_ingest_db'), + 'username' => env('DB_USERNAME2', 'risk_ingest_user'), + 'password' => env('DB_PASSWORD2', ''), + 'charset' => env('DB_CHARSET', 'utf8'), + 'prefix' => '', + 'prefix_indexes' => true, + 'search_path' => 'public', + 'sslmode' => 'prefer', + ], + + 'backend' => [ + 'driver' => env('DB_CONNECTION2'), + 'host' => env('DB_HOST2', '127.0.0.1'), + 'port' => env('DB_PORT2', '5433'), + 'database' => env('DB_DATABASE2', 'laravel'), + 'username' => env('DB_USERNAME2', 'root'), + 'password' => env('DB_PASSWORD2', ''), + 'charset' => env('DB_CHARSET', 'utf8'), + 'prefix' => '', + 'prefix_indexes' => true, + 'search_path' => 'backend', // Backend Schema (gleiche DB wie pgsql_second, anderes Schema) + 'sslmode' => 'prefer', + ], + 'pgsql_second' => [ 'driver' => env('DB_CONNECTION2'), 'host' => env('DB_HOST2', '127.0.0.1'), @@ -146,7 +175,7 @@ return [ 'options' => [ 'cluster' => env('REDIS_CLUSTER', 'redis'), - 'prefix' => env('REDIS_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_').'_database_'), + 'prefix' => env('REDIS_PREFIX', Str::slug(env('APP_NAME', 'laravel'), '_') . '_database_'), 'persistent' => env('REDIS_PERSISTENT', false), ], diff --git a/misc/ETL_PIPELINE_MIGRATION_PLAN.md b/misc/ETL_PIPELINE_MIGRATION_PLAN.md new file mode 100644 index 0000000..90f9df9 --- /dev/null +++ b/misc/ETL_PIPELINE_MIGRATION_PLAN.md @@ -0,0 +1,1414 @@ +# ETL-Pipeline Migration: CSV → KI Workflow → Frontend + +## System-Architektur (Klargestellt) + +### Aktueller Zustand (Nicht optimal) +``` +┌─────────────────────────────────────────────────────┐ +│ Laravel Frontend │ +│ │ +│ CSV Upload → Manual Processing │ +│ ↓ │ +│ public.companies (11 Felder) │ +│ ↓ │ +│ public.transactions (49 Felder, komplex) │ +│ - Core-Felder (14) │ +│ - Enrichment-Felder (35) │ +│ │ +└─────────────────────────────────────────────────────┘ +``` + +**Probleme**: +- ❌ Companies und Transactions vermischt +- ❌ Enrichment-Logik direkt in Laravel +- ❌ Keine klare Trennung: Raw Data vs Processed Data +- ❌ Skalierbarkeit begrenzt + +--- + +### Ziel-Zustand (Backend-Pipeline) +``` +┌──────────────────────────────────────────────────────────────┐ +│ CSV Upload │ +│ ↓ │ +│ backend.transactions (14 Felder) │ +│ - corporate_entity │ +│ - corporate_counterparty │ +│ - tx_date, tx_amount, tx_currency │ +│ - raw_payload (Original-CSV) │ +│ - status: 'pending' │ +└─────────────────────────┬────────────────────────────────────┘ + │ + ↓ +┌──────────────────────────────────────────────────────────────┐ +│ KI Workflow (Python/MCP) │ +│ │ +│ Für jede Transaction: │ +│ 1. Analyse: Risk Assessment │ +│ 2. Enrichment: External APIs │ +│ 3. Output: Strukturierte Ergebnisse │ +│ │ +│ Nutzt: backend.prompt_templates │ +│ backend.prompt_runs │ +└─────────────────────────┬────────────────────────────────────┘ + │ + ↓ +┌──────────────────────────────────────────────────────────────┐ +│ backend.transaction_outputs (5 Felder) │ +│ │ +│ - transaction_id (FK → backend.transactions.id) │ +│ - prompt_id (FK → backend.prompt_templates.id) │ +│ - output_key (z.B. 'risk_assessment', 'sanctions_check') │ +│ - content (JSON mit Ergebnissen) │ +│ - run_id (Batch-Tracking) │ +│ │ +│ Beispiel-Outputs: │ +│ • 'risk_assessment' → {score: 85, level: 'high'} │ +│ • 'sanctions_check' → {found: true, matches: [...]} │ +│ • 'pep_check' → {is_pep: false} │ +│ • 'company_info' → {name, sector, country} │ +└─────────────────────────┬────────────────────────────────────┘ + │ + ↓ +┌──────────────────────────────────────────────────────────────┐ +│ Laravel Frontend │ +│ │ +│ Liest von: │ +│ - backend.transactions (Raw Data) │ +│ - backend.transaction_outputs (Processed Results) │ +│ │ +│ Zeigt an: │ +│ - Transaction Details │ +│ - Risk Score & Assessment │ +│ - Enrichment Results (Sanctions, PEP, etc.) │ +│ - Review Queue │ +└──────────────────────────────────────────────────────────────┘ +``` + +--- + +## Mapping: Alt → Neu + +### 1. Companies Table → ENTFÄLLT + +**Warum?** +- Companies sind **keine separate Entität** mehr +- Firmeninformationen kommen aus: + 1. `backend.transactions.corporate_entity` (Name aus CSV) + 2. `backend.transaction_outputs` mit `output_key='company_info'` (KI-Enrichment) + +**Migration**: +```sql +-- Bestehende Companies werden zu Transaction Outputs +INSERT INTO backend.transaction_outputs ( + transaction_id, + prompt_id, + output_key, + content +) +SELECT + bt.id as transaction_id, + 1 as prompt_id, -- Default prompt für Company Info + 'company_info' as output_key, + jsonb_build_object( + 'name', c.name, + 'legal_name', c.legal_name, + 'ticker', c.ticker, + 'sector', c.sector, + 'country', c.country, + 'headquarters', c.headquarters, + 'kyc_risk_level', c.kyc_risk_level, + 'summary', c.summary + ) as content +FROM public.companies c +JOIN public.transactions t ON t.company_id = c.id +JOIN backend.transactions bt ON bt.corporate_entity = c.name; +``` + +### 2. Transactions Table → transaction_outputs + +**public.transactions (49 Felder)**: +``` +Core-Felder (14): +- company_id, reference, amount, currency +- counterparty, executed_at, status +- risk_score, requires_review +→ Gespeichert in backend.transactions + +Enrichment-Felder (35): +- registry_data, sanctions_data, pep_data, ... +→ Jedes wird zu einem Output in transaction_outputs +``` + +**Mapping-Strategie**: + +| public.transactions Feld | Ziel | output_key | +|--------------------------|------|------------| +| amount, currency, counterparty | backend.transactions | - | +| risk_score, requires_review | transaction_outputs | 'risk_assessment' | +| registry_data | transaction_outputs | 'registry' | +| sanctions_data | transaction_outputs | 'sanctions' | +| pep_data | transaction_outputs | 'pep' | +| gleif_data | transaction_outputs | 'gleif' | +| ... | ... | ... | + +--- + +## Phase 1: Backend Models & Relationships (Tag 1-2) + +### 1.1 Database Config + +**config/database.php**: +```php +'backend' => [ + 'driver' => 'pgsql', + 'host' => env('DB_HOST', '127.0.0.1'), + 'port' => env('DB_PORT', '5432'), + 'database' => env('DB_DATABASE', 'forge'), + 'username' => env('DB_USERNAME', 'forge'), + 'password' => env('DB_PASSWORD', ''), + 'charset' => 'utf8', + 'prefix' => '', + 'search_path' => 'backend', + 'sslmode' => 'prefer', +], +``` + +### 1.2 Backend\Transaction Model + +**app/Models/Backend/Transaction.php**: +```php + 'decimal:2', + // tx_date ist noch text - später zu timestamp migrieren + ]; + + /** + * KI-generierte Outputs für diese Transaktion + */ + public function outputs(): HasMany + { + return $this->hasMany(TransactionOutput::class, 'transaction_id'); + } + + /** + * Hole spezifischen Output-Typ + */ + public function getOutput(string $key): ?array + { + return $this->outputs() + ->where('output_key', $key) + ->first() + ?->content; + } + + /** + * Hole alle Outputs als Key-Value Array + */ + public function getOutputsArray(): array + { + return $this->outputs() + ->get() + ->pluck('content', 'output_key') + ->toArray(); + } + + /** + * Company Info aus Outputs + */ + public function getCompanyInfo(): ?array + { + return $this->getOutput('company_info'); + } + + /** + * Risk Assessment aus Outputs + */ + public function getRiskAssessment(): ?array + { + return $this->getOutput('risk_assessment'); + } + + /** + * Helper: Hat diese Transaction einen bestimmten Output? + */ + public function hasOutput(string $key): bool + { + return $this->outputs()->where('output_key', $key)->exists(); + } + + /** + * Ist KI-Verarbeitung abgeschlossen? + */ + public function isProcessed(): bool + { + return $this->status === self::STATUS_COMPLETED; + } + + /** + * Benötigt Review? + */ + public function requiresReview(): bool + { + $risk = $this->getRiskAssessment(); + + if (!$risk) { + return true; // Keine Risk-Assessment → Review + } + + return ($risk['score'] ?? 0) >= 70 || ($risk['requires_review'] ?? false); + } +} +``` + +### 1.3 Backend\TransactionOutput Model + +**app/Models/Backend/TransactionOutput.php**: +```php + 'array', // Automatisch JSON encode/decode + ]; + + /** + * Transaction zu der dieser Output gehört + */ + public function transaction(): BelongsTo + { + return $this->belongsTo(Transaction::class, 'transaction_id'); + } + + /** + * Prompt Template das diesen Output erzeugt hat + */ + public function promptTemplate(): BelongsTo + { + return $this->belongsTo(PromptTemplate::class, 'prompt_id'); + } + + /** + * Output-Key Konstanten für Type Safety + */ + public const KEY_COMPANY_INFO = 'company_info'; + public const KEY_RISK_ASSESSMENT = 'risk_assessment'; + public const KEY_SANCTIONS = 'sanctions'; + public const KEY_PEP = 'pep'; + public const KEY_REGISTRY = 'registry'; + public const KEY_GLEIF = 'gleif'; + public const KEY_INSOLVENCY = 'insolvency'; + public const KEY_BUNDESANZEIGER = 'bundesanzeiger'; + public const KEY_RSS = 'rss'; + public const KEY_EU_SANCTIONS = 'eu_sanctions'; + public const KEY_HANDELSREGISTER = 'handelsregister'; + + /** + * Alle verfügbaren Output-Keys + */ + public static function availableKeys(): array + { + return [ + self::KEY_COMPANY_INFO, + self::KEY_RISK_ASSESSMENT, + self::KEY_SANCTIONS, + self::KEY_PEP, + self::KEY_REGISTRY, + self::KEY_GLEIF, + self::KEY_INSOLVENCY, + self::KEY_BUNDESANZEIGER, + self::KEY_RSS, + self::KEY_EU_SANCTIONS, + self::KEY_HANDELSREGISTER, + ]; + } +} +``` + +### 1.4 Backend\PromptTemplate Model (optional) + +**app/Models/Backend/PromptTemplate.php**: +```php +hasMany(TransactionOutput::class, 'prompt_id'); + } +} +``` + +--- + +## Phase 2: CSV Upload Service (Tag 3-4) + +### 2.1 CSV Upload Controller + +**app/Http/Controllers/TransactionUploadController.php**: +```php +validate([ + 'csv_file' => 'required|file|mimes:csv,txt|max:10240', // 10MB + ]); + + try { + $result = $this->uploadService->process($validated['csv_file']); + + return redirect() + ->route('transactions.index') + ->with('success', "Imported {$result['count']} transactions. Processing started."); + } catch (\Exception $e) { + return back() + ->withErrors(['csv_file' => $e->getMessage()]) + ->withInput(); + } + } +} +``` + +### 2.2 Upload Service + +**app/Services/TransactionUploadService.php**: +```php +storeFile($file); + + // 2. Parse CSV + $rows = $this->parseCsv($file); + + // 3. Validiere Daten + $this->validate($rows); + + // 4. Importiere zu backend.transactions + $transactions = $this->import($rows, $fileName); + + // 5. Starte KI-Verarbeitung (async) + $this->queueAiProcessing($transactions); + + return [ + 'count' => count($transactions), + 'file' => $fileName, + ]; + } + + /** + * Speichere Original-CSV für Audit + */ + private function storeFile(UploadedFile $file): string + { + return Storage::disk('local')->putFileAs( + 'uploads/transactions', + $file, + date('Y-m-d_His') . '_' . $file->getClientOriginalName() + ); + } + + /** + * Parse CSV-Datei + */ + private function parseCsv(UploadedFile $file): array + { + $handle = fopen($file->getRealPath(), 'r'); + $headers = fgetcsv($handle); // Erste Zeile = Header + + $rows = []; + while (($data = fgetcsv($handle)) !== false) { + $rows[] = array_combine($headers, $data); + } + + fclose($handle); + + return $rows; + } + + /** + * Validiere CSV-Daten + */ + private function validate(array $rows): void + { + if (empty($rows)) { + throw new \InvalidArgumentException('CSV file is empty'); + } + + $requiredColumns = [ + 'corporate_entity', + 'corporate_counterparty', + 'tx_date', + 'tx_amount', + ]; + + $headers = array_keys($rows[0]); + $missing = array_diff($requiredColumns, $headers); + + if (!empty($missing)) { + throw new \InvalidArgumentException( + 'Missing required columns: ' . implode(', ', $missing) + ); + } + } + + /** + * Importiere Transaktionen + */ + private function import(array $rows, string $fileName): array + { + $transactions = []; + + DB::connection('backend')->transaction(function () use ($rows, $fileName, &$transactions) { + foreach ($rows as $row) { + $transactions[] = Transaction::create([ + 'corporate_entity' => $row['corporate_entity'], + 'corporate_counterparty' => $row['corporate_counterparty'], + 'tx_date' => $row['tx_date'], + 'tx_amount' => (float) $row['tx_amount'], + 'tx_currency' => $row['tx_currency'] ?? 'EUR', + 'tx_purpose' => $row['tx_purpose'] ?? null, + 'tx_country_outgoing' => $row['tx_country_outgoing'] ?? null, + 'tx_country_incoming' => $row['tx_country_incoming'] ?? null, + 'source_file' => $fileName, + 'raw_payload' => json_encode($row), + 'status' => Transaction::STATUS_PENDING, + ]); + } + }); + + return $transactions; + } + + /** + * Starte KI-Verarbeitung für alle Transaktionen + */ + private function queueAiProcessing(array $transactions): void + { + foreach ($transactions as $transaction) { + ProcessTransactionWithAI::dispatch($transaction); + } + } +} +``` + +--- + +## Phase 3: KI Workflow Integration (Tag 5-7) + +### 3.1 AI Processing Job + +**app/Jobs/ProcessTransactionWithAI.php**: +```php +transaction->update(['status' => Transaction::STATUS_PROCESSING]); + + try { + // 1. Company Info Enrichment + $companyInfo = $aiService->enrichCompanyInfo($this->transaction); + $this->storeOutput('company_info', $companyInfo); + + // 2. Risk Assessment + $riskAssessment = $aiService->assessRisk($this->transaction); + $this->storeOutput('risk_assessment', $riskAssessment); + + // 3. Sanctions Check + $sanctionsCheck = $aiService->checkSanctions($this->transaction); + $this->storeOutput('sanctions', $sanctionsCheck); + + // 4. PEP Check + $pepCheck = $aiService->checkPep($this->transaction); + $this->storeOutput('pep', $pepCheck); + + // 5. Weitere Checks (conditional) + if ($riskAssessment['score'] >= 50) { + $registryData = $aiService->checkRegistry($this->transaction); + $this->storeOutput('registry', $registryData); + } + + // Fertig + $this->transaction->update(['status' => Transaction::STATUS_COMPLETED]); + + } catch (\Exception $e) { + $this->transaction->update([ + 'status' => Transaction::STATUS_FAILED, + ]); + + // Store Error Output + $this->storeOutput('error', [ + 'message' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + + throw $e; + } + } + + private function storeOutput(string $key, array $data): void + { + TransactionOutput::create([ + 'transaction_id' => $this->transaction->id, + 'prompt_id' => 1, // TODO: Map zu echtem Prompt Template + 'output_key' => $key, + 'content' => $data, + 'run_id' => null, // TODO: Track Batch-Runs + ]); + } +} +``` + +### 3.2 AI Workflow Service (Stub) + +**app/Services/AiWorkflowService.php**: +```php + $transaction->corporate_entity, + 'legal_name' => null, + 'country' => $transaction->tx_country_outgoing, + 'sector' => 'Unknown', + 'kyc_risk_level' => 'medium', + 'enriched_at' => now()->toIso8601String(), + ]; + } + + /** + * Risk Assessment + */ + public function assessRisk(Transaction $transaction): array + { + // TODO: Echte Risk-Logik via KI + + $amount = (float) $transaction->tx_amount; + $score = 0; + + // Einfache Heuristik + if ($amount > 100000) { + $score += 30; + } + if ($transaction->tx_country_incoming !== 'DE') { + $score += 20; + } + + return [ + 'score' => $score, + 'level' => $score >= 70 ? 'high' : ($score >= 40 ? 'medium' : 'low'), + 'requires_review' => $score >= 70, + 'factors' => [ + 'high_amount' => $amount > 100000, + 'foreign_country' => $transaction->tx_country_incoming !== 'DE', + ], + 'assessed_at' => now()->toIso8601String(), + ]; + } + + /** + * Sanctions Check + */ + public function checkSanctions(Transaction $transaction): array + { + // TODO: API zu Sanktionslisten + + return [ + 'checked' => true, + 'found' => false, + 'matches' => [], + 'sources' => ['EU', 'OFAC'], + 'checked_at' => now()->toIso8601String(), + ]; + } + + /** + * PEP Check + */ + public function checkPep(Transaction $transaction): array + { + // TODO: PEP Database Check + + return [ + 'is_pep' => false, + 'confidence' => 0.0, + 'matches' => [], + 'checked_at' => now()->toIso8601String(), + ]; + } + + /** + * Registry Check + */ + public function checkRegistry(Transaction $transaction): array + { + // TODO: Handelsregister API + + return [ + 'found' => false, + 'company_number' => null, + 'match_score' => 0.0, + 'data' => null, + 'checked_at' => now()->toIso8601String(), + ]; + } +} +``` + +--- + +## Phase 4: Frontend Anpassung (Tag 8-10) + +### 4.1 Transaction Repository + +**app/Repositories/TransactionRepository.php**: +```php +orderBy('created_at', 'desc') + ->get(); + } + + /** + * Transaktionen die Review benötigen + */ + public function requiresReview(): Collection + { + return Transaction::with('outputs') + ->where('status', Transaction::STATUS_COMPLETED) + ->get() + ->filter(fn($t) => $t->requiresReview()); + } + + /** + * High-Risk Transaktionen + */ + public function highRisk(int $threshold = 70): Collection + { + return $this->allWithOutputs() + ->filter(function ($transaction) use ($threshold) { + $risk = $transaction->getRiskAssessment(); + return ($risk['score'] ?? 0) >= $threshold; + }); + } + + /** + * Transaktionen nach Firma + */ + public function byCompany(string $companyName): Collection + { + return Transaction::with('outputs') + ->where('corporate_entity', $companyName) + ->orderBy('tx_date', 'desc') + ->get(); + } + + /** + * Statistiken + */ + public function stats(): array + { + $total = Transaction::count(); + $pending = Transaction::where('status', Transaction::STATUS_PENDING)->count(); + $processing = Transaction::where('status', Transaction::STATUS_PROCESSING)->count(); + $completed = Transaction::where('status', Transaction::STATUS_COMPLETED)->count(); + $failed = Transaction::where('status', Transaction::STATUS_FAILED)->count(); + + return compact('total', 'pending', 'processing', 'completed', 'failed'); + } +} +``` + +### 4.2 Transaction Controller + +**app/Http/Controllers/TransactionController.php**: +```php +transactions->allWithOutputs(); + $stats = $this->transactions->stats(); + + return view('transactions.index', compact('transactions', 'stats')); + } + + public function show(int $id) + { + $transaction = $this->transactions->find($id); + + if (!$transaction) { + abort(404); + } + + return view('transactions.show', compact('transaction')); + } + + public function review() + { + $transactions = $this->transactions->requiresReview(); + + return view('transactions.review', compact('transactions')); + } +} +``` + +### 4.3 Volt Component für Transaction List + +**resources/views/pages/transactions/index.blade.php**: +```php + $transactionRepo->allWithOutputs()); +$stats = computed(fn() => $transactionRepo->stats()); + +?> + +
+ Transactions + + {{-- Stats Cards --}} +
+ +
Total
+
{{ $this->stats['total'] }}
+ + + +
Pending
+
{{ $this->stats['pending'] }}
+ + + +
Processing
+
{{ $this->stats['processing'] }}
+ + + +
Completed
+
{{ $this->stats['completed'] }}
+ +
+ + {{-- Transaction List --}} + + + + + Date + Company + Counterparty + Amount + Risk Score + Status + Actions + + + + @foreach($this->transactions as $transaction) + @php + $risk = $transaction->getRiskAssessment(); + $riskScore = $risk['score'] ?? 0; + $riskColor = $riskScore >= 70 ? 'red' : ($riskScore >= 40 ? 'yellow' : 'green'); + @endphp + + {{ $transaction->tx_date }} + {{ $transaction->corporate_entity }} + {{ $transaction->corporate_counterparty }} + {{ number_format($transaction->tx_amount, 2) }} {{ $transaction->tx_currency }} + + + {{ $riskScore }} + + + + + {{ $transaction->status }} + + + + + View + + + + @endforeach + + + +
+``` + +### 4.4 Transaction Detail View + +**resources/views/pages/transactions/show.blade.php**: +```php +findOrFail($id); +$companyInfo = $transaction->getCompanyInfo(); +$riskAssessment = $transaction->getRiskAssessment(); +$sanctionsCheck = $transaction->getOutput('sanctions'); +$pepCheck = $transaction->getOutput('pep'); + +?> + +
+ Transaction Details + + {{-- Transaction Info --}} + + Transaction Information + +
+
+
Company
+
{{ $transaction->corporate_entity }}
+
+
+
Counterparty
+
{{ $transaction->corporate_counterparty }}
+
+
+
Amount
+
{{ number_format($transaction->tx_amount, 2) }} {{ $transaction->tx_currency }}
+
+
+
Date
+
{{ $transaction->tx_date }}
+
+
+
Status
+
+ + {{ $transaction->status }} + +
+
+
+ + + {{-- Company Info --}} + @if($companyInfo) + + Company Information + +
+
+
Name
+
{{ $companyInfo['name'] }}
+
+
+
Country
+
{{ $companyInfo['country'] }}
+
+
+
Sector
+
{{ $companyInfo['sector'] ?? 'Unknown' }}
+
+
+
KYC Risk Level
+
+ {{ $companyInfo['kyc_risk_level'] ?? 'medium' }} +
+
+
+ + @endif + + {{-- Risk Assessment --}} + @if($riskAssessment) + + Risk Assessment + +
+
+ Risk Score + + {{ $riskAssessment['score'] }} / 100 + +
+
+
Risk Level
+
{{ ucfirst($riskAssessment['level']) }}
+
+
+
Requires Review
+
{{ $riskAssessment['requires_review'] ? 'Yes' : 'No' }}
+
+
+ + @endif + + {{-- Sanctions Check --}} + @if($sanctionsCheck) + + Sanctions Check + +
+ + {{ $sanctionsCheck['found'] ? 'Matches Found' : 'Clear' }} + +
+ + @endif + + {{-- All Outputs (Debug) --}} + + All AI Outputs + +
+ @foreach($transaction->outputs as $output) +
+ {{ $output->output_key }} + 
{{ json_encode($output->content, JSON_PRETTY_PRINT) }}
+
+ @endforeach +
+ +
+``` + +--- + +## Phase 5: Datenmigration (Tag 11-15) + +### 5.1 Migration: public.companies → transaction_outputs + +**database/migrations/2025_11_12_migrate_companies_to_outputs.php**: +```php += 70 THEN 'high' + WHEN t.risk_score >= 40 THEN 'medium' + ELSE 'low' + END, + 'requires_review', t.requires_review, + 'flagged_by', t.flagged_by, + 'flagged_reason', t.flagged_reason, + 'signals', t.signals + ) + FROM public.transactions t + JOIN public.companies c ON t.company_id = c.id + JOIN backend.transactions bt ON ( + bt.corporate_entity = c.name + AND bt.tx_date = t.executed_at::text + AND bt.tx_amount = t.amount + ) + "); + + // 3. Enrichment Outputs (Loop through all sources) + $sources = [ + 'registry' => ['registry_data', 'registry_last_refreshed_at'], + 'sanctions' => ['sanctions_data', 'sanctions_last_refreshed_at'], + 'pep' => ['pep_data', 'pep_last_refreshed_at'], + 'gleif' => ['gleif_data', 'gleif_last_refreshed_at'], + // ... weitere + ]; + + foreach ($sources as $key => [$dataCol, $refreshCol]) { + DB::statement(" + INSERT INTO backend.transaction_outputs ( + transaction_id, + prompt_id, + output_key, + content + ) + SELECT + bt.id, + 1, + '{$key}', + t.{$dataCol} + FROM public.transactions t + JOIN public.companies c ON t.company_id = c.id + JOIN backend.transactions bt ON ( + bt.corporate_entity = c.name + AND bt.tx_date = t.executed_at::text + AND bt.tx_amount = t.amount + ) + WHERE t.{$dataCol} IS NOT NULL + "); + } + } + + public function down(): void + { + // Rollback: Lösche migrierte Daten + DB::statement(" + DELETE FROM backend.transactions + WHERE status = 'completed' + AND EXISTS ( + SELECT 1 FROM backend.transaction_outputs + WHERE transaction_id = backend.transactions.id + ) + "); + } +}; +``` + +--- + +## Phase 6: Testing & Rollout + +### 6.1 Feature Flag basierter Rollout + +**.env**: +```env +# Phase 1: Backend verfügbar, aber inaktiv +FEATURE_USE_BACKEND_TRANSACTIONS=false + +# Phase 2: Neue Uploads gehen zu Backend +FEATURE_BACKEND_CSV_UPLOAD=true + +# Phase 3: Frontend liest von Backend +FEATURE_USE_BACKEND_TRANSACTIONS=true + +# Phase 4: Public deprecated +FEATURE_DEPRECATE_PUBLIC_SCHEMA=true +``` + +### 6.2 Monitoring + +**app/Console/Commands/MonitorTransactionProcessing.php**: +```php + Transaction::where('status', Transaction::STATUS_PENDING)->count(), + 'processing' => Transaction::where('status', Transaction::STATUS_PROCESSING)->count(), + 'completed' => Transaction::where('status', Transaction::STATUS_COMPLETED)->count(), + 'failed' => Transaction::where('status', Transaction::STATUS_FAILED)->count(), + ]; + + $this->table( + ['Status', 'Count'], + collect($stats)->map(fn($count, $status) => [$status, $count])->values() + ); + + // Alert bei vielen Failed + if ($stats['failed'] > 10) { + $this->error("⚠️ Warning: {$stats['failed']} failed transactions!"); + } + } +} +``` + +--- + +## Zeitplan + +| Phase | Beschreibung | Dauer | +|-------|--------------|-------| +| 1 | Backend Models & DB Config | 1-2 Tage | +| 2 | CSV Upload Service | 2-3 Tage | +| 3 | KI Workflow Integration | 3-5 Tage | +| 4 | Frontend Anpassung | 3-4 Tage | +| 5 | Datenmigration | 3-5 Tage | +| 6 | Testing & Rollout | 2-3 Tage | + +**Total**: 14-22 Tage (3-4 Wochen) + +--- + +## Nächste Schritte + +### Sofort starten: +1. ✅ Database Config erweitern +2. ✅ Backend Models erstellen +3. ✅ Ersten Upload-Test durchführen + +Soll ich mit der **Implementierung von Phase 1** beginnen? diff --git a/misc/IMPLEMENTATION_SUMMARY.md b/misc/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 0000000..826a5cf --- /dev/null +++ b/misc/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,415 @@ +# Implementation Summary: Backend Schema Integration + +## ✅ Was wurde implementiert + +### 1. Database Configuration +**Datei**: [config/database.php](config/database.php) + +Zwei Verbindungen hinzugefügt: +- `pgsql` → public Schema (Default) +- `backend` → backend Schema (search_path: 'backend') + +```php +'default' => env('DB_CONNECTION', 'pgsql'), // Geändert von sqlite zu pgsql + +'pgsql' => [ + 'search_path' => 'public', +], + +'backend' => [ + 'search_path' => 'backend', // Zugriff auf backend Schema +], +``` + +### 2. Backend Models + +#### Backend\Transaction Model +**Datei**: [app/Models/Backend/Transaction.php](app/Models/Backend/Transaction.php) + +**Eigenschaften**: +- Connection: `backend` +- Table: `transactions` +- 14 Felder (Rohdaten aus CSV Upload) + +**Relationships**: +- `outputs()` - HasMany zu TransactionOutput + +**Helper Methods**: +```php +// Outputs abrufen +getOutput(string $key): ?array +getOutputsArray(): array +getCompanyInfo(): ?array +getRiskAssessment(): ?array +getSanctionsCheck(): ?array +getPepCheck(): ?array + +// Status & Review +hasOutput(string $key): bool +isProcessed(): bool +requiresReview(): bool + +// Display Helpers +getCompanyName(): string +getRiskScore(): int +getRiskLevel(): string +``` + +#### Backend\TransactionOutput Model +**Datei**: [app/Models/Backend/TransactionOutput.php](app/Models/Backend/TransactionOutput.php) + +**Eigenschaften**: +- Connection: `backend` +- Table: `transaction_outputs` +- 5 Felder (KI-generierte Outputs) + +**Relationships**: +- `transaction()` - BelongsTo Transaction + +**Konstanten** (Output-Keys): +```php +KEY_COMPANY_INFO = 'company_info' +KEY_RISK_ASSESSMENT = 'risk_assessment' +KEY_SANCTIONS = 'sanctions' +KEY_PEP = 'pep' +KEY_REGISTRY = 'registry' +KEY_GLEIF = 'gleif' +KEY_INSOLVENCY = 'insolvency' +KEY_BUNDESANZEIGER = 'bundesanzeiger' +KEY_RSS = 'rss' +KEY_EU_SANCTIONS = 'eu_sanctions' +KEY_HANDELSREGISTER = 'handelsregister' +KEY_GENESIS = 'genesis' +KEY_GOVDATA = 'govdata' +``` + +**Helper Methods**: +```php +availableKeys(): array +getKeyLabel(string $key): string +``` + +### 3. Repository Layer + +**Datei**: [app/Repositories/TransactionRepository.php](app/Repositories/TransactionRepository.php) + +**Methoden**: +```php +// Basic CRUD +all(): Collection +find(int $id): ?Transaction +paginated(int $perPage = 20) + +// Filtering +requiresReview(): Collection +highRisk(int $threshold = 70): Collection +byCompany(string $companyName): Collection +byStatus(string $status): Collection +byDateRange(string $startDate, string $endDate): Collection +search(string $query): Collection + +// Status-specific +pending(): Collection +processing(): Collection +completed(): Collection +failed(): Collection + +// Statistics +stats(): array +dashboardData(): array +getAllCompanies(): Collection +``` + +### 4. Tests + +**Datei**: [tests/Feature/BackendModelsTest.php](tests/Feature/BackendModelsTest.php) + +11 Tests geschrieben (3 laufen ohne DB-Verbindung): +- ✅ Connection Tests +- ✅ Model Relationship Tests +- ✅ Helper Method Tests +- ✅ Constant Tests + +--- + +## 📋 Nächste Schritte: Component-Migration + +### Betroffene Components + +#### 1. transaction-review.blade.php +**Aktuell**: Verwendet `App\Models\Transaction` und `App\Models\Company` + +**Änderungen**: +```php +// Alt +use App\Models\Transaction; +use App\Models\Company; + +// Neu +use App\Models\Backend\Transaction; +use App\Repositories\TransactionRepository; + +// Repository verwenden statt direkte Model-Queries +public function __construct( + private TransactionRepository $transactions +) {} + +$transactions = $this->transactions->all(); +``` + +**Mapping**: +| Alt (public.transactions) | Neu (backend) | +|---------------------------|---------------| +| `$transaction->company->name` | `$transaction->getCompanyName()` | +| `$transaction->company->legal_name` | `$transaction->getCompanyInfo()['legal_name']` | +| `$transaction->company->sector` | `$transaction->getCompanyInfo()['sector']` | +| `$transaction->risk_score` | `$transaction->getRiskScore()` | +| `$transaction->requires_review` | `$transaction->requiresReview()` | +| `$transaction->amount` | `$transaction->tx_amount` | +| `$transaction->counterparty` | `$transaction->corporate_counterparty` | +| `$transaction->executed_at` | `$transaction->tx_date` (⚠️ ist text) | + +#### 2. companies/transactions.blade.php +**Aktuell**: Verwendet `App\Models\Company` Parameter + +**Änderungen**: +- Company-Daten kommen jetzt aus `transaction_outputs` +- Grouping nach `corporate_entity` statt `company_id` + +**Neuer Ansatz**: +```php +// Alle Transaktionen für eine Firma +$companyName = 'Siemens AG'; // Aus Route oder Parameter +$transactions = $this->transactions->byCompany($companyName); + +// Company-Info aus erster Transaction +$companyInfo = $transactions->first()?->getCompanyInfo(); +``` + +#### 3. company-search.blade.php +**Änderungen**: +- Suche jetzt in `backend.transactions.corporate_entity` +- Keine separate Company-Tabelle mehr + +```php +// Repository Method +public function getAllCompanies(): Collection +{ + return DB::connection('backend') + ->table('transactions') + ->select('corporate_entity as name') + ->distinct() + ->orderBy('corporate_entity') + ->get(); +} +``` + +--- + +## 🔄 Migrationsplan (Step-by-Step) + +### Phase 1: Testen ohne Breaking Changes (empfohlen) + +**Option A: Feature Flag** +```php +// config/features.php +'use_backend_schema' => env('FEATURE_USE_BACKEND_SCHEMA', false), + +// In Components +if (config('features.use_backend_schema')) { + $transactions = app(TransactionRepository::class)->all(); +} else { + $transactions = Transaction::all(); // Alt +} +``` + +**Option B: Neue Routes** (empfohlen für parallele Tests) +```php +// routes/web.php +Route::get('/beta/transactions', ...); // Nutzt Backend-Schema +Route::get('/transactions', ...); // Alte Implementation +``` + +### Phase 2: Direkte Migration (schneller, aber riskanter) + +1. **Alle `use App\Models\Transaction` ersetzen**: +```bash +find resources/views/livewire -type f -name "*.php" -exec sed -i '' 's/use App\\Models\\Transaction/use App\\Models\\Backend\\Transaction/g' {} + +``` + +2. **Alle `use App\Models\Company` entfernen**: +```bash +find resources/views/livewire -type f -name "*.php" -exec sed -i '' 's/use App\\Models\\Company;//g' {} + +``` + +3. **Component für Component anpassen**: + - transaction-review.blade.php + - companies/transactions.blade.php + - company-search.blade.php + +--- + +## 🗺️ Daten-Mapping Referenz + +### Transaction Fields + +| public.transactions | backend.transactions | Typ-Unterschied | +|---------------------|----------------------|-----------------| +| id | id | bigint → integer | +| company_id | - (via corporate_entity) | Relationship entfällt | +| reference | - | Neu: auto-generated | +| amount | tx_amount | numeric → double | +| currency | tx_currency | ✓ | +| counterparty | corporate_counterparty | ✓ | +| counterparty_country | tx_country_incoming | ✓ | +| channel | - | Nicht in backend | +| executed_at | tx_date | **timestamp → text!** | +| risk_score | - (in outputs) | Via getRiskScore() | +| status | status | ✓ | +| requires_review | - (computed) | Via requiresReview() | +| flagged_by | - | Nicht in backend | +| flagged_reason | tx_purpose | Ähnlich | +| signals | - | Nicht in backend | + +### Company Fields (jetzt in transaction_outputs) + +| public.companies | transaction_outputs (key='company_info') | +|------------------|-------------------------------------------| +| name | content['name'] | +| legal_name | content['legal_name'] | +| ticker | content['ticker'] | +| sector | content['sector'] | +| country | content['country'] | +| headquarters | content['headquarters'] | +| kyc_risk_level | content['kyc_risk_level'] | +| summary | content['summary'] | + +### Enrichment Fields (in transaction_outputs) + +Alle 35 Enrichment-Felder aus `public.transactions` sind jetzt separate Outputs: + +```php +// Alt +$transaction->sanctions_data // JSON + +// Neu +$transaction->getOutput('sanctions') // Array +``` + +--- + +## ⚠️ Breaking Changes + +### 1. Timestamps sind Text +```php +// Alt +$transaction->executed_at->format('d.m.Y') + +// Neu (ACHTUNG: tx_date ist Text!) +$transaction->tx_date // Already string, no format() +``` + +**Empfehlung**: Migration hinzufügen um `tx_date` von `text` zu `timestamp` zu ändern. + +### 2. Company Relationship entfällt +```php +// Alt +$transaction->company->name +$transaction->company()->where(...) + +// Neu +$transaction->getCompanyName() +$transaction->getCompanyInfo()['name'] +// Kein Relationship mehr verfügbar +``` + +### 3. Feld-Namen ändern sich +```php +// Alt → Neu +amount → tx_amount +counterparty → corporate_counterparty +executed_at → tx_date +``` + +**Empfehlung**: Accessor in Model für Backward-Compatibility: + +```php +// In Backend\Transaction Model +protected $appends = ['amount', 'counterparty', 'executed_at']; + +public function getAmountAttribute(): float +{ + return $this->tx_amount; +} + +public function getCounterpartyAttribute(): string +{ + return $this->corporate_counterparty; +} + +public function getExecutedAtAttribute(): string +{ + return $this->tx_date; +} +``` + +--- + +## 🎯 Empfohlenes Vorgehen + +### Option 1: Schrittweise mit Feature Flags (Sicher, 2-3 Wochen) +1. ✅ Backend Models & Repository erstellt +2. ⬜ Feature Flag System einrichten +3. ⬜ Parallele Routes `/beta/*` erstellen +4. ⬜ Einen Component nach dem anderen migrieren +5. ⬜ Testen mit echten Nutzern (10%) +6. ⬜ Gradual Rollout +7. ⬜ Alte Components entfernen + +### Option 2: Direkte Migration (Schnell, 3-5 Tage) +1. ✅ Backend Models & Repository erstellt +2. ⬜ Alle Components in einem PR umstellen +3. ⬜ Accessor für Backward-Compatibility hinzufügen +4. ⬜ Intensives Testing +5. ⬜ Deploy mit Rollback-Plan + +### Option 3: Hybrid (Empfohlen, 1 Woche) +1. ✅ Backend Models & Repository erstellt +2. ⬜ Accessor für Backward-Compatibility in Backend Models +3. ⬜ `transaction-review` Component migrieren (wichtigster) +4. ⬜ 1-2 Tage Testing +5. ⬜ Restliche Components migrieren +6. ⬜ Deploy + +--- + +## 📝 Checkliste für Component-Migration + +Für jeden Component: + +- [ ] Import `App\Models\Transaction` → `App\Models\Backend\Transaction` +- [ ] Import `App\Models\Company` entfernen +- [ ] Repository injecten statt direkte Model-Queries +- [ ] `company->` zu `getCompanyInfo()` ändern +- [ ] Field-Namen anpassen (`amount` → `tx_amount`, etc.) +- [ ] `executed_at` zu `tx_date` ändern +- [ ] `->format()` Calls bei `tx_date` entfernen (ist schon Text) +- [ ] Tests schreiben/anpassen +- [ ] Manuell testen +- [ ] PR erstellen + +--- + +## 🚀 Los geht's! + +Möchten Sie: + +**A)** Dass ich jetzt `transaction-review.blade.php` auf Backend-Schema umstelle? + +**B)** Erst Accessors für Backward-Compatibility hinzufügen? + +**C)** Ein Feature-Flag-System einrichten? + +**D)** Etwas anderes? + +Was ist Ihr bevorzugter Ansatz? diff --git a/misc/INCREMENTAL_MIGRATION_PLAN.md b/misc/INCREMENTAL_MIGRATION_PLAN.md new file mode 100644 index 0000000..029ff17 --- /dev/null +++ b/misc/INCREMENTAL_MIGRATION_PLAN.md @@ -0,0 +1,965 @@ +# Inkrementelle Migrations-Strategie: Parallele Schema-Integration + +## Konzept: Strangler Fig Pattern + +Diese Strategie nutzt das **Strangler Fig Pattern** - das Backend-Schema wird parallel integriert und schrittweise übernimmt die Logik, während das alte System weiterläuft. Kein Big Bang, keine Breaking Changes. + +``` +Phase 1: Beide Systeme parallel +┌─────────────────┐ ┌─────────────────┐ +│ Public Schema │ │ Backend Schema │ +│ (Aktiv) │ │ (Read-Only) │ +└─────────────────┘ └─────────────────┘ + +Phase 2: Dual-Write Pattern +┌─────────────────┐ ┌─────────────────┐ +│ Public Schema │────▶│ Backend Schema │ +│ (Primary) │ │ (Secondary) │ +└─────────────────┘ └─────────────────┘ + +Phase 3: Umstellung +┌─────────────────┐ ┌─────────────────┐ +│ Public Schema │◀────│ Backend Schema │ +│ (Read-Only) │ │ (Primary) │ +└─────────────────┘ └─────────────────┘ + +Phase 4: Deprecation + ┌─────────────────┐ + │ Backend Schema │ + │ (Einzige Quelle)│ + └─────────────────┘ +``` + +--- + +## Phase 1: Dual-Connection Setup (1-2 Tage) + +### 1.1 Database-Konfiguration erweitern + +**config/database.php**: +```php + env('DB_CONNECTION', 'pgsql'), + + 'connections' => [ + // Bestehende Public-Schema Verbindung + 'pgsql' => [ + 'driver' => 'pgsql', + 'host' => env('DB_HOST', '127.0.0.1'), + 'port' => env('DB_PORT', '5432'), + 'database' => env('DB_DATABASE', 'forge'), + 'username' => env('DB_USERNAME', 'forge'), + 'password' => env('DB_PASSWORD', ''), + 'charset' => 'utf8', + 'prefix' => '', + 'prefix_indexes' => true, + 'search_path' => 'public', + 'sslmode' => 'prefer', + ], + + // Neue Backend-Schema Verbindung + 'backend' => [ + 'driver' => 'pgsql', + 'host' => env('DB_HOST', '127.0.0.1'), + 'port' => env('DB_PORT', '5432'), + 'database' => env('DB_DATABASE', 'forge'), // Gleiche DB + 'username' => env('DB_USERNAME', 'forge'), + 'password' => env('DB_PASSWORD', ''), + 'charset' => 'utf8', + 'prefix' => '', + 'prefix_indexes' => true, + 'search_path' => 'backend', // Unterschiedliches Schema! + 'sslmode' => 'prefer', + ], + ], +]; +``` + +**.env**: +```env +# Keine Änderung nötig - beide Connections nutzen gleiche Credentials +DB_CONNECTION=pgsql +DB_HOST=127.0.0.1 +DB_PORT=5432 +DB_DATABASE=your_database +DB_USERNAME=your_user +DB_PASSWORD=your_password +``` + +### 1.2 Backend Models erstellen + +**app/Models/Backend/Company.php**: +```php +hasMany(Transaction::class); + } + + /** + * Sync zu public.companies (während Migration) + */ + public function syncToPublic(): \App\Models\Company + { + return \App\Models\Company::updateOrCreate( + ['id' => $this->id], + $this->only($this->fillable) + ); + } +} +``` + +**app/Models/Backend/Transaction.php**: +```php + 'decimal:2', + // tx_date ist als TEXT gespeichert - später migrieren zu timestamp + ]; + + public function company(): BelongsTo + { + return $this->belongsTo(Company::class); + } + + /** + * Konvertiere zu public.transactions Format + */ + public function toPublicFormat(): array + { + return [ + 'company_id' => $this->company_id, + 'reference' => $this->id, // oder generiere unique reference + 'amount' => $this->tx_amount, + 'currency' => $this->tx_currency, + 'counterparty' => $this->corporate_counterparty, + 'counterparty_country' => $this->tx_country_incoming, + 'executed_at' => $this->tx_date, + 'status' => $this->status, + 'requires_review' => true, + ]; + } +} +``` + +### 1.3 Test der Dual-Connection + +```php +toBeGreaterThan(0); + + // Backend Schema + $backendCount = BackendCompany::count(); + expect($backendCount)->toBeGreaterThanOrEqual(0); +}); + +test('connections are isolated', function () { + $public = PublicCompany::first(); + $backend = BackendCompany::first(); + + // Verschiedene Connections + expect($public->getConnectionName())->toBe('pgsql'); + expect($backend->getConnectionName())->toBe('backend'); +}); +``` + +--- + +## Phase 2: Abstraction Layer (2-3 Tage) + +### 2.1 Repository Pattern mit Feature Flags + +**app/Repositories/CompanyRepository.php**: +```php +useBackendSchema = config('features.use_backend_schema', false); + } + + public function all(): Collection + { + return $this->useBackendSchema + ? BackendCompany::all() + : PublicCompany::all(); + } + + public function find(int $id): PublicCompany|BackendCompany|null + { + return $this->useBackendSchema + ? BackendCompany::find($id) + : PublicCompany::find($id); + } + + public function create(array $data): PublicCompany|BackendCompany + { + if ($this->useBackendSchema) { + $company = BackendCompany::create($data); + + // Dual-Write: Sync zu Public während Übergangsphase + if (config('features.dual_write', true)) { + $company->syncToPublic(); + } + + return $company; + } + + return PublicCompany::create($data); + } + + public function update(int $id, array $data): bool + { + $company = $this->find($id); + + if (!$company) { + return false; + } + + $result = $company->update($data); + + // Dual-Write + if ($this->useBackendSchema && config('features.dual_write', true)) { + $company->syncToPublic(); + } + + return $result; + } + + public function delete(int $id): bool + { + $company = $this->find($id); + + if (!$company) { + return false; + } + + // Dual-Delete + if ($this->useBackendSchema && config('features.dual_write', true)) { + PublicCompany::destroy($id); + } + + return $company->delete(); + } + + /** + * Helper: Get model class + */ + public function getModelClass(): string + { + return $this->useBackendSchema + ? BackendCompany::class + : PublicCompany::class; + } +} +``` + +**app/Repositories/TransactionRepository.php**: +```php +useBackendSchema = config('features.use_backend_schema', false); + } + + public function forCompany(int $companyId): Collection + { + return $this->useBackendSchema + ? BackendTransaction::where('company_id', $companyId)->get() + : PublicTransaction::where('company_id', $companyId)->get(); + } + + public function requiresReview(): Collection + { + if ($this->useBackendSchema) { + // Backend hat kein requires_review Feld - nutze status + return BackendTransaction::where('status', 'pending')->get(); + } + + return PublicTransaction::where('requires_review', true)->get(); + } + + public function highRisk(int $threshold = 70): Collection + { + if ($this->useBackendSchema) { + // Backend hat keinen risk_score - Alternative Logik + return BackendTransaction::where('status', 'flagged')->get(); + } + + return PublicTransaction::where('risk_score', '>=', $threshold)->get(); + } +} +``` + +### 2.2 Feature Flag Konfiguration + +**config/features.php**: +```php + env('FEATURE_USE_BACKEND_SCHEMA', false), + + // Dual-Write: Schreibe in beide Schemas während Migration + 'dual_write' => env('FEATURE_DUAL_WRITE', true), + + // Read-Verification: Vergleiche Reads aus beiden Schemas (Logging) + 'verify_reads' => env('FEATURE_VERIFY_READS', false), + + // Schrittweise Migration pro Bereich + 'backend_schema_areas' => [ + 'companies' => env('FEATURE_BACKEND_COMPANIES', false), + 'transactions' => env('FEATURE_BACKEND_TRANSACTIONS', false), + 'reports' => env('FEATURE_BACKEND_REPORTS', false), + ], +]; +``` + +**.env** (für schrittweise Aktivierung): +```env +# Phase 1: Beide Schemas verfügbar, aber Public aktiv +FEATURE_USE_BACKEND_SCHEMA=false +FEATURE_DUAL_WRITE=false + +# Phase 2: Dual-Write aktivieren +# FEATURE_USE_BACKEND_SCHEMA=false +# FEATURE_DUAL_WRITE=true + +# Phase 3: Backend als Primary, Public als Fallback +# FEATURE_USE_BACKEND_SCHEMA=true +# FEATURE_DUAL_WRITE=true + +# Phase 4: Nur Backend +# FEATURE_USE_BACKEND_SCHEMA=true +# FEATURE_DUAL_WRITE=false +``` + +### 2.3 Service Provider für Dependency Injection + +**app/Providers/RepositoryServiceProvider.php**: +```php +app->singleton(CompanyRepository::class, function ($app) { + return new CompanyRepository( + useBackendSchema: config('features.use_backend_schema', false) + ); + }); + + $this->app->singleton(TransactionRepository::class, function ($app) { + return new TransactionRepository( + useBackendSchema: config('features.use_backend_schema', false) + ); + }); + } +} +``` + +**bootstrap/providers.php**: +```php +get(); + + return view('companies.index', compact('companies')); + } + + public function store(Request $request) + { + $validated = $request->validate([ + 'name' => 'required|string', + 'country' => 'required|string|size:2', + // ... + ]); + + $company = Company::create($validated); + + return redirect()->route('companies.show', $company); + } +} +``` + +**Nachher**: +```php +companies->all(); + + return view('companies.index', compact('companies')); + } + + public function store(Request $request) + { + $validated = $request->validate([ + 'name' => 'required|string', + 'country' => 'required|string|size:2', + // ... + ]); + + $company = $this->companies->create($validated); + + return redirect()->route('companies.show', $company); + } +} +``` + +### 3.2 Volt/Livewire Components anpassen + +**Vorher** (resources/views/pages/companies/index.blade.php): +```php +with('transactions') + ->orderBy('name') + ->get(); + +?> + +
+ @foreach($companies as $company) + {{ $company->name }} + @endforeach +
+``` + +**Nachher**: +```php +all(); + +?> + +
+ @foreach($companies as $company) + {{ $company->name }} + @endforeach +
+``` + +--- + +## Phase 4: Monitoring & Verification (Parallel zu Phase 3) + +### 4.1 Dual-Read Verification Middleware + +**app/Http/Middleware/VerifyDualSchemaReads.php**: +```php +listen(function ($query) use (&$publicQueries) { + $publicQueries[] = $query->sql; + }); + + \DB::connection('backend')->listen(function ($query) use (&$backendQueries) { + $backendQueries[] = $query->sql; + }); + + $response = $next($request); + + // Log für Analyse + if (!empty($publicQueries) || !empty($backendQueries)) { + Log::channel('migration')->info('Dual Schema Access', [ + 'route' => $request->path(), + 'public_queries' => count($publicQueries), + 'backend_queries' => count($backendQueries), + ]); + } + + return $response; + } +} +``` + +### 4.2 Health Check Command + +**app/Console/Commands/VerifySchemaConsistency.php**: +```php +info('Checking schema consistency...'); + + // Company Count + $publicCount = PublicCompany::count(); + $backendCount = BackendCompany::count(); + + $this->table( + ['Schema', 'Companies', 'Status'], + [ + ['Public', $publicCount, '✓'], + ['Backend', $backendCount, $backendCount === $publicCount ? '✓' : '⚠'], + ] + ); + + if ($backendCount !== $publicCount) { + $this->warn("Company count mismatch: Public={$publicCount}, Backend={$backendCount}"); + } + + // Sample Data Verification + $sampleSize = min(10, $publicCount); + $publicSample = PublicCompany::take($sampleSize)->get(); + $mismatches = 0; + + foreach ($publicSample as $publicCompany) { + $backendCompany = BackendCompany::find($publicCompany->id); + + if (!$backendCompany) { + $this->warn("Company {$publicCompany->id} missing in backend"); + $mismatches++; + continue; + } + + if ($publicCompany->name !== $backendCompany->name) { + $this->warn("Company {$publicCompany->id} name mismatch"); + $mismatches++; + } + } + + if ($mismatches === 0) { + $this->info('✓ All consistency checks passed!'); + return self::SUCCESS; + } + + $this->error("✗ Found {$mismatches} inconsistencies"); + return self::FAILURE; + } +} +``` + +--- + +## Phase 5: Graduelle Umstellung (2-4 Wochen) + +### Woche 1: Read-Only Access +```env +FEATURE_USE_BACKEND_SCHEMA=false +FEATURE_DUAL_WRITE=false +FEATURE_VERIFY_READS=true +``` + +- ✅ Backend-Schema ist verfügbar +- ✅ Monitoring läuft +- ✅ Keine Produktions-Daten betroffen + +### Woche 2: Dual-Write aktivieren +```env +FEATURE_USE_BACKEND_SCHEMA=false # Lesen: Public +FEATURE_DUAL_WRITE=true # Schreiben: Beide +FEATURE_VERIFY_READS=true +``` + +- ✅ Neue Daten gehen in beide Schemas +- ✅ Public bleibt Primary +- ⚠️ Monitor auf Sync-Errors + +### Woche 3: Backend als Primary (Canary) +```env +# Nur für 10% Traffic oder spezifische Routes +FEATURE_BACKEND_COMPANIES=true # Companies von Backend lesen +FEATURE_BACKEND_TRANSACTIONS=false # Transactions noch von Public +FEATURE_DUAL_WRITE=true +``` + +- ✅ Schrittweise Umstellung pro Feature +- ✅ A/B Testing möglich +- ✅ Rollback jederzeit möglich + +### Woche 4: Full Switchover +```env +FEATURE_USE_BACKEND_SCHEMA=true # Lesen: Backend +FEATURE_DUAL_WRITE=true # Schreiben: Beide (Sicherheit) +``` + +- ✅ Backend ist Primary +- ✅ Public als Safety Net + +### Nach 2 Wochen stabiler Betrieb: +```env +FEATURE_USE_BACKEND_SCHEMA=true +FEATURE_DUAL_WRITE=false # Public wird deprecated +``` + +--- + +## Phase 6: Testing-Strategie + +### 6.1 Feature Tests mit Feature Flags + +```php + false]); + + $response = $this->post('/companies', [ + 'name' => 'Test Corp', + 'country' => 'DE', + ]); + + $response->assertRedirect(); + expect(PublicCompany::where('name', 'Test Corp')->exists())->toBeTrue(); +}); + +test('can create company with backend schema', function () { + config(['features.use_backend_schema' => true]); + + $response = $this->post('/companies', [ + 'name' => 'Test Corp Backend', + 'country' => 'DE', + ]); + + $response->assertRedirect(); + expect(BackendCompany::where('corporate_entity', 'Test Corp Backend')->exists())->toBeTrue(); +}); + +test('dual write creates in both schemas', function () { + config([ + 'features.use_backend_schema' => true, + 'features.dual_write' => true, + ]); + + $response = $this->post('/companies', [ + 'name' => 'Dual Write Test', + 'country' => 'DE', + ]); + + $response->assertRedirect(); + + expect(BackendCompany::where('corporate_entity', 'Dual Write Test')->exists())->toBeTrue(); + expect(PublicCompany::where('name', 'Dual Write Test')->exists())->toBeTrue(); +}); +``` + +### 6.2 Performance Tests + +```php + false]); + app(CompanyRepository::class)->all(); + $publicTime = microtime(true) - $start; + + // Backend Schema + $start = microtime(true); + config(['features.use_backend_schema' => true]); + app(CompanyRepository::class)->all(); + $backendTime = microtime(true) - $start; + + // Backend sollte nicht mehr als 20% langsamer sein + expect($backendTime)->toBeLessThan($publicTime * 1.2); +}); +``` + +--- + +## Phase 7: Cleanup & Optimization (Nach 4-6 Wochen) + +### 7.1 Repository vereinfachen + +Wenn Backend-Schema stabil läuft: + +```php +>> DB::connection('backend')->select('SELECT 1') +``` + +### Schritt 2: Backend Models (heute, 1-2 Std) +```bash +# Models erstellen +mkdir -p app/Models/Backend +# Company.php & Transaction.php erstellen +``` + +### Schritt 3: Erster Test (heute, 30 Min) +```bash +php artisan test --filter=DualConnectionTest +``` + +### Schritt 4: Feature Flags (morgen, 1-2 Std) +```bash +# config/features.php erstellen +# Repository Pattern implementieren +``` + +Möchten Sie, dass ich mit **Schritt 1-2 beginne** und die konkrete Implementierung starte? + diff --git a/misc/MCP_SERVER_TESTS.md b/misc/MCP_SERVER_TESTS.md new file mode 100644 index 0000000..8acc6d6 --- /dev/null +++ b/misc/MCP_SERVER_TESTS.md @@ -0,0 +1,86 @@ +# MCP-Server Aktivierung und Tests + +## Aktivierung der globalen MCP-Server + +### Methode 1: @-Erwähnung im Chat +Schreibe einfach den Server-Namen mit @ in deiner Nachricht: +- `@sqlite` +- `@postgresql` +- `@bear` +- `@MCP_DOCKER` +- `@Ref` + +### Methode 2: CLI Flag beim Start +```bash +claude --mcp-config ~/.claude.json +``` + +### Methode 3: Beide Configs kombinieren +```bash +claude --mcp-config .mcp.json ~/.claude.json +``` + +## Test-Befehle für jeden Server + +### 1. SQLite Server +**Datenbank:** `/Users/sebastianfrohlich/Downloads/company.db` + +Nach Aktivierung mit `@sqlite`: +``` +Bitte zeige mir alle Tabellen in der SQLite-Datenbank +``` + +### 2. PostgreSQL Server +**Verbindung:** localhost:5433, DB: risk_ingest_db + +Nach Aktivierung mit `@postgresql`: +``` +Bitte zeige mir das Schema der PostgreSQL-Datenbank risk_ingest_db +``` + +### 3. Bear Notes Server +**Pfad:** `/Users/sebastianfrohlich/Projekte/bear-notes-mcp` + +Nach Aktivierung mit `@bear`: +``` +Erstelle eine neue Bear-Notiz mit dem Titel "Test MCP Server" +``` + +### 4. MCP_DOCKER Server +**Command:** `docker mcp gateway run` + +Nach Aktivierung mit `@MCP_DOCKER`: +``` +Zeige mir die verfügbaren Docker-Container +``` + +### 5. Ref.tools Server +**URL:** https://api.ref.tools/mcp + +Nach Aktivierung mit `@Ref`: +``` +Nutze Ref.tools um [spezifische Aufgabe] +``` + +## Debugging + +### Server-Status prüfen +```bash +claude mcp list +``` + +### MCP-Debug-Modus aktivieren +```bash +claude --mcp-debug +``` + +### Server-Logs anzeigen +Prüfe die Logs in: +- `~/.claude/logs/` + +## Hinweise + +- Globale Server aus `~/.claude.json` sind standardmäßig nicht in jeder Session geladen +- Projekt-Server aus `.mcp.json` werden automatisch geladen +- @-Erwähnung ist die einfachste Methode zur Ad-hoc-Aktivierung +- Einige Server benötigen laufende Dienste (z.B. PostgreSQL muss auf Port 5433 laufen) diff --git a/misc/MIGRATION_PLAN.md b/misc/MIGRATION_PLAN.md new file mode 100644 index 0000000..c4953f9 --- /dev/null +++ b/misc/MIGRATION_PLAN.md @@ -0,0 +1,801 @@ +# Migrationsplan: Schema-Restrukturierung + +## Zielsetzung + +### Aktueller Zustand +``` +public.companies (11 Spalten) + ↓ 1:N +public.transactions (49 Spalten) - Angereicherte Produktionsdaten + +backend.transactions (14 Spalten) - Rohdaten + ↓ N:M +backend.transaction_outputs (5 Spalten) - AI-generierte Outputs +``` + +### Ziel-Zustand +``` +backend.transactions (14 Spalten) - Ersetzt public.companies + ↓ 1:N +public.transactions (5 Spalten, ähnlich backend.transaction_outputs) - Vereinfacht +``` + +--- + +## ⚠️ Klärungsfragen (KRITISCH) + +Bevor wir fortfahren, müssen folgende Fragen geklärt werden: + +### 1. Companies → backend.transactions Mapping +**Problem**: Die Strukturen sind sehr unterschiedlich + +| public.companies | backend.transactions | Kompatibilität | +|------------------|---------------------|----------------| +| id (bigint) | id (integer) | ⚠️ Typ-Unterschied | +| name | corporate_entity | ✅ Ähnlich | +| legal_name | - | ❌ Fehlt | +| ticker | - | ❌ Fehlt | +| sector | - | ❌ Fehlt | +| country | tx_country_outgoing? | ⚠️ Unklar | +| headquarters | - | ❌ Fehlt | +| kyc_risk_level | - | ❌ Fehlt | +| summary | - | ❌ Fehlt | +| - | corporate_counterparty | ❌ Neu | +| - | tx_date, tx_amount, tx_currency | ❌ Neu | +| - | tx_purpose | ❌ Neu | +| - | source_file, raw_payload | ❌ Neu | +| - | status | ❌ Neu | + +**Frage**: +- Soll `backend.transactions` erweitert werden, um Company-Felder aufzunehmen? +- Oder sollen Companies als einzelne Zeilen ohne Transaktionsdaten gespeichert werden? +- Wie wird `corporate_entity` zu `companies.name` gemappt? + +### 2. public.transactions → transaction_outputs Mapping +**Problem**: Drastischer Datenverlust bei Vereinfachung + +| public.transactions (49 Felder) | backend.transaction_outputs (5 Felder) | +|---------------------------------|----------------------------------------| +| Alle Core-Felder (14) | ❌ Verloren | +| Alle Enrichment-Felder (34) | ❌ Verloren | +| - | transaction_id (Foreign Key) | +| - | prompt_id (Welcher?) | +| - | output_key (Welcher Typ?) | +| - | content (Wie strukturiert?) | +| - | run_id (Optional) | + +**Frage**: +- Welche Daten aus den 49 Feldern sollen in `content` serialisiert werden? +- Welchen `output_key` verwenden wir? (z.B. "transaction_data", "risk_assessment"?) +- Welchen `prompt_id` verwenden wir? (Muss in `backend.prompt_templates` existieren) +- Was passiert mit den 11 Enrichment-Datenquellen? + +### 3. Relationship & Foreign Keys +**Problem**: Beziehungen ändern sich fundamental + +**Aktuell**: +``` +companies.id → transactions.company_id (1:N) +``` + +**Ziel** (unklar): +``` +backend.transactions.id → public.transactions.transaction_id (1:N)? +``` + +**Frage**: +- Bleibt die 1:N Beziehung erhalten? +- Wie wird `transaction_id` in der neuen `public.transactions` gemappt? + +--- + +## Vorgeschlagene Alternative: Erweiterte Migration + +Ich schlage eine modifizierte Zielstruktur vor, die Datenverlust minimiert: + +### Option A: Erweitere backend.transactions (Empfohlen) + +```sql +backend.companies (neue Tabelle) + - id (integer) + - name (text) + - legal_name (text, nullable) + - country (text) + - kyc_risk_level (text) + - ... weitere Company-Felder + +backend.transactions (erweitert, bleibt) + - id (integer) + - company_id (integer FK → backend.companies) + - corporate_counterparty (text) + - tx_date (text → sollte timestamp werden) + - tx_amount (double precision) + - tx_currency (text) + - ... bestehende Felder + +public.transaction_enrichments (neue Tabelle) + - id (bigint) + - transaction_id (integer FK → backend.transactions) + - enrichment_type (varchar) -- 'registry', 'sanctions', etc. + - data (jsonb) + - last_refreshed_at (timestamp) + - created_at, updated_at +``` + +**Vorteile**: +- ✅ Kein Datenverlust +- ✅ Klare Trennung: Companies, Transactions, Enrichments +- ✅ Backend-Schema behält Rohdaten +- ✅ Public-Schema behält angereicherte Daten +- ✅ Laravel-Logik kann schrittweise migriert werden + +### Option B: Vollständiger Umzug zu backend Schema + +```sql +backend.companies (neu) + - Alle Felder von public.companies + +backend.transactions (bleibt) + - Bestehende Struktur + +backend.transaction_enrichments (neu) + - Alle Enrichment-Daten aus public.transactions + +public.* (deprecated, später löschen) +``` + +**Vorteile**: +- ✅ Alles im backend Schema +- ✅ Klare Schema-Trennung +- ❌ Laravel-App muss komplett umgeschrieben werden +- ❌ Größere Breaking Changes + +--- + +## Migrationsplan (nach Klärung) + +### Phase 1: Vorbereitung (1-2 Tage) + +#### 1.1 Backup erstellen +```bash +# Vollständiges Backup +pg_dump -h localhost -U username -d database_name > backup_$(date +%Y%m%d_%H%M%S).sql + +# Schema-spezifische Backups +pg_dump -h localhost -U username -d database_name -n public > backup_public_$(date +%Y%m%d).sql +pg_dump -h localhost -U username -d database_name -n backend > backup_backend_$(date +%Y%m%d).sql +``` + +#### 1.2 Datenanalyse +```sql +-- Anzahl Companies +SELECT COUNT(*) FROM public.companies; + +-- Anzahl Transactions +SELECT COUNT(*) FROM public.transactions; + +-- Datenintegrität prüfen +SELECT + COUNT(*) as total_transactions, + COUNT(DISTINCT company_id) as unique_companies, + COUNT(*) FILTER (WHERE company_id NOT IN (SELECT id FROM public.companies)) as orphaned_transactions +FROM public.transactions; + +-- Enrichment-Daten Analyse +SELECT + COUNT(*) FILTER (WHERE registry_data IS NOT NULL) as has_registry, + COUNT(*) FILTER (WHERE sanctions_data IS NOT NULL) as has_sanctions, + COUNT(*) FILTER (WHERE pep_data IS NOT NULL) as has_pep +FROM public.transactions; +``` + +#### 1.3 Test-Umgebung aufsetzen +```bash +# Kopie der Datenbank für Tests +createdb -T production_db test_migration_db +``` + +### Phase 2: Schema-Erweiterung (2-3 Tage) + +#### 2.1 backend.companies erstellen + +**Laravel Migration**: +```php +create('companies', function (Blueprint $table) { + $table->id(); + $table->string('name'); + $table->string('legal_name')->nullable(); + $table->string('ticker')->nullable(); + $table->string('sector')->nullable(); + $table->string('country', 2)->default('DE'); + $table->string('headquarters')->nullable(); + $table->string('kyc_risk_level')->default('medium'); + $table->text('summary')->nullable(); + $table->timestamps(); + + $table->index('name'); + }); + } + + public function down(): void + { + Schema::connection('backend')->dropIfExists('companies'); + } +}; +``` + +#### 2.2 backend.transactions erweitern + +**SQL Migration** (wenn Laravel Multi-Schema-Support limitiert ist): +```sql +-- Füge company_id zu backend.transactions hinzu +ALTER TABLE backend.transactions +ADD COLUMN company_id INTEGER REFERENCES backend.companies(id) ON DELETE CASCADE; + +-- Index für Performance +CREATE INDEX idx_backend_transactions_company_id ON backend.transactions(company_id); +``` + +#### 2.3 public.transaction_enrichments erstellen + +**Laravel Migration**: +```php +id(); + $table->integer('backend_transaction_id'); // FK zu backend.transactions + $table->string('enrichment_type', 50); // 'registry', 'sanctions', etc. + $table->jsonb('data'); + $table->timestamp('last_refreshed_at')->nullable(); + $table->timestamps(); + + $table->index(['backend_transaction_id', 'enrichment_type']); + $table->unique(['backend_transaction_id', 'enrichment_type']); + }); + } + + public function down(): void + { + Schema::dropIfExists('transaction_enrichments'); + } +}; +``` + +### Phase 3: Datenmigration (3-5 Tage) + +#### 3.1 Companies migrieren + +```php + ['registry_data', 'registry_last_refreshed_at'], + 'genesis' => ['genesis_context', 'genesis_last_refreshed_at'], + 'govdata' => ['govdata_data', 'govdata_last_refreshed_at'], + 'bundesanzeiger' => ['bundesanzeiger_data', 'bundesanzeiger_last_refreshed_at'], + 'insolvency' => ['insolvency_data', 'insolvency_last_refreshed_at'], + 'rss' => ['rss_alerts', 'rss_last_refreshed_at'], + 'sanctions' => ['sanctions_data', 'sanctions_last_refreshed_at'], + 'pep' => ['pep_data', 'pep_last_refreshed_at'], + 'gleif' => ['gleif_data', 'gleif_last_refreshed_at'], + 'eu_sanctions' => ['eu_sanctions_data', 'eu_sanctions_last_refreshed_at'], + 'handelsregister' => ['handelsregister_data', 'handelsregister_last_refreshed_at'], + ]; + + public function up(): void + { + foreach ($this->enrichmentSources as $type => [$dataField, $refreshField]) { + DB::statement(" + INSERT INTO public.transaction_enrichments ( + backend_transaction_id, + enrichment_type, + data, + last_refreshed_at, + created_at, + updated_at + ) + SELECT + bt.id as backend_transaction_id, + '{$type}' as enrichment_type, + t.{$dataField} as data, + t.{$refreshField} as last_refreshed_at, + NOW(), + NOW() + FROM public.transactions t + JOIN backend.transactions bt ON ( + bt.tx_date = t.executed_at::text + AND bt.tx_amount = t.amount + ) + WHERE t.{$dataField} IS NOT NULL + ON CONFLICT (backend_transaction_id, enrichment_type) DO NOTHING + "); + } + } + + public function down(): void + { + DB::statement("TRUNCATE public.transaction_enrichments"); + } +}; +``` + +### Phase 4: Model-Anpassung (2-3 Tage) + +#### 4.1 Neue Models erstellen + +**Backend\Company Model**: +```php +hasMany(Transaction::class); + } +} +``` + +**Backend\Transaction Model**: +```php + 'decimal:2', + ]; + + public function company(): BelongsTo + { + return $this->belongsTo(Company::class); + } + + public function enrichments(): HasMany + { + return $this->hasMany(TransactionEnrichment::class, 'backend_transaction_id'); + } + + public function getEnrichment(string $type): ?array + { + return $this->enrichments() + ->where('enrichment_type', $type) + ->first() + ?->data; + } +} +``` + +**TransactionEnrichment Model**: +```php + 'array', + 'last_refreshed_at' => 'datetime' + ]; + + public function transaction(): BelongsTo + { + return $this->belongsTo(Transaction::class, 'backend_transaction_id'); + } +} +``` + +### Phase 5: Code-Refactoring (5-7 Tage) + +#### 5.1 Controller anpassen + +**Vorher**: +```php +use App\Models\Transaction; + +$transactions = Transaction::with('company') + ->where('risk_score', '>', 50) + ->get(); +``` + +**Nachher**: +```php +use App\Models\Backend\Transaction; + +$transactions = Transaction::with(['company', 'enrichments']) + ->where('risk_score', '>', 50) + ->get(); + +// Enrichment-Daten abrufen +foreach ($transactions as $transaction) { + $sanctionsData = $transaction->getEnrichment('sanctions'); + $pepData = $transaction->getEnrichment('pep'); +} +``` + +#### 5.2 Compatibility Layer (Optional) + +```php +getEnrichment($type); + } + + return parent::__get($key); + } +} +``` + +### Phase 6: Testing (3-5 Tage) + +#### 6.1 Unit Tests + +```php +create(); + + TransactionEnrichment::factory()->create([ + 'backend_transaction_id' => $transaction->id, + 'enrichment_type' => 'sanctions', + 'data' => ['status' => 'clear'] + ]); + + expect($transaction->enrichments)->toHaveCount(1); + expect($transaction->getEnrichment('sanctions'))->toBe(['status' => 'clear']); +}); +``` + +#### 6.2 Integration Tests + +```php +count(); + $newCount = DB::table('backend.transactions')->count(); + + expect($newCount)->toBeGreaterThanOrEqual($oldCount); + + // Vergleiche Enrichment-Daten + $oldEnrichments = DB::table('public.transactions') + ->whereNotNull('sanctions_data') + ->count(); + + $newEnrichments = DB::table('public.transaction_enrichments') + ->where('enrichment_type', 'sanctions') + ->count(); + + expect($newEnrichments)->toBe($oldEnrichments); +}); +``` + +### Phase 7: Deployment (1-2 Tage) + +#### 7.1 Deployment-Schritte + +```bash +# 1. Backup +php artisan backup:database + +# 2. Migrations ausführen (in Reihenfolge!) +php artisan migrate --path=database/migrations/2025_11_12_create_backend_companies_table.php +php artisan migrate --path=database/migrations/2025_11_12_create_transaction_enrichments_table.php +php artisan migrate --path=database/migrations/2025_11_12_migrate_companies_data.php +php artisan migrate --path=database/migrations/2025_11_12_migrate_transaction_core_data.php +php artisan migrate --path=database/migrations/2025_11_12_migrate_enrichment_data.php + +# 3. Verification +php artisan tinker +>>> DB::table('backend.companies')->count() +>>> DB::table('backend.transactions')->count() +>>> DB::table('public.transaction_enrichments')->count() + +# 4. Clear caches +php artisan cache:clear +php artisan config:clear +php artisan route:clear +php artisan view:clear + +# 5. Run tests +php artisan test --filter=TransactionMigration +``` + +#### 7.2 Rollback-Plan + +```bash +# Falls etwas schief geht +php artisan migrate:rollback --step=5 + +# Restore from backup +psql -U username -d database_name < backup_20251112_120000.sql +``` + +### Phase 8: Cleanup (nach 2-4 Wochen Monitoring) + +```sql +-- Wenn alles stabil läuft, alte Tabellen entfernen +DROP TABLE public.transactions CASCADE; +DROP TABLE public.companies CASCADE; + +-- Views für Backward-Compatibility (optional) +CREATE VIEW public.companies AS +SELECT * FROM backend.companies; + +CREATE VIEW public.transactions AS +SELECT + bt.id, + bt.company_id, + bt.corporate_counterparty as counterparty, + bt.tx_amount as amount, + bt.tx_currency as currency, + bt.tx_date::timestamp as executed_at, + bt.status, + bt.created_at::timestamp, + bt.last_modified_at::timestamp as updated_at +FROM backend.transactions bt; +``` + +--- + +## Zeitplan + +| Phase | Dauer | Abhängigkeiten | +|-------|-------|----------------| +| 1. Vorbereitung | 1-2 Tage | - | +| 2. Schema-Erweiterung | 2-3 Tage | Phase 1 | +| 3. Datenmigration | 3-5 Tage | Phase 2 | +| 4. Model-Anpassung | 2-3 Tage | Phase 3 | +| 5. Code-Refactoring | 5-7 Tage | Phase 4 | +| 6. Testing | 3-5 Tage | Phase 5 | +| 7. Deployment | 1-2 Tage | Phase 6 | +| 8. Cleanup | Nach 2-4 Wochen | Phase 7 | + +**Gesamtdauer**: 17-27 Arbeitstage (3-5 Wochen) + +--- + +## Risiken & Mitigation + +### Risiko 1: Datenverlust +**Mitigation**: +- Vollständige Backups vor jedem Schritt +- Test-Migration in Staging-Umgebung +- Datenvalidierung nach jeder Phase + +### Risiko 2: Downtime +**Mitigation**: +- Migrations während Wartungsfenster +- Blue-Green Deployment +- Read-Replica für Zero-Downtime + +### Risiko 3: Performance-Probleme +**Mitigation**: +- Indizes auf Foreign Keys +- Batch-Processing für große Datasets +- Query-Optimierung mit EXPLAIN ANALYZE + +### Risiko 4: Code-Inkompatibilität +**Mitigation**: +- Compatibility Layer +- Schrittweises Refactoring +- Feature-Flags für graduelle Umstellung + +--- + +## Nächste Schritte + +1. ✅ **Klärung der Fragen oben** +2. ⬜ Detaillierte Datenanalyse durchführen +3. ⬜ Test-Umgebung aufsetzen +4. ⬜ Erste Migration in Staging testen +5. ⬜ Review & Approval vom Team +6. ⬜ Production-Migration planen + +--- + +*Erstellt am: 2025-11-12* +*Status: ENTWURF - Wartet auf Klärung der kritischen Fragen* diff --git a/misc/database-tables-detailed-description.md b/misc/database-tables-detailed-description.md new file mode 100644 index 0000000..2c9bc45 --- /dev/null +++ b/misc/database-tables-detailed-description.md @@ -0,0 +1,332 @@ +# Detaillierte Beschreibung der Datenbank-Tabellen + +## **Backend Schema** + +### **1. backend.transactions** + +**Zweck**: Rohdaten-Tabelle für eingehende Transaktionen aus verschiedenen Quellen (vermutlich CSV/Excel-Uploads oder API-Imports) + +**Struktur**: 14 Spalten + +#### Identifikation +- **id** (integer, NOT NULL, AUTO_INCREMENT) + - Primärschlüssel + - Sequenz: `backend.transactions_id_seq` + +#### Transaktions-Stammdaten +- **corporate_entity** (text, NOT NULL) + - Name der durchführenden Firma/Entität + - Kein Foreign Key - als Textfeld gespeichert + +- **corporate_counterparty** (text, NOT NULL) + - Name der Gegenpartei/Empfänger + - Freitext, keine Normalisierung + +- **tx_date** (text, NOT NULL) + - Transaktionsdatum + - ⚠️ Als Text gespeichert (nicht als DATE/TIMESTAMP) + - Wahrscheinlich verschiedene Formate möglich + +- **tx_amount** (double precision, NOT NULL) + - Transaktionsbetrag + - Fließkommazahl für Währungsbeträge + +- **tx_currency** (text, nullable) + - Währungscode (z.B. EUR, USD) + - Optional + +- **tx_purpose** (text, nullable) + - Verwendungszweck/Beschreibung der Transaktion + - Freitextfeld + +#### Geografische Informationen +- **tx_country_outgoing** (text, nullable) + - Herkunftsland der Zahlung + +- **tx_country_incoming** (text, nullable) + - Zielland der Zahlung + +#### Metadaten & Verarbeitung +- **source_file** (text, nullable) + - Name/Pfad der Quelldatei + - Für Nachverfolgbarkeit der Datenherkunft + +- **raw_payload** (text, nullable) + - Rohdaten im Originalformat + - Ermöglicht Reprocessing bei Bedarf + +- **status** (text, NOT NULL) + - Verarbeitungsstatus (z.B. "pending", "processed", "error") + +- **created_at** (text, NOT NULL) + - Erstellungszeitpunkt + - ⚠️ Als Text gespeichert (nicht als TIMESTAMP) + +- **last_modified_at** (text, NOT NULL) + - Letzte Änderung + - ⚠️ Als Text gespeichert (nicht als TIMESTAMP) + +**Charakteristik**: ETL-/Staging-Tabelle mit lockerer Typisierung für maximale Flexibilität beim Import + +--- + +### **2. backend.transaction_outputs** + +**Zweck**: Speichert generierte Outputs/Ergebnisse aus Prompt-Verarbeitung für Transaktionen (vermutlich KI/LLM-generierte Analysen) + +**Struktur**: 5 Spalten + +#### Primärschlüssel (zusammengesetzt) +- **transaction_id** (integer, NOT NULL) + - Foreign Key zu `backend.transactions.id` + - Referenziert die analysierte Transaktion + +- **prompt_id** (integer, NOT NULL) + - Foreign Key zu `backend.prompt_templates` (vermutlich) + - Identifiziert welcher Prompt verwendet wurde + +- **output_key** (text, NOT NULL) + - Schlüssel für den Output-Typ + - Beispiele: "risk_assessment", "compliance_check", "summary", "recommendations" + +#### Output-Daten +- **content** (text, NOT NULL) + - Der generierte Inhalt/Ergebnis + - Kann strukturierter Text, JSON oder Markdown sein + +#### Verknüpfung +- **run_id** (integer, nullable) + - Foreign Key zu `backend.prompt_runs` (vermutlich) + - Gruppiert Outputs aus demselben Batch/Durchlauf + - Optional für ad-hoc Generierungen + +**Charakteristik**: N:M-Mapping zwischen Transaktionen und Prompts mit flexiblen Output-Keys + +--- + +## **Public Schema** + +### **3. public.companies** + +**Zweck**: Normalisierte Firmenstammdaten für KYC (Know Your Customer) und Compliance + +**Struktur**: 11 Spalten + +#### Identifikation +- **id** (bigint, NOT NULL, AUTO_INCREMENT) + - Primärschlüssel + - Sequenz: `companies_id_seq` + +#### Firmenidentifikation +- **name** (varchar, NOT NULL) + - Primärer Firmenname (Kurzform/Handelsname) + +- **legal_name** (varchar, nullable) + - Offizieller rechtlicher Name + - Kann vom Handelsnamen abweichen + +- **ticker** (varchar, nullable) + - Börsenticker-Symbol (z.B. "AAPL", "MSFT") + - Nur für börsennotierte Unternehmen + +#### Klassifikation & Lokalisierung +- **sector** (varchar, nullable) + - Wirtschaftssektor/Branche + - Z.B. "Technology", "Finance", "Manufacturing" + +- **country** (varchar, NOT NULL, default: 'DE') + - Ländercode (ISO 2-Letter) + - Standard: Deutschland + +- **headquarters** (varchar, nullable) + - Hauptsitz/Firmenzentrale + - Stadt oder Stadt + Land + +#### Risk & Compliance +- **kyc_risk_level** (varchar, NOT NULL, default: 'medium') + - KYC-Risikoeinstufung + - Mögliche Werte: "low", "medium", "high" + - Default: mittleres Risiko + +#### Zusatzinformationen +- **summary** (text, nullable) + - Firmenbeschreibung/Zusammenfassung + - Freitextfeld für Kontext + +#### Zeitstempel +- **created_at** (timestamp, nullable) + - Erstellungszeitpunkt + +- **updated_at** (timestamp, nullable) + - Letzte Aktualisierung + - Laravel-Standard für Timestamps + +**Charakteristik**: Saubere, normalisierte Stammdatentabelle mit KYC-Fokus + +--- + +### **4. public.transactions** + +**Zweck**: Produktive Transaktionsdaten mit umfassender Anreicherung aus externen Datenquellen und Risikoanalyse + +**Struktur**: 54 Spalten (!) + +#### Identifikation +- **id** (bigint, NOT NULL, AUTO_INCREMENT) - 9x aufgelistet (⚠️ Schema-Anomalie!) + - Primärschlüssel + - Sequenz: `transactions_id_seq` + +#### Transaktions-Basis +- **company_id** (bigint, NOT NULL, default: 1) + - Foreign Key zu `public.companies.id` + - Zuordnung zur durchführenden Firma + +- **reference** (varchar, NOT NULL) + - Transaktionsreferenz/Buchungsnummer + - Eindeutiger Identifier + +- **amount** (numeric, NOT NULL) + - Transaktionsbetrag + - Numeric für präzise Währungsbeträge + +- **currency** (varchar, NOT NULL, default: 'EUR') + - Währungscode + - Standard: Euro + +- **counterparty** (varchar, NOT NULL) + - Name der Gegenpartei + +- **counterparty_country** (varchar, nullable) + - Land der Gegenpartei + +- **channel** (varchar, nullable) + - Transaktionskanal (z.B. "wire", "sepa", "swift") + +- **executed_at** (timestamp, NOT NULL) + - Ausführungszeitpunkt der Transaktion + +#### Risk Management +- **risk_score** (smallint, NOT NULL, default: 0) + - Risikobewertung (0-100 oder ähnlich) + +- **status** (varchar, NOT NULL) + - Transaktionsstatus (z.B. "pending", "approved", "flagged") + +- **requires_review** (boolean, NOT NULL, default: true) + - Manuelles Review erforderlich? + +- **flagged_by** (varchar, nullable) + - System/User der die Transaktion markiert hat + +- **flagged_reason** (text, nullable) + - Grund für Markierung + +- **signals** (json, nullable) + - Risikosignale/Trigger als JSON + - Strukturierte Risikoindikatoren + +#### Externe Datenquellen (11 Integrationen) + +**1. Registry (Handelsregister Basic)** +- **registry_company_number** (text) +- **registry_source** (text) - Quelle (z.B. "Handelsregister") +- **registry_match_score** (double precision) - Matching-Genauigkeit +- **registry_data** (jsonb) - Registrierungsdaten +- **registry_last_refreshed_at** (timestamp) + +**2. Genesis (Statistisches Bundesamt)** +- **genesis_context** (jsonb) +- **genesis_last_refreshed_at** (timestamp) + +**3. GovData (Offene Verwaltungsdaten)** +- **govdata_data** (jsonb) +- **govdata_last_refreshed_at** (timestamp) + +**4. Bundesanzeiger** +- **bundesanzeiger_data** (jsonb) +- **bundesanzeiger_last_refreshed_at** (timestamp) + +**5. Insolvency (Insolvenzregister)** +- **insolvency_data** (jsonb) +- **insolvency_last_refreshed_at** (timestamp) + +**6. RSS Alerts (News/Medien)** +- **rss_alerts** (jsonb) +- **rss_last_refreshed_at** (timestamp) + +**7. Sanctions (Sanktionslisten)** +- **sanctions_data** (jsonb) +- **sanctions_last_refreshed_at** (timestamp) + +**8. PEP (Politically Exposed Persons)** +- **pep_data** (jsonb) +- **pep_last_refreshed_at** (timestamp) + +**9. GLEIF (Legal Entity Identifier)** +- **gleif_lei** (text) - LEI-Nummer +- **gleif_data** (json) +- **gleif_last_refreshed_at** (timestamp) + +**10. EU Sanctions** +- **eu_sanctions_data** (jsonb) +- **eu_sanctions_last_refreshed_at** (timestamp) + +**11. Handelsregister (Extended)** +- **handelsregister_data** (jsonb) +- **handelsregister_last_refreshed_at** (timestamp) +- **handelsregister_status** (text) +- **handelsregister_entity_id** (bigint) + +#### Zeitstempel +- **created_at** (timestamp, nullable) +- **updated_at** (timestamp, nullable) + +**Charakteristik**: Hochgradig angereichertes Data Warehouse für Compliance und Risk Management mit Multi-Source-Integration + +--- + +## Zusammenfassung der Architektur + +``` +┌─────────────────────────────────────┐ +│ Backend Schema (Staging) │ +├─────────────────────────────────────┤ +│ • Rohdaten-Import │ +│ • Lockere Typisierung (text) │ +│ • Source-Tracking │ +│ • Prompt/AI-Integration │ +└────────────┬────────────────────────┘ + │ + │ ETL/Processing + ↓ +┌─────────────────────────────────────┐ +│ Public Schema (Production) │ +├─────────────────────────────────────┤ +│ • Normalisierte Daten │ +│ • Strikte Typisierung │ +│ • Multi-Source-Enrichment │ +│ • Risk & Compliance Features │ +└─────────────────────────────────────┘ +``` + +## Datenfluss-Hypothese + +1. **Import**: Rohdaten landen in `backend.transactions` +2. **AI-Verarbeitung**: Prompts generieren Outputs in `backend.transaction_outputs` +3. **Enrichment**: Externe Datenquellen werden abgefragt +4. **Normalisierung**: Daten werden nach `public.companies` und `public.transactions` übertragen +5. **Risk Assessment**: Risikoscores und Flags werden berechnet +6. **Review**: Transaktionen mit `requires_review=true` landen in der Queue + +## Technische Hinweise + +### Probleme +- ⚠️ `public.transactions` hat 9x duplizierte `id` Spalte im Schema +- ⚠️ `backend.transactions` speichert Timestamps als TEXT statt TIMESTAMP +- ⚠️ Keine expliziten Foreign Key Constraints sichtbar zwischen den Schemas + +### Empfehlungen +1. Schema-Anomalie in `public.transactions` untersuchen +2. Datum-Felder in `backend.transactions` zu echten TIMESTAMP-Typen migrieren +3. Indizes auf häufig genutzte JOIN/WHERE Spalten prüfen +4. Foreign Key Constraints zwischen den Schemas dokumentieren diff --git a/misc/database-tables-overview.md b/misc/database-tables-overview.md new file mode 100644 index 0000000..e54877d --- /dev/null +++ b/misc/database-tables-overview.md @@ -0,0 +1,56 @@ +# Datenbank-Tabellen Übersicht + +## Alle Tabellen in deiner PostgreSQL-Datenbank (Schema: public) + +**Insgesamt: 41 Tabellen** + +### ✅ Von Laravel-Migrationen erstellt (13 Tabellen): +1. `cache` +2. `cache_locks` +3. `companies` +4. `failed_jobs` +5. `job_batches` +6. `jobs` +7. `migrations` (Laravel-interne Tracking-Tabelle) +8. `password_reset_tokens` +9. `sessions` +10. `transactions` +11. `users` + +*(Die `users`-Tabelle wurde zusätzlich durch Migration [2025_09_02_075243_add_two_factor_columns_to_users_table.php](database/migrations/2025_09_02_075243_add_two_factor_columns_to_users_table.php) um 2FA-Spalten erweitert)* + +--- + +### ❌ NICHT von Laravel-Migrationen erstellt (28 Tabellen): +1. `alembic_version` (Python Alembic Migrations) +2. `bundesanzeiger_cache` +3. `companies_view` (PostgreSQL View) +4. `company_gleif_cache` +5. `company_master_data` +6. `company_master_data_links` +7. `company_opencorporates_cache` +8. `company_registry_cache` +9. `dpma_cache` +10. `entity_corporate_context` +11. `eu_sanctions_cache` +12. `evidence_registry` +13. `genesis_cache` +14. `govdata_cache` +15. `handelsregister_cache` +16. `handelsregister_document_links` +17. `handelsregister_documents` +18. `handelsregister_entities` +19. `handelsregister_entity_transactions` +20. `handelsregister_relations` +21. `insolvency_cache` +22. `pep_cache` +23. `prompt_runs` +24. `prompt_templates` +25. `rss_cache` +26. `sanctions_cache` +27. `test_transaction_llm` +28. `transaction` (Singular-Version, eventuell Legacy?) +29. `transaction_outputs` +30. `transactions_enriched` + +Die meisten dieser externen Tabellen scheinen Cache-Tabellen für verschiedene Datenquellen (Handelsregister, Sanctions, GLEIF, etc.) und Enrichment-Daten zu sein. Die `alembic_version`-Tabelle deutet darauf hin, dass möglicherweise ein Python-Backend parallel läuft. diff --git a/misc/laravel-models-migrations-analysis.md b/misc/laravel-models-migrations-analysis.md new file mode 100644 index 0000000..af7466b --- /dev/null +++ b/misc/laravel-models-migrations-analysis.md @@ -0,0 +1,610 @@ +# Analyse: Laravel Models vs Migrations vs Datenbank + +## Übersicht + +Diese Analyse vergleicht die Laravel Eloquent Models mit den entsprechenden Migration-Dateien und der tatsächlichen Datenbankstruktur. + +--- + +## **1. Company Model & Migration** + +### ✅ **PERFEKT SYNCHRON** + +#### Migration +**Datei**: [database/migrations/2025_10_20_181750_create_companies_table.php](database/migrations/2025_10_20_181750_create_companies_table.php) + +```php +Schema::create('companies', function (Blueprint $table) { + $table->id(); + $table->string('name')->unique(); + $table->string('legal_name')->nullable(); + $table->string('ticker')->nullable(); + $table->string('sector')->nullable(); + $table->string('country', 2)->default('DE'); + $table->string('headquarters')->nullable(); + $table->string('kyc_risk_level')->default('medium'); + $table->text('summary')->nullable(); + $table->timestamps(); +}); +``` + +**Felder**: +- `id` (auto-increment) +- `name` (string, unique) +- `legal_name` (string, nullable) +- `ticker` (string, nullable) +- `sector` (string, nullable) +- `country` (string(2), default: 'DE') +- `headquarters` (string, nullable) +- `kyc_risk_level` (string, default: 'medium') +- `summary` (text, nullable) +- `timestamps` (created_at, updated_at) + +#### Model +**Datei**: [app/Models/Company.php](app/Models/Company.php) + +```php +protected $fillable = [ + 'name', + 'legal_name', + 'ticker', + 'sector', + 'country', + 'headquarters', + 'kyc_risk_level', + 'summary', +]; + +public function transactions(): HasMany +{ + return $this->hasMany(Transaction::class); +} +``` + +#### Datenbank-Status +- ✅ Alle Felder vorhanden +- ✅ Datentypen stimmen überein +- ✅ Defaults korrekt gesetzt +- ✅ Unique Constraint auf `name` +- ✅ Relationship `hasMany(Transaction::class)` definiert + +--- + +## **2. Transaction Model & Migration** + +### ⚠️ **TEILWEISE DISKREPANZEN** + +#### Migration +**Datei**: [database/migrations/2025_10_20_181753_create_transactions_table.php](database/migrations/2025_10_20_181753_create_transactions_table.php) + +**Gesamt**: 49 Spalten (ohne timestamps) + +##### Core Felder (15 Spalten) +```php +$table->id(); +$table->foreignId('company_id')->constrained()->cascadeOnDelete(); +$table->string('reference')->unique(); +$table->decimal('amount', 16, 2); +$table->string('currency', 3)->default('EUR'); +$table->string('counterparty'); +$table->string('counterparty_country', 2)->nullable(); +$table->string('channel')->nullable(); +$table->dateTime('executed_at'); +$table->unsignedTinyInteger('risk_score')->default(0); +$table->string('status', 32)->index(); +$table->boolean('requires_review')->default(true); +$table->string('flagged_by')->nullable(); +$table->text('flagged_reason')->nullable(); +$table->json('signals')->nullable(); +``` + +##### Enrichment-Felder (11 Datenquellen, 34 Spalten) + +**1. Registry (5 Felder)** +```php +$table->text('registry_company_number')->nullable(); +$table->text('registry_source')->nullable(); +$table->double('registry_match_score')->nullable(); +$table->jsonb('registry_data')->nullable(); +$table->dateTime('registry_last_refreshed_at')->nullable(); +``` + +**2. Genesis (2 Felder)** +```php +$table->jsonb('genesis_context')->nullable(); +$table->dateTime('genesis_last_refreshed_at')->nullable(); +``` + +**3. GovData (2 Felder)** +```php +$table->jsonb('govdata_data')->nullable(); +$table->dateTime('govdata_last_refreshed_at')->nullable(); +``` + +**4. Bundesanzeiger (2 Felder)** +```php +$table->jsonb('bundesanzeiger_data')->nullable(); +$table->dateTime('bundesanzeiger_last_refreshed_at')->nullable(); +``` + +**5. Insolvency (2 Felder)** +```php +$table->jsonb('insolvency_data')->nullable(); +$table->dateTime('insolvency_last_refreshed_at')->nullable(); +``` + +**6. RSS Alerts (2 Felder)** +```php +$table->jsonb('rss_alerts')->nullable(); +$table->dateTime('rss_last_refreshed_at')->nullable(); +``` + +**7. Sanctions (2 Felder)** +```php +$table->jsonb('sanctions_data')->nullable(); +$table->dateTime('sanctions_last_refreshed_at')->nullable(); +``` + +**8. PEP (2 Felder)** +```php +$table->jsonb('pep_data')->nullable(); +$table->dateTime('pep_last_refreshed_at')->nullable(); +``` + +**9. GLEIF (3 Felder)** +```php +$table->text('gleif_lei')->nullable(); +$table->json('gleif_data')->nullable(); +$table->dateTime('gleif_last_refreshed_at')->nullable(); +``` + +**10. EU Sanctions (2 Felder)** +```php +$table->jsonb('eu_sanctions_data')->nullable(); +$table->dateTime('eu_sanctions_last_refreshed_at')->nullable(); +``` + +**11. Handelsregister (4 Felder)** +```php +$table->jsonb('handelsregister_data')->nullable(); +$table->dateTime('handelsregister_last_refreshed_at')->nullable(); +$table->text('handelsregister_status')->nullable(); +$table->bigInteger('handelsregister_entity_id')->nullable(); +``` + +#### Model +**Datei**: [app/Models/Transaction.php](app/Models/Transaction.php) + +```php +// Status-Konstanten +public const STATUS_TRUE_POSITIVE = 'true_positive'; +public const STATUS_FALSE_POSITIVE = 'false_positive'; +public const STATUS_CLEARED = 'cleared'; + +// Fillable (nur Core-Felder!) +protected $fillable = [ + 'company_id', + 'reference', + 'amount', + 'currency', + 'counterparty', + 'counterparty_country', + 'channel', + 'executed_at', + 'risk_score', + 'status', + 'requires_review', + 'flagged_by', + 'flagged_reason', + 'signals', +]; + +// Casts +protected $casts = [ + 'executed_at' => 'datetime', + 'requires_review' => 'boolean', + 'signals' => 'array', +]; + +// Relationship +public function company(): BelongsTo +{ + return $this->belongsTo(Company::class); +} + +// Helper-Methode +public function statusLabel(): string +{ + return match ($this->status) { + self::STATUS_TRUE_POSITIVE => __('Bestätigter Treffer'), + self::STATUS_FALSE_POSITIVE => __('Fehlalarm'), + default => __('Freigegeben'), + }; +} +``` + +### ⚠️ **FEHLENDE FELDER IM MODEL** + +Das Transaction Model hat **NUR 14 Core-Felder** im `$fillable` Array, aber die Migration definiert **49 Felder** (exkl. timestamps). + +#### Fehlende Enrichment-Felder (34 Spalten): + +**Registry-Felder:** +- `registry_company_number` +- `registry_source` +- `registry_match_score` +- `registry_data` +- `registry_last_refreshed_at` + +**Genesis-Felder:** +- `genesis_context` +- `genesis_last_refreshed_at` + +**GovData-Felder:** +- `govdata_data` +- `govdata_last_refreshed_at` + +**Bundesanzeiger-Felder:** +- `bundesanzeiger_data` +- `bundesanzeiger_last_refreshed_at` + +**Insolvency-Felder:** +- `insolvency_data` +- `insolvency_last_refreshed_at` + +**RSS-Felder:** +- `rss_alerts` +- `rss_last_refreshed_at` + +**Sanctions-Felder:** +- `sanctions_data` +- `sanctions_last_refreshed_at` + +**PEP-Felder:** +- `pep_data` +- `pep_last_refreshed_at` + +**GLEIF-Felder:** +- `gleif_lei` +- `gleif_data` +- `gleif_last_refreshed_at` + +**EU Sanctions-Felder:** +- `eu_sanctions_data` +- `eu_sanctions_last_refreshed_at` + +**Handelsregister-Felder:** +- `handelsregister_data` +- `handelsregister_last_refreshed_at` +- `handelsregister_status` +- `handelsregister_entity_id` + +### ⚠️ **FEHLENDE CASTS** + +Das Model sollte Casts für alle zeitbasierten und JSON-Felder haben: + +**Fehlende DateTime-Casts:** +- `registry_last_refreshed_at` +- `genesis_last_refreshed_at` +- `govdata_last_refreshed_at` +- `bundesanzeiger_last_refreshed_at` +- `insolvency_last_refreshed_at` +- `rss_last_refreshed_at` +- `sanctions_last_refreshed_at` +- `pep_last_refreshed_at` +- `gleif_last_refreshed_at` +- `eu_sanctions_last_refreshed_at` +- `handelsregister_last_refreshed_at` + +**Fehlende JSON/Array-Casts:** +- `registry_data` +- `genesis_context` +- `govdata_data` +- `bundesanzeiger_data` +- `insolvency_data` +- `rss_alerts` +- `sanctions_data` +- `pep_data` +- `gleif_data` +- `eu_sanctions_data` +- `handelsregister_data` + +--- + +## **3. Vergleich: Datenbank vs Migration** + +### public.companies + +**Status**: ✅ **100% Übereinstimmung** + +| Feature | Migration | Datenbank | Status | +|---------|-----------|-----------|--------| +| Spalten | 11 | 11 | ✅ | +| Unique Constraint | `name` | `name` | ✅ | +| Defaults | `country='DE'`, `kyc_risk_level='medium'` | Identisch | ✅ | + +### public.transactions + +**Status**: ✅ **Migration deckt alle DB-Felder ab** + +#### Constraints & Indizes +| Constraint | Migration | Datenbank | Status | +|------------|-----------|-----------|--------| +| Foreign Key | `company_id → companies.id` | Vorhanden | ✅ | +| Cascade Delete | `cascadeOnDelete()` | Implementiert | ✅ | +| Unique | `reference` | Vorhanden | ✅ | +| Index | `status` | Vorhanden | ✅ | + +#### Datentypen-Vergleich + +| Feld | Migration | Datenbank | Status | +|------|-----------|-----------|--------| +| id | `id()` | bigint | ✅ | +| company_id | `foreignId()` | bigint | ✅ | +| amount | `decimal(16,2)` | numeric | ✅ | +| currency | `string(3)` | varchar | ✅ | +| counterparty_country | `string(2)` | varchar | ✅ | +| risk_score | `unsignedTinyInteger` | smallint | ⚠️* | +| status | `string(32)` | varchar | ✅ | +| requires_review | `boolean` | boolean | ✅ | +| signals | `json` | json | ✅ | +| *_data | `jsonb` | jsonb | ✅ | +| gleif_data | `json` | json | ✅ | +| executed_at | `dateTime` | timestamp | ✅ | +| *_last_refreshed_at | `dateTime` | timestamp | ✅ | + +*`unsignedTinyInteger` (0-255) vs `smallint` (-32768 bis 32767) sind funktional kompatibel + +--- + +## Zusammenfassung + +### ✅ **Stärken** + +1. **Migration-Dateien sind vollständig** + - Alle Datenbank-Felder korrekt definiert + - Foreign Key Constraints implementiert + - Indizes sinnvoll gesetzt + +2. **Core-Model-Felder stimmen überein** + - Basis-Transaktionsfelder vollständig + - Relationships sauber definiert + +3. **Datenbank-Konsistenz** + - Migrations wurden korrekt ausgeführt + - Constraints sind aktiv + +--- + +## ⚠️ **Probleme & Empfehlungen** + +### Problem 1: Transaction Model ist unvollständig + +**Problem**: +- Das Model definiert nur 14 von 49 Feldern im `$fillable` Array +- Alle 34 Enrichment-Felder fehlen + +**Auswirkungen**: +- ❌ Enrichment-Felder können nicht via Mass Assignment gesetzt werden +- ❌ Keine automatischen Type Casts für externe Datenfelder +- ❌ Potenzielle Fehler beim Zugriff auf nicht-gecastete JSON-Daten +- ❌ DateTime-Felder werden als Strings zurückgegeben + +**Lösungsvorschläge**: + +**Option 1**: Alle Felder zu `$fillable` hinzufügen +```php +protected $fillable = [ + // Core fields + 'company_id', 'reference', 'amount', 'currency', + 'counterparty', 'counterparty_country', 'channel', + 'executed_at', 'risk_score', 'status', + 'requires_review', 'flagged_by', 'flagged_reason', 'signals', + + // Registry + 'registry_company_number', 'registry_source', 'registry_match_score', + 'registry_data', 'registry_last_refreshed_at', + + // Genesis + 'genesis_context', 'genesis_last_refreshed_at', + + // ... alle weiteren Felder +]; +``` + +**Option 2**: `$guarded` verwenden (empfohlen für interne Anwendungen) +```php +protected $guarded = ['id']; +``` + +**Option 3**: Separate Accessor/Mutator für Enrichment-Felder +```php +public function registryData(): Attribute +{ + return Attribute::make( + get: fn ($value) => json_decode($value, true), + set: fn ($value) => json_encode($value), + ); +} +``` + +### Problem 2: Fehlende Casts für Enrichment-Felder + +**Problem**: +- Keine Casts für `*_last_refreshed_at` Felder +- Keine Casts für `*_data` JSON-Felder + +**Auswirkungen**: +- DateTime-Felder werden als Strings zurückgegeben (kein Carbon-Objekt) +- JSON-Felder müssen manuell dekodiert werden + +**Lösung**: +```php +protected $casts = [ + // Existing + 'executed_at' => 'datetime', + 'requires_review' => 'boolean', + 'signals' => 'array', + + // DateTime casts for all refresh timestamps + 'registry_last_refreshed_at' => 'datetime', + 'genesis_last_refreshed_at' => 'datetime', + 'govdata_last_refreshed_at' => 'datetime', + 'bundesanzeiger_last_refreshed_at' => 'datetime', + 'insolvency_last_refreshed_at' => 'datetime', + 'rss_last_refreshed_at' => 'datetime', + 'sanctions_last_refreshed_at' => 'datetime', + 'pep_last_refreshed_at' => 'datetime', + 'gleif_last_refreshed_at' => 'datetime', + 'eu_sanctions_last_refreshed_at' => 'datetime', + 'handelsregister_last_refreshed_at' => 'datetime', + + // JSON/Array casts for all data fields + 'registry_data' => 'array', + 'genesis_context' => 'array', + 'govdata_data' => 'array', + 'bundesanzeiger_data' => 'array', + 'insolvency_data' => 'array', + 'rss_alerts' => 'array', + 'sanctions_data' => 'array', + 'pep_data' => 'array', + 'gleif_data' => 'array', + 'eu_sanctions_data' => 'array', + 'handelsregister_data' => 'array', +]; +``` + +### Problem 3: Datenbank-Schema-Anomalie + +**Problem**: +- Die `public.transactions` Tabelle zeigt 9x duplizierte `id` Spalten im describe_table Output + +**Mögliche Ursachen**: +- Korruptes Schema-Metadaten +- Mehrfache Migration-Ausführungen ohne Rollback +- PostgreSQL-Katalog-Problem + +**Lösung**: +1. Schema inspizieren: `\d+ transactions` in psql +2. Bei Bedarf Migration neu ausführen +3. Oder manuelles ALTER TABLE zur Bereinigung + +--- + +## Nächste Schritte + +### Empfohlene Reihenfolge: + +1. ✅ **Transaction Model aktualisieren** + - Alle fehlenden Felder zu `$fillable` hinzufügen + - Alle fehlenden Casts definieren + +2. ✅ **Tests schreiben** + - Unit-Tests für Model-Casts + - Feature-Tests für Enrichment-Datenfluss + +3. ⚠️ **Datenbank-Anomalie untersuchen** + - PostgreSQL-Schema inspizieren + - Ggf. Migration neu ausführen + +4. 📝 **Dokumentation erweitern** + - Enrichment-Pipeline dokumentieren + - API für externe Datenquellen dokumentieren + +--- + +## Checkliste + +### Companies +- [x] Migration vollständig +- [x] Model synchron mit Migration +- [x] Datenbank korrekt strukturiert +- [x] Relationships definiert +- [x] Casts korrekt + +### Transactions +- [x] Migration vollständig +- [ ] Model synchron mit Migration ⚠️ +- [x] Datenbank korrekt strukturiert +- [x] Relationships definiert +- [ ] Casts vollständig ⚠️ +- [ ] Schema-Anomalie behoben ⚠️ + +--- + +## Anhang: Vollständige Feldliste Transaction Model + +### Core Felder (14) +✅ Im Model vorhanden + +1. company_id +2. reference +3. amount +4. currency +5. counterparty +6. counterparty_country +7. channel +8. executed_at +9. risk_score +10. status +11. requires_review +12. flagged_by +13. flagged_reason +14. signals + +### Enrichment Felder (34) +❌ Im Model fehlend + +**Registry (5)** +15. registry_company_number +16. registry_source +17. registry_match_score +18. registry_data +19. registry_last_refreshed_at + +**Genesis (2)** +20. genesis_context +21. genesis_last_refreshed_at + +**GovData (2)** +22. govdata_data +23. govdata_last_refreshed_at + +**Bundesanzeiger (2)** +24. bundesanzeiger_data +25. bundesanzeiger_last_refreshed_at + +**Insolvency (2)** +26. insolvency_data +27. insolvency_last_refreshed_at + +**RSS (2)** +28. rss_alerts +29. rss_last_refreshed_at + +**Sanctions (2)** +30. sanctions_data +31. sanctions_last_refreshed_at + +**PEP (2)** +32. pep_data +33. pep_last_refreshed_at + +**GLEIF (3)** +34. gleif_lei +35. gleif_data +36. gleif_last_refreshed_at + +**EU Sanctions (2)** +37. eu_sanctions_data +38. eu_sanctions_last_refreshed_at + +**Handelsregister (4)** +39. handelsregister_data +40. handelsregister_last_refreshed_at +41. handelsregister_status +42. handelsregister_entity_id + +--- + +*Analysiert am: 2025-11-12* diff --git a/misc/migrate-tables-to-archive.sql b/misc/migrate-tables-to-archive.sql new file mode 100644 index 0000000..77fede0 --- /dev/null +++ b/misc/migrate-tables-to-archive.sql @@ -0,0 +1,68 @@ +-- ===================================================== +-- Migration Script: Move Non-Laravel Tables to Archive +-- ===================================================== +-- This script creates a new schema 'public_archiv' and +-- moves all tables that were not created by Laravel +-- migrations from the 'public' schema to 'public_archiv'. +-- ===================================================== + +-- Step 1: Create new schema +CREATE SCHEMA IF NOT EXISTS public_archiv; + +-- Step 2: Move tables to public_archiv schema +-- (28 tables that were not created by Laravel migrations) + +ALTER TABLE public.alembic_version SET SCHEMA public_archiv; +ALTER TABLE public.bundesanzeiger_cache SET SCHEMA public_archiv; +ALTER TABLE public.companies_view SET SCHEMA public_archiv; +ALTER TABLE public.company_gleif_cache SET SCHEMA public_archiv; +ALTER TABLE public.company_master_data SET SCHEMA public_archiv; +ALTER TABLE public.company_master_data_links SET SCHEMA public_archiv; +ALTER TABLE public.company_opencorporates_cache SET SCHEMA public_archiv; +ALTER TABLE public.company_registry_cache SET SCHEMA public_archiv; +ALTER TABLE public.dpma_cache SET SCHEMA public_archiv; +ALTER TABLE public.entity_corporate_context SET SCHEMA public_archiv; +ALTER TABLE public.eu_sanctions_cache SET SCHEMA public_archiv; +ALTER TABLE public.evidence_registry SET SCHEMA public_archiv; +ALTER TABLE public.genesis_cache SET SCHEMA public_archiv; +ALTER TABLE public.govdata_cache SET SCHEMA public_archiv; +ALTER TABLE public.handelsregister_cache SET SCHEMA public_archiv; +ALTER TABLE public.handelsregister_document_links SET SCHEMA public_archiv; +ALTER TABLE public.handelsregister_documents SET SCHEMA public_archiv; +ALTER TABLE public.handelsregister_entities SET SCHEMA public_archiv; +ALTER TABLE public.handelsregister_entity_transactions SET SCHEMA public_archiv; +ALTER TABLE public.handelsregister_relations SET SCHEMA public_archiv; +ALTER TABLE public.insolvency_cache SET SCHEMA public_archiv; +ALTER TABLE public.pep_cache SET SCHEMA public_archiv; +ALTER TABLE public.prompt_runs SET SCHEMA public_archiv; +ALTER TABLE public.prompt_templates SET SCHEMA public_archiv; +ALTER TABLE public.rss_cache SET SCHEMA public_archiv; +ALTER TABLE public.sanctions_cache SET SCHEMA public_archiv; +ALTER TABLE public.test_transaction_llm SET SCHEMA public_archiv; +ALTER TABLE public.transaction SET SCHEMA public_archiv; +ALTER TABLE public.transaction_outputs SET SCHEMA public_archiv; +ALTER TABLE public.transactions_enriched SET SCHEMA public_archiv; + +-- ===================================================== +-- Verification Queries +-- ===================================================== + +-- Check tables in public_archiv schema +SELECT table_name +FROM information_schema.tables +WHERE table_schema = 'public_archiv' +ORDER BY table_name; + +-- Check remaining tables in public schema (should only be Laravel tables) +SELECT table_name +FROM information_schema.tables +WHERE table_schema = 'public' +ORDER BY table_name; + +-- Count tables per schema +SELECT + table_schema, + COUNT(*) as table_count +FROM information_schema.tables +WHERE table_schema IN ('public', 'public_archiv') +GROUP BY table_schema; diff --git a/misc/playground.sql b/misc/playground.sql new file mode 100644 index 0000000..7f0b2b8 --- /dev/null +++ b/misc/playground.sql @@ -0,0 +1,18 @@ +SELECT * FROM public.test_transaction_llm +ORDER BY id ASC ; + + +select * from backend.transactions; + +Select * from +backend.transactions as t +left join backend.transaction_outputs as tout on t.id = tout.transaction_id +left join backend.prompt_templates as prt on tout.prompt_id = prt.prompt_id +--where t.id = 2 +order by transaction_id, tout.prompt_id; + +select * from backend.prompt_templates; + +Select * from public.companies; + +select * from public.transactions; \ No newline at end of file diff --git a/resources/views/components/layouts/app/sidebar.blade.php b/resources/views/components/layouts/app/sidebar.blade.php index 2a933e7..624b57d 100644 --- a/resources/views/components/layouts/app/sidebar.blade.php +++ b/resources/views/components/layouts/app/sidebar.blade.php @@ -16,6 +16,7 @@ {{ __('Übersicht') }} {{ __('Unternehmensauskunft') }} {{ __('Transaktionsprüfung') }} + {{ __('Datei-Upload') }} diff --git a/resources/views/livewire/upload/index.blade.php b/resources/views/livewire/upload/index.blade.php new file mode 100644 index 0000000..187bf0a --- /dev/null +++ b/resources/views/livewire/upload/index.blade.php @@ -0,0 +1,141 @@ + + +
+
+
+ {{ __('File Upload') }} + {{ __('Upload files to the processing endpoint') }} + +
+
+
+ + + +

+
+ +
+ + + +
+ +
+ + {{ __('Upload File') }} + {{ __('Uploading...') }} + + + + {{ __('Clear') }} + +
+ +
+
+
+
+

+ {{ __('Please wait while your file is being uploaded...') }} +

+
+
+
+ + + +
+

{{ __('Information:') }}

+
    +
  • {{ __('Maximum file size: 50 MB') }}
    • +
  • {{ __('Upload endpoint: https://upload.trai.mcs.local/api/upload') }}
    • +
+
+
+
+
diff --git a/routes/web.php b/routes/web.php index 90c1ecf..6a08f9e 100644 --- a/routes/web.php +++ b/routes/web.php @@ -43,6 +43,10 @@ Volt::route('companies/{company}/transactions', 'companies.transactions') ->middleware(['auth', 'verified']) ->name('company.transactions'); +Volt::route('upload', 'upload.index') + ->middleware(['auth', 'verified']) + ->name('upload'); + Route::middleware(['auth'])->group(function () { Route::redirect('settings', 'settings/profile'); diff --git a/tests/Feature/BackendModelsTest.php b/tests/Feature/BackendModelsTest.php new file mode 100644 index 0000000..d51293a --- /dev/null +++ b/tests/Feature/BackendModelsTest.php @@ -0,0 +1,128 @@ +toBeGreaterThanOrEqual(0); +}); + +test('can read transactions from backend schema', function () { + $transaction = Transaction::first(); + + if ($transaction) { + expect($transaction)->toBeInstanceOf(Transaction::class); + expect($transaction->corporate_entity)->toBeString(); + expect($transaction->corporate_counterparty)->toBeString(); + expect($transaction->tx_amount)->not->toBeNull(); + } else { + expect(true)->toBeTrue(); // No transactions yet + } +}); + +test('can read transaction outputs', function () { + $output = TransactionOutput::first(); + + if ($output) { + expect($output)->toBeInstanceOf(TransactionOutput::class); + expect($output->transaction_id)->toBeInt(); + expect($output->output_key)->toBeString(); + expect($output->content)->toBeArray(); + } else { + expect(true)->toBeTrue(); // No outputs yet + } +}); + +test('transaction has outputs relationship', function () { + $transaction = Transaction::with('outputs')->first(); + + if ($transaction) { + expect($transaction->outputs)->toBeInstanceOf(\Illuminate\Database\Eloquent\Collection::class); + } else { + expect(true)->toBeTrue(); // No transactions yet + } +}); + +test('can get specific output by key', function () { + $transaction = Transaction::with('outputs')->first(); + + if ($transaction && $transaction->outputs->isNotEmpty()) { + $firstKey = $transaction->outputs->first()->output_key; + $output = $transaction->getOutput($firstKey); + + expect($output)->toBeArray(); + } else { + expect(true)->toBeTrue(); // No transactions with outputs yet + } +}); + +test('can get company info from transaction', function () { + $transaction = Transaction::with('outputs')->first(); + + if ($transaction) { + $companyInfo = $transaction->getCompanyInfo(); + + // Kann null sein wenn kein company_info output existiert + expect($companyInfo)->toBeIn([null, 'array']); + } else { + expect(true)->toBeTrue(); + } +}); + +test('can get risk assessment from transaction', function () { + $transaction = Transaction::with('outputs')->first(); + + if ($transaction) { + $risk = $transaction->getRiskAssessment(); + + // Kann null sein wenn kein risk_assessment output existiert + expect($risk)->toBeIn([null, 'array']); + + if ($risk) { + expect($risk)->toHaveKey('score'); + } + } else { + expect(true)->toBeTrue(); + } +}); + +test('can check if transaction requires review', function () { + $transaction = Transaction::with('outputs')->first(); + + if ($transaction) { + $requiresReview = $transaction->requiresReview(); + + expect($requiresReview)->toBeBool(); + } else { + expect(true)->toBeTrue(); + } +}); + +test('transaction output keys constants exist', function () { + expect(TransactionOutput::KEY_COMPANY_INFO)->toBe('company_info'); + expect(TransactionOutput::KEY_RISK_ASSESSMENT)->toBe('risk_assessment'); + expect(TransactionOutput::KEY_SANCTIONS)->toBe('sanctions'); + expect(TransactionOutput::KEY_PEP)->toBe('pep'); +}); + +test('can get all available output keys', function () { + $keys = TransactionOutput::availableKeys(); + + expect($keys)->toBeArray(); + expect($keys)->toContain('company_info'); + expect($keys)->toContain('risk_assessment'); +}); + +test('can get label for output key', function () { + $label = TransactionOutput::getKeyLabel('company_info'); + + expect($label)->toBe('Company Information'); + + $label = TransactionOutput::getKeyLabel('risk_assessment'); + + expect($label)->toBe('Risk Assessment'); +}); diff --git a/tests/Feature/Livewire/Upload/IndexTest.php b/tests/Feature/Livewire/Upload/IndexTest.php new file mode 100644 index 0000000..d111afe --- /dev/null +++ b/tests/Feature/Livewire/Upload/IndexTest.php @@ -0,0 +1,105 @@ +get('/upload'); + + $response->assertRedirect('/login'); +}); + +it('can render upload page when authenticated', function () { + $user = User::factory()->create(); + + $this->actingAs($user); + + $component = Volt::test('upload.index'); + + $component + ->assertSee('File Upload') + ->assertSee('Select File') + ->assertSee('Maximum file size: 50 MB') + ->assertSee('https://upload.trai.mcs.local/api/upload'); +}); + +it('can upload file successfully', function () { + Http::fake([ + 'https://upload.trai.mcs.local/api/upload' => Http::response('Success', 200), + ]); + + $user = User::factory()->create(); + $this->actingAs($user); + + Storage::fake('local'); + + $file = UploadedFile::fake()->create('document.pdf', 100); + + Volt::test('upload.index') + ->set('file', $file) + ->call('upload') + ->assertSet('uploadStatus', 'success') + ->assertSet('uploadMessage', 'File uploaded successfully!') + ->assertSet('file', null); +}); + +it('validates file is required', function () { + $user = User::factory()->create(); + $this->actingAs($user); + + Volt::test('upload.index') + ->set('file', null) + ->call('upload') + ->assertHasErrors(['file' => 'required']); +}); + +it('validates file size limit', function () { + $user = User::factory()->create(); + $this->actingAs($user); + + Storage::fake('local'); + + $file = UploadedFile::fake()->create('large.pdf', 51201); // Exceeds 50MB + + Volt::test('upload.index') + ->set('file', $file) + ->call('upload') + ->assertHasErrors('file'); +}); + +it('handles upload errors gracefully', function () { + Http::fake([ + 'https://upload.trai.mcs.local/api/upload' => Http::response('Server Error', 500), + ]); + + $user = User::factory()->create(); + $this->actingAs($user); + + Storage::fake('local'); + + $file = UploadedFile::fake()->create('document.pdf', 100); + + Volt::test('upload.index') + ->set('file', $file) + ->call('upload') + ->assertSet('uploadStatus', 'error') + ->assertSee('Upload failed'); +}); + +it('can clear selected file', function () { + $user = User::factory()->create(); + $this->actingAs($user); + + Storage::fake('local'); + + $file = UploadedFile::fake()->create('document.pdf', 100); + + Volt::test('upload.index') + ->set('file', $file) + ->assertSet('file', fn ($value) => $value !== null) + ->set('file', null) + ->assertSet('file', null); +}); diff --git a/tests/Feature/NavigationTest.php b/tests/Feature/NavigationTest.php new file mode 100644 index 0000000..73c3ebd --- /dev/null +++ b/tests/Feature/NavigationTest.php @@ -0,0 +1,12 @@ +create(); + + $this->actingAs($user) + ->get('/dashboard') + ->assertSee('Datei-Upload') + ->assertSee(route('upload')); +});