fix: added new command - complete rebuild
This commit is contained in:
cbazza committed
1 parent
1505975ebb
commit
a36d2b4cc4
4 files changed
+486
-8
No files matched your search
@@ -1,30 +1,90 @@
|
|||||||
<?php
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
namespace App\Console\Commands;
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use App\Jobs\SyncBackendDataPool;
|
||||||
|
use App\Jobs\TransformDataPoolToProduction;
|
||||||
use Illuminate\Console\Command;
|
use Illuminate\Console\Command;
|
||||||
|
|
||||||
|
use function Laravel\Prompts\confirm;
|
||||||
|
use function Laravel\Prompts\info;
|
||||||
|
use function Laravel\Prompts\spin;
|
||||||
|
use function Laravel\Prompts\warning;
|
||||||
|
|
||||||
class RebuildFromDataPoolCommand extends Command
|
class RebuildFromDataPoolCommand extends Command
|
||||||
{
|
{
|
||||||
/**
|
/**
|
||||||
* The name and signature of the console command.
|
* The name and signature of the console command.
|
||||||
*
|
|
||||||
* @var string
|
|
||||||
*/
|
*/
|
||||||
protected $signature = 'app:rebuild-from-data-pool-command';
|
protected $signature = 'backend:rebuild
|
||||||
|
{--batch-size=1000 : Number of records to process per batch}
|
||||||
|
{--transform-batch-size=100 : Number of records to process per batch for transformation}
|
||||||
|
{--queue : Dispatch jobs to the queue instead of running synchronously}';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The console command description.
|
* The console command description.
|
||||||
*
|
|
||||||
* @var string
|
|
||||||
*/
|
*/
|
||||||
protected $description = 'Command description';
|
protected $description = 'Complete rebuild: Full sync from backend + transformation to production tables';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute the console command.
|
* Execute the console command.
|
||||||
*/
|
*/
|
||||||
public function handle()
|
public function handle(): int
|
||||||
{
|
{
|
||||||
//
|
info('Preparing for COMPLETE REBUILD');
|
||||||
|
warning('This will:');
|
||||||
|
warning('1. Truncate and rebuild backend_data_pool (full sync)');
|
||||||
|
warning('2. Transform all data into companies and transactions');
|
||||||
|
|
||||||
|
if (! $this->option('no-interaction') && ! confirm('Do you want to continue?', true)) {
|
||||||
|
info('Rebuild cancelled.');
|
||||||
|
|
||||||
|
return self::SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
$batchSize = (int) $this->option('batch-size');
|
||||||
|
$transformBatchSize = (int) $this->option('transform-batch-size');
|
||||||
|
$useQueue = $this->option('queue');
|
||||||
|
|
||||||
|
// Step 1: Full Sync
|
||||||
|
info('Step 1/2: Full Sync from Backend');
|
||||||
|
$syncJob = new SyncBackendDataPool(
|
||||||
|
fullSync: true,
|
||||||
|
batchSize: $batchSize
|
||||||
|
);
|
||||||
|
|
||||||
|
if ($useQueue) {
|
||||||
|
info('Dispatching sync job to queue...');
|
||||||
|
dispatch($syncJob);
|
||||||
|
info('Sync job dispatched!');
|
||||||
|
warning('Please wait for the sync job to complete before transformation.');
|
||||||
|
|
||||||
|
return self::SUCCESS;
|
||||||
|
} else {
|
||||||
|
spin(
|
||||||
|
fn () => $syncJob->handle(),
|
||||||
|
'Syncing data from backend...'
|
||||||
|
);
|
||||||
|
info('✓ Sync completed!');
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->newLine();
|
||||||
|
|
||||||
|
// Step 2: Transform Data
|
||||||
|
info('Step 2/2: Transform Data Pool to Production');
|
||||||
|
$transformJob = new TransformDataPoolToProduction(batchSize: $transformBatchSize);
|
||||||
|
|
||||||
|
spin(
|
||||||
|
fn () => $transformJob->handle(),
|
||||||
|
'Transforming data...'
|
||||||
|
);
|
||||||
|
info('✓ Transformation completed!');
|
||||||
|
|
||||||
|
$this->newLine();
|
||||||
|
info('Complete rebuild finished successfully!');
|
||||||
|
|
||||||
|
return self::SUCCESS;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,401 @@
|
|||||||
|
# Backend Sync & Transform Commands
|
||||||
|
|
||||||
|
Dokumentation für die manuellen Artisan Commands zum Synchronisieren und Transformieren von Backend-Daten.
|
||||||
|
|
||||||
|
## Übersicht
|
||||||
|
|
||||||
|
Das System verfügt über 3 Haupt-Commands zur Datensynchronisation:
|
||||||
|
|
||||||
|
1. **`backend:sync-data-pool`** - Synchronisiert Daten vom Backend-Schema in den Data Pool
|
||||||
|
2. **`backend:transform-data-pool`** - Transformiert Data Pool in Companies + Transactions
|
||||||
|
3. **`backend:rebuild`** - Kompletter Rebuild (Sync + Transform in einem Schritt)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Backend Sync Command
|
||||||
|
|
||||||
|
Synchronisiert Daten von `backend.transactions` + `backend.transaction_outputs` nach `public.backend_data_pool`.
|
||||||
|
|
||||||
|
### Verwendung
|
||||||
|
|
||||||
|
#### Incremental Sync (Standard)
|
||||||
|
Nur neue oder geänderte Datensätze seit dem letzten Sync:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --incremental
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Full Sync
|
||||||
|
Kompletter Rebuild - löscht alle Daten im Data Pool und baut sie neu auf:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --full
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Statistiken anzeigen
|
||||||
|
Zeigt nur Statistiken an, ohne Sync durchzuführen:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --stats
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Im Hintergrund über Queue
|
||||||
|
Führt den Sync asynchron über die Queue aus:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --full --queue
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Batch-Größe anpassen
|
||||||
|
Definiert, wie viele Records pro Batch verarbeitet werden (Standard: 1000):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --full --batch-size=2000
|
||||||
|
```
|
||||||
|
|
||||||
|
### Optionen
|
||||||
|
|
||||||
|
| Option | Beschreibung | Default |
|
||||||
|
|--------|--------------|---------|
|
||||||
|
| `--full` | Kompletter Sync (truncate + rebuild) | false |
|
||||||
|
| `--incremental` | Nur neue/geänderte Records | false |
|
||||||
|
| `--batch-size` | Records pro Batch | 1000 |
|
||||||
|
| `--queue` | Im Hintergrund ausführen | false |
|
||||||
|
| `--stats` | Nur Statistiken anzeigen | false |
|
||||||
|
|
||||||
|
### Datenbankverbindungen
|
||||||
|
|
||||||
|
- **Source:** `DB::connection('pgsql')` → `backend.transactions`, `backend.transaction_outputs`
|
||||||
|
- **Target:** `DB::table('backend_data_pool')` → `public.backend_data_pool`
|
||||||
|
|
||||||
|
### Was passiert beim Sync?
|
||||||
|
|
||||||
|
1. **Full Sync:**
|
||||||
|
- Truncate `backend_data_pool`
|
||||||
|
- Alle `done` Transactions vom Backend laden
|
||||||
|
- Mit `transaction_outputs` joinen
|
||||||
|
- In Batches in Data Pool schreiben
|
||||||
|
|
||||||
|
2. **Incremental Sync:**
|
||||||
|
- Letzten Sync-Zeitstempel ermitteln
|
||||||
|
- Nur Records mit `last_modified_at > last_synced_at` laden
|
||||||
|
- In Batches in Data Pool schreiben
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Transform Command
|
||||||
|
|
||||||
|
Transformiert Daten aus `backend_data_pool` in die Production-Tables `companies` und `transactions`.
|
||||||
|
|
||||||
|
### Verwendung
|
||||||
|
|
||||||
|
#### Standard Transformation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:transform-data-pool
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Statistiken anzeigen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:transform-data-pool --stats
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Im Hintergrund über Queue
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:transform-data-pool --queue
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Batch-Größe anpassen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:transform-data-pool --batch-size=200
|
||||||
|
```
|
||||||
|
|
||||||
|
### Optionen
|
||||||
|
|
||||||
|
| Option | Beschreibung | Default |
|
||||||
|
|--------|--------------|---------|
|
||||||
|
| `--batch-size` | Records pro Batch | 100 |
|
||||||
|
| `--queue` | Im Hintergrund ausführen | false |
|
||||||
|
| `--stats` | Nur Statistiken anzeigen | false |
|
||||||
|
|
||||||
|
### Datenbankverbindungen
|
||||||
|
|
||||||
|
- **Source:** `DB::table('backend_data_pool')` → `public.backend_data_pool`
|
||||||
|
- **Target:** Eloquent Models → `public.companies`, `public.transactions`
|
||||||
|
|
||||||
|
### Was passiert bei der Transformation?
|
||||||
|
|
||||||
|
1. Alle eindeutigen `transaction_id` aus Data Pool laden
|
||||||
|
2. Für jede Transaction:
|
||||||
|
- **Company erstellen/aktualisieren:**
|
||||||
|
- Name: `corporate_entity`
|
||||||
|
- Country, Sector, HQ, Summary, Legal Name extrahieren
|
||||||
|
- KYC Risk Level aus `risk_score` berechnen
|
||||||
|
- **Transaction erstellen/aktualisieren:**
|
||||||
|
- Reference, Amount, Currency, Counterparty
|
||||||
|
- Risk Score & Level mapping
|
||||||
|
- Status basierend auf Risk Level
|
||||||
|
- Alle `output_keys` als JSON-Felder speichern
|
||||||
|
|
||||||
|
### Risk Score Mapping
|
||||||
|
|
||||||
|
Backend verwendet eine Skala von **-100 bis 100**:
|
||||||
|
|
||||||
|
| Backend Score | Risk Level | Transaction Status |
|
||||||
|
|---------------|------------|-------------------|
|
||||||
|
| -100 bis 20 | `critical` | True Positive |
|
||||||
|
| 20 bis 80 | `high` | False Positive |
|
||||||
|
| 80 bis 100 | `low` | Cleared |
|
||||||
|
|
||||||
|
Für das Frontend wird der Score auf **0-255** normalisiert:
|
||||||
|
```
|
||||||
|
normalized = ((score + 100) / 200) * 255
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Complete Rebuild Command
|
||||||
|
|
||||||
|
Führt Full Sync + Transformation in einem Schritt aus.
|
||||||
|
|
||||||
|
### Verwendung
|
||||||
|
|
||||||
|
#### Standard Rebuild
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:rebuild
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Mit angepassten Batch-Größen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:rebuild --batch-size=2000 --transform-batch-size=150
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Im Hintergrund über Queue
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:rebuild --queue
|
||||||
|
```
|
||||||
|
|
||||||
|
**Hinweis:** Bei `--queue` wird nur der Sync-Job in die Queue geschickt. Die Transformation muss manuell gestartet werden, nachdem der Sync abgeschlossen ist.
|
||||||
|
|
||||||
|
### Optionen
|
||||||
|
|
||||||
|
| Option | Beschreibung | Default |
|
||||||
|
|--------|--------------|---------|
|
||||||
|
| `--batch-size` | Records pro Batch (Sync) | 1000 |
|
||||||
|
| `--transform-batch-size` | Records pro Batch (Transform) | 100 |
|
||||||
|
| `--queue` | Im Hintergrund ausführen | false |
|
||||||
|
|
||||||
|
### Ablauf
|
||||||
|
|
||||||
|
1. **Step 1/2:** Full Sync from Backend
|
||||||
|
- Truncate `backend_data_pool`
|
||||||
|
- Sync alle `done` Transactions
|
||||||
|
2. **Step 2/2:** Transform Data Pool
|
||||||
|
- Companies erstellen/aktualisieren
|
||||||
|
- Transactions erstellen/aktualisieren
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Automatisierung mit Laravel Scheduler
|
||||||
|
|
||||||
|
Die Commands sind bereits im Laravel Scheduler konfiguriert ([routes/console.php](routes/console.php)):
|
||||||
|
|
||||||
|
### Aktuelle Schedule
|
||||||
|
|
||||||
|
```php
|
||||||
|
// Incremental Sync: Alle 6 Stunden (0:00, 6:00, 12:00, 18:00)
|
||||||
|
Schedule::job(new SyncBackendDataPool(fullSync: false, batchSize: 1000))
|
||||||
|
->everySixHours()
|
||||||
|
->name('sync-backend-data-pool-incremental')
|
||||||
|
->withoutOverlapping(1800);
|
||||||
|
|
||||||
|
// Full Sync: Jeden Sonntag um 3:00 Uhr
|
||||||
|
Schedule::job(new SyncBackendDataPool(fullSync: true, batchSize: 1000))
|
||||||
|
->weeklyOn(0, '03:00')
|
||||||
|
->name('sync-backend-data-pool-full')
|
||||||
|
->withoutOverlapping(3600);
|
||||||
|
|
||||||
|
// Transformation: 30 Minuten nach jedem Sync (0:30, 6:30, 12:30, 18:30)
|
||||||
|
Schedule::job(new TransformDataPoolToProduction(batchSize: 100))
|
||||||
|
->cron('30 */6 * * *')
|
||||||
|
->name('transform-data-pool-to-production')
|
||||||
|
->withoutOverlapping(1800);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scheduler aktivieren
|
||||||
|
|
||||||
|
#### Für lokale Entwicklung
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan schedule:work
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Für Produktion (Crontab)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
crontab -e
|
||||||
|
```
|
||||||
|
|
||||||
|
Folgende Zeile hinzufügen:
|
||||||
|
```
|
||||||
|
* * * * * cd /Users/sebastianfrohlich/Herd/frontend && php artisan schedule:run >> /dev/null 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scheduled Tasks anzeigen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan schedule:list
|
||||||
|
```
|
||||||
|
|
||||||
|
### Scheduled Task testen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan schedule:test sync-backend-data-pool-incremental
|
||||||
|
php artisan schedule:test sync-backend-data-pool-full
|
||||||
|
php artisan schedule:test transform-data-pool-to-production
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
### Alle Commands bieten
|
||||||
|
|
||||||
|
✅ **Interaktive Bestätigung** - Sicherheitsfrage vor kritischen Operationen
|
||||||
|
✅ **Statistiken vorher/nachher** - Direkter Vergleich der Datenänderungen
|
||||||
|
✅ **Batch-Processing** - Performante Verarbeitung großer Datenmengen
|
||||||
|
✅ **Queue-Support** - Ausführung im Hintergrund möglich
|
||||||
|
✅ **Laravel Prompts** - Moderne, benutzerfreundliche CLI-Oberfläche
|
||||||
|
✅ **Error Handling** - Robuste Fehlerbehandlung mit Logging
|
||||||
|
✅ **Timeout Protection** - 1 Stunde Timeout pro Job
|
||||||
|
✅ **Retry Logic** - 3 Versuche bei Fehlern
|
||||||
|
✅ **Overlap Prevention** - Verhindert gleichzeitige Ausführung
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Datenfluss
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────┐
|
||||||
|
│ backend.transactions │
|
||||||
|
│ backend.transaction_│
|
||||||
|
│ outputs │
|
||||||
|
└──────────┬───────────┘
|
||||||
|
│
|
||||||
|
│ SyncBackendDataPool
|
||||||
|
│ (backend:sync-data-pool)
|
||||||
|
▼
|
||||||
|
┌──────────────────────┐
|
||||||
|
│ public.backend_data_ │
|
||||||
|
│ pool │
|
||||||
|
└──────────┬───────────┘
|
||||||
|
│
|
||||||
|
│ TransformDataPoolToProduction
|
||||||
|
│ (backend:transform-data-pool)
|
||||||
|
▼
|
||||||
|
┌──────────────────────┐
|
||||||
|
│ public.companies │
|
||||||
|
│ public.transactions │
|
||||||
|
└──────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Command funktioniert nicht
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Commands neu registrieren
|
||||||
|
php artisan optimize:clear
|
||||||
|
|
||||||
|
# Artisan Cache leeren
|
||||||
|
php artisan cache:clear
|
||||||
|
```
|
||||||
|
|
||||||
|
### Queue-Jobs laufen nicht
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Queue Worker starten
|
||||||
|
php artisan queue:work
|
||||||
|
|
||||||
|
# Queue Status prüfen
|
||||||
|
php artisan queue:monitor
|
||||||
|
```
|
||||||
|
|
||||||
|
### Logs prüfen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Application Logs
|
||||||
|
tail -f storage/logs/laravel.log
|
||||||
|
|
||||||
|
# Nur Sync-Logs
|
||||||
|
tail -f storage/logs/laravel.log | grep SyncBackendDataPool
|
||||||
|
|
||||||
|
# Nur Transform-Logs
|
||||||
|
tail -f storage/logs/laravel.log | grep TransformDataPoolToProduction
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Beispiele
|
||||||
|
|
||||||
|
### Kompletter Rebuild mit Custom Settings
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:rebuild \
|
||||||
|
--batch-size=5000 \
|
||||||
|
--transform-batch-size=250
|
||||||
|
```
|
||||||
|
|
||||||
|
### Nur neuen Daten synchronisieren und transformieren
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --incremental
|
||||||
|
php artisan backend:transform-data-pool
|
||||||
|
```
|
||||||
|
|
||||||
|
### Statistiken aller Systeme anzeigen
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --stats
|
||||||
|
php artisan backend:transform-data-pool --stats
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Performance-Tipps
|
||||||
|
|
||||||
|
1. **Batch-Größe anpassen:**
|
||||||
|
- Kleine Batches (100-500): Weniger RAM, langsamer
|
||||||
|
- Große Batches (2000-5000): Mehr RAM, schneller
|
||||||
|
|
||||||
|
2. **Queue nutzen für lange Jobs:**
|
||||||
|
```bash
|
||||||
|
php artisan backend:sync-data-pool --full --queue
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Incremental Sync bevorzugen:**
|
||||||
|
- Full Sync nur wenn wirklich nötig
|
||||||
|
- Incremental ist viel schneller
|
||||||
|
|
||||||
|
4. **Off-Peak Zeiten nutzen:**
|
||||||
|
- Full Sync: Sonntag 3:00 Uhr (bereits konfiguriert)
|
||||||
|
- Incremental: Alle 6 Stunden (bereits konfiguriert)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Weiterführende Links
|
||||||
|
|
||||||
|
- [routes/console.php](routes/console.php) - Scheduler Configuration
|
||||||
|
- [app/Jobs/SyncBackendDataPool.php](app/Jobs/SyncBackendDataPool.php) - Sync Job
|
||||||
|
- [app/Jobs/TransformDataPoolToProduction.php](app/Jobs/TransformDataPoolToProduction.php) - Transform Job
|
||||||
|
- [app/Console/Commands/SyncBackendDataPoolCommand.php](app/Console/Commands/SyncBackendDataPoolCommand.php)
|
||||||
|
- [app/Console/Commands/TransformDataPoolCommand.php](app/Console/Commands/TransformDataPoolCommand.php)
|
||||||
|
- [app/Console/Commands/RebuildFromDataPoolCommand.php](app/Console/Commands/RebuildFromDataPoolCommand.php)
|
||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
;;;;;
|
||||||
|
Hauptebene Titel;Unterebene Titel;Daten;Prompt ID;Überschriften je Prompt ID;prompt_id: output_key = DB Feld transaction
|
||||||
|
Unternehmens-Status;;;;;
|
||||||
|
Unternehmens-Status;Liquidation/Stilllegung;;29;Liquidation;"29: entity_corporate_liquidation
|
||||||
|
29: counterparty_corporate_liquidation
|
||||||
|
"
|
||||||
|
Unternehmens-Status;Insolvenzeröffnungen;;28;Insolvenzverfahren;"28: entity_corporate_insolvency
|
||||||
|
28: counterparty_corporate_insolvency
|
||||||
|
"
|
||||||
|
Unternehmens-Status;Rechtliche /gerichtliche Verfahren;;89;Rechtsverletzungen;"89: entity_corporate_courtcases
|
||||||
|
89: counterparty_corporate_courtcases
|
||||||
|
"
|
||||||
|
Unternehmens-Status;M&A;;90;M&A;"90: entity_corporate_manda
|
||||||
|
90: counterparty_corporate_manda
|
||||||
|
"
|
||||||
|
Unternehmens-Status;Betrug;;91;Transaktion Betrugstypologie;"91: tranx_fakepurposecheck
|
||||||
|
"
|
||||||
|
Reference in new issue
Block a user