fix: added new command - complete rebuild
This commit is contained in:
cbazza committed
1 parent
1505975ebb
commit
a36d2b4cc4
4 files changed
+486
-8
No files matched your search
@@ -1,30 +1,90 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\SyncBackendDataPool;
|
||||
use App\Jobs\TransformDataPoolToProduction;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
use function Laravel\Prompts\confirm;
|
||||
use function Laravel\Prompts\info;
|
||||
use function Laravel\Prompts\spin;
|
||||
use function Laravel\Prompts\warning;
|
||||
|
||||
class RebuildFromDataPoolCommand extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:rebuild-from-data-pool-command';
|
||||
protected $signature = 'backend:rebuild
|
||||
{--batch-size=1000 : Number of records to process per batch}
|
||||
{--transform-batch-size=100 : Number of records to process per batch for transformation}
|
||||
{--queue : Dispatch jobs to the queue instead of running synchronously}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Command description';
|
||||
protected $description = 'Complete rebuild: Full sync from backend + transformation to production tables';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
public function handle(): int
|
||||
{
|
||||
//
|
||||
info('Preparing for COMPLETE REBUILD');
|
||||
warning('This will:');
|
||||
warning('1. Truncate and rebuild backend_data_pool (full sync)');
|
||||
warning('2. Transform all data into companies and transactions');
|
||||
|
||||
if (! $this->option('no-interaction') && ! confirm('Do you want to continue?', true)) {
|
||||
info('Rebuild cancelled.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$batchSize = (int) $this->option('batch-size');
|
||||
$transformBatchSize = (int) $this->option('transform-batch-size');
|
||||
$useQueue = $this->option('queue');
|
||||
|
||||
// Step 1: Full Sync
|
||||
info('Step 1/2: Full Sync from Backend');
|
||||
$syncJob = new SyncBackendDataPool(
|
||||
fullSync: true,
|
||||
batchSize: $batchSize
|
||||
);
|
||||
|
||||
if ($useQueue) {
|
||||
info('Dispatching sync job to queue...');
|
||||
dispatch($syncJob);
|
||||
info('Sync job dispatched!');
|
||||
warning('Please wait for the sync job to complete before transformation.');
|
||||
|
||||
return self::SUCCESS;
|
||||
} else {
|
||||
spin(
|
||||
fn () => $syncJob->handle(),
|
||||
'Syncing data from backend...'
|
||||
);
|
||||
info('✓ Sync completed!');
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
|
||||
// Step 2: Transform Data
|
||||
info('Step 2/2: Transform Data Pool to Production');
|
||||
$transformJob = new TransformDataPoolToProduction(batchSize: $transformBatchSize);
|
||||
|
||||
spin(
|
||||
fn () => $transformJob->handle(),
|
||||
'Transforming data...'
|
||||
);
|
||||
info('✓ Transformation completed!');
|
||||
|
||||
$this->newLine();
|
||||
info('Complete rebuild finished successfully!');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
# Backend Sync & Transform Commands
|
||||
|
||||
Dokumentation für die manuellen Artisan Commands zum Synchronisieren und Transformieren von Backend-Daten.
|
||||
|
||||
## Übersicht
|
||||
|
||||
Das System verfügt über 3 Haupt-Commands zur Datensynchronisation:
|
||||
|
||||
1. **`backend:sync-data-pool`** - Synchronisiert Daten vom Backend-Schema in den Data Pool
|
||||
2. **`backend:transform-data-pool`** - Transformiert Data Pool in Companies + Transactions
|
||||
3. **`backend:rebuild`** - Kompletter Rebuild (Sync + Transform in einem Schritt)
|
||||
|
||||
---
|
||||
|
||||
## 1. Backend Sync Command
|
||||
|
||||
Synchronisiert Daten von `backend.transactions` + `backend.transaction_outputs` nach `public.backend_data_pool`.
|
||||
|
||||
### Verwendung
|
||||
|
||||
#### Incremental Sync (Standard)
|
||||
Nur neue oder geänderte Datensätze seit dem letzten Sync:
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --incremental
|
||||
```
|
||||
|
||||
#### Full Sync
|
||||
Kompletter Rebuild - löscht alle Daten im Data Pool und baut sie neu auf:
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --full
|
||||
```
|
||||
|
||||
#### Statistiken anzeigen
|
||||
Zeigt nur Statistiken an, ohne Sync durchzuführen:
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --stats
|
||||
```
|
||||
|
||||
#### Im Hintergrund über Queue
|
||||
Führt den Sync asynchron über die Queue aus:
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --full --queue
|
||||
```
|
||||
|
||||
#### Batch-Größe anpassen
|
||||
Definiert, wie viele Records pro Batch verarbeitet werden (Standard: 1000):
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --full --batch-size=2000
|
||||
```
|
||||
|
||||
### Optionen
|
||||
|
||||
| Option | Beschreibung | Default |
|
||||
|--------|--------------|---------|
|
||||
| `--full` | Kompletter Sync (truncate + rebuild) | false |
|
||||
| `--incremental` | Nur neue/geänderte Records | false |
|
||||
| `--batch-size` | Records pro Batch | 1000 |
|
||||
| `--queue` | Im Hintergrund ausführen | false |
|
||||
| `--stats` | Nur Statistiken anzeigen | false |
|
||||
|
||||
### Datenbankverbindungen
|
||||
|
||||
- **Source:** `DB::connection('pgsql')` → `backend.transactions`, `backend.transaction_outputs`
|
||||
- **Target:** `DB::table('backend_data_pool')` → `public.backend_data_pool`
|
||||
|
||||
### Was passiert beim Sync?
|
||||
|
||||
1. **Full Sync:**
|
||||
- Truncate `backend_data_pool`
|
||||
- Alle `done` Transactions vom Backend laden
|
||||
- Mit `transaction_outputs` joinen
|
||||
- In Batches in Data Pool schreiben
|
||||
|
||||
2. **Incremental Sync:**
|
||||
- Letzten Sync-Zeitstempel ermitteln
|
||||
- Nur Records mit `last_modified_at > last_synced_at` laden
|
||||
- In Batches in Data Pool schreiben
|
||||
|
||||
---
|
||||
|
||||
## 2. Transform Command
|
||||
|
||||
Transformiert Daten aus `backend_data_pool` in die Production-Tables `companies` und `transactions`.
|
||||
|
||||
### Verwendung
|
||||
|
||||
#### Standard Transformation
|
||||
|
||||
```bash
|
||||
php artisan backend:transform-data-pool
|
||||
```
|
||||
|
||||
#### Statistiken anzeigen
|
||||
|
||||
```bash
|
||||
php artisan backend:transform-data-pool --stats
|
||||
```
|
||||
|
||||
#### Im Hintergrund über Queue
|
||||
|
||||
```bash
|
||||
php artisan backend:transform-data-pool --queue
|
||||
```
|
||||
|
||||
#### Batch-Größe anpassen
|
||||
|
||||
```bash
|
||||
php artisan backend:transform-data-pool --batch-size=200
|
||||
```
|
||||
|
||||
### Optionen
|
||||
|
||||
| Option | Beschreibung | Default |
|
||||
|--------|--------------|---------|
|
||||
| `--batch-size` | Records pro Batch | 100 |
|
||||
| `--queue` | Im Hintergrund ausführen | false |
|
||||
| `--stats` | Nur Statistiken anzeigen | false |
|
||||
|
||||
### Datenbankverbindungen
|
||||
|
||||
- **Source:** `DB::table('backend_data_pool')` → `public.backend_data_pool`
|
||||
- **Target:** Eloquent Models → `public.companies`, `public.transactions`
|
||||
|
||||
### Was passiert bei der Transformation?
|
||||
|
||||
1. Alle eindeutigen `transaction_id` aus Data Pool laden
|
||||
2. Für jede Transaction:
|
||||
- **Company erstellen/aktualisieren:**
|
||||
- Name: `corporate_entity`
|
||||
- Country, Sector, HQ, Summary, Legal Name extrahieren
|
||||
- KYC Risk Level aus `risk_score` berechnen
|
||||
- **Transaction erstellen/aktualisieren:**
|
||||
- Reference, Amount, Currency, Counterparty
|
||||
- Risk Score & Level mapping
|
||||
- Status basierend auf Risk Level
|
||||
- Alle `output_keys` als JSON-Felder speichern
|
||||
|
||||
### Risk Score Mapping
|
||||
|
||||
Backend verwendet eine Skala von **-100 bis 100**:
|
||||
|
||||
| Backend Score | Risk Level | Transaction Status |
|
||||
|---------------|------------|-------------------|
|
||||
| -100 bis 20 | `critical` | True Positive |
|
||||
| 20 bis 80 | `high` | False Positive |
|
||||
| 80 bis 100 | `low` | Cleared |
|
||||
|
||||
Für das Frontend wird der Score auf **0-255** normalisiert:
|
||||
```
|
||||
normalized = ((score + 100) / 200) * 255
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Complete Rebuild Command
|
||||
|
||||
Führt Full Sync + Transformation in einem Schritt aus.
|
||||
|
||||
### Verwendung
|
||||
|
||||
#### Standard Rebuild
|
||||
|
||||
```bash
|
||||
php artisan backend:rebuild
|
||||
```
|
||||
|
||||
#### Mit angepassten Batch-Größen
|
||||
|
||||
```bash
|
||||
php artisan backend:rebuild --batch-size=2000 --transform-batch-size=150
|
||||
```
|
||||
|
||||
#### Im Hintergrund über Queue
|
||||
|
||||
```bash
|
||||
php artisan backend:rebuild --queue
|
||||
```
|
||||
|
||||
**Hinweis:** Bei `--queue` wird nur der Sync-Job in die Queue geschickt. Die Transformation muss manuell gestartet werden, nachdem der Sync abgeschlossen ist.
|
||||
|
||||
### Optionen
|
||||
|
||||
| Option | Beschreibung | Default |
|
||||
|--------|--------------|---------|
|
||||
| `--batch-size` | Records pro Batch (Sync) | 1000 |
|
||||
| `--transform-batch-size` | Records pro Batch (Transform) | 100 |
|
||||
| `--queue` | Im Hintergrund ausführen | false |
|
||||
|
||||
### Ablauf
|
||||
|
||||
1. **Step 1/2:** Full Sync from Backend
|
||||
- Truncate `backend_data_pool`
|
||||
- Sync alle `done` Transactions
|
||||
2. **Step 2/2:** Transform Data Pool
|
||||
- Companies erstellen/aktualisieren
|
||||
- Transactions erstellen/aktualisieren
|
||||
|
||||
---
|
||||
|
||||
## Automatisierung mit Laravel Scheduler
|
||||
|
||||
Die Commands sind bereits im Laravel Scheduler konfiguriert ([routes/console.php](routes/console.php)):
|
||||
|
||||
### Aktuelle Schedule
|
||||
|
||||
```php
|
||||
// Incremental Sync: Alle 6 Stunden (0:00, 6:00, 12:00, 18:00)
|
||||
Schedule::job(new SyncBackendDataPool(fullSync: false, batchSize: 1000))
|
||||
->everySixHours()
|
||||
->name('sync-backend-data-pool-incremental')
|
||||
->withoutOverlapping(1800);
|
||||
|
||||
// Full Sync: Jeden Sonntag um 3:00 Uhr
|
||||
Schedule::job(new SyncBackendDataPool(fullSync: true, batchSize: 1000))
|
||||
->weeklyOn(0, '03:00')
|
||||
->name('sync-backend-data-pool-full')
|
||||
->withoutOverlapping(3600);
|
||||
|
||||
// Transformation: 30 Minuten nach jedem Sync (0:30, 6:30, 12:30, 18:30)
|
||||
Schedule::job(new TransformDataPoolToProduction(batchSize: 100))
|
||||
->cron('30 */6 * * *')
|
||||
->name('transform-data-pool-to-production')
|
||||
->withoutOverlapping(1800);
|
||||
```
|
||||
|
||||
### Scheduler aktivieren
|
||||
|
||||
#### Für lokale Entwicklung
|
||||
|
||||
```bash
|
||||
php artisan schedule:work
|
||||
```
|
||||
|
||||
#### Für Produktion (Crontab)
|
||||
|
||||
```bash
|
||||
crontab -e
|
||||
```
|
||||
|
||||
Folgende Zeile hinzufügen:
|
||||
```
|
||||
* * * * * cd /Users/sebastianfrohlich/Herd/frontend && php artisan schedule:run >> /dev/null 2>&1
|
||||
```
|
||||
|
||||
### Scheduled Tasks anzeigen
|
||||
|
||||
```bash
|
||||
php artisan schedule:list
|
||||
```
|
||||
|
||||
### Scheduled Task testen
|
||||
|
||||
```bash
|
||||
php artisan schedule:test sync-backend-data-pool-incremental
|
||||
php artisan schedule:test sync-backend-data-pool-full
|
||||
php artisan schedule:test transform-data-pool-to-production
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
### Alle Commands bieten
|
||||
|
||||
✅ **Interaktive Bestätigung** - Sicherheitsfrage vor kritischen Operationen
|
||||
✅ **Statistiken vorher/nachher** - Direkter Vergleich der Datenänderungen
|
||||
✅ **Batch-Processing** - Performante Verarbeitung großer Datenmengen
|
||||
✅ **Queue-Support** - Ausführung im Hintergrund möglich
|
||||
✅ **Laravel Prompts** - Moderne, benutzerfreundliche CLI-Oberfläche
|
||||
✅ **Error Handling** - Robuste Fehlerbehandlung mit Logging
|
||||
✅ **Timeout Protection** - 1 Stunde Timeout pro Job
|
||||
✅ **Retry Logic** - 3 Versuche bei Fehlern
|
||||
✅ **Overlap Prevention** - Verhindert gleichzeitige Ausführung
|
||||
|
||||
---
|
||||
|
||||
## Datenfluss
|
||||
|
||||
```
|
||||
┌──────────────────────┐
|
||||
│ backend.transactions │
|
||||
│ backend.transaction_│
|
||||
│ outputs │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
│ SyncBackendDataPool
|
||||
│ (backend:sync-data-pool)
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ public.backend_data_ │
|
||||
│ pool │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
│ TransformDataPoolToProduction
|
||||
│ (backend:transform-data-pool)
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ public.companies │
|
||||
│ public.transactions │
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Command funktioniert nicht
|
||||
|
||||
```bash
|
||||
# Commands neu registrieren
|
||||
php artisan optimize:clear
|
||||
|
||||
# Artisan Cache leeren
|
||||
php artisan cache:clear
|
||||
```
|
||||
|
||||
### Queue-Jobs laufen nicht
|
||||
|
||||
```bash
|
||||
# Queue Worker starten
|
||||
php artisan queue:work
|
||||
|
||||
# Queue Status prüfen
|
||||
php artisan queue:monitor
|
||||
```
|
||||
|
||||
### Logs prüfen
|
||||
|
||||
```bash
|
||||
# Application Logs
|
||||
tail -f storage/logs/laravel.log
|
||||
|
||||
# Nur Sync-Logs
|
||||
tail -f storage/logs/laravel.log | grep SyncBackendDataPool
|
||||
|
||||
# Nur Transform-Logs
|
||||
tail -f storage/logs/laravel.log | grep TransformDataPoolToProduction
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Beispiele
|
||||
|
||||
### Kompletter Rebuild mit Custom Settings
|
||||
|
||||
```bash
|
||||
php artisan backend:rebuild \
|
||||
--batch-size=5000 \
|
||||
--transform-batch-size=250
|
||||
```
|
||||
|
||||
### Nur neuen Daten synchronisieren und transformieren
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --incremental
|
||||
php artisan backend:transform-data-pool
|
||||
```
|
||||
|
||||
### Statistiken aller Systeme anzeigen
|
||||
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --stats
|
||||
php artisan backend:transform-data-pool --stats
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance-Tipps
|
||||
|
||||
1. **Batch-Größe anpassen:**
|
||||
- Kleine Batches (100-500): Weniger RAM, langsamer
|
||||
- Große Batches (2000-5000): Mehr RAM, schneller
|
||||
|
||||
2. **Queue nutzen für lange Jobs:**
|
||||
```bash
|
||||
php artisan backend:sync-data-pool --full --queue
|
||||
```
|
||||
|
||||
3. **Incremental Sync bevorzugen:**
|
||||
- Full Sync nur wenn wirklich nötig
|
||||
- Incremental ist viel schneller
|
||||
|
||||
4. **Off-Peak Zeiten nutzen:**
|
||||
- Full Sync: Sonntag 3:00 Uhr (bereits konfiguriert)
|
||||
- Incremental: Alle 6 Stunden (bereits konfiguriert)
|
||||
|
||||
---
|
||||
|
||||
## Weiterführende Links
|
||||
|
||||
- [routes/console.php](routes/console.php) - Scheduler Configuration
|
||||
- [app/Jobs/SyncBackendDataPool.php](app/Jobs/SyncBackendDataPool.php) - Sync Job
|
||||
- [app/Jobs/TransformDataPoolToProduction.php](app/Jobs/TransformDataPoolToProduction.php) - Transform Job
|
||||
- [app/Console/Commands/SyncBackendDataPoolCommand.php](app/Console/Commands/SyncBackendDataPoolCommand.php)
|
||||
- [app/Console/Commands/TransformDataPoolCommand.php](app/Console/Commands/TransformDataPoolCommand.php)
|
||||
- [app/Console/Commands/RebuildFromDataPoolCommand.php](app/Console/Commands/RebuildFromDataPoolCommand.php)
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
;;;;;
|
||||
Hauptebene Titel;Unterebene Titel;Daten;Prompt ID;Überschriften je Prompt ID;prompt_id: output_key = DB Feld transaction
|
||||
Unternehmens-Status;;;;;
|
||||
Unternehmens-Status;Liquidation/Stilllegung;;29;Liquidation;"29: entity_corporate_liquidation
|
||||
29: counterparty_corporate_liquidation
|
||||
"
|
||||
Unternehmens-Status;Insolvenzeröffnungen;;28;Insolvenzverfahren;"28: entity_corporate_insolvency
|
||||
28: counterparty_corporate_insolvency
|
||||
"
|
||||
Unternehmens-Status;Rechtliche /gerichtliche Verfahren;;89;Rechtsverletzungen;"89: entity_corporate_courtcases
|
||||
89: counterparty_corporate_courtcases
|
||||
"
|
||||
Unternehmens-Status;M&A;;90;M&A;"90: entity_corporate_manda
|
||||
90: counterparty_corporate_manda
|
||||
"
|
||||
Unternehmens-Status;Betrug;;91;Transaktion Betrugstypologie;"91: tranx_fakepurposecheck
|
||||
"
|
||||
|
Reference in new issue
Block a user