376 lines
11 KiB
PHP
376 lines
11 KiB
PHP
|
<?php
|
||
|
|
|
||
|
|
declare(strict_types=1);
|
||
|
|
|
||
|
|
namespace App\Jobs;
|
||
|
|
|
||
|
|
use App\Models\Company;
|
||
|
|
use App\Models\Transaction;
|
||
|
|
use App\Services\KycRiskCalculator;
|
||
|
|
use Carbon\Carbon;
|
||
|
|
use Illuminate\Bus\Queueable;
|
||
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
||
|
|
use Illuminate\Queue\InteractsWithQueue;
|
||
|
|
use Illuminate\Queue\SerializesModels;
|
||
|
|
use Illuminate\Support\Facades\DB;
|
||
|
|
use Illuminate\Support\Facades\Log;
|
||
|
|
|
||
|
|
class TransformDataPoolToProduction implements ShouldQueue
|
||
|
|
{
|
||
|
|
use Dispatchable;
|
||
|
|
use InteractsWithQueue;
|
||
|
|
use Queueable;
|
||
|
|
use SerializesModels;
|
||
|
|
|
||
|
|
public int $timeout = 3600; // 1 hour timeout
|
||
|
|
|
||
|
|
public int $tries = 3;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Create a new job instance.
|
||
|
|
*/
|
||
|
|
public function __construct(
|
||
|
|
public ?int $batchSize = 100,
|
||
|
|
) {
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Execute the job.
|
||
|
|
*/
|
||
|
|
public function handle(KycRiskCalculator $kycCalculator): void
|
||
|
|
{
|
||
|
|
$startTime = now();
|
||
|
|
|
||
|
|
Log::info('TransformDataPoolToProduction: Starting transformation', [
|
||
|
|
'batch_size' => $this->batchSize,
|
||
|
|
]);
|
||
|
|
|
||
|
|
try {
|
||
|
|
DB::transaction(function () use ($kycCalculator) {
|
||
|
|
$this->transformData($kycCalculator);
|
||
|
|
});
|
||
|
|
|
||
|
|
$duration = now()->diffInSeconds($startTime);
|
||
|
|
|
||
|
|
Log::info('TransformDataPoolToProduction: Transformation completed successfully', [
|
||
|
|
'duration_seconds' => $duration,
|
||
|
|
]);
|
||
|
|
} catch (\Exception $e) {
|
||
|
|
Log::error('TransformDataPoolToProduction: Transformation failed', [
|
||
|
|
'error' => $e->getMessage(),
|
||
|
|
'trace' => $e->getTraceAsString(),
|
||
|
|
]);
|
||
|
|
|
||
|
|
throw $e;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Transform data from backend_data_pool to companies and transactions
|
||
|
|
*/
|
||
|
|
private function transformData(KycRiskCalculator $kycCalculator): void
|
||
|
|
{
|
||
|
|
// Get all unique transaction IDs from data pool
|
||
|
|
$transactionIds = DB::table('backend_data_pool')
|
||
|
|
->select('transaction_id')
|
||
|
|
->distinct()
|
||
|
|
->pluck('transaction_id');
|
||
|
|
|
||
|
|
Log::info('TransformDataPoolToProduction: Found transactions to process', [
|
||
|
|
'count' => $transactionIds->count(),
|
||
|
|
]);
|
||
|
|
|
||
|
|
$processedCompanies = 0;
|
||
|
|
$processedTransactions = 0;
|
||
|
|
|
||
|
|
foreach ($transactionIds as $transactionId) {
|
||
|
|
// Get all outputs for this transaction
|
||
|
|
$outputs = DB::table('backend_data_pool')
|
||
|
|
->where('transaction_id', $transactionId)
|
||
|
|
->get();
|
||
|
|
|
||
|
|
if ($outputs->isEmpty()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
$firstOutput = $outputs->first();
|
||
|
|
|
||
|
|
// 1. Create or update Company
|
||
|
|
$company = $this->createOrUpdateCompany($firstOutput, $outputs, $kycCalculator);
|
||
|
|
if ($company->wasRecentlyCreated) {
|
||
|
|
$processedCompanies++;
|
||
|
|
}
|
||
|
|
|
||
|
|
// 2. Create or update Transaction
|
||
|
|
$transaction = $this->createOrUpdateTransaction($company, $firstOutput, $outputs);
|
||
|
|
if ($transaction->wasRecentlyCreated) {
|
||
|
|
$processedTransactions++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
Log::info('TransformDataPoolToProduction: Processing completed', [
|
||
|
|
'new_companies' => $processedCompanies,
|
||
|
|
'new_transactions' => $processedTransactions,
|
||
|
|
]);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Create or update a Company from data pool outputs
|
||
|
|
*/
|
||
|
|
private function createOrUpdateCompany($firstOutput, $outputs, KycRiskCalculator $kycCalculator): Company
|
||
|
|
{
|
||
|
|
$corporateEntity = $firstOutput->corporate_entity;
|
||
|
|
|
||
|
|
// Calculate KYC Risk Level
|
||
|
|
$kycRiskLevel = $kycCalculator->calculateKycRiskLevel(collect([$outputs])->first());
|
||
|
|
|
||
|
|
// Extract company data from outputs
|
||
|
|
$companyData = [
|
||
|
|
'country' => $this->extractCountry($firstOutput, $outputs),
|
||
|
|
'kyc_risk_level' => $kycRiskLevel,
|
||
|
|
'sector' => $this->extractField($outputs, 'corporate_sector', 255),
|
||
|
|
'headquarters' => $this->extractField($outputs, 'corporate_HQ', 255),
|
||
|
|
'summary' => $this->extractField($outputs, 'corporate_summary', 5000), // text field, longer OK
|
||
|
|
'legal_name' => $this->extractField($outputs, 'corporate_name', 255),
|
||
|
|
];
|
||
|
|
|
||
|
|
return Company::updateOrCreate(
|
||
|
|
['name' => $corporateEntity],
|
||
|
|
$companyData
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Create or update a Transaction from data pool outputs
|
||
|
|
*/
|
||
|
|
private function createOrUpdateTransaction(Company $company, $firstOutput, $outputs): Transaction
|
||
|
|
{
|
||
|
|
$reference = 'MIGRATED-' . $firstOutput->transaction_id;
|
||
|
|
|
||
|
|
// Map KYC Risk Level to Transaction Status
|
||
|
|
$status = match ($company->kyc_risk_level) {
|
||
|
|
'critical' => Transaction::STATUS_TRUE_POSITIVE,
|
||
|
|
'high' => Transaction::STATUS_FALSE_POSITIVE,
|
||
|
|
'low' => Transaction::STATUS_CLEARED,
|
||
|
|
default => Transaction::STATUS_FALSE_POSITIVE, // Default to high risk
|
||
|
|
};
|
||
|
|
|
||
|
|
// Core transaction data
|
||
|
|
$transactionData = [
|
||
|
|
'company_id' => $company->id,
|
||
|
|
'amount' => $firstOutput->tx_amount,
|
||
|
|
'currency' => $firstOutput->tx_currency ?? 'EUR',
|
||
|
|
'counterparty' => $firstOutput->corporate_counterparty,
|
||
|
|
'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming),
|
||
|
|
'channel' => $firstOutput->source_file,
|
||
|
|
'executed_at' => $this->parseDate($firstOutput->tx_date),
|
||
|
|
'risk_score' => $this->extractRiskScore($outputs),
|
||
|
|
'status' => $status,
|
||
|
|
'requires_review' => true,
|
||
|
|
'flagged_reason' => $firstOutput->tx_purpose,
|
||
|
|
];
|
||
|
|
|
||
|
|
// Map all output_keys to their respective columns
|
||
|
|
foreach ($outputs as $output) {
|
||
|
|
$columnName = $output->output_key;
|
||
|
|
|
||
|
|
// Skip if column doesn't exist in transactions table
|
||
|
|
if (! $this->columnExists($columnName)) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Decode JSON content, store NULL if empty or invalid
|
||
|
|
$content = json_decode($output->content, true);
|
||
|
|
$transactionData[$columnName] = $content ?: null;
|
||
|
|
}
|
||
|
|
|
||
|
|
return Transaction::updateOrCreate(
|
||
|
|
['reference' => $reference],
|
||
|
|
$transactionData
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Extract country code from data
|
||
|
|
*/
|
||
|
|
private function extractCountry($firstOutput, $outputs): string
|
||
|
|
{
|
||
|
|
// Try to get from tx_country_incoming
|
||
|
|
if (! empty($firstOutput->tx_country_incoming)) {
|
||
|
|
$country = $this->extractCountryCode($firstOutput->tx_country_incoming);
|
||
|
|
if ($country) {
|
||
|
|
return $country;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Try to extract from corporate_HQ
|
||
|
|
$hq = $this->extractField($outputs, 'corporate_HQ');
|
||
|
|
if ($hq && is_array($hq) && isset($hq['answer'])) {
|
||
|
|
// Simple pattern matching for country codes
|
||
|
|
if (preg_match('/\b([A-Z]{2})\b/', $hq['answer'], $matches)) {
|
||
|
|
return $matches[1];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return 'DE'; // Default
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Extract country code (convert to ISO 2-letter if needed)
|
||
|
|
*/
|
||
|
|
private function extractCountryCode(?string $country): ?string
|
||
|
|
{
|
||
|
|
if (empty($country)) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
// If already 2 letters, return uppercase
|
||
|
|
if (strlen($country) === 2) {
|
||
|
|
return strtoupper($country);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Simple country name mapping (extend as needed)
|
||
|
|
$countryMap = [
|
||
|
|
'Germany' => 'DE',
|
||
|
|
'Deutschland' => 'DE',
|
||
|
|
'United States' => 'US',
|
||
|
|
'USA' => 'US',
|
||
|
|
'United Kingdom' => 'GB',
|
||
|
|
'UK' => 'GB',
|
||
|
|
'France' => 'FR',
|
||
|
|
'Spain' => 'ES',
|
||
|
|
'Italy' => 'IT',
|
||
|
|
'Netherlands' => 'NL',
|
||
|
|
'Belgium' => 'BE',
|
||
|
|
'Austria' => 'AT',
|
||
|
|
'Switzerland' => 'CH',
|
||
|
|
'Poland' => 'PL',
|
||
|
|
'Czech Republic' => 'CZ',
|
||
|
|
'Denmark' => 'DK',
|
||
|
|
'Sweden' => 'SE',
|
||
|
|
'Norway' => 'NO',
|
||
|
|
'Finland' => 'FI',
|
||
|
|
];
|
||
|
|
|
||
|
|
return $countryMap[$country] ?? null;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Extract risk score from tranx_score output
|
||
|
|
*/
|
||
|
|
private function extractRiskScore($outputs): int
|
||
|
|
{
|
||
|
|
$scoreOutput = $outputs->firstWhere('output_key', 'tranx_score');
|
||
|
|
|
||
|
|
if (! $scoreOutput) {
|
||
|
|
return 128; // Default: Medium risk (255/2)
|
||
|
|
}
|
||
|
|
|
||
|
|
$scoreData = json_decode($scoreOutput->content, true);
|
||
|
|
|
||
|
|
if (is_array($scoreData) && isset($scoreData['score'])) {
|
||
|
|
// Assume score is 0-100, convert to 0-255
|
||
|
|
$score = (int) $scoreData['score'];
|
||
|
|
|
||
|
|
return (int) round(($score / 100) * 255);
|
||
|
|
}
|
||
|
|
|
||
|
|
return 128;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Extract a field from outputs
|
||
|
|
*/
|
||
|
|
private function extractField($outputs, string $outputKey, int $maxLength = 255): mixed
|
||
|
|
{
|
||
|
|
$output = $outputs->firstWhere('output_key', $outputKey);
|
||
|
|
|
||
|
|
if (! $output) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
$data = json_decode($output->content, true);
|
||
|
|
|
||
|
|
// Extract 'answer' field if it exists
|
||
|
|
if (is_array($data) && isset($data['answer'])) {
|
||
|
|
$answer = $data['answer'];
|
||
|
|
|
||
|
|
// If answer is an object/array, convert to string
|
||
|
|
if (is_array($answer)) {
|
||
|
|
$answer = json_encode($answer);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Truncate to max length if needed
|
||
|
|
return $this->truncateString((string) $answer, $maxLength);
|
||
|
|
}
|
||
|
|
|
||
|
|
// If data is an object/array, convert to JSON string
|
||
|
|
if (is_array($data)) {
|
||
|
|
return $this->truncateString(json_encode($data), $maxLength);
|
||
|
|
}
|
||
|
|
|
||
|
|
return $this->truncateString((string) $data, $maxLength);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Truncate string to max length
|
||
|
|
*/
|
||
|
|
private function truncateString(string $text, int $maxLength): string
|
||
|
|
{
|
||
|
|
if (strlen($text) <= $maxLength) {
|
||
|
|
return $text;
|
||
|
|
}
|
||
|
|
|
||
|
|
return substr($text, 0, $maxLength - 3) . '...';
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Parse date string to Carbon instance
|
||
|
|
*/
|
||
|
|
private function parseDate(?string $date): ?Carbon
|
||
|
|
{
|
||
|
|
if (empty($date)) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
return Carbon::parse($date);
|
||
|
|
} catch (\Exception $e) {
|
||
|
|
Log::warning('Failed to parse date', ['date' => $date]);
|
||
|
|
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Check if column exists in transactions table
|
||
|
|
*/
|
||
|
|
private function columnExists(string $columnName): bool
|
||
|
|
{
|
||
|
|
static $columns = null;
|
||
|
|
|
||
|
|
if ($columns === null) {
|
||
|
|
$columns = DB::getSchemaBuilder()->getColumnListing('transactions');
|
||
|
|
}
|
||
|
|
|
||
|
|
return in_array($columnName, $columns);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Get statistics about the transformation
|
||
|
|
*/
|
||
|
|
public function getStats(): array
|
||
|
|
{
|
||
|
|
return [
|
||
|
|
'data_pool_transactions' => DB::table('backend_data_pool')
|
||
|
|
->select('transaction_id')
|
||
|
|
->distinct()
|
||
|
|
->count(),
|
||
|
|
'companies_count' => Company::count(),
|
||
|
|
'transactions_count' => Transaction::count(),
|
||
|
|
'migrated_transactions' => Transaction::where('reference', 'like', 'MIGRATED-%')->count(),
|
||
|
|
];
|
||
|
|
}
|
||
|
|
}
|