617 lines
20 KiB
PHP
617 lines
20 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Models\Company;
|
|
use App\Models\Transaction;
|
|
use Carbon\Carbon;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
class TransformDataPoolToProduction implements ShouldQueue
|
|
{
|
|
use Dispatchable;
|
|
use InteractsWithQueue;
|
|
use Queueable;
|
|
use SerializesModels;
|
|
|
|
public int $timeout = 3600; // 1 hour timeout
|
|
|
|
public int $tries = 3;
|
|
|
|
/**
|
|
* Create a new job instance.
|
|
*/
|
|
public function __construct(
|
|
public ?int $batchSize = 100,
|
|
) {}
|
|
|
|
/**
|
|
* Execute the job.
|
|
*/
|
|
public function handle(): void
|
|
{
|
|
$startTime = now();
|
|
|
|
Log::info('TransformDataPoolToProduction: Starting transformation', [
|
|
'batch_size' => $this->batchSize,
|
|
]);
|
|
|
|
try {
|
|
DB::transaction(function () {
|
|
$this->transformData();
|
|
});
|
|
|
|
$duration = now()->diffInSeconds($startTime);
|
|
|
|
Log::info('TransformDataPoolToProduction: Transformation completed successfully', [
|
|
'duration_seconds' => $duration,
|
|
]);
|
|
} catch (\Exception $e) {
|
|
Log::error('TransformDataPoolToProduction: Transformation failed', [
|
|
'error' => $e->getMessage(),
|
|
'trace' => $e->getTraceAsString(),
|
|
]);
|
|
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Transform data from backend_data_pool to companies and transactions
|
|
*/
|
|
private function transformData(): void
|
|
{
|
|
// Get all unique transaction IDs from data pool
|
|
$transactionIds = DB::table('backend_data_pool')
|
|
->select('transaction_id')
|
|
->distinct()
|
|
->pluck('transaction_id');
|
|
|
|
Log::info('TransformDataPoolToProduction: Found transactions to process', [
|
|
'count' => $transactionIds->count(),
|
|
]);
|
|
|
|
$processedCompanies = 0;
|
|
$processedTransactions = 0;
|
|
|
|
foreach ($transactionIds as $transactionId) {
|
|
// Get all outputs for this transaction
|
|
$outputs = DB::table('backend_data_pool')
|
|
->where('transaction_id', $transactionId)
|
|
->get();
|
|
|
|
if ($outputs->isEmpty()) {
|
|
continue;
|
|
}
|
|
|
|
$firstOutput = $outputs->first();
|
|
|
|
// 1. Create or update Company
|
|
$company = $this->createOrUpdateCompany($firstOutput, $outputs);
|
|
if ($company->wasRecentlyCreated) {
|
|
$processedCompanies++;
|
|
}
|
|
|
|
// 2. Create or update Transaction
|
|
$transaction = $this->createOrUpdateTransaction($company, $firstOutput, $outputs);
|
|
if ($transaction->wasRecentlyCreated) {
|
|
$processedTransactions++;
|
|
}
|
|
}
|
|
|
|
Log::info('TransformDataPoolToProduction: Processing completed', [
|
|
'new_companies' => $processedCompanies,
|
|
'new_transactions' => $processedTransactions,
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* Create or update a Company from data pool outputs
|
|
*/
|
|
private function createOrUpdateCompany($firstOutput, $outputs): Company
|
|
{
|
|
$corporateEntity = $firstOutput->corporate_entity;
|
|
|
|
// Use backend risk_score and map to risk_level
|
|
$backendRiskScore = $firstOutput->risk_score ?? 50;
|
|
$kycRiskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore);
|
|
|
|
// Extract company data from outputs
|
|
$companyData = [
|
|
'country' => $this->extractCountry($firstOutput, $outputs),
|
|
'kyc_risk_level' => $kycRiskLevel,
|
|
'sector' => $this->extractLeanField($outputs, 'entity_corporate_sector', 255),
|
|
'headquarters' => $this->extractLeanField($outputs, 'entity_corporate_HQ', 255),
|
|
'summary' => $this->extractField($outputs, 'entity_corporate_summary', 5000), // text field, longer OK
|
|
'legal_name' => $this->extractLegalName($outputs, $corporateEntity),
|
|
'ticker' => null, // Ticker is not available in backend_data_pool
|
|
];
|
|
|
|
return Company::updateOrCreate(
|
|
['name' => $corporateEntity],
|
|
$companyData
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Create or update a Transaction from data pool outputs
|
|
*/
|
|
private function createOrUpdateTransaction(Company $company, $firstOutput, $outputs): Transaction
|
|
{
|
|
// Generate better reference number
|
|
$reference = $this->generateTransactionReference($firstOutput);
|
|
|
|
// Use risk_score from backend (leading field)
|
|
$backendRiskScore = $firstOutput->risk_score ?? 50; // Default to 50 (high risk) if not set
|
|
|
|
// Map backend risk_score to risk_level
|
|
// -100 to 20 = Critical
|
|
// 20 to 80 = High
|
|
// 80 to 100 = Low
|
|
$riskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore);
|
|
|
|
// Convert to 0-255 scale for frontend transactions table
|
|
$riskScore = $this->mapBackendRiskScoreTo255Scale($backendRiskScore);
|
|
|
|
// Map risk level to Transaction Status
|
|
$status = match ($riskLevel) {
|
|
'critical' => Transaction::STATUS_TRUE_POSITIVE,
|
|
'high' => Transaction::STATUS_FALSE_POSITIVE,
|
|
'low' => Transaction::STATUS_CLEARED,
|
|
default => Transaction::STATUS_FALSE_POSITIVE,
|
|
};
|
|
|
|
// Core transaction data
|
|
$transactionData = [
|
|
'company_id' => $company->id,
|
|
'amount' => $firstOutput->tx_amount,
|
|
'currency' => $firstOutput->tx_currency ?? 'EUR',
|
|
'counterparty' => $firstOutput->corporate_counterparty,
|
|
'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming, $firstOutput->corporate_counterparty),
|
|
'counterparty_kyc_risk_level' => $riskLevel,
|
|
'channel' => $this->determineTransactionType($firstOutput),
|
|
'executed_at' => $this->parseDate($firstOutput->tx_date),
|
|
'risk_score' => $riskScore,
|
|
'status' => $status,
|
|
'requires_review' => $this->shouldRequireReview($riskLevel),
|
|
'flagged_reason' => $firstOutput->tx_purpose,
|
|
// Store backend risk data
|
|
'entity_risk_breakdown' => $firstOutput->risk_details_json,
|
|
'counterparty_risk_breakdown' => null,
|
|
'transaction_plausibility_breakdown' => null,
|
|
'combined_risk_breakdown' => null,
|
|
];
|
|
|
|
// Map all output_keys to their respective columns
|
|
foreach ($outputs as $output) {
|
|
$columnName = $output->output_key;
|
|
|
|
// Skip if column doesn't exist in transactions table
|
|
if (! $this->columnExists($columnName)) {
|
|
continue;
|
|
}
|
|
|
|
// Decode JSON content, store NULL if empty or invalid
|
|
$content = json_decode($output->content, true);
|
|
$transactionData[$columnName] = $content ?: null;
|
|
}
|
|
|
|
return Transaction::updateOrCreate(
|
|
['reference' => $reference],
|
|
$transactionData
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Extract country code from data
|
|
*/
|
|
private function extractCountry($firstOutput, $outputs): string
|
|
{
|
|
// Try to get from tx_country_incoming
|
|
if (! empty($firstOutput->tx_country_incoming)) {
|
|
$country = $this->extractCountryCode($firstOutput->tx_country_incoming);
|
|
if ($country) {
|
|
return $country;
|
|
}
|
|
}
|
|
|
|
// Try to extract from corporate_HQ
|
|
$hq = $this->extractField($outputs, 'corporate_HQ');
|
|
if ($hq && is_array($hq) && isset($hq['answer'])) {
|
|
// Simple pattern matching for country codes
|
|
if (preg_match('/\b([A-Z]{2})\b/', $hq['answer'], $matches)) {
|
|
return $matches[1];
|
|
}
|
|
}
|
|
|
|
return 'DE'; // Default
|
|
}
|
|
|
|
/**
|
|
* Extract country code (convert to ISO 2-letter if needed)
|
|
* Also checks counterparty name for country indicators
|
|
*/
|
|
private function extractCountryCode(?string $country, ?string $counterpartyName = null): ?string
|
|
{
|
|
// First, try to extract from counterparty name (highest priority for tax havens)
|
|
if (! empty($counterpartyName)) {
|
|
$extractedFromName = $this->extractCountryFromText($counterpartyName);
|
|
if ($extractedFromName) {
|
|
return $extractedFromName;
|
|
}
|
|
}
|
|
|
|
if (empty($country)) {
|
|
return null;
|
|
}
|
|
|
|
// If already 2 letters, check if it's a suspicious/wrong code
|
|
if (strlen($country) === 2) {
|
|
$suspiciousCodes = ['IO']; // British Indian Ocean Territory often wrong for Virgin Islands
|
|
if (in_array(strtoupper($country), $suspiciousCodes)) {
|
|
return null; // Reject suspicious codes
|
|
}
|
|
|
|
return strtoupper($country);
|
|
}
|
|
|
|
// Simple country name mapping (extend as needed)
|
|
$countryMap = [
|
|
'Germany' => 'DE',
|
|
'Deutschland' => 'DE',
|
|
'United States' => 'US',
|
|
'USA' => 'US',
|
|
'United Kingdom' => 'GB',
|
|
'UK' => 'GB',
|
|
'France' => 'FR',
|
|
'Spain' => 'ES',
|
|
'Italy' => 'IT',
|
|
'Netherlands' => 'NL',
|
|
'Belgium' => 'BE',
|
|
'Austria' => 'AT',
|
|
'Switzerland' => 'CH',
|
|
'Poland' => 'PL',
|
|
'Czech Republic' => 'CZ',
|
|
'Denmark' => 'DK',
|
|
'Sweden' => 'SE',
|
|
'Norway' => 'NO',
|
|
'Finland' => 'FI',
|
|
];
|
|
|
|
return $countryMap[$country] ?? null;
|
|
}
|
|
|
|
/**
|
|
* Extract country code from text (e.g., company name)
|
|
*/
|
|
private function extractCountryFromText(string $text): ?string
|
|
{
|
|
$countryPatterns = [
|
|
'/\b(?:British )?Virgin Islands\b/i' => 'VG',
|
|
'/\bCayman Islands\b/i' => 'KY',
|
|
'/\bBermuda\b/i' => 'BM',
|
|
'/\bBahamas\b/i' => 'BS',
|
|
'/\bPanama\b/i' => 'PA',
|
|
'/\bLiechtenstein\b/i' => 'LI',
|
|
'/\bMonaco\b/i' => 'MC',
|
|
'/\bLuxembourg\b/i' => 'LU',
|
|
'/\bSwitzerland\b/i' => 'CH',
|
|
'/\bAzerbaijan\b/i' => 'AZ',
|
|
'/\bRussia\b/i' => 'RU',
|
|
'/\bChina\b/i' => 'CN',
|
|
'/\bGermany\b/i' => 'DE',
|
|
];
|
|
|
|
foreach ($countryPatterns as $pattern => $code) {
|
|
if (preg_match($pattern, $text)) {
|
|
return $code;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Map backend risk_score (-100 to 100) to risk_level
|
|
* -100 to 20 = Critical
|
|
* 20 to 80 = High
|
|
* 80 to 100 = Low
|
|
*/
|
|
private function mapBackendRiskScoreToLevel(int $backendRiskScore): string
|
|
{
|
|
if ($backendRiskScore <= 20) {
|
|
return 'critical';
|
|
}
|
|
|
|
if ($backendRiskScore <= 80) {
|
|
return 'high';
|
|
}
|
|
|
|
return 'low';
|
|
}
|
|
|
|
/**
|
|
* Map backend risk_score (-100 to 100) to 0-255 scale
|
|
*/
|
|
private function mapBackendRiskScoreTo255Scale(int $backendRiskScore): int
|
|
{
|
|
// Normalize -100 to 100 → 0 to 255
|
|
// Formula: ((score + 100) / 200) * 255
|
|
$normalized = (($backendRiskScore + 100) / 200) * 255;
|
|
|
|
return (int) round(max(0, min(255, $normalized)));
|
|
}
|
|
|
|
/**
|
|
* Determine if transaction requires review based on risk level
|
|
*/
|
|
private function shouldRequireReview(string $riskLevel): bool
|
|
{
|
|
return match ($riskLevel) {
|
|
'critical' => true,
|
|
'high' => true,
|
|
'low' => false,
|
|
default => true,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Extract a field from outputs
|
|
*/
|
|
private function extractField($outputs, string $outputKey, int $maxLength = 255): mixed
|
|
{
|
|
$output = $outputs->firstWhere('output_key', $outputKey);
|
|
|
|
if (! $output) {
|
|
return null;
|
|
}
|
|
|
|
$data = json_decode($output->content, true);
|
|
|
|
// Extract 'answer' field if it exists
|
|
if (is_array($data) && isset($data['answer'])) {
|
|
$answer = $data['answer'];
|
|
|
|
// If answer is an object/array, convert to string
|
|
if (is_array($answer)) {
|
|
$answer = json_encode($answer);
|
|
}
|
|
|
|
// Truncate to max length if needed
|
|
return $this->truncateString((string) $answer, $maxLength);
|
|
}
|
|
|
|
// If data is an object/array, convert to JSON string
|
|
if (is_array($data)) {
|
|
return $this->truncateString(json_encode($data), $maxLength);
|
|
}
|
|
|
|
return $this->truncateString((string) $data, $maxLength);
|
|
}
|
|
|
|
/**
|
|
* Extract a lean/concise field from outputs (extracts first sentence only)
|
|
*/
|
|
private function extractLeanField($outputs, string $outputKey, int $maxLength = 255): mixed
|
|
{
|
|
$fullText = $this->extractField($outputs, $outputKey, 5000);
|
|
|
|
if (! $fullText) {
|
|
return null;
|
|
}
|
|
|
|
// Extract first sentence or up to first period
|
|
$firstSentence = preg_split('/\.(?:\s|$)/', $fullText, 2)[0];
|
|
|
|
if ($firstSentence) {
|
|
$firstSentence = trim($firstSentence).'.';
|
|
|
|
return $this->truncateString($firstSentence, $maxLength);
|
|
}
|
|
|
|
return $this->truncateString($fullText, $maxLength);
|
|
}
|
|
|
|
/**
|
|
* Extract legal name from corporate name field
|
|
*/
|
|
private function extractLegalName($outputs, string $fallbackName): ?string
|
|
{
|
|
$fullText = $this->extractField($outputs, 'entity_corporate_name', 5000);
|
|
|
|
if (! $fullText) {
|
|
return $fallbackName;
|
|
}
|
|
|
|
// Pattern 1: "is exactly as shown in an official register: [NAME]"
|
|
if (preg_match('/is exactly as shown in an official register:\s*([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
|
|
return trim($matches[1]);
|
|
}
|
|
|
|
// Pattern 2: "is [NAME]" - matches various patterns like "legal name is NAME", "company is NAME"
|
|
if (preg_match('/(?:legal name|name|company) (?:of .+ )?is\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
|
|
$name = trim($matches[1]);
|
|
// Remove trailing explanatory text
|
|
$name = preg_replace('/\s+(?:is exactly|exactly as shown|with former|former names).*$/i', '', $name);
|
|
|
|
return trim($name);
|
|
}
|
|
|
|
// Pattern 3: "of [NAME]" - matches "legal name of Deutsche Bank AG"
|
|
if (preg_match('/legal name of\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
|
|
$name = trim($matches[1]);
|
|
$name = preg_replace('/\s+(?:is exactly|with former).*$/i', '', $name);
|
|
|
|
return trim($name);
|
|
}
|
|
|
|
// Pattern 4: Extract company name with suffix at start of sentence
|
|
if (preg_match('/^(?:The current legal name (?:of|is) )?([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
|
|
return trim($matches[1]);
|
|
}
|
|
|
|
// If no pattern matches, use the fallback name
|
|
return $fallbackName;
|
|
}
|
|
|
|
/**
|
|
* Truncate string to max length
|
|
*/
|
|
private function truncateString(string $text, int $maxLength): string
|
|
{
|
|
if (strlen($text) <= $maxLength) {
|
|
return $text;
|
|
}
|
|
|
|
return substr($text, 0, $maxLength - 3).'...';
|
|
}
|
|
|
|
/**
|
|
* Parse date string to Carbon instance
|
|
*/
|
|
private function parseDate(?string $date): ?Carbon
|
|
{
|
|
if (empty($date)) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
return Carbon::parse($date);
|
|
} catch (\Exception $e) {
|
|
Log::warning('Failed to parse date', ['date' => $date]);
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate a meaningful transaction reference
|
|
*/
|
|
private function generateTransactionReference($firstOutput): string
|
|
{
|
|
// Try to extract reference from raw_payload
|
|
if (! empty($firstOutput->raw_payload)) {
|
|
$payload = json_decode($firstOutput->raw_payload, true);
|
|
|
|
// Look for common reference fields
|
|
$referenceFields = [
|
|
'Transaction Reference',
|
|
'Reference',
|
|
'Transaction ID',
|
|
'Payment Reference',
|
|
'Transfer Reference',
|
|
];
|
|
|
|
foreach ($referenceFields as $field) {
|
|
if (isset($payload[$field]) && ! empty($payload[$field])) {
|
|
return (string) $payload[$field];
|
|
}
|
|
}
|
|
}
|
|
|
|
// Generate a date-based reference if no reference found
|
|
$date = $this->parseDate($firstOutput->tx_date);
|
|
$dateStr = $date ? $date->format('Ymd') : date('Ymd');
|
|
|
|
return sprintf('TXN-%s-%05d', $dateStr, $firstOutput->transaction_id);
|
|
}
|
|
|
|
/**
|
|
* Determine transaction type from available data
|
|
*/
|
|
private function determineTransactionType($firstOutput): string
|
|
{
|
|
// Try to determine from raw_payload first
|
|
if (! empty($firstOutput->raw_payload)) {
|
|
$payload = json_decode($firstOutput->raw_payload, true);
|
|
|
|
// Check for explicit transaction type field
|
|
if (isset($payload['Transaction Type']) && ! empty($payload['Transaction Type'])) {
|
|
return $payload['Transaction Type'];
|
|
}
|
|
|
|
if (isset($payload['Payment Type']) && ! empty($payload['Payment Type'])) {
|
|
return $payload['Payment Type'];
|
|
}
|
|
}
|
|
|
|
// Determine based on amount and countries
|
|
$amount = (float) $firstOutput->tx_amount;
|
|
$isInternational = $firstOutput->tx_country_outgoing !== $firstOutput->tx_country_incoming;
|
|
|
|
// Large international transactions are likely SWIFT
|
|
if ($isInternational && $amount > 100000) {
|
|
return 'SWIFT Transfer';
|
|
}
|
|
|
|
// European transfers
|
|
if ($isInternational && $this->isEuropeanCountry($firstOutput->tx_country_incoming)) {
|
|
return 'SEPA Transfer';
|
|
}
|
|
|
|
// Domestic high-value
|
|
if (! $isInternational && $amount > 50000) {
|
|
return 'Wire Transfer';
|
|
}
|
|
|
|
// Default
|
|
return 'Bank Transfer';
|
|
}
|
|
|
|
/**
|
|
* Check if country is in Europe
|
|
*/
|
|
private function isEuropeanCountry(?string $countryCode): bool
|
|
{
|
|
if (empty($countryCode)) {
|
|
return false;
|
|
}
|
|
|
|
$europeanCountries = [
|
|
'AT', 'BE', 'BG', 'HR', 'CY', 'CZ', 'DK', 'EE', 'FI', 'FR',
|
|
'DE', 'GR', 'HU', 'IE', 'IT', 'LV', 'LT', 'LU', 'MT', 'NL',
|
|
'PL', 'PT', 'RO', 'SK', 'SI', 'ES', 'SE', 'CH', 'NO', 'IS',
|
|
];
|
|
|
|
return in_array(strtoupper($countryCode), $europeanCountries);
|
|
}
|
|
|
|
/**
|
|
* Check if column exists in transactions table
|
|
*/
|
|
private function columnExists(string $columnName): bool
|
|
{
|
|
static $columns = null;
|
|
|
|
if ($columns === null) {
|
|
$columns = DB::getSchemaBuilder()->getColumnListing('transactions');
|
|
}
|
|
|
|
return in_array($columnName, $columns);
|
|
}
|
|
|
|
/**
|
|
* Get statistics about the transformation
|
|
*/
|
|
public function getStats(): array
|
|
{
|
|
return [
|
|
'data_pool_transactions' => DB::table('backend_data_pool')
|
|
->select('transaction_id')
|
|
->distinct()
|
|
->count(),
|
|
'companies_count' => Company::count(),
|
|
'transactions_count' => Transaction::count(),
|
|
'migrated_transactions' => Transaction::where('reference', 'like', 'MIGRATED-%')->count(),
|
|
];
|
|
}
|
|
}
|