$this->batchSize, ]); try { DB::transaction(function () { $this->transformData(); }); $duration = now()->diffInSeconds($startTime); Log::info('TransformDataPoolToProduction: Transformation completed successfully', [ 'duration_seconds' => $duration, ]); } catch (\Exception $e) { Log::error('TransformDataPoolToProduction: Transformation failed', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), ]); throw $e; } } /** * Transform data from backend_data_pool to companies and transactions */ private function transformData(): void { // Get all unique transaction IDs from data pool $transactionIds = DB::table('backend_data_pool') ->select('transaction_id') ->distinct() ->pluck('transaction_id'); Log::info('TransformDataPoolToProduction: Found transactions to process', [ 'count' => $transactionIds->count(), ]); $processedCompanies = 0; $processedTransactions = 0; foreach ($transactionIds as $transactionId) { // Get all outputs for this transaction $outputs = DB::table('backend_data_pool') ->where('transaction_id', $transactionId) ->get(); if ($outputs->isEmpty()) { continue; } $firstOutput = $outputs->first(); // 1. Create or update Company $company = $this->createOrUpdateCompany($firstOutput, $outputs); if ($company->wasRecentlyCreated) { $processedCompanies++; } // 2. Create or update Transaction $transaction = $this->createOrUpdateTransaction($company, $firstOutput, $outputs); if ($transaction->wasRecentlyCreated) { $processedTransactions++; } } Log::info('TransformDataPoolToProduction: Processing completed', [ 'new_companies' => $processedCompanies, 'new_transactions' => $processedTransactions, ]); } /** * Create or update a Company from data pool outputs */ private function createOrUpdateCompany($firstOutput, $outputs): Company { $corporateEntity = $firstOutput->corporate_entity; // Use backend risk_score and map to risk_level $backendRiskScore = $firstOutput->risk_score ?? 50; $kycRiskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore); // Extract company data from outputs $companyData = [ 'country' => $this->extractCountry($firstOutput, $outputs), 'kyc_risk_level' => $kycRiskLevel, 'sector' => $this->extractLeanField($outputs, 'entity_corporate_sector', 255), 'headquarters' => $this->extractLeanField($outputs, 'entity_corporate_HQ', 255), 'summary' => $this->extractField($outputs, 'entity_corporate_summary', 5000), // text field, longer OK 'legal_name' => $this->extractLegalName($outputs, $corporateEntity), 'ticker' => null, // Ticker is not available in backend_data_pool ]; return Company::updateOrCreate( ['name' => $corporateEntity], $companyData ); } /** * Create or update a Transaction from data pool outputs */ private function createOrUpdateTransaction(Company $company, $firstOutput, $outputs): Transaction { // Generate better reference number $reference = $this->generateTransactionReference($firstOutput); // Use risk_score from backend (leading field) $backendRiskScore = $firstOutput->risk_score ?? 50; // Default to 50 (high risk) if not set // Map backend risk_score to risk_level // -100 to 20 = Critical // 20 to 80 = High // 80 to 100 = Low $riskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore); // Convert to 0-255 scale for frontend transactions table $riskScore = $this->mapBackendRiskScoreTo255Scale($backendRiskScore); // Map risk level to Transaction Status $status = match ($riskLevel) { 'critical' => Transaction::STATUS_TRUE_POSITIVE, 'high' => Transaction::STATUS_FALSE_POSITIVE, 'low' => Transaction::STATUS_CLEARED, default => Transaction::STATUS_FALSE_POSITIVE, }; // Core transaction data $transactionData = [ 'company_id' => $company->id, 'amount' => $firstOutput->tx_amount, 'currency' => $firstOutput->tx_currency ?? 'EUR', 'counterparty' => $firstOutput->corporate_counterparty, 'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming, $firstOutput->corporate_counterparty), 'counterparty_kyc_risk_level' => $riskLevel, 'channel' => $this->determineTransactionType($firstOutput), 'executed_at' => $this->parseDate($firstOutput->tx_date), 'risk_score' => $riskScore, 'status' => $status, 'requires_review' => $this->shouldRequireReview($riskLevel), 'flagged_reason' => $firstOutput->tx_purpose, // Store backend risk data 'entity_risk_breakdown' => $firstOutput->risk_details_json, 'counterparty_risk_breakdown' => null, 'transaction_plausibility_breakdown' => null, 'combined_risk_breakdown' => null, ]; // Map all output_keys to their respective columns foreach ($outputs as $output) { $columnName = $output->output_key; // Skip if column doesn't exist in transactions table if (! $this->columnExists($columnName)) { continue; } // Decode JSON content, store NULL if empty or invalid $content = json_decode($output->content, true); $transactionData[$columnName] = $content ?: null; } return Transaction::updateOrCreate( ['reference' => $reference], $transactionData ); } /** * Extract country code from data */ private function extractCountry($firstOutput, $outputs): string { // Try to get from tx_country_incoming if (! empty($firstOutput->tx_country_incoming)) { $country = $this->extractCountryCode($firstOutput->tx_country_incoming); if ($country) { return $country; } } // Try to extract from corporate_HQ $hq = $this->extractField($outputs, 'corporate_HQ'); if ($hq && is_array($hq) && isset($hq['answer'])) { // Simple pattern matching for country codes if (preg_match('/\b([A-Z]{2})\b/', $hq['answer'], $matches)) { return $matches[1]; } } return 'DE'; // Default } /** * Extract country code (convert to ISO 2-letter if needed) * Also checks counterparty name for country indicators */ private function extractCountryCode(?string $country, ?string $counterpartyName = null): ?string { // First, try to extract from counterparty name (highest priority for tax havens) if (! empty($counterpartyName)) { $extractedFromName = $this->extractCountryFromText($counterpartyName); if ($extractedFromName) { return $extractedFromName; } } if (empty($country)) { return null; } // If already 2 letters, check if it's a suspicious/wrong code if (strlen($country) === 2) { $suspiciousCodes = ['IO']; // British Indian Ocean Territory often wrong for Virgin Islands if (in_array(strtoupper($country), $suspiciousCodes)) { return null; // Reject suspicious codes } return strtoupper($country); } // Simple country name mapping (extend as needed) $countryMap = [ 'Germany' => 'DE', 'Deutschland' => 'DE', 'United States' => 'US', 'USA' => 'US', 'United Kingdom' => 'GB', 'UK' => 'GB', 'France' => 'FR', 'Spain' => 'ES', 'Italy' => 'IT', 'Netherlands' => 'NL', 'Belgium' => 'BE', 'Austria' => 'AT', 'Switzerland' => 'CH', 'Poland' => 'PL', 'Czech Republic' => 'CZ', 'Denmark' => 'DK', 'Sweden' => 'SE', 'Norway' => 'NO', 'Finland' => 'FI', ]; return $countryMap[$country] ?? null; } /** * Extract country code from text (e.g., company name) */ private function extractCountryFromText(string $text): ?string { $countryPatterns = [ '/\b(?:British )?Virgin Islands\b/i' => 'VG', '/\bCayman Islands\b/i' => 'KY', '/\bBermuda\b/i' => 'BM', '/\bBahamas\b/i' => 'BS', '/\bPanama\b/i' => 'PA', '/\bLiechtenstein\b/i' => 'LI', '/\bMonaco\b/i' => 'MC', '/\bLuxembourg\b/i' => 'LU', '/\bSwitzerland\b/i' => 'CH', '/\bAzerbaijan\b/i' => 'AZ', '/\bRussia\b/i' => 'RU', '/\bChina\b/i' => 'CN', '/\bGermany\b/i' => 'DE', ]; foreach ($countryPatterns as $pattern => $code) { if (preg_match($pattern, $text)) { return $code; } } return null; } /** * Map backend risk_score (-100 to 100) to risk_level * -100 to 20 = Critical * 20 to 80 = High * 80 to 100 = Low */ private function mapBackendRiskScoreToLevel(int $backendRiskScore): string { if ($backendRiskScore <= 20) { return 'critical'; } if ($backendRiskScore <= 80) { return 'high'; } return 'low'; } /** * Map backend risk_score (-100 to 100) to 0-255 scale */ private function mapBackendRiskScoreTo255Scale(int $backendRiskScore): int { // Normalize -100 to 100 → 0 to 255 // Formula: ((score + 100) / 200) * 255 $normalized = (($backendRiskScore + 100) / 200) * 255; return (int) round(max(0, min(255, $normalized))); } /** * Determine if transaction requires review based on risk level */ private function shouldRequireReview(string $riskLevel): bool { return match ($riskLevel) { 'critical' => true, 'high' => true, 'low' => false, default => true, }; } /** * Extract a field from outputs */ private function extractField($outputs, string $outputKey, int $maxLength = 255): mixed { $output = $outputs->firstWhere('output_key', $outputKey); if (! $output) { return null; } $data = json_decode($output->content, true); // Extract 'answer' field if it exists if (is_array($data) && isset($data['answer'])) { $answer = $data['answer']; // If answer is an object/array, convert to string if (is_array($answer)) { $answer = json_encode($answer); } // Truncate to max length if needed return $this->truncateString((string) $answer, $maxLength); } // If data is an object/array, convert to JSON string if (is_array($data)) { return $this->truncateString(json_encode($data), $maxLength); } return $this->truncateString((string) $data, $maxLength); } /** * Extract a lean/concise field from outputs (extracts first sentence only) */ private function extractLeanField($outputs, string $outputKey, int $maxLength = 255): mixed { $fullText = $this->extractField($outputs, $outputKey, 5000); if (! $fullText) { return null; } // Extract first sentence or up to first period $firstSentence = preg_split('/\.(?:\s|$)/', $fullText, 2)[0]; if ($firstSentence) { $firstSentence = trim($firstSentence).'.'; return $this->truncateString($firstSentence, $maxLength); } return $this->truncateString($fullText, $maxLength); } /** * Extract legal name from corporate name field */ private function extractLegalName($outputs, string $fallbackName): ?string { $fullText = $this->extractField($outputs, 'entity_corporate_name', 5000); if (! $fullText) { return $fallbackName; } // Pattern 1: "is exactly as shown in an official register: [NAME]" if (preg_match('/is exactly as shown in an official register:\s*([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) { return trim($matches[1]); } // Pattern 2: "is [NAME]" - matches various patterns like "legal name is NAME", "company is NAME" if (preg_match('/(?:legal name|name|company) (?:of .+ )?is\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) { $name = trim($matches[1]); // Remove trailing explanatory text $name = preg_replace('/\s+(?:is exactly|exactly as shown|with former|former names).*$/i', '', $name); return trim($name); } // Pattern 3: "of [NAME]" - matches "legal name of Deutsche Bank AG" if (preg_match('/legal name of\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) { $name = trim($matches[1]); $name = preg_replace('/\s+(?:is exactly|with former).*$/i', '', $name); return trim($name); } // Pattern 4: Extract company name with suffix at start of sentence if (preg_match('/^(?:The current legal name (?:of|is) )?([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) { return trim($matches[1]); } // If no pattern matches, use the fallback name return $fallbackName; } /** * Truncate string to max length */ private function truncateString(string $text, int $maxLength): string { if (strlen($text) <= $maxLength) { return $text; } return substr($text, 0, $maxLength - 3).'...'; } /** * Parse date string to Carbon instance */ private function parseDate(?string $date): ?Carbon { if (empty($date)) { return null; } try { return Carbon::parse($date); } catch (\Exception $e) { Log::warning('Failed to parse date', ['date' => $date]); return null; } } /** * Generate a meaningful transaction reference */ private function generateTransactionReference($firstOutput): string { // Try to extract reference from raw_payload if (! empty($firstOutput->raw_payload)) { $payload = json_decode($firstOutput->raw_payload, true); // Look for common reference fields $referenceFields = [ 'Transaction Reference', 'Reference', 'Transaction ID', 'Payment Reference', 'Transfer Reference', ]; foreach ($referenceFields as $field) { if (isset($payload[$field]) && ! empty($payload[$field])) { return (string) $payload[$field]; } } } // Generate a date-based reference if no reference found $date = $this->parseDate($firstOutput->tx_date); $dateStr = $date ? $date->format('Ymd') : date('Ymd'); return sprintf('TXN-%s-%05d', $dateStr, $firstOutput->transaction_id); } /** * Determine transaction type from available data */ private function determineTransactionType($firstOutput): string { // Try to determine from raw_payload first if (! empty($firstOutput->raw_payload)) { $payload = json_decode($firstOutput->raw_payload, true); // Check for explicit transaction type field if (isset($payload['Transaction Type']) && ! empty($payload['Transaction Type'])) { return $payload['Transaction Type']; } if (isset($payload['Payment Type']) && ! empty($payload['Payment Type'])) { return $payload['Payment Type']; } } // Determine based on amount and countries $amount = (float) $firstOutput->tx_amount; $isInternational = $firstOutput->tx_country_outgoing !== $firstOutput->tx_country_incoming; // Large international transactions are likely SWIFT if ($isInternational && $amount > 100000) { return 'SWIFT Transfer'; } // European transfers if ($isInternational && $this->isEuropeanCountry($firstOutput->tx_country_incoming)) { return 'SEPA Transfer'; } // Domestic high-value if (! $isInternational && $amount > 50000) { return 'Wire Transfer'; } // Default return 'Bank Transfer'; } /** * Check if country is in Europe */ private function isEuropeanCountry(?string $countryCode): bool { if (empty($countryCode)) { return false; } $europeanCountries = [ 'AT', 'BE', 'BG', 'HR', 'CY', 'CZ', 'DK', 'EE', 'FI', 'FR', 'DE', 'GR', 'HU', 'IE', 'IT', 'LV', 'LT', 'LU', 'MT', 'NL', 'PL', 'PT', 'RO', 'SK', 'SI', 'ES', 'SE', 'CH', 'NO', 'IS', ]; return in_array(strtoupper($countryCode), $europeanCountries); } /** * Check if column exists in transactions table */ private function columnExists(string $columnName): bool { static $columns = null; if ($columns === null) { $columns = DB::getSchemaBuilder()->getColumnListing('transactions'); } return in_array($columnName, $columns); } /** * Get statistics about the transformation */ public function getStats(): array { return [ 'data_pool_transactions' => DB::table('backend_data_pool') ->select('transaction_id') ->distinct() ->count(), 'companies_count' => Company::count(), 'transactions_count' => Transaction::count(), 'migrated_transactions' => Transaction::where('reference', 'like', 'MIGRATED-%')->count(), ]; } }