$this->batchSize, ]); try { DB::transaction(function () use ($kycCalculator) { $this->transformData($kycCalculator); }); $duration = now()->diffInSeconds($startTime); Log::info('TransformDataPoolToProduction: Transformation completed successfully', [ 'duration_seconds' => $duration, ]); } catch (\Exception $e) { Log::error('TransformDataPoolToProduction: Transformation failed', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), ]); throw $e; } } /** * Transform data from backend_data_pool to companies and transactions */ private function transformData(KycRiskCalculator $kycCalculator): void { // Get all unique transaction IDs from data pool $transactionIds = DB::table('backend_data_pool') ->select('transaction_id') ->distinct() ->pluck('transaction_id'); Log::info('TransformDataPoolToProduction: Found transactions to process', [ 'count' => $transactionIds->count(), ]); $processedCompanies = 0; $processedTransactions = 0; foreach ($transactionIds as $transactionId) { // Get all outputs for this transaction $outputs = DB::table('backend_data_pool') ->where('transaction_id', $transactionId) ->get(); if ($outputs->isEmpty()) { continue; } $firstOutput = $outputs->first(); // 1. Create or update Company $company = $this->createOrUpdateCompany($firstOutput, $outputs, $kycCalculator); if ($company->wasRecentlyCreated) { $processedCompanies++; } // 2. Create or update Transaction $transaction = $this->createOrUpdateTransaction($company, $firstOutput, $outputs); if ($transaction->wasRecentlyCreated) { $processedTransactions++; } } Log::info('TransformDataPoolToProduction: Processing completed', [ 'new_companies' => $processedCompanies, 'new_transactions' => $processedTransactions, ]); } /** * Create or update a Company from data pool outputs */ private function createOrUpdateCompany($firstOutput, $outputs, KycRiskCalculator $kycCalculator): Company { $corporateEntity = $firstOutput->corporate_entity; // Calculate KYC Risk Level $kycRiskLevel = $kycCalculator->calculateKycRiskLevel(collect([$outputs])->first()); // Extract company data from outputs $companyData = [ 'country' => $this->extractCountry($firstOutput, $outputs), 'kyc_risk_level' => $kycRiskLevel, 'sector' => $this->extractField($outputs, 'corporate_sector', 255), 'headquarters' => $this->extractField($outputs, 'corporate_HQ', 255), 'summary' => $this->extractField($outputs, 'corporate_summary', 5000), // text field, longer OK 'legal_name' => $this->extractField($outputs, 'corporate_name', 255), ]; return Company::updateOrCreate( ['name' => $corporateEntity], $companyData ); } /** * Create or update a Transaction from data pool outputs */ private function createOrUpdateTransaction(Company $company, $firstOutput, $outputs): Transaction { $reference = 'MIGRATED-' . $firstOutput->transaction_id; // Map KYC Risk Level to Transaction Status $status = match ($company->kyc_risk_level) { 'critical' => Transaction::STATUS_TRUE_POSITIVE, 'high' => Transaction::STATUS_FALSE_POSITIVE, 'low' => Transaction::STATUS_CLEARED, default => Transaction::STATUS_FALSE_POSITIVE, // Default to high risk }; // Core transaction data $transactionData = [ 'company_id' => $company->id, 'amount' => $firstOutput->tx_amount, 'currency' => $firstOutput->tx_currency ?? 'EUR', 'counterparty' => $firstOutput->corporate_counterparty, 'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming), 'channel' => $firstOutput->source_file, 'executed_at' => $this->parseDate($firstOutput->tx_date), 'risk_score' => $this->extractRiskScore($outputs), 'status' => $status, 'requires_review' => true, 'flagged_reason' => $firstOutput->tx_purpose, ]; // Map all output_keys to their respective columns foreach ($outputs as $output) { $columnName = $output->output_key; // Skip if column doesn't exist in transactions table if (! $this->columnExists($columnName)) { continue; } // Decode JSON content, store NULL if empty or invalid $content = json_decode($output->content, true); $transactionData[$columnName] = $content ?: null; } return Transaction::updateOrCreate( ['reference' => $reference], $transactionData ); } /** * Extract country code from data */ private function extractCountry($firstOutput, $outputs): string { // Try to get from tx_country_incoming if (! empty($firstOutput->tx_country_incoming)) { $country = $this->extractCountryCode($firstOutput->tx_country_incoming); if ($country) { return $country; } } // Try to extract from corporate_HQ $hq = $this->extractField($outputs, 'corporate_HQ'); if ($hq && is_array($hq) && isset($hq['answer'])) { // Simple pattern matching for country codes if (preg_match('/\b([A-Z]{2})\b/', $hq['answer'], $matches)) { return $matches[1]; } } return 'DE'; // Default } /** * Extract country code (convert to ISO 2-letter if needed) */ private function extractCountryCode(?string $country): ?string { if (empty($country)) { return null; } // If already 2 letters, return uppercase if (strlen($country) === 2) { return strtoupper($country); } // Simple country name mapping (extend as needed) $countryMap = [ 'Germany' => 'DE', 'Deutschland' => 'DE', 'United States' => 'US', 'USA' => 'US', 'United Kingdom' => 'GB', 'UK' => 'GB', 'France' => 'FR', 'Spain' => 'ES', 'Italy' => 'IT', 'Netherlands' => 'NL', 'Belgium' => 'BE', 'Austria' => 'AT', 'Switzerland' => 'CH', 'Poland' => 'PL', 'Czech Republic' => 'CZ', 'Denmark' => 'DK', 'Sweden' => 'SE', 'Norway' => 'NO', 'Finland' => 'FI', ]; return $countryMap[$country] ?? null; } /** * Extract risk score from tranx_score output */ private function extractRiskScore($outputs): int { $scoreOutput = $outputs->firstWhere('output_key', 'tranx_score'); if (! $scoreOutput) { return 128; // Default: Medium risk (255/2) } $scoreData = json_decode($scoreOutput->content, true); if (is_array($scoreData) && isset($scoreData['score'])) { // Assume score is 0-100, convert to 0-255 $score = (int) $scoreData['score']; return (int) round(($score / 100) * 255); } return 128; } /** * Extract a field from outputs */ private function extractField($outputs, string $outputKey, int $maxLength = 255): mixed { $output = $outputs->firstWhere('output_key', $outputKey); if (! $output) { return null; } $data = json_decode($output->content, true); // Extract 'answer' field if it exists if (is_array($data) && isset($data['answer'])) { $answer = $data['answer']; // If answer is an object/array, convert to string if (is_array($answer)) { $answer = json_encode($answer); } // Truncate to max length if needed return $this->truncateString((string) $answer, $maxLength); } // If data is an object/array, convert to JSON string if (is_array($data)) { return $this->truncateString(json_encode($data), $maxLength); } return $this->truncateString((string) $data, $maxLength); } /** * Truncate string to max length */ private function truncateString(string $text, int $maxLength): string { if (strlen($text) <= $maxLength) { return $text; } return substr($text, 0, $maxLength - 3) . '...'; } /** * Parse date string to Carbon instance */ private function parseDate(?string $date): ?Carbon { if (empty($date)) { return null; } try { return Carbon::parse($date); } catch (\Exception $e) { Log::warning('Failed to parse date', ['date' => $date]); return null; } } /** * Check if column exists in transactions table */ private function columnExists(string $columnName): bool { static $columns = null; if ($columns === null) { $columns = DB::getSchemaBuilder()->getColumnListing('transactions'); } return in_array($columnName, $columns); } /** * Get statistics about the transformation */ public function getStats(): array { return [ 'data_pool_transactions' => DB::table('backend_data_pool') ->select('transaction_id') ->distinct() ->count(), 'companies_count' => Company::count(), 'transactions_count' => Transaction::count(), 'migrated_transactions' => Transaction::where('reference', 'like', 'MIGRATED-%')->count(), ]; } }