AFC-Demo/app/Jobs/TransformDataPoolToProduction.php

<?php

declare(strict_types=1);

namespace App\Jobs;

use App\Models\Company;
use App\Models\Transaction;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;

class TransformDataPoolToProduction implements ShouldQueue
{
    use Dispatchable;
    use InteractsWithQueue;
    use Queueable;
    use SerializesModels;

    public int $timeout = 3600; // 1 hour timeout

    public int $tries = 3;

    /**
     * Create a new job instance.
     */
    public function __construct(
        public ?int $batchSize = 100,
    ) {}

    /**
     * Execute the job.
     */
    public function handle(): void
    {
        $startTime = now();

        Log::info('TransformDataPoolToProduction: Starting transformation', [
            'batch_size' => $this->batchSize,
        ]);

        try {
            DB::transaction(function () {
                $this->transformData();
            });

            $duration = now()->diffInSeconds($startTime);

            Log::info('TransformDataPoolToProduction: Transformation completed successfully', [
                'duration_seconds' => $duration,
            ]);
        } catch (\Exception $e) {
            Log::error('TransformDataPoolToProduction: Transformation failed', [
                'error' => $e->getMessage(),
                'trace' => $e->getTraceAsString(),
            ]);

            throw $e;
        }
    }

    /**
     * Transform data from backend_data_pool to companies and transactions
     */
    private function transformData(): void
    {
        // Get all unique transaction IDs from data pool
        $transactionIds = DB::table('backend_data_pool')
            ->select('transaction_id')
            ->distinct()
            ->pluck('transaction_id');

        Log::info('TransformDataPoolToProduction: Found transactions to process', [
            'count' => $transactionIds->count(),
        ]);

        $processedCompanies = 0;
        $processedTransactions = 0;

        foreach ($transactionIds as $transactionId) {
            // Get all outputs for this transaction
            $outputs = DB::table('backend_data_pool')
                ->where('transaction_id', $transactionId)
                ->get();

            if ($outputs->isEmpty()) {
                continue;
            }

            $firstOutput = $outputs->first();

            // 1. Create or update Company
            $company = $this->createOrUpdateCompany($firstOutput, $outputs);
            if ($company->wasRecentlyCreated) {
                $processedCompanies++;
            }

            // 2. Create or update Transaction
            $transaction = $this->createOrUpdateTransaction($company, $firstOutput, $outputs);
            if ($transaction->wasRecentlyCreated) {
                $processedTransactions++;
            }
        }

        Log::info('TransformDataPoolToProduction: Processing completed', [
            'new_companies' => $processedCompanies,
            'new_transactions' => $processedTransactions,
        ]);
    }

    /**
     * Create or update a Company from data pool outputs
     */
    private function createOrUpdateCompany($firstOutput, $outputs): Company
    {
        $corporateEntity = $firstOutput->corporate_entity;

        // Use backend risk_score and map to risk_level
        $backendRiskScore = $firstOutput->risk_score ?? 50;
        $kycRiskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore);

        // Extract company data from outputs
        $companyData = [
            'country' => $this->extractCountry($firstOutput, $outputs),
            'kyc_risk_level' => $kycRiskLevel,
            'sector' => $this->extractLeanField($outputs, 'entity_corporate_sector', 255),
            'headquarters' => $this->extractLeanField($outputs, 'entity_corporate_HQ', 255),
            'summary' => $this->extractField($outputs, 'entity_corporate_summary', 5000), // text field, longer OK
            'legal_name' => $this->extractLegalName($outputs, $corporateEntity),
            'ticker' => null, // Ticker is not available in backend_data_pool
        ];

        return Company::updateOrCreate(
            ['name' => $corporateEntity],
            $companyData
        );
    }

    /**
     * Create or update a Transaction from data pool outputs
     */
    private function createOrUpdateTransaction(Company $company, $firstOutput, $outputs): Transaction
    {
        // Generate better reference number
        $reference = $this->generateTransactionReference($firstOutput);

        // Use risk_score from backend (leading field)
        $backendRiskScore = $firstOutput->risk_score ?? 50; // Default to 50 (high risk) if not set

        // Map backend risk_score to risk_level
        // -100 to 20 = Critical
        // 20 to 80 = High
        // 80 to 100 = Low
        $riskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore);

        // Convert to 0-255 scale for frontend transactions table
        $riskScore = $this->mapBackendRiskScoreTo255Scale($backendRiskScore);

        // Map risk level to Transaction Status
        $status = match ($riskLevel) {
            'critical' => Transaction::STATUS_TRUE_POSITIVE,
            'high' => Transaction::STATUS_FALSE_POSITIVE,
            'low' => Transaction::STATUS_CLEARED,
            default => Transaction::STATUS_FALSE_POSITIVE,
        };

        // Core transaction data
        $transactionData = [
            'company_id' => $company->id,
            'amount' => $firstOutput->tx_amount,
            'currency' => $firstOutput->tx_currency ?? 'EUR',
            'counterparty' => $firstOutput->corporate_counterparty,
            'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming, $firstOutput->corporate_counterparty),
            'counterparty_kyc_risk_level' => $riskLevel,
            'channel' => $this->determineTransactionType($firstOutput),
            'executed_at' => $this->parseDate($firstOutput->tx_date),
            'risk_score' => $riskScore,
            'status' => $status,
            'requires_review' => $this->shouldRequireReview($riskLevel),
            'flagged_reason' => $firstOutput->tx_purpose,
            // Store backend risk data
            'entity_risk_breakdown' => $firstOutput->risk_details_json,
            'counterparty_risk_breakdown' => null,
            'transaction_plausibility_breakdown' => null,
            'combined_risk_breakdown' => null,
        ];

        // Map all output_keys to their respective columns
        foreach ($outputs as $output) {
            $columnName = $output->output_key;

            // Skip if column doesn't exist in transactions table
            if (! $this->columnExists($columnName)) {
                continue;
            }

            // Decode JSON content, store NULL if empty or invalid
            $content = json_decode($output->content, true);
            $transactionData[$columnName] = $content ?: null;
        }

        return Transaction::updateOrCreate(
            ['reference' => $reference],
            $transactionData
        );
    }

    /**
     * Extract country code from data
     */
    private function extractCountry($firstOutput, $outputs): string
    {
        // Try to get from tx_country_incoming
        if (! empty($firstOutput->tx_country_incoming)) {
            $country = $this->extractCountryCode($firstOutput->tx_country_incoming);
            if ($country) {
                return $country;
            }
        }

        // Try to extract from corporate_HQ
        $hq = $this->extractField($outputs, 'corporate_HQ');
        if ($hq && is_array($hq) && isset($hq['answer'])) {
            // Simple pattern matching for country codes
            if (preg_match('/\b([A-Z]{2})\b/', $hq['answer'], $matches)) {
                return $matches[1];
            }
        }

        return 'DE'; // Default
    }

    /**
     * Extract country code (convert to ISO 2-letter if needed)
     * Also checks counterparty name for country indicators
     */
    private function extractCountryCode(?string $country, ?string $counterpartyName = null): ?string
    {
        // First, try to extract from counterparty name (highest priority for tax havens)
        if (! empty($counterpartyName)) {
            $extractedFromName = $this->extractCountryFromText($counterpartyName);
            if ($extractedFromName) {
                return $extractedFromName;
            }
        }

        if (empty($country)) {
            return null;
        }

        // If already 2 letters, check if it's a suspicious/wrong code
        if (strlen($country) === 2) {
            $suspiciousCodes = ['IO']; // British Indian Ocean Territory often wrong for Virgin Islands
            if (in_array(strtoupper($country), $suspiciousCodes)) {
                return null; // Reject suspicious codes
            }

            return strtoupper($country);
        }

        // Simple country name mapping (extend as needed)
        $countryMap = [
            'Germany' => 'DE',
            'Deutschland' => 'DE',
            'United States' => 'US',
            'USA' => 'US',
            'United Kingdom' => 'GB',
            'UK' => 'GB',
            'France' => 'FR',
            'Spain' => 'ES',
            'Italy' => 'IT',
            'Netherlands' => 'NL',
            'Belgium' => 'BE',
            'Austria' => 'AT',
            'Switzerland' => 'CH',
            'Poland' => 'PL',
            'Czech Republic' => 'CZ',
            'Denmark' => 'DK',
            'Sweden' => 'SE',
            'Norway' => 'NO',
            'Finland' => 'FI',
        ];

        return $countryMap[$country] ?? null;
    }

    /**
     * Extract country code from text (e.g., company name)
     */
    private function extractCountryFromText(string $text): ?string
    {
        $countryPatterns = [
            '/\b(?:British )?Virgin Islands\b/i' => 'VG',
            '/\bCayman Islands\b/i' => 'KY',
            '/\bBermuda\b/i' => 'BM',
            '/\bBahamas\b/i' => 'BS',
            '/\bPanama\b/i' => 'PA',
            '/\bLiechtenstein\b/i' => 'LI',
            '/\bMonaco\b/i' => 'MC',
            '/\bLuxembourg\b/i' => 'LU',
            '/\bSwitzerland\b/i' => 'CH',
            '/\bAzerbaijan\b/i' => 'AZ',
            '/\bRussia\b/i' => 'RU',
            '/\bChina\b/i' => 'CN',
            '/\bGermany\b/i' => 'DE',
        ];

        foreach ($countryPatterns as $pattern => $code) {
            if (preg_match($pattern, $text)) {
                return $code;
            }
        }

        return null;
    }

    /**
     * Map backend risk_score (-100 to 100) to risk_level
     * -100 to 20 = Critical
     * 20 to 80 = High
     * 80 to 100 = Low
     */
    private function mapBackendRiskScoreToLevel(int $backendRiskScore): string
    {
        if ($backendRiskScore <= 20) {
            return 'critical';
        }

        if ($backendRiskScore <= 80) {
            return 'high';
        }

        return 'low';
    }

    /**
     * Map backend risk_score (-100 to 100) to 0-255 scale
     */
    private function mapBackendRiskScoreTo255Scale(int $backendRiskScore): int
    {
        // Normalize -100 to 100 → 0 to 255
        // Formula: ((score + 100) / 200) * 255
        $normalized = (($backendRiskScore + 100) / 200) * 255;

        return (int) round(max(0, min(255, $normalized)));
    }

    /**
     * Determine if transaction requires review based on risk level
     */
    private function shouldRequireReview(string $riskLevel): bool
    {
        return match ($riskLevel) {
            'critical' => true,
            'high' => true,
            'low' => false,
            default => true,
        };
    }

    /**
     * Extract a field from outputs
     */
    private function extractField($outputs, string $outputKey, int $maxLength = 255): mixed
    {
        $output = $outputs->firstWhere('output_key', $outputKey);

        if (! $output) {
            return null;
        }

        $data = json_decode($output->content, true);

        // Extract 'answer' field if it exists
        if (is_array($data) && isset($data['answer'])) {
            $answer = $data['answer'];

            // If answer is an object/array, convert to string
            if (is_array($answer)) {
                $answer = json_encode($answer);
            }

            // Truncate to max length if needed
            return $this->truncateString((string) $answer, $maxLength);
        }

        // If data is an object/array, convert to JSON string
        if (is_array($data)) {
            return $this->truncateString(json_encode($data), $maxLength);
        }

        return $this->truncateString((string) $data, $maxLength);
    }

    /**
     * Extract a lean/concise field from outputs (extracts first sentence only)
     */
    private function extractLeanField($outputs, string $outputKey, int $maxLength = 255): mixed
    {
        $fullText = $this->extractField($outputs, $outputKey, 5000);

        if (! $fullText) {
            return null;
        }

        // Extract first sentence or up to first period
        $firstSentence = preg_split('/\.(?:\s|$)/', $fullText, 2)[0];

        if ($firstSentence) {
            $firstSentence = trim($firstSentence).'.';

            return $this->truncateString($firstSentence, $maxLength);
        }

        return $this->truncateString($fullText, $maxLength);
    }

    /**
     * Extract legal name from corporate name field
     */
    private function extractLegalName($outputs, string $fallbackName): ?string
    {
        $fullText = $this->extractField($outputs, 'entity_corporate_name', 5000);

        if (! $fullText) {
            return $fallbackName;
        }

        // Pattern 1: "is exactly as shown in an official register: [NAME]"
        if (preg_match('/is exactly as shown in an official register:\s*([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
            return trim($matches[1]);
        }

        // Pattern 2: "is [NAME]" - matches various patterns like "legal name is NAME", "company is NAME"
        if (preg_match('/(?:legal name|name|company) (?:of .+ )?is\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
            $name = trim($matches[1]);
            // Remove trailing explanatory text
            $name = preg_replace('/\s+(?:is exactly|exactly as shown|with former|former names).*$/i', '', $name);

            return trim($name);
        }

        // Pattern 3: "of [NAME]" - matches "legal name of Deutsche Bank AG"
        if (preg_match('/legal name of\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
            $name = trim($matches[1]);
            $name = preg_replace('/\s+(?:is exactly|with former).*$/i', '', $name);

            return trim($name);
        }

        // Pattern 4: Extract company name with suffix at start of sentence
        if (preg_match('/^(?:The current legal name (?:of|is) )?([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
            return trim($matches[1]);
        }

        // If no pattern matches, use the fallback name
        return $fallbackName;
    }

    /**
     * Truncate string to max length
     */
    private function truncateString(string $text, int $maxLength): string
    {
        if (strlen($text) <= $maxLength) {
            return $text;
        }

        return substr($text, 0, $maxLength - 3).'...';
    }

    /**
     * Parse date string to Carbon instance
     */
    private function parseDate(?string $date): ?Carbon
    {
        if (empty($date)) {
            return null;
        }

        try {
            return Carbon::parse($date);
        } catch (\Exception $e) {
            Log::warning('Failed to parse date', ['date' => $date]);

            return null;
        }
    }

    /**
     * Generate a meaningful transaction reference
     */
    private function generateTransactionReference($firstOutput): string
    {
        // Try to extract reference from raw_payload
        if (! empty($firstOutput->raw_payload)) {
            $payload = json_decode($firstOutput->raw_payload, true);

            // Look for common reference fields
            $referenceFields = [
                'Transaction Reference',
                'Reference',
                'Transaction ID',
                'Payment Reference',
                'Transfer Reference',
            ];

            foreach ($referenceFields as $field) {
                if (isset($payload[$field]) && ! empty($payload[$field])) {
                    return (string) $payload[$field];
                }
            }
        }

        // Generate a date-based reference if no reference found
        $date = $this->parseDate($firstOutput->tx_date);
        $dateStr = $date ? $date->format('Ymd') : date('Ymd');

        return sprintf('TXN-%s-%05d', $dateStr, $firstOutput->transaction_id);
    }

    /**
     * Determine transaction type from available data
     */
    private function determineTransactionType($firstOutput): string
    {
        // Try to determine from raw_payload first
        if (! empty($firstOutput->raw_payload)) {
            $payload = json_decode($firstOutput->raw_payload, true);

            // Check for explicit transaction type field
            if (isset($payload['Transaction Type']) && ! empty($payload['Transaction Type'])) {
                return $payload['Transaction Type'];
            }

            if (isset($payload['Payment Type']) && ! empty($payload['Payment Type'])) {
                return $payload['Payment Type'];
            }
        }

        // Determine based on amount and countries
        $amount = (float) $firstOutput->tx_amount;
        $isInternational = $firstOutput->tx_country_outgoing !== $firstOutput->tx_country_incoming;

        // Large international transactions are likely SWIFT
        if ($isInternational && $amount > 100000) {
            return 'SWIFT Transfer';
        }

        // European transfers
        if ($isInternational && $this->isEuropeanCountry($firstOutput->tx_country_incoming)) {
            return 'SEPA Transfer';
        }

        // Domestic high-value
        if (! $isInternational && $amount > 50000) {
            return 'Wire Transfer';
        }

        // Default
        return 'Bank Transfer';
    }

    /**
     * Check if country is in Europe
     */
    private function isEuropeanCountry(?string $countryCode): bool
    {
        if (empty($countryCode)) {
            return false;
        }

        $europeanCountries = [
            'AT', 'BE', 'BG', 'HR', 'CY', 'CZ', 'DK', 'EE', 'FI', 'FR',
            'DE', 'GR', 'HU', 'IE', 'IT', 'LV', 'LT', 'LU', 'MT', 'NL',
            'PL', 'PT', 'RO', 'SK', 'SI', 'ES', 'SE', 'CH', 'NO', 'IS',
        ];

        return in_array(strtoupper($countryCode), $europeanCountries);
    }

    /**
     * Check if column exists in transactions table
     */
    private function columnExists(string $columnName): bool
    {
        static $columns = null;

        if ($columns === null) {
            $columns = DB::getSchemaBuilder()->getColumnListing('transactions');
        }

        return in_array($columnName, $columns);
    }

    /**
     * Get statistics about the transformation
     */
    public function getStats(): array
    {
        return [
            'data_pool_transactions' => DB::table('backend_data_pool')
                ->select('transaction_id')
                ->distinct()
                ->count(),
            'companies_count' => Company::count(),
            'transactions_count' => Transaction::count(),
            'migrated_transactions' => Transaction::where('reference', 'like', 'MIGRATED-%')->count(),
        ];
    }
}