cbazza/AFC-Demo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removed KycRiskCalculator, added 3 new field for the database table data_pool_backend, added simple risk level and mapping funcion for these new risk factor

Browse Source
This commit is contained in:
cbazza
2025-11-19 12:23:27 +01:00
parent 976af0f2e6
commit 85e4fce229
10 changed files with 1722 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Jobs/SyncBackendDataPool.php
+6
View File
@@ -138,6 +138,9 @@ class SyncBackendDataPool implements ShouldQueue
'status' => $record->status,
'created_at' => $record->created_at,
'last_modified_at' => $record->last_modified_at,
'risk_score' => $record->risk_score,
'risk_level' => $record->risk_level,
'risk_details_json' => $record->risk_details_json,
'prompt_id' => $record->prompt_id,
'output_key' => $record->output_key,
'content' => $record->content,
@@ -191,6 +194,9 @@ class SyncBackendDataPool implements ShouldQueue
't.status',
't.created_at',
't.last_modified_at',
't.risk_score',
't.risk_level',
't.risk_details_json',
'tout.prompt_id',
'tout.output_key',
'tout.content',
app/Jobs/TransformDataPoolToProduction.php
+279 -38
View File
@@ -6,7 +6,6 @@ namespace App\Jobs;
use App\Models\Company;
use App\Models\Transaction;
use App\Services\KycRiskCalculator;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
@@ -32,13 +31,12 @@ class TransformDataPoolToProduction implements ShouldQueue
*/
public function __construct(
public ?int $batchSize = 100,
) {
}
) {}
/**
* Execute the job.
*/
public function handle(KycRiskCalculator $kycCalculator): void
public function handle(): void
{
$startTime = now();
@@ -47,8 +45,8 @@ class TransformDataPoolToProduction implements ShouldQueue
]);
try {
DB::transaction(function () use ($kycCalculator) {
$this->transformData($kycCalculator);
DB::transaction(function () {
$this->transformData();
});
$duration = now()->diffInSeconds($startTime);
@@ -69,7 +67,7 @@ class TransformDataPoolToProduction implements ShouldQueue
/**
* Transform data from backend_data_pool to companies and transactions
*/
private function transformData(KycRiskCalculator $kycCalculator): void
private function transformData(): void
{
// Get all unique transaction IDs from data pool
$transactionIds = DB::table('backend_data_pool')
@@ -97,7 +95,7 @@ class TransformDataPoolToProduction implements ShouldQueue
$firstOutput = $outputs->first();
// 1. Create or update Company
$company = $this->createOrUpdateCompany($firstOutput, $outputs, $kycCalculator);
$company = $this->createOrUpdateCompany($firstOutput, $outputs);
if ($company->wasRecentlyCreated) {
$processedCompanies++;
}
@@ -118,21 +116,23 @@ class TransformDataPoolToProduction implements ShouldQueue
/**
* Create or update a Company from data pool outputs
*/
private function createOrUpdateCompany($firstOutput, $outputs, KycRiskCalculator $kycCalculator): Company
private function createOrUpdateCompany($firstOutput, $outputs): Company
{
$corporateEntity = $firstOutput->corporate_entity;
// Calculate KYC Risk Level
$kycRiskLevel = $kycCalculator->calculateKycRiskLevel(collect([$outputs])->first());
// Use backend risk_score and map to risk_level
$backendRiskScore = $firstOutput->risk_score ?? 50;
$kycRiskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore);
// Extract company data from outputs
$companyData = [
'country' => $this->extractCountry($firstOutput, $outputs),
'kyc_risk_level' => $kycRiskLevel,
'sector' => $this->extractField($outputs, 'corporate_sector', 255),
'headquarters' => $this->extractField($outputs, 'corporate_HQ', 255),
'summary' => $this->extractField($outputs, 'corporate_summary', 5000), // text field, longer OK
'legal_name' => $this->extractField($outputs, 'corporate_name', 255),
'sector' => $this->extractLeanField($outputs, 'entity_corporate_sector', 255),
'headquarters' => $this->extractLeanField($outputs, 'entity_corporate_HQ', 255),
'summary' => $this->extractField($outputs, 'entity_corporate_summary', 5000), // text field, longer OK
'legal_name' => $this->extractLegalName($outputs, $corporateEntity),
'ticker' => null, // Ticker is not available in backend_data_pool
];
return Company::updateOrCreate(
@@ -146,14 +146,27 @@ class TransformDataPoolToProduction implements ShouldQueue
*/
private function createOrUpdateTransaction(Company $company, $firstOutput, $outputs): Transaction
{
$reference = 'MIGRATED-' . $firstOutput->transaction_id;
// Generate better reference number
$reference = $this->generateTransactionReference($firstOutput);
// Map KYC Risk Level to Transaction Status
$status = match ($company->kyc_risk_level) {
// Use risk_score from backend (leading field)
$backendRiskScore = $firstOutput->risk_score ?? 50; // Default to 50 (high risk) if not set
// Map backend risk_score to risk_level
// -100 to 20 = Critical
// 20 to 80 = High
// 80 to 100 = Low
$riskLevel = $this->mapBackendRiskScoreToLevel($backendRiskScore);
// Convert to 0-255 scale for frontend transactions table
$riskScore = $this->mapBackendRiskScoreTo255Scale($backendRiskScore);
// Map risk level to Transaction Status
$status = match ($riskLevel) {
'critical' => Transaction::STATUS_TRUE_POSITIVE,
'high' => Transaction::STATUS_FALSE_POSITIVE,
'low' => Transaction::STATUS_CLEARED,
default => Transaction::STATUS_FALSE_POSITIVE, // Default to high risk
default => Transaction::STATUS_FALSE_POSITIVE,
};
// Core transaction data
@@ -162,13 +175,19 @@ class TransformDataPoolToProduction implements ShouldQueue
'amount' => $firstOutput->tx_amount,
'currency' => $firstOutput->tx_currency ?? 'EUR',
'counterparty' => $firstOutput->corporate_counterparty,
'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming),
'channel' => $firstOutput->source_file,
'counterparty_country' => $this->extractCountryCode($firstOutput->tx_country_incoming, $firstOutput->corporate_counterparty),
'counterparty_kyc_risk_level' => $riskLevel,
'channel' => $this->determineTransactionType($firstOutput),
'executed_at' => $this->parseDate($firstOutput->tx_date),
'risk_score' => $this->extractRiskScore($outputs),
'risk_score' => $riskScore,
'status' => $status,
'requires_review' => true,
'requires_review' => $this->shouldRequireReview($riskLevel),
'flagged_reason' => $firstOutput->tx_purpose,
// Store backend risk data
'entity_risk_breakdown' => $firstOutput->risk_details_json,
'counterparty_risk_breakdown' => null,
'transaction_plausibility_breakdown' => null,
'combined_risk_breakdown' => null,
];
// Map all output_keys to their respective columns
@@ -218,15 +237,29 @@ class TransformDataPoolToProduction implements ShouldQueue
/**
* Extract country code (convert to ISO 2-letter if needed)
* Also checks counterparty name for country indicators
*/
private function extractCountryCode(?string $country): ?string
private function extractCountryCode(?string $country, ?string $counterpartyName = null): ?string
{
// First, try to extract from counterparty name (highest priority for tax havens)
if (! empty($counterpartyName)) {
$extractedFromName = $this->extractCountryFromText($counterpartyName);
if ($extractedFromName) {
return $extractedFromName;
}
}
if (empty($country)) {
return null;
}
// If already 2 letters, return uppercase
// If already 2 letters, check if it's a suspicious/wrong code
if (strlen($country) === 2) {
$suspiciousCodes = ['IO']; // British Indian Ocean Territory often wrong for Virgin Islands
if (in_array(strtoupper($country), $suspiciousCodes)) {
return null; // Reject suspicious codes
}
return strtoupper($country);
}
@@ -257,26 +290,77 @@ class TransformDataPoolToProduction implements ShouldQueue
}
/**
* Extract risk score from tranx_score output
* Extract country code from text (e.g., company name)
*/
private function extractRiskScore($outputs): int
private function extractCountryFromText(string $text): ?string
{
$scoreOutput = $outputs->firstWhere('output_key', 'tranx_score');
$countryPatterns = [
'/\b(?:British )?Virgin Islands\b/i' => 'VG',
'/\bCayman Islands\b/i' => 'KY',
'/\bBermuda\b/i' => 'BM',
'/\bBahamas\b/i' => 'BS',
'/\bPanama\b/i' => 'PA',
'/\bLiechtenstein\b/i' => 'LI',
'/\bMonaco\b/i' => 'MC',
'/\bLuxembourg\b/i' => 'LU',
'/\bSwitzerland\b/i' => 'CH',
'/\bAzerbaijan\b/i' => 'AZ',
'/\bRussia\b/i' => 'RU',
'/\bChina\b/i' => 'CN',
'/\bGermany\b/i' => 'DE',
];
if (! $scoreOutput) {
return 128; // Default: Medium risk (255/2)
foreach ($countryPatterns as $pattern => $code) {
if (preg_match($pattern, $text)) {
return $code;
}
}
$scoreData = json_decode($scoreOutput->content, true);
return null;
}
if (is_array($scoreData) && isset($scoreData['score'])) {
// Assume score is 0-100, convert to 0-255
$score = (int) $scoreData['score'];
return (int) round(($score / 100) * 255);
/**
* Map backend risk_score (-100 to 100) to risk_level
* -100 to 20 = Critical
* 20 to 80 = High
* 80 to 100 = Low
*/
private function mapBackendRiskScoreToLevel(int $backendRiskScore): string
{
if ($backendRiskScore <= 20) {
return 'critical';
}
return 128;
if ($backendRiskScore <= 80) {
return 'high';
}
return 'low';
}
/**
* Map backend risk_score (-100 to 100) to 0-255 scale
*/
private function mapBackendRiskScoreTo255Scale(int $backendRiskScore): int
{
// Normalize -100 to 100 → 0 to 255
// Formula: ((score + 100) / 200) * 255
$normalized = (($backendRiskScore + 100) / 200) * 255;
return (int) round(max(0, min(255, $normalized)));
}
/**
* Determine if transaction requires review based on risk level
*/
private function shouldRequireReview(string $riskLevel): bool
{
return match ($riskLevel) {
'critical' => true,
'high' => true,
'low' => false,
default => true,
};
}
/**
@@ -313,6 +397,71 @@ class TransformDataPoolToProduction implements ShouldQueue
return $this->truncateString((string) $data, $maxLength);
}
/**
* Extract a lean/concise field from outputs (extracts first sentence only)
*/
private function extractLeanField($outputs, string $outputKey, int $maxLength = 255): mixed
{
$fullText = $this->extractField($outputs, $outputKey, 5000);
if (! $fullText) {
return null;
}
// Extract first sentence or up to first period
$firstSentence = preg_split('/\.(?:\s|$)/', $fullText, 2)[0];
if ($firstSentence) {
$firstSentence = trim($firstSentence).'.';
return $this->truncateString($firstSentence, $maxLength);
}
return $this->truncateString($fullText, $maxLength);
}
/**
* Extract legal name from corporate name field
*/
private function extractLegalName($outputs, string $fallbackName): ?string
{
$fullText = $this->extractField($outputs, 'entity_corporate_name', 5000);
if (! $fullText) {
return $fallbackName;
}
// Pattern 1: "is exactly as shown in an official register: [NAME]"
if (preg_match('/is exactly as shown in an official register:\s*([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
return trim($matches[1]);
}
// Pattern 2: "is [NAME]" - matches various patterns like "legal name is NAME", "company is NAME"
if (preg_match('/(?:legal name|name|company) (?:of .+ )?is\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
$name = trim($matches[1]);
// Remove trailing explanatory text
$name = preg_replace('/\s+(?:is exactly|exactly as shown|with former|former names).*$/i', '', $name);
return trim($name);
}
// Pattern 3: "of [NAME]" - matches "legal name of Deutsche Bank AG"
if (preg_match('/legal name of\s+([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
$name = trim($matches[1]);
$name = preg_replace('/\s+(?:is exactly|with former).*$/i', '', $name);
return trim($name);
}
// Pattern 4: Extract company name with suffix at start of sentence
if (preg_match('/^(?:The current legal name (?:of|is) )?([A-Z][^.,]+(?:AG|SE|GmbH|Inc\.|Ltd\.|Corp\.|N\.V\.|S\.A\.|Aktiengesellschaft))/i', $fullText, $matches)) {
return trim($matches[1]);
}
// If no pattern matches, use the fallback name
return $fallbackName;
}
/**
* Truncate string to max length
*/
@@ -322,7 +471,7 @@ class TransformDataPoolToProduction implements ShouldQueue
return $text;
}
return substr($text, 0, $maxLength - 3) . '...';
return substr($text, 0, $maxLength - 3).'...';
}
/**
@@ -343,6 +492,98 @@ class TransformDataPoolToProduction implements ShouldQueue
}
}
/**
* Generate a meaningful transaction reference
*/
private function generateTransactionReference($firstOutput): string
{
// Try to extract reference from raw_payload
if (! empty($firstOutput->raw_payload)) {
$payload = json_decode($firstOutput->raw_payload, true);
// Look for common reference fields
$referenceFields = [
'Transaction Reference',
'Reference',
'Transaction ID',
'Payment Reference',
'Transfer Reference',
];
foreach ($referenceFields as $field) {
if (isset($payload[$field]) && ! empty($payload[$field])) {
return (string) $payload[$field];
}
}
}
// Generate a date-based reference if no reference found
$date = $this->parseDate($firstOutput->tx_date);
$dateStr = $date ? $date->format('Ymd') : date('Ymd');
return sprintf('TXN-%s-%05d', $dateStr, $firstOutput->transaction_id);
}
/**
* Determine transaction type from available data
*/
private function determineTransactionType($firstOutput): string
{
// Try to determine from raw_payload first
if (! empty($firstOutput->raw_payload)) {
$payload = json_decode($firstOutput->raw_payload, true);
// Check for explicit transaction type field
if (isset($payload['Transaction Type']) && ! empty($payload['Transaction Type'])) {
return $payload['Transaction Type'];
}
if (isset($payload['Payment Type']) && ! empty($payload['Payment Type'])) {
return $payload['Payment Type'];
}
}
// Determine based on amount and countries
$amount = (float) $firstOutput->tx_amount;
$isInternational = $firstOutput->tx_country_outgoing !== $firstOutput->tx_country_incoming;
// Large international transactions are likely SWIFT
if ($isInternational && $amount > 100000) {
return 'SWIFT Transfer';
}
// European transfers
if ($isInternational && $this->isEuropeanCountry($firstOutput->tx_country_incoming)) {
return 'SEPA Transfer';
}
// Domestic high-value
if (! $isInternational && $amount > 50000) {
return 'Wire Transfer';
}
// Default
return 'Bank Transfer';
}
/**
* Check if country is in Europe
*/
private function isEuropeanCountry(?string $countryCode): bool
{
if (empty($countryCode)) {
return false;
}
$europeanCountries = [
'AT', 'BE', 'BG', 'HR', 'CY', 'CZ', 'DK', 'EE', 'FI', 'FR',
'DE', 'GR', 'HU', 'IE', 'IT', 'LV', 'LT', 'LU', 'MT', 'NL',
'PL', 'PT', 'RO', 'SK', 'SI', 'ES', 'SE', 'CH', 'NO', 'IS',
];
return in_array(strtoupper($countryCode), $europeanCountries);
}
/**
* Check if column exists in transactions table
*/
app/Services/KycRiskCalculator.php
+851 -46
View File
File diff suppressed because it is too large Load Diff