From 38c1ae4daf92c7d87c827a96f222f5ed1723866a Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 16 Jul 2026 08:20:58 -0400 Subject: [PATCH] feat(19-01): implement classifyCampaign orchestrator + evidence gathering (GREEN) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - gatherCampaignEvidence: bulk-fetches reports (earliest-first, joined to contacts for requester email) -> messages (report_id = ANY) -> indicators (message_id = ANY), parses messages.headers JSONB into bounded ParsedMessage fields, and runs one getBlastRadius() lookup keyed off the earliest report's sender/subject/±24h window (research A6); synthesizes unavailable/not_configured with no Mimecast call when no report is linked - evaluateThreatTier (D-03): blastRadius.status==='ok' AND (delivered>0 OR clicked>0) AND (hasHardAuthFail via effectiveAuthResults OR hasKnownBadIndicatorMatch — same attachment_hash/url value spanning >=2 distinct messages, cross-report correlation only, no external reputation lookup per research A4) - evaluateSpamVsUnwanted (D-04): UNWANTED when any attachment/url indicator matches or delivery is contained to the reporter(s) only; SPAM otherwise - classifyCampaign: D-06 simulation short-circuit -> D-03 -> D-04 -> computeConfidence -> mapVerdictToActions -> computeRequiresApproval -> append-only INSERT into classifications (D-02, no ON CONFLICT), wrapped in try/catch logging [CAMPAIGN-CLASSIFIER] + err.message and rethrowing - isKnownSimulationSender relaxed to a narrower SenderIdentity shape so both the full NormalizedMessage fixtures and the bounded ParsedMessage type can share it - All 39 tests green; tsc clean; full `npm test` suite green except 2 pre-existing, unrelated itglue-search.test.ts failures (see deferred-items.md) --- lib/services/campaign-classifier.ts | 351 +++++++++++++++++++++++++++- 1 file changed, 349 insertions(+), 2 deletions(-) diff --git a/lib/services/campaign-classifier.ts b/lib/services/campaign-classifier.ts index 472aab3..48605c4 100644 --- a/lib/services/campaign-classifier.ts +++ b/lib/services/campaign-classifier.ts @@ -14,7 +14,9 @@ * functions those steps compose (Task 1). */ -import type { NormalizedMessage, AuthResults } from './eml-parser'; +import type { AuthResults } from './eml-parser'; +import { postgresClient } from './postgres-client'; +import { getBlastRadius, type BlastRadiusResult } from './mimecast-blast-radius'; // ============================================================================= // D-06/D-07: KnowBe4 / Breach Secure Now simulation sender-domain allowlist @@ -53,12 +55,24 @@ export function domainMatchesAllowlist(domain: string): boolean { ); } +/** + * Narrow shape shared by both the full Phase 16 `NormalizedMessage` (used + * directly by unit tests/fixtures) and this module's own bounded + * `ParsedMessage` (built from `messages.headers` JSONB in + * `gatherCampaignEvidence`) — only the sender-identity fields the allowlist + * check needs. + */ +export interface SenderIdentity { + from: { domain: string | null }; + returnPath: string | null; +} + /** * Checks BOTH the From domain and the Return-Path domain (Pitfall 3 — From * may lack a visible email address on some real report tickets, e.g. the * Breach Secure Now fixture). */ -export function isKnownSimulationSender(message: NormalizedMessage): boolean { +export function isKnownSimulationSender(message: SenderIdentity): boolean { const fromDomain = message.from.domain; const returnPathDomain = message.returnPath?.split('@')[1] ?? null; return [fromDomain, returnPathDomain] @@ -175,3 +189,336 @@ export function mapVerdictToActions(verdict: Verdict, evidence: ActionEvidence): export function computeRequiresApproval(actions: string[]): boolean { return actions.some((action) => DESTRUCTIVE_ACTIONS.has(action)); } + +// ============================================================================= +// Evidence gathering — gatherCampaignEvidence (CLASSIFY-06 / T-19-03 bounded) +// ============================================================================= + +/** Cap on sample arrays exposed for human-readable evidence — never the full set (T-19-03). */ +const MAX_SAMPLE_SIZE = 10; + +interface ReportDbRow { + id: string; + title: string | null; + created_at: string; + requester_email: string | null; +} + +interface MessageDbRow { + id: string; + report_id: string; + headers: Record | null; +} + +interface IndicatorDbRow { + id: string; + message_id: string; + indicator_type: string; + value: string; +} + +/** + * Bounded, structured per-message fields needed by the rule engine — parsed + * out of `messages.headers` JSONB (the Phase 16 `NormalizedMessage` object). + * Deliberately narrower than the full `NormalizedMessage` shape: never + * carries `bodyPreview`/raw content into the classifier or its output + * (CLASSIFY-06). + */ +export interface ParsedMessage extends SenderIdentity { + id: string; + reportId: string; + from: { domain: string | null; email: string | null }; + authResults: AuthResults; + authResultsOriginal: AuthResults | null; + subject: string | null; +} + +export interface CampaignIndicator { + id: string; + messageId: string; + indicatorType: string; + value: string; +} + +export interface CampaignReportSummary { + id: string; + title: string | null; + createdAt: string; + requesterEmail: string | null; +} + +export interface CampaignEvidence { + campaignId: string; + reportCount: number; + messageCount: number; + indicatorCount: number; + /** Capped at MAX_SAMPLE_SIZE — for human-readable evidence only (T-19-03). */ + reportSample: CampaignReportSummary[]; + messages: ParsedMessage[]; + indicators: CampaignIndicator[]; + blastRadius: BlastRadiusResult; +} + +/** + * Gathers a bounded evidence payload for one campaign: linked reports + * (earliest-first), their parsed messages, cross-message indicators, and a + * single `getBlastRadius()` lookup keyed off the EARLIEST report (research + * A6 — "the original" convention, mirroring campaign-grouping-service.ts's + * own `ORDER BY r.created_at ASC`). When the campaign has no linked reports, + * synthesizes `unavailable`/`not_configured` without calling Mimecast. + */ +export async function gatherCampaignEvidence(campaignId: string): Promise { + // Bulk-fetch linked reports (+ join contacts for requester email — mirrors + // app/api/phishing/campaigns/[id]/route.ts's existing bulk-fetch shape). + const reportsRes = await postgresClient.query( + `SELECT r.id::text AS id, r.title, r.created_at::text AS created_at, + c.email_address AS requester_email + FROM reports r + LEFT JOIN contacts c ON c.id = r.requester_contact_id + WHERE r.campaign_id = $1 + ORDER BY r.created_at ASC`, + [campaignId] + ); + const reports: CampaignReportSummary[] = reportsRes.rows.map((r) => ({ + id: r.id, + title: r.title, + createdAt: r.created_at, + requesterEmail: r.requester_email, + })); + const reportIds = reports.map((r) => r.id); + + const messagesRes = reportIds.length + ? await postgresClient.query( + `SELECT id::text AS id, report_id::text AS report_id, headers + FROM messages WHERE report_id = ANY($1::uuid[])`, + [reportIds] + ) + : { rows: [] as MessageDbRow[] }; + const messages: ParsedMessage[] = messagesRes.rows.map((m) => { + const headers = (m.headers ?? {}) as { + from?: { domain?: string | null; email?: string | null }; + returnPath?: string | null; + authResults?: AuthResults; + authResultsOriginal?: AuthResults | null; + subject?: string | null; + }; + return { + id: m.id, + reportId: m.report_id, + from: { domain: headers.from?.domain ?? null, email: headers.from?.email ?? null }, + returnPath: headers.returnPath ?? null, + authResults: headers.authResults ?? {}, + authResultsOriginal: headers.authResultsOriginal ?? null, + subject: headers.subject ?? null, + }; + }); + const messageIds = messages.map((m) => m.id); + + const indicatorsRes = messageIds.length + ? await postgresClient.query( + `SELECT id::text AS id, message_id::text AS message_id, indicator_type, value + FROM indicators WHERE message_id = ANY($1::uuid[])`, + [messageIds] + ) + : { rows: [] as IndicatorDbRow[] }; + const indicators: CampaignIndicator[] = indicatorsRes.rows.map((i) => ({ + id: i.id, + messageId: i.message_id, + indicatorType: i.indicator_type, + value: i.value, + })); + + const primaryReport = reports[0] ?? null; + let blastRadius: BlastRadiusResult; + if (primaryReport) { + const primaryMessage = messages.find((m) => m.reportId === primaryReport.id) ?? null; + const senderIndicator = indicators.find( + (i) => i.messageId === primaryMessage?.id && i.indicatorType === 'sender' + ); + const createdAt = new Date(primaryReport.createdAt); + blastRadius = await getBlastRadius({ + sender: senderIndicator?.value ?? primaryMessage?.from.email ?? '', + recipient: primaryReport.requesterEmail ?? '', + subject: primaryMessage?.subject ?? primaryReport.title ?? '', + dateWindow: { + start: new Date(createdAt.getTime() - 24 * 60 * 60 * 1000), + end: new Date(createdAt.getTime() + 24 * 60 * 60 * 1000), + }, + }); + } else { + // No report ever linked to this campaign — nothing to look up (research A6). + blastRadius = { status: 'unavailable', reason: 'not_configured' }; + } + + return { + campaignId, + reportCount: reports.length, + messageCount: messages.length, + indicatorCount: indicators.length, + reportSample: reports.slice(0, MAX_SAMPLE_SIZE), + messages, + indicators, + blastRadius, + }; +} + +// ============================================================================= +// D-03/D-04: Verdict tier evaluation +// ============================================================================= + +/** + * Known-bad indicator match (D-03 second signal, research A4): cross-report + * correlation only — same attachment_hash/url value spanning >=2 distinct + * messages in the campaign. No external reputation lookup. + */ +function hasKnownBadIndicatorMatch(indicators: CampaignIndicator[]): boolean { + const messagesByValue = new Map>(); + for (const indicator of indicators) { + if (indicator.indicatorType !== 'attachment_hash' && indicator.indicatorType !== 'url') continue; + const key = `${indicator.indicatorType}:${indicator.value}`; + const set = messagesByValue.get(key) ?? new Set(); + set.add(indicator.messageId); + messagesByValue.set(key, set); + } + return Array.from(messagesByValue.values()).some((set) => set.size >= 2); +} + +/** + * D-03: THREAT requires BOTH (a) evidence the message reached someone + * (blast-radius delivered>0 or clicked>0) AND (b) a malicious signal — a + * hard SPF/DKIM/DMARC fail (via effectiveAuthResults' authResultsOriginal + * precedence) OR a known-bad indicator match. Either signal alone is not + * enough — a contained blast radius isn't a realized threat yet. + */ +function evaluateThreatTier(evidence: CampaignEvidence): boolean { + if (evidence.blastRadius.status !== 'ok') return false; + const { delivered, clicked } = evidence.blastRadius; + if (delivered <= 0 && clicked <= 0) return false; + + const hasAuthFail = evidence.messages.some((message) => hasHardAuthFail(effectiveAuthResults(message))); + const hasIndicatorMatch = hasKnownBadIndicatorMatch(evidence.indicators); + return hasAuthFail || hasIndicatorMatch; +} + +/** + * D-04: SPAM = no suspicious signal at all. UNWANTED = a suspicious signal + * present (an attachment/url indicator match, even a single one, or + * delivery contained to the reporter(s) only) but below the THREAT bar. + */ +function evaluateSpamVsUnwanted(evidence: CampaignEvidence): 'SPAM' | 'UNWANTED' { + const hasAnyIndicator = evidence.indicators.some( + (indicator) => indicator.indicatorType === 'attachment_hash' || indicator.indicatorType === 'url' + ); + + const requesterEmails = evidence.reportSample + .map((report) => report.requesterEmail) + .filter((email): email is string => email !== null); + + const deliveryContainedToReporter = + evidence.blastRadius.status === 'ok' && + evidence.blastRadius.delivered > 0 && + evidence.blastRadius.perRecipient + .filter((recipient) => recipient.status === 'delivered') + .every((recipient) => requesterEmails.includes(recipient.recipient)); + + return hasAnyIndicator || deliveryContainedToReporter ? 'UNWANTED' : 'SPAM'; +} + +// ============================================================================= +// classifyCampaign — orchestrator (CLASSIFY-01, D-02 append-only INSERT) +// ============================================================================= + +export interface ClassifyResult { + id: string; + campaignId: string; + verdict: Verdict; + confidence: number; + summary: string; + reasons: string[]; + recommendedActions: string[]; + requiresApproval: boolean; + createdAt: string; +} + +/** + * Single exported orchestrator: gather evidence -> D-06 simulation + * short-circuit -> D-03 THREAT tier -> D-04 SPAM/UNWANTED split -> D-05 + * confidence -> D-08 recommended actions -> append-only INSERT into + * `classifications` (D-02, no ON CONFLICT — history is never overwritten). + */ +export async function classifyCampaign(campaignId: string): Promise { + try { + const evidence = await gatherCampaignEvidence(campaignId); + + // D-06: allowlist match short-circuits before THREAT tier evaluation, + // regardless of other signals — D-03/D-04 still decide which of SPAM/ + // UNWANTED applies. + const isSimulation = evidence.messages.some((message) => isKnownSimulationSender(message)); + + let verdict: Verdict; + const reasons: string[] = []; + if (isSimulation) { + verdict = evaluateSpamVsUnwanted(evidence); + reasons.push( + 'Sender domain matches a known phishing-simulation vendor allowlist (KnowBe4/Breach Secure Now) — THREAT tier skipped' + ); + } else if (evaluateThreatTier(evidence)) { + verdict = 'THREAT'; + } else { + verdict = evaluateSpamVsUnwanted(evidence); + } + + const clicked = evidence.blastRadius.status === 'ok' ? evidence.blastRadius.clicked : 0; + const recommendedActions = mapVerdictToActions(verdict, { clicked }); + const requiresApproval = computeRequiresApproval(recommendedActions); + + const { confidence, reasons: confidenceReasons } = computeConfidence({ + hasAnyMessage: evidence.messageCount > 0, + blastRadiusStatus: evidence.blastRadius.status === 'ok' ? 'ok' : 'unavailable', + hasAttachmentOrUrlIndicators: evidence.indicators.some( + (indicator) => indicator.indicatorType === 'attachment_hash' || indicator.indicatorType === 'url' + ), + }); + reasons.push(...confidenceReasons); + reasons.push( + `Verdict ${verdict} determined from ${evidence.reportCount} linked report(s) and ${evidence.messageCount} parsed message(s)` + ); + + const summary = `${verdict} (confidence ${confidence}): ${reasons[0]}`; + + // D-02: append-only INSERT — no ON CONFLICT. Each classify call is a new + // history row; "current" verdict is the most recent by created_at. + const insertResult = await postgresClient.query<{ id: string; created_at: string }>( + `INSERT INTO classifications ( + campaign_id, verdict, confidence, summary, reasons, recommended_actions, requires_approval + ) VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7) + RETURNING id::text AS id, created_at::text AS created_at`, + [ + campaignId, + verdict, + confidence, + summary, + JSON.stringify(reasons), + JSON.stringify(recommendedActions), + requiresApproval, + ] + ); + const row = insertResult.rows[0]; + + return { + id: row.id, + campaignId, + verdict, + confidence, + summary, + reasons, + recommendedActions, + requiresApproval, + createdAt: row.created_at, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error('[CAMPAIGN-CLASSIFIER] Failed to classify campaign', campaignId, message); + throw error; + } +}