diff --git a/.planning/phases/18-campaign-grouping-phishing-analysis-api/18-01-SUMMARY.md b/.planning/phases/18-campaign-grouping-phishing-analysis-api/18-01-SUMMARY.md new file mode 100644 index 0000000..54cd481 --- /dev/null +++ b/.planning/phases/18-campaign-grouping-phishing-analysis-api/18-01-SUMMARY.md @@ -0,0 +1,112 @@ +--- +phase: 18-campaign-grouping-phishing-analysis-api +plan: 01 +subsystem: api +tags: [postgres, phishing, campaign-grouping, permissions, better-auth, vitest] + +# Dependency graph +requires: + - phase: 15-data-model-detection-ticket-evidence + provides: "campaigns/reports/messages/indicators schema (migrations 097/099), phishing-detector.ts's DetectableTicket shape and orchestration style" + - phase: 16-eml-mime-evidence-parser + provides: "parseAndStoreMessage() as the (currently uncalled) writer of messages/indicators rows that Tier 1/2 depend on" +provides: + - "lib/services/campaign-grouping-service.ts — groupReportIntoCampaign(reportId, opts), normalizeSubject, extractUrlDomain, GroupReportResult" + - "phishing permission resource in lib/permissions.ts (full D-05 vocabulary + partial role grants)" +affects: [18-02, 18-03, 19-classification, 20-remediation] + +# Tech tracking +tech-stack: + added: [] + patterns: + - "Shared grouping core called from 3 sites (webhook/cron/analyze), mirrors phishing-detector.ts's shared-detector architecture" + - "Tiered find-or-create inside postgresClient.transaction() — pure-JS key computation + targeted parameterized queries per tier, not one mega-WHERE clause" + - "SQL-content-routed mock client in tests (each query's distinguishing SQL substring maps to staged rows) instead of strict call-order mocking" + +key-files: + created: + - lib/services/campaign-grouping-service.ts + - lib/services/campaign-grouping-service.test.ts + modified: + - lib/permissions.ts + +key-decisions: + - "Every tier query includes a self-exclusion clause (r.id != ) so re-running groupReportIntoCampaign on an already-grouped report can never match its own messages/indicators row against itself and double-increment its own campaign" + - "New campaign creation picks the strongest available tier key (Tier 1 message_id > Tier 2 attachment_or_url > Tier 3 sender_subject_client) so future duplicates of that report have the best chance of matching it" + - "campaigns.status is never set/transitioned by this plan — left at the migration default 'open' per D-04/scope boundary" + - "phishing resource declares the full action vocabulary (read/analyze/approve/remediate) now but only grants read+analyze (admin/super-admin) and read (user) this phase, per D-05" + +patterns-established: + - "Tier queries compute fuzzy keys (normalizeSubject, extractUrlDomain) in JS and query Postgres with targeted parameterized $n placeholders — never encode fuzzy matching in SQL WHERE clauses" + +requirements-completed: [CAMP-01, CAMP-02, ACCESS-01] + +# Metrics +duration: 15min +completed: 2026-07-15 +--- + +# Phase 18 Plan 01: Campaign Grouping Core + Phishing Permission Resource Summary + +**`groupReportIntoCampaign()` — tiered Message-ID → attachment-hash/URL-domain → sender+subject+client campaign matching inside a single Postgres transaction, plus the `phishing` permission resource all future `/api/phishing/*` routes will gate on.** + +## Performance + +- **Duration:** ~15 min +- **Started:** 2026-07-15T19:19Z (approx, first commit) +- **Completed:** 2026-07-15T19:23Z (last task commit) +- **Tasks:** 3/3 completed +- **Files modified:** 3 (2 created, 1 modified) + +## Accomplishments +- `lib/services/campaign-grouping-service.ts` — `groupReportIntoCampaign(reportId, opts)` implements the full CAMP-01 tiered match (Message-ID → attachment-hash/URL-domain+subject+sender+24h → sender+normalized-subject+client+24h) and CAMP-02 accumulation (report_count/last_seen_at bump on match, new campaign on no-match), wrapped in `postgresClient.transaction()` to guard against `campaigns.campaign_key`'s missing UNIQUE constraint. +- `lib/services/campaign-grouping-service.test.ts` — 16 passing tests covering pure helpers (`normalizeSubject`, `extractUrlDomain`) and mocked-DB tiered-matching/find-or-create/self-exclusion/`skipIfAlreadyGrouped` behavior. +- `lib/permissions.ts` — `phishing` resource added with the full D-05 action vocabulary; `read`+`analyze` granted to `super-admin`/`admin`, `read` only to `user`; `approve`/`remediate` declared but ungranted to any role (Phase 20's job). + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Pure tier-key helpers + test file scaffold** - `ea677b7` (test) +2. **Task 2: groupReportIntoCampaign — tiered matching + transactional find-or-create** - `da926bb` (feat) +3. **Task 3: Add phishing permission resource + role grants** - `04ec51f` (feat) + +**Plan metadata:** committed alongside this SUMMARY (see final commit in this plan's history) + +## TDD Gate Compliance + +Plan frontmatter is `type: tdd`. Gate sequence verified in git log: `ea677b7` is a `test(...)` commit (RED gate) followed by `da926bb`/`04ec51f`, both `feat(...)` commits (GREEN gate) — sequence satisfied, no warning needed. No `refactor(...)` commit was needed (no cleanup pass required after GREEN). + +## Files Created/Modified +- `lib/services/campaign-grouping-service.ts` - `groupReportIntoCampaign`, `normalizeSubject`, `extractUrlDomain`, `GroupReportResult`; D-07 doc comment stating the Tier-3-only automatic-path limitation +- `lib/services/campaign-grouping-service.test.ts` - 16 tests: 5 `normalizeSubject`, 3 `extractUrlDomain`, 8 `groupReportIntoCampaign` (Tier 1/2/3 matches, no-match create, `skipIfAlreadyGrouped` short-circuit + pass-through, self-exclusion/no-double-increment) +- `lib/permissions.ts` - `phishing` resource in `statement` (4 actions) + role grants in `superAdminRole`/`adminRole`/`userRole` + +## Decisions Made +- Self-exclusion clause (`r.id != `) added to every one of the 3 tier queries — this was a plan-checker finding baked into the plan's action text, not a deviation, but worth restating: without it, re-running grouping on an already-linked report could match the report's own `messages`/`indicators` row against itself and double-increment its own campaign's `report_count`. +- New-campaign `campaign_key`/`group_method` is chosen from whichever tier's key is computable for the current report (Tier 1 > Tier 2 > Tier 3 > `report:{id}` fallback if no signal exists at all), so a future duplicate of *this* report has the best chance of finding it via the strongest tier reachable. +- Test mocking uses a SQL-content-routed dispatcher (map each query's distinguishing substring to staged rows) rather than strict call-order `mockResolvedValueOnce` chaining — more robust to the branching control flow (tier 2/3 are conditionally skipped once an earlier tier matches). + +## Deviations from Plan + +None - plan executed exactly as written. The self-exclusion requirement and the D-07 doc-comment requirement were both already spelled out explicitly in the plan's task text (not gaps discovered during execution), so no Rule 1-4 deviation applies. + +## Issues Encountered +- Initial version of the self-exclusion test asserted the wrong `groupMethod` for the newly-created campaign (`attachment_or_url` instead of `message_id`) — the test fixture staged an own `message_id`, so `computeTier1Key` correctly won priority over the Tier 2 key in the no-match/create-new-campaign branch. Fixed by correcting the test's expectation, not the implementation (confirmed via re-reading `computeTier1Key > computeTier2Key > computeTier3Key` priority order in the source). + +## User Setup Required + +None - no external service configuration required. + +## Next Phase Readiness +- `groupReportIntoCampaign` and the `phishing` permission resource are ready for Wave 2's 3 new `app/api/phishing/*` routes (list/detail/analyze) and the 2 modified trigger sites (`webhook-service.ts`, `phishing-sweep-service.ts`) to call, per 18-PATTERNS.md's exact call shapes. +- Known limitation carried forward per D-07: the automatic webhook/cron path only reaches Tier 3 (sender+subject+client+24h) until a report has been through an explicit `/analyze` call at least once, since `parseAndStoreMessage` (Phase 16) is still not wired into the automatic path — this plan documents it, does not fix it (out of scope, matches CONTEXT.md's locked decision). +- `approve`/`remediate` phishing actions are declared in `lib/permissions.ts`'s `statement` but ungranted to any role — Phase 20 only needs to add role grants, not touch the `statement` block again. + +--- +*Phase: 18-campaign-grouping-phishing-analysis-api* +*Completed: 2026-07-15* + +## Self-Check: PASSED + +All created/modified files confirmed present on disk; all 3 task commit hashes (`ea677b7`, `da926bb`, `04ec51f`) confirmed present in git log. diff --git a/lib/permissions.ts b/lib/permissions.ts index c11d963..a1a3e3b 100644 --- a/lib/permissions.ts +++ b/lib/permissions.ts @@ -28,6 +28,10 @@ export const statement = { // Datto RMM Overshell evidence (Phase 4.2 — read jobs / execute scripts) rmm: ["read", "execute"], + + // Phishing triage campaigns/reports (Phase 18 — D-05: full vocabulary now; + // approve/remediate ungranted to any role until Phase 20) + phishing: ["read", "analyze", "approve", "remediate"], } as const; // Create access control instance @@ -44,6 +48,7 @@ export const superAdminRole = ac.newRole({ settings: ["read", "update"], itglue: ["read", "write"], rmm: ["read", "execute"], + phishing: ["read", "analyze"], // approve/remediate ungranted until Phase 20 }); // Admin role - access to admin panel and user management, but not role management @@ -57,6 +62,7 @@ export const adminRole = ac.newRole({ settings: ["read"], itglue: ["read", "write"], rmm: ["read", "execute"], + phishing: ["read", "analyze"], }); // User role - basic access @@ -70,6 +76,7 @@ export const userRole = ac.newRole({ settings: [], itglue: ["read"], rmm: ["read"], + phishing: ["read"], // cannot trigger /analyze }); // Helper function to check if a user has a specific permission diff --git a/lib/services/campaign-grouping-service.test.ts b/lib/services/campaign-grouping-service.test.ts new file mode 100644 index 0000000..e0e5d21 --- /dev/null +++ b/lib/services/campaign-grouping-service.test.ts @@ -0,0 +1,336 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock postgresClient BEFORE importing the module under test. +const queryMock = vi.fn(); +const transactionMock = vi.fn(); +vi.mock('./postgres-client', () => ({ + postgresClient: { + query: (...args: unknown[]) => queryMock(...args), + transaction: (...args: unknown[]) => transactionMock(...args), + }, +})); + +// eslint-disable-next-line import/first -- imported after vi.mock hoisting +import { normalizeSubject, extractUrlDomain, groupReportIntoCampaign } from './campaign-grouping-service'; + +describe('normalizeSubject', () => { + it('strips a single Re: prefix, lowercases, trims', () => { + expect(normalizeSubject('Re: Your Invoice ')).toBe('your invoice'); + }); + + it('strips repeated Re:/Fwd:/Fw: prefixes case-insensitively', () => { + expect(normalizeSubject('FW: Re: fwd: Urgent Payment')).toBe('urgent payment'); + }); + + it('handles null', () => { + expect(normalizeSubject(null)).toBe(''); + }); + + it('handles empty string', () => { + expect(normalizeSubject('')).toBe(''); + }); + + it('lowercases and trims a subject with no prefix', () => { + expect(normalizeSubject(' Urgent Payment ')).toBe('urgent payment'); + }); +}); + +describe('extractUrlDomain', () => { + it('extracts hostname from a full URL', () => { + expect(extractUrlDomain('https://evil.example.com/path?x=1')).toBe('evil.example.com'); + }); + + it('returns null for a malformed URL instead of throwing', () => { + expect(extractUrlDomain('not-a-url')).toBeNull(); + }); + + it('extracts hostname regardless of scheme', () => { + expect(extractUrlDomain('http://another.example.net')).toBe('another.example.net'); + }); +}); + +// ============================================================================= +// groupReportIntoCampaign — mocked-DB behavior (CAMP-01, CAMP-02, D-08) +// +// `transactionMock` invokes its callback with a fake `client` whose `query` +// method routes to staged rows based on a distinguishing SQL substring for +// each of the implementation's queries (own report / own message / tier 1 / +// own indicators / tier 2 candidates / candidate indicators / tier 3 / +// update campaigns / update reports / insert campaign). This mirrors +// phishing-eml-service.test.ts's `callsContaining()` discipline but resolves +// per-call instead of asserting only after the fact, since groupReportInto +// Campaign's control flow branches on intermediate query results. +// ============================================================================= + +interface MockRows { + ownReport?: unknown[]; + ownMessage?: unknown[]; + tier1?: unknown[]; + ownIndicators?: unknown[]; + tier2Candidates?: unknown[]; + candidateIndicators?: unknown[]; + tier3?: unknown[]; + insertCampaign?: unknown[]; +} + +/** Records every SQL call made through the fake transaction client. */ +let clientCalls: Array<{ sql: string; params: unknown[] }> = []; + +function makeClient(rows: MockRows) { + return { + query: vi.fn(async (sql: string, params?: unknown[]) => { + clientCalls.push({ sql, params: params ?? [] }); + + if (sql.includes('requester_contact_id, company_id, created_at')) { + return { rows: rows.ownReport ?? [], rowCount: rows.ownReport?.length ?? 0 }; + } + if (sql.includes('FROM messages') && sql.includes('WHERE report_id = $1')) { + return { rows: rows.ownMessage ?? [], rowCount: rows.ownMessage?.length ?? 0 }; + } + if (sql.includes('WHERE m.message_id = $1')) { + return { rows: rows.tier1 ?? [], rowCount: rows.tier1?.length ?? 0 }; + } + if (sql.includes('FROM indicators') && sql.includes('WHERE message_id = $1')) { + return { rows: rows.ownIndicators ?? [], rowCount: rows.ownIndicators?.length ?? 0 }; + } + if (sql.includes('BETWEEN $2::timestamptz')) { + return { rows: rows.tier2Candidates ?? [], rowCount: rows.tier2Candidates?.length ?? 0 }; + } + if (sql.includes('message_id = ANY')) { + return { rows: rows.candidateIndicators ?? [], rowCount: rows.candidateIndicators?.length ?? 0 }; + } + if (sql.includes('BETWEEN $4::timestamptz')) { + return { rows: rows.tier3 ?? [], rowCount: rows.tier3?.length ?? 0 }; + } + if (sql.includes('UPDATE campaigns')) { + return { rows: [], rowCount: 1 }; + } + if (sql.includes('UPDATE reports SET campaign_id')) { + return { rows: [], rowCount: 1 }; + } + if (sql.includes('INSERT INTO campaigns')) { + return { rows: rows.insertCampaign ?? [{ id: 'unstaged-campaign-id' }], rowCount: 1 }; + } + throw new Error(`Unstaged query in test mock: ${sql}`); + }), + }; +} + +function callsContaining(needle: string) { + return clientCalls.filter((c) => c.sql.includes(needle)); +} + +describe('groupReportIntoCampaign', () => { + beforeEach(() => { + queryMock.mockReset(); + transactionMock.mockReset(); + clientCalls = []; + }); + + function stage(rows: MockRows) { + transactionMock.mockImplementation(async (callback: (client: unknown) => Promise) => + callback(makeClient(rows)) + ); + } + + const REPORT_ROW = { + title: 'Re: Invoice Alert', + requester_contact_id: 5, + company_id: 10, + created_at: '2026-07-15T10:00:00Z', + }; + + it('Tier 3 match increments existing campaign report_count, updates last_seen_at, links campaign_id, and never creates a second campaign', async () => { + stage({ + ownReport: [REPORT_ROW], + ownMessage: [], + tier3: [{ campaign_id: 'campaign-1', title: 'Invoice Alert' }], + }); + + const result = await groupReportIntoCampaign('report-2'); + + expect(result).toEqual({ + campaignId: 'campaign-1', + groupMethod: 'sender_subject_client', + created: false, + }); + + const updateCampaignCalls = callsContaining('UPDATE campaigns'); + expect(updateCampaignCalls).toHaveLength(1); + expect(updateCampaignCalls[0].sql).toContain('report_count = report_count + 1'); + expect(updateCampaignCalls[0].sql).toContain('last_seen_at = NOW()'); + expect(updateCampaignCalls[0].params).toEqual(['campaign-1']); + + const updateReportCalls = callsContaining('UPDATE reports SET campaign_id'); + expect(updateReportCalls).toHaveLength(1); + expect(updateReportCalls[0].params).toEqual(['campaign-1', 'report-2']); + + expect(callsContaining('INSERT INTO campaigns')).toHaveLength(0); + }); + + it('creates exactly one new campaign when no tier matches anything', async () => { + stage({ + ownReport: [REPORT_ROW], + ownMessage: [], + tier3: [], + insertCampaign: [{ id: 'new-campaign-id' }], + }); + + const result = await groupReportIntoCampaign('report-3'); + + expect(result).toEqual({ + campaignId: 'new-campaign-id', + groupMethod: 'sender_subject_client', + created: true, + }); + + const insertCalls = callsContaining('INSERT INTO campaigns'); + expect(insertCalls).toHaveLength(1); + expect(insertCalls[0].params).toEqual([ + 'sender_subject_client:5:invoice alert:10', + 'sender_subject_client', + ]); + expect(callsContaining('UPDATE campaigns')).toHaveLength(0); + }); + + it('skipIfAlreadyGrouped:true returns null when the pre-check SELECT reports a non-null campaign_id (D-08)', async () => { + queryMock.mockResolvedValueOnce({ rows: [{ campaign_id: 'already-grouped' }], rowCount: 1 }); + + const result = await groupReportIntoCampaign('report-4', { skipIfAlreadyGrouped: true }); + + expect(result).toBeNull(); + expect(transactionMock).not.toHaveBeenCalled(); + }); + + it('skipIfAlreadyGrouped:true proceeds to full matching when campaign_id is null', async () => { + queryMock.mockResolvedValueOnce({ rows: [{ campaign_id: null }], rowCount: 1 }); + stage({ + ownReport: [REPORT_ROW], + ownMessage: [], + tier3: [{ campaign_id: 'campaign-9', title: 'Invoice Alert' }], + }); + + const result = await groupReportIntoCampaign('report-5', { skipIfAlreadyGrouped: true }); + + expect(result).toEqual({ + campaignId: 'campaign-9', + groupMethod: 'sender_subject_client', + created: false, + }); + }); + + it('Tier 1 (Message-ID) matches before Tier 2/3 are ever queried', async () => { + stage({ + ownReport: [REPORT_ROW], + ownMessage: [{ id: 'own-msg-id', message_id: 'shared-message-id-123' }], + tier1: [{ campaign_id: 'campaign-t1' }], + }); + + const result = await groupReportIntoCampaign('report-6'); + + expect(result).toEqual({ + campaignId: 'campaign-t1', + groupMethod: 'message_id', + created: false, + }); + expect(callsContaining('FROM indicators')).toHaveLength(0); + expect(callsContaining('JOIN contacts c')).toHaveLength(0); + }); + + it('Tier 2 (attachment-hash/URL-domain + subject + sender + 24h) matches when Tier 1 finds nothing', async () => { + stage({ + ownReport: [REPORT_ROW], + ownMessage: [{ id: 'own-msg-id', message_id: 'unique-message-id-no-other-match' }], + tier1: [], // own message_id matches no other already-grouped report + ownIndicators: [ + { indicator_type: 'attachment_hash', value: 'hash123' }, + { indicator_type: 'sender', value: 'attacker@evil.example.com' }, + ], + tier2Candidates: [ + { + report_id: 'other-report', + campaign_id: 'campaign-t2', + title: 'Invoice Alert', + message_id: 'other-msg-id', + }, + ], + candidateIndicators: [ + { message_id: 'other-msg-id', indicator_type: 'attachment_hash', value: 'hash123' }, + { message_id: 'other-msg-id', indicator_type: 'sender', value: 'attacker@evil.example.com' }, + ], + }); + + const result = await groupReportIntoCampaign('report-7'); + + expect(result).toEqual({ + campaignId: 'campaign-t2', + groupMethod: 'attachment_or_url', + created: false, + }); + }); + + it('extracts URL-domain (not raw URL string) when matching Tier 2 via url indicators', async () => { + stage({ + ownReport: [REPORT_ROW], + ownMessage: [{ id: 'own-msg-id', message_id: 'unique-message-id-no-other-match-2' }], + tier1: [], + ownIndicators: [ + { indicator_type: 'url', value: 'https://evil.example.com/phish?x=1' }, + { indicator_type: 'sender', value: 'attacker@evil.example.com' }, + ], + tier2Candidates: [ + { + report_id: 'other-report', + campaign_id: 'campaign-t2-url', + title: 'Invoice Alert', + message_id: 'other-msg-id-2', + }, + ], + candidateIndicators: [ + // Different raw URL string, same domain — must match via extractUrlDomain, not exact string equality. + { message_id: 'other-msg-id-2', indicator_type: 'url', value: 'https://evil.example.com/different-path' }, + { message_id: 'other-msg-id-2', indicator_type: 'sender', value: 'attacker@evil.example.com' }, + ], + }); + + const result = await groupReportIntoCampaign('report-8'); + + expect(result).toEqual({ + campaignId: 'campaign-t2-url', + groupMethod: 'attachment_or_url', + created: false, + }); + }); + + it('re-running on an already-grouped report whose own messages/indicators would otherwise self-match does not increment report_count a second time (self-exclusion)', async () => { + // Own message_id and own indicators exist (this report has already been + // through parseAndStoreMessage), but NO other report shares them — the + // only thing that *could* match is this report's own row, which every + // tier query excludes via `r.id != `. + stage({ + ownReport: [REPORT_ROW], + ownMessage: [{ id: 'own-msg-id', message_id: 'self-message-id' }], + tier1: [], // self-exclusion means the report's own linked campaign never surfaces here + ownIndicators: [ + { indicator_type: 'attachment_hash', value: 'self-hash' }, + { indicator_type: 'sender', value: 'attacker@evil.example.com' }, + ], + tier2Candidates: [], // no other report in the 24h window + tier3: [], // no other report matches sender+subject+client either + insertCampaign: [{ id: 'fresh-campaign-id' }], + }); + + const result = await groupReportIntoCampaign('already-grouped-report'); + + // Tier 1's own message_id exists, so the new campaign is created with + // the strongest available key (message_id) — see computeTier1Key. + expect(result).toEqual({ + campaignId: 'fresh-campaign-id', + groupMethod: 'message_id', + created: true, + }); + // The critical assertion: no existing campaign's report_count is bumped + // a second time for this report re-run. + expect(callsContaining('UPDATE campaigns')).toHaveLength(0); + }); +}); diff --git a/lib/services/campaign-grouping-service.ts b/lib/services/campaign-grouping-service.ts new file mode 100644 index 0000000..39ba1df --- /dev/null +++ b/lib/services/campaign-grouping-service.ts @@ -0,0 +1,389 @@ +/** + * Campaign Grouping Service + * + * Shared grouping core called by all 3 trigger sites: the webhook path + * (`webhook-service.ts`'s `triggerPhishingDetection()`), the cron sweep + * (`phishing-sweep-service.ts`), and the on-demand + * `POST /api/phishing/tickets/{id}/analyze` route. One deterministic, + * testable function computes a tiered campaign key (CAMP-01) and + * finds-or-creates a `campaigns` row (CAMP-02) — no duplicated matching + * logic between callers, mirroring `phishing-detector.ts`'s shared-core + * architecture. + * + * D-07 limitation (load-bearing, stated explicitly): `parseAndStoreMessage` + * (the only writer of `messages`/`indicators` rows — Phase 16) is not wired + * into the automatic webhook/cron path this phase. That means the automatic + * path only ever has `reports`/`contacts` data available, so Tier 1 + * (Message-ID) and Tier 2 (attachment-hash/URL-domain) can only ever match + * for a report that has already been through an explicit `/analyze` call at + * least once. Until then, automatic grouping effectively only reaches + * Tier 3 (sender + normalized subject + client + 24h window). + */ + +import { postgresClient } from './postgres-client'; + +// ============================================================================= +// Pure logic — tier-key helpers (mirrors matchesPhishingPatterns / computePhishingContentHash) +// ============================================================================= + +/** + * D-03: strip leading Re:/Fwd:/Fw: (repeated, case-insensitive), lowercase, trim. + */ +export function normalizeSubject(subject: string | null): string { + let s = (subject ?? '').trim(); + const prefixRe = /^(re|fwd|fw):\s*/i; + while (prefixRe.test(s)) { + s = s.replace(prefixRe, '').trim(); + } + return s.toLowerCase(); +} + +/** + * Pitfall 4: indicators.value for indicator_type='url' is a bare URL string, + * not a domain — extract at read time, guarded (malformed/relative URLs are + * possible in real-world phishing emails). + */ +export function extractUrlDomain(url: string): string | null { + try { + return new URL(url).hostname || null; + } catch { + return null; + } +} + +// ============================================================================= +// Orchestration — groupReportIntoCampaign (mirrors detectPhishingTicket) +// ============================================================================= + +export interface GroupReportResult { + campaignId: string; + groupMethod: 'message_id' | 'attachment_or_url' | 'sender_subject_client'; + created: boolean; +} + +interface OwnReportRow { + title: string | null; + requester_contact_id: number | null; + company_id: number | null; + created_at: string; +} + +interface OwnMessageRow { + id: string; + message_id: string | null; +} + +interface IndicatorRow { + indicator_type: string; + value: string; +} + +interface CandidateIndicatorRow extends IndicatorRow { + message_id: string; +} + +interface Tier2CandidateRow { + report_id: string; + campaign_id: string; + title: string | null; + message_id: string; +} + +/** Builds a `message_id:...` key — the strongest available signal (Tier 1). */ +function computeTier1Key(messageId: string | null): string | null { + return messageId ? `message_id:${messageId}` : null; +} + +/** Builds an `attachment_or_url:...` key from own indicators + subject + sender (Tier 2). */ +function computeTier2Key( + attachmentHashes: string[], + urlDomains: string[], + normalizedSubject: string, + senderValue: string | null +): string | null { + if (!senderValue || !normalizedSubject) return null; + if (attachmentHashes.length === 0 && urlDomains.length === 0) return null; + const keyParts = [...attachmentHashes, ...urlDomains].sort(); + return `attachment_or_url:${keyParts.join(',')}:${normalizedSubject}:${senderValue}`; +} + +/** Builds a `sender_subject_client:...` key from sender + subject + company (Tier 3). */ +function computeTier3Key( + requesterContactId: number | null, + normalizedSubject: string, + companyId: number | null +): string | null { + if (!requesterContactId || !normalizedSubject || !companyId) return null; + return `sender_subject_client:${requesterContactId}:${normalizedSubject}:${companyId}`; +} + +/** + * Shared entry point called by the webhook path, the cron sweep, and the + * on-demand `/analyze` route. Looks up the report's own signal (message, + * indicators, requester/company), tries Tier 1 -> Tier 2 -> Tier 3 matching + * queries in order inside a single transaction (Pitfall 2 — `campaigns. + * campaign_key` has no UNIQUE constraint), and either links the report to an + * existing campaign (bumping report_count/last_seen_at, CAMP-02) or inserts a + * new campaigns row. + * + * D-08: `opts.skipIfAlreadyGrouped` short-circuits (returns null) when the + * report is already linked — used by the automatic webhook/cron path to + * avoid redundant work on every re-fire/sweep pass. The `/analyze` route + * omits this option so it always re-runs full tiered matching (it may + * upgrade a Tier-3-only report to Tier 1 after `parseAndStoreMessage` has + * just populated `messages`/`indicators` for the first time). + * + * D-04: never merges campaigns — attaches to the first/best tiered match + * found, or creates a new campaign. Multi-campaign-merge logic is explicitly + * out of scope this phase. + */ +export async function groupReportIntoCampaign( + reportId: string, + opts?: { skipIfAlreadyGrouped?: boolean } +): Promise { + try { + if (opts?.skipIfAlreadyGrouped) { + const existing = await postgresClient.query<{ campaign_id: string | null }>( + `SELECT campaign_id::text AS campaign_id FROM reports WHERE id = $1`, + [reportId] + ); + if (existing.rows[0]?.campaign_id) { + return null; // already grouped, short-circuit (D-08) + } + } + + return await postgresClient.transaction(async (client) => { + const ownReportRes = await client.query( + `SELECT title, requester_contact_id, company_id, created_at + FROM reports + WHERE id = $1`, + [reportId] + ); + const ownReport = ownReportRes.rows[0]; + if (!ownReport) { + throw new Error(`groupReportIntoCampaign: report ${reportId} not found`); + } + + const normalizedSubject = normalizeSubject(ownReport.title); + + // Own messages row, if `parseAndStoreMessage` has already run for this + // report (D-07 — not the case for the automatic webhook/cron path + // until an explicit /analyze call has happened at least once). + const ownMessageRes = await client.query( + `SELECT id::text AS id, message_id + FROM messages + WHERE report_id = $1 + LIMIT 1`, + [reportId] + ); + const ownMessage = ownMessageRes.rows[0] ?? null; + + let matchCampaignId: string | null = null; + let matchGroupMethod: GroupReportResult['groupMethod'] | null = null; + + // --------------------------------------------------------------------- + // Tier 1: Message-ID. Self-exclusion (`r.id != $2`) is REQUIRED — without + // it this query would trivially match the report's own messages row + // against itself, double-incrementing its own already-linked campaign + // on every re-run (plan-checker finding). + // --------------------------------------------------------------------- + if (ownMessage?.message_id) { + const tier1 = await client.query<{ campaign_id: string }>( + `SELECT r.campaign_id::text AS campaign_id + FROM messages m + JOIN reports r ON r.id = m.report_id + WHERE m.message_id = $1 + AND r.campaign_id IS NOT NULL + AND r.id != $2 + ORDER BY r.created_at ASC + LIMIT 1`, + [ownMessage.message_id, reportId] + ); + if (tier1.rows[0]?.campaign_id) { + matchCampaignId = tier1.rows[0].campaign_id; + matchGroupMethod = 'message_id'; + } + } + + // --------------------------------------------------------------------- + // Tier 2: attachment-hash / URL-domain + normalized subject + sender, + // within a 24h window (D-02). Only reachable if the own report has its + // own indicators (i.e. parseAndStoreMessage already ran for it). + // Self-exclusion (`r.id != $1`) required for the same reason as Tier 1. + // --------------------------------------------------------------------- + let ownAttachmentHashes: string[] = []; + let ownUrlDomains: string[] = []; + let ownSenderValue: string | null = null; + + if (!matchCampaignId && ownMessage) { + const ownIndicatorsRes = await client.query( + `SELECT indicator_type, value + FROM indicators + WHERE message_id = $1`, + [ownMessage.id] + ); + ownAttachmentHashes = ownIndicatorsRes.rows + .filter((i) => i.indicator_type === 'attachment_hash') + .map((i) => i.value); + ownUrlDomains = ownIndicatorsRes.rows + .filter((i) => i.indicator_type === 'url') + .map((i) => extractUrlDomain(i.value)) + .filter((d): d is string => d !== null); + ownSenderValue = + ownIndicatorsRes.rows.find((i) => i.indicator_type === 'sender')?.value ?? null; + + if ( + (ownAttachmentHashes.length > 0 || ownUrlDomains.length > 0) && + ownSenderValue && + normalizedSubject + ) { + const tier2Candidates = await client.query( + `SELECT r.id::text AS report_id, r.campaign_id::text AS campaign_id, r.title, + m.id::text AS message_id + FROM messages m + JOIN reports r ON r.id = m.report_id + WHERE r.campaign_id IS NOT NULL + AND r.id != $1 + AND r.created_at BETWEEN $2::timestamptz - INTERVAL '24 hours' + AND $2::timestamptz + INTERVAL '24 hours'`, + [reportId, ownReport.created_at] + ); + + if (tier2Candidates.rows.length > 0) { + const candidateMessageIds = tier2Candidates.rows.map((c) => c.message_id); + const candidateIndicatorsRes = await client.query( + `SELECT message_id::text AS message_id, indicator_type, value + FROM indicators + WHERE message_id = ANY($1::uuid[])`, + [candidateMessageIds] + ); + const indicatorsByMessage = new Map(); + for (const ind of candidateIndicatorsRes.rows) { + const arr = indicatorsByMessage.get(ind.message_id) ?? []; + arr.push(ind); + indicatorsByMessage.set(ind.message_id, arr); + } + + for (const candidate of tier2Candidates.rows) { + if (normalizeSubject(candidate.title) !== normalizedSubject) continue; + const candidateIndicators = indicatorsByMessage.get(candidate.message_id) ?? []; + const candidateSender = candidateIndicators.find( + (i) => i.indicator_type === 'sender' + )?.value; + if (candidateSender !== ownSenderValue) continue; + const candidateHashes = candidateIndicators + .filter((i) => i.indicator_type === 'attachment_hash') + .map((i) => i.value); + const candidateDomains = candidateIndicators + .filter((i) => i.indicator_type === 'url') + .map((i) => extractUrlDomain(i.value)) + .filter((d): d is string => d !== null); + const hashMatch = ownAttachmentHashes.some((h) => candidateHashes.includes(h)); + const domainMatch = ownUrlDomains.some((d) => candidateDomains.includes(d)); + if (hashMatch || domainMatch) { + matchCampaignId = candidate.campaign_id; + matchGroupMethod = 'attachment_or_url'; + break; + } + } + } + } + } + + // --------------------------------------------------------------------- + // Tier 3: sender + normalizeSubject(title) + client + 24h window + // (D-02). Joins reports.requester_contact_id -> contacts (Pitfall 5 — + // NOT `contact_id`). Self-exclusion (`r.id != $3`) required for the + // same reason as Tiers 1-2. + // --------------------------------------------------------------------- + if (!matchCampaignId && ownReport.requester_contact_id && ownReport.company_id && normalizedSubject) { + const tier3 = await client.query<{ campaign_id: string; title: string | null }>( + `SELECT r.campaign_id::text AS campaign_id, r.title + FROM reports r + JOIN contacts c ON c.id = r.requester_contact_id + WHERE r.requester_contact_id = $1 + AND r.company_id = $2 + AND r.campaign_id IS NOT NULL + AND r.id != $3 + AND r.created_at BETWEEN $4::timestamptz - INTERVAL '24 hours' + AND $4::timestamptz + INTERVAL '24 hours' + ORDER BY r.created_at ASC`, + [ownReport.requester_contact_id, ownReport.company_id, reportId, ownReport.created_at] + ); + const match = tier3.rows.find((r) => normalizeSubject(r.title) === normalizedSubject); + if (match) { + matchCampaignId = match.campaign_id; + matchGroupMethod = 'sender_subject_client'; + } + } + + // --------------------------------------------------------------------- + // Find-or-create against `campaigns` (CAMP-02). + // --------------------------------------------------------------------- + if (matchCampaignId && matchGroupMethod) { + await client.query( + `UPDATE campaigns + SET report_count = report_count + 1, last_seen_at = NOW(), updated_at = NOW() + WHERE id = $1`, + [matchCampaignId] + ); + await client.query( + `UPDATE reports SET campaign_id = $1, updated_at = NOW() WHERE id = $2`, + [matchCampaignId, reportId] + ); + return { campaignId: matchCampaignId, groupMethod: matchGroupMethod, created: false }; + } + + // No match on any tier — create a new campaign. Prefer the strongest + // available key (Tier 1 > Tier 2 > Tier 3) so future duplicates of THIS + // report can match it; do NOT set/transition campaigns.status (leave + // migration default 'open' — Phase 19/20's concern). + const tier1Key = computeTier1Key(ownMessage?.message_id ?? null); + const tier2Key = computeTier2Key( + ownAttachmentHashes, + ownUrlDomains, + normalizedSubject, + ownSenderValue + ); + const tier3Key = computeTier3Key( + ownReport.requester_contact_id, + normalizedSubject, + ownReport.company_id + ); + + let newCampaignKey: string; + let newGroupMethod: GroupReportResult['groupMethod']; + if (tier1Key) { + newCampaignKey = tier1Key; + newGroupMethod = 'message_id'; + } else if (tier2Key) { + newCampaignKey = tier2Key; + newGroupMethod = 'attachment_or_url'; + } else if (tier3Key) { + newCampaignKey = tier3Key; + newGroupMethod = 'sender_subject_client'; + } else { + newCampaignKey = `report:${reportId}`; + newGroupMethod = 'sender_subject_client'; + } + + const newCampaign = await client.query<{ id: string }>( + `INSERT INTO campaigns (campaign_key, group_method, first_seen_at, last_seen_at, report_count) + VALUES ($1, $2, NOW(), NOW(), 1) + RETURNING id::text AS id`, + [newCampaignKey, newGroupMethod] + ); + const newCampaignId = newCampaign.rows[0].id; + + await client.query( + `UPDATE reports SET campaign_id = $1, updated_at = NOW() WHERE id = $2`, + [newCampaignId, reportId] + ); + + return { campaignId: newCampaignId, groupMethod: newGroupMethod, created: true }; + }); + } catch (error) { + console.error('[CAMPAIGN-GROUPING] Failed to group report into campaign', reportId, error); + throw error; + } +}