chore: merge executor worktree (worktree-agent-a7caddbc3f0fc1d30)
This commit is contained in:
commit
19f05dd564
4 changed files with 844 additions and 0 deletions
|
|
@ -0,0 +1,112 @@
|
|||
---
|
||||
phase: 18-campaign-grouping-phishing-analysis-api
|
||||
plan: 01
|
||||
subsystem: api
|
||||
tags: [postgres, phishing, campaign-grouping, permissions, better-auth, vitest]
|
||||
|
||||
# Dependency graph
|
||||
requires:
|
||||
- phase: 15-data-model-detection-ticket-evidence
|
||||
provides: "campaigns/reports/messages/indicators schema (migrations 097/099), phishing-detector.ts's DetectableTicket shape and orchestration style"
|
||||
- phase: 16-eml-mime-evidence-parser
|
||||
provides: "parseAndStoreMessage() as the (currently uncalled) writer of messages/indicators rows that Tier 1/2 depend on"
|
||||
provides:
|
||||
- "lib/services/campaign-grouping-service.ts — groupReportIntoCampaign(reportId, opts), normalizeSubject, extractUrlDomain, GroupReportResult"
|
||||
- "phishing permission resource in lib/permissions.ts (full D-05 vocabulary + partial role grants)"
|
||||
affects: [18-02, 18-03, 19-classification, 20-remediation]
|
||||
|
||||
# Tech tracking
|
||||
tech-stack:
|
||||
added: []
|
||||
patterns:
|
||||
- "Shared grouping core called from 3 sites (webhook/cron/analyze), mirrors phishing-detector.ts's shared-detector architecture"
|
||||
- "Tiered find-or-create inside postgresClient.transaction() — pure-JS key computation + targeted parameterized queries per tier, not one mega-WHERE clause"
|
||||
- "SQL-content-routed mock client in tests (each query's distinguishing SQL substring maps to staged rows) instead of strict call-order mocking"
|
||||
|
||||
key-files:
|
||||
created:
|
||||
- lib/services/campaign-grouping-service.ts
|
||||
- lib/services/campaign-grouping-service.test.ts
|
||||
modified:
|
||||
- lib/permissions.ts
|
||||
|
||||
key-decisions:
|
||||
- "Every tier query includes a self-exclusion clause (r.id != <this report's id>) so re-running groupReportIntoCampaign on an already-grouped report can never match its own messages/indicators row against itself and double-increment its own campaign"
|
||||
- "New campaign creation picks the strongest available tier key (Tier 1 message_id > Tier 2 attachment_or_url > Tier 3 sender_subject_client) so future duplicates of that report have the best chance of matching it"
|
||||
- "campaigns.status is never set/transitioned by this plan — left at the migration default 'open' per D-04/scope boundary"
|
||||
- "phishing resource declares the full action vocabulary (read/analyze/approve/remediate) now but only grants read+analyze (admin/super-admin) and read (user) this phase, per D-05"
|
||||
|
||||
patterns-established:
|
||||
- "Tier queries compute fuzzy keys (normalizeSubject, extractUrlDomain) in JS and query Postgres with targeted parameterized $n placeholders — never encode fuzzy matching in SQL WHERE clauses"
|
||||
|
||||
requirements-completed: [CAMP-01, CAMP-02, ACCESS-01]
|
||||
|
||||
# Metrics
|
||||
duration: 15min
|
||||
completed: 2026-07-15
|
||||
---
|
||||
|
||||
# Phase 18 Plan 01: Campaign Grouping Core + Phishing Permission Resource Summary
|
||||
|
||||
**`groupReportIntoCampaign()` — tiered Message-ID → attachment-hash/URL-domain → sender+subject+client campaign matching inside a single Postgres transaction, plus the `phishing` permission resource all future `/api/phishing/*` routes will gate on.**
|
||||
|
||||
## Performance
|
||||
|
||||
- **Duration:** ~15 min
|
||||
- **Started:** 2026-07-15T19:19Z (approx, first commit)
|
||||
- **Completed:** 2026-07-15T19:23Z (last task commit)
|
||||
- **Tasks:** 3/3 completed
|
||||
- **Files modified:** 3 (2 created, 1 modified)
|
||||
|
||||
## Accomplishments
|
||||
- `lib/services/campaign-grouping-service.ts` — `groupReportIntoCampaign(reportId, opts)` implements the full CAMP-01 tiered match (Message-ID → attachment-hash/URL-domain+subject+sender+24h → sender+normalized-subject+client+24h) and CAMP-02 accumulation (report_count/last_seen_at bump on match, new campaign on no-match), wrapped in `postgresClient.transaction()` to guard against `campaigns.campaign_key`'s missing UNIQUE constraint.
|
||||
- `lib/services/campaign-grouping-service.test.ts` — 16 passing tests covering pure helpers (`normalizeSubject`, `extractUrlDomain`) and mocked-DB tiered-matching/find-or-create/self-exclusion/`skipIfAlreadyGrouped` behavior.
|
||||
- `lib/permissions.ts` — `phishing` resource added with the full D-05 action vocabulary; `read`+`analyze` granted to `super-admin`/`admin`, `read` only to `user`; `approve`/`remediate` declared but ungranted to any role (Phase 20's job).
|
||||
|
||||
## Task Commits
|
||||
|
||||
Each task was committed atomically:
|
||||
|
||||
1. **Task 1: Pure tier-key helpers + test file scaffold** - `ea677b7` (test)
|
||||
2. **Task 2: groupReportIntoCampaign — tiered matching + transactional find-or-create** - `da926bb` (feat)
|
||||
3. **Task 3: Add phishing permission resource + role grants** - `04ec51f` (feat)
|
||||
|
||||
**Plan metadata:** committed alongside this SUMMARY (see final commit in this plan's history)
|
||||
|
||||
## TDD Gate Compliance
|
||||
|
||||
Plan frontmatter is `type: tdd`. Gate sequence verified in git log: `ea677b7` is a `test(...)` commit (RED gate) followed by `da926bb`/`04ec51f`, both `feat(...)` commits (GREEN gate) — sequence satisfied, no warning needed. No `refactor(...)` commit was needed (no cleanup pass required after GREEN).
|
||||
|
||||
## Files Created/Modified
|
||||
- `lib/services/campaign-grouping-service.ts` - `groupReportIntoCampaign`, `normalizeSubject`, `extractUrlDomain`, `GroupReportResult`; D-07 doc comment stating the Tier-3-only automatic-path limitation
|
||||
- `lib/services/campaign-grouping-service.test.ts` - 16 tests: 5 `normalizeSubject`, 3 `extractUrlDomain`, 8 `groupReportIntoCampaign` (Tier 1/2/3 matches, no-match create, `skipIfAlreadyGrouped` short-circuit + pass-through, self-exclusion/no-double-increment)
|
||||
- `lib/permissions.ts` - `phishing` resource in `statement` (4 actions) + role grants in `superAdminRole`/`adminRole`/`userRole`
|
||||
|
||||
## Decisions Made
|
||||
- Self-exclusion clause (`r.id != <this report's own id>`) added to every one of the 3 tier queries — this was a plan-checker finding baked into the plan's action text, not a deviation, but worth restating: without it, re-running grouping on an already-linked report could match the report's own `messages`/`indicators` row against itself and double-increment its own campaign's `report_count`.
|
||||
- New-campaign `campaign_key`/`group_method` is chosen from whichever tier's key is computable for the current report (Tier 1 > Tier 2 > Tier 3 > `report:{id}` fallback if no signal exists at all), so a future duplicate of *this* report has the best chance of finding it via the strongest tier reachable.
|
||||
- Test mocking uses a SQL-content-routed dispatcher (map each query's distinguishing substring to staged rows) rather than strict call-order `mockResolvedValueOnce` chaining — more robust to the branching control flow (tier 2/3 are conditionally skipped once an earlier tier matches).
|
||||
|
||||
## Deviations from Plan
|
||||
|
||||
None - plan executed exactly as written. The self-exclusion requirement and the D-07 doc-comment requirement were both already spelled out explicitly in the plan's task text (not gaps discovered during execution), so no Rule 1-4 deviation applies.
|
||||
|
||||
## Issues Encountered
|
||||
- Initial version of the self-exclusion test asserted the wrong `groupMethod` for the newly-created campaign (`attachment_or_url` instead of `message_id`) — the test fixture staged an own `message_id`, so `computeTier1Key` correctly won priority over the Tier 2 key in the no-match/create-new-campaign branch. Fixed by correcting the test's expectation, not the implementation (confirmed via re-reading `computeTier1Key > computeTier2Key > computeTier3Key` priority order in the source).
|
||||
|
||||
## User Setup Required
|
||||
|
||||
None - no external service configuration required.
|
||||
|
||||
## Next Phase Readiness
|
||||
- `groupReportIntoCampaign` and the `phishing` permission resource are ready for Wave 2's 3 new `app/api/phishing/*` routes (list/detail/analyze) and the 2 modified trigger sites (`webhook-service.ts`, `phishing-sweep-service.ts`) to call, per 18-PATTERNS.md's exact call shapes.
|
||||
- Known limitation carried forward per D-07: the automatic webhook/cron path only reaches Tier 3 (sender+subject+client+24h) until a report has been through an explicit `/analyze` call at least once, since `parseAndStoreMessage` (Phase 16) is still not wired into the automatic path — this plan documents it, does not fix it (out of scope, matches CONTEXT.md's locked decision).
|
||||
- `approve`/`remediate` phishing actions are declared in `lib/permissions.ts`'s `statement` but ungranted to any role — Phase 20 only needs to add role grants, not touch the `statement` block again.
|
||||
|
||||
---
|
||||
*Phase: 18-campaign-grouping-phishing-analysis-api*
|
||||
*Completed: 2026-07-15*
|
||||
|
||||
## Self-Check: PASSED
|
||||
|
||||
All created/modified files confirmed present on disk; all 3 task commit hashes (`ea677b7`, `da926bb`, `04ec51f`) confirmed present in git log.
|
||||
|
|
@ -28,6 +28,10 @@ export const statement = {
|
|||
|
||||
// Datto RMM Overshell evidence (Phase 4.2 — read jobs / execute scripts)
|
||||
rmm: ["read", "execute"],
|
||||
|
||||
// Phishing triage campaigns/reports (Phase 18 — D-05: full vocabulary now;
|
||||
// approve/remediate ungranted to any role until Phase 20)
|
||||
phishing: ["read", "analyze", "approve", "remediate"],
|
||||
} as const;
|
||||
|
||||
// Create access control instance
|
||||
|
|
@ -44,6 +48,7 @@ export const superAdminRole = ac.newRole({
|
|||
settings: ["read", "update"],
|
||||
itglue: ["read", "write"],
|
||||
rmm: ["read", "execute"],
|
||||
phishing: ["read", "analyze"], // approve/remediate ungranted until Phase 20
|
||||
});
|
||||
|
||||
// Admin role - access to admin panel and user management, but not role management
|
||||
|
|
@ -57,6 +62,7 @@ export const adminRole = ac.newRole({
|
|||
settings: ["read"],
|
||||
itglue: ["read", "write"],
|
||||
rmm: ["read", "execute"],
|
||||
phishing: ["read", "analyze"],
|
||||
});
|
||||
|
||||
// User role - basic access
|
||||
|
|
@ -70,6 +76,7 @@ export const userRole = ac.newRole({
|
|||
settings: [],
|
||||
itglue: ["read"],
|
||||
rmm: ["read"],
|
||||
phishing: ["read"], // cannot trigger /analyze
|
||||
});
|
||||
|
||||
// Helper function to check if a user has a specific permission
|
||||
|
|
|
|||
336
lib/services/campaign-grouping-service.test.ts
Normal file
336
lib/services/campaign-grouping-service.test.ts
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// Mock postgresClient BEFORE importing the module under test.
|
||||
const queryMock = vi.fn();
|
||||
const transactionMock = vi.fn();
|
||||
vi.mock('./postgres-client', () => ({
|
||||
postgresClient: {
|
||||
query: (...args: unknown[]) => queryMock(...args),
|
||||
transaction: (...args: unknown[]) => transactionMock(...args),
|
||||
},
|
||||
}));
|
||||
|
||||
// eslint-disable-next-line import/first -- imported after vi.mock hoisting
|
||||
import { normalizeSubject, extractUrlDomain, groupReportIntoCampaign } from './campaign-grouping-service';
|
||||
|
||||
describe('normalizeSubject', () => {
|
||||
it('strips a single Re: prefix, lowercases, trims', () => {
|
||||
expect(normalizeSubject('Re: Your Invoice ')).toBe('your invoice');
|
||||
});
|
||||
|
||||
it('strips repeated Re:/Fwd:/Fw: prefixes case-insensitively', () => {
|
||||
expect(normalizeSubject('FW: Re: fwd: Urgent Payment')).toBe('urgent payment');
|
||||
});
|
||||
|
||||
it('handles null', () => {
|
||||
expect(normalizeSubject(null)).toBe('');
|
||||
});
|
||||
|
||||
it('handles empty string', () => {
|
||||
expect(normalizeSubject('')).toBe('');
|
||||
});
|
||||
|
||||
it('lowercases and trims a subject with no prefix', () => {
|
||||
expect(normalizeSubject(' Urgent Payment ')).toBe('urgent payment');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractUrlDomain', () => {
|
||||
it('extracts hostname from a full URL', () => {
|
||||
expect(extractUrlDomain('https://evil.example.com/path?x=1')).toBe('evil.example.com');
|
||||
});
|
||||
|
||||
it('returns null for a malformed URL instead of throwing', () => {
|
||||
expect(extractUrlDomain('not-a-url')).toBeNull();
|
||||
});
|
||||
|
||||
it('extracts hostname regardless of scheme', () => {
|
||||
expect(extractUrlDomain('http://another.example.net')).toBe('another.example.net');
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// groupReportIntoCampaign — mocked-DB behavior (CAMP-01, CAMP-02, D-08)
|
||||
//
|
||||
// `transactionMock` invokes its callback with a fake `client` whose `query`
|
||||
// method routes to staged rows based on a distinguishing SQL substring for
|
||||
// each of the implementation's queries (own report / own message / tier 1 /
|
||||
// own indicators / tier 2 candidates / candidate indicators / tier 3 /
|
||||
// update campaigns / update reports / insert campaign). This mirrors
|
||||
// phishing-eml-service.test.ts's `callsContaining()` discipline but resolves
|
||||
// per-call instead of asserting only after the fact, since groupReportInto
|
||||
// Campaign's control flow branches on intermediate query results.
|
||||
// =============================================================================
|
||||
|
||||
interface MockRows {
|
||||
ownReport?: unknown[];
|
||||
ownMessage?: unknown[];
|
||||
tier1?: unknown[];
|
||||
ownIndicators?: unknown[];
|
||||
tier2Candidates?: unknown[];
|
||||
candidateIndicators?: unknown[];
|
||||
tier3?: unknown[];
|
||||
insertCampaign?: unknown[];
|
||||
}
|
||||
|
||||
/** Records every SQL call made through the fake transaction client. */
|
||||
let clientCalls: Array<{ sql: string; params: unknown[] }> = [];
|
||||
|
||||
function makeClient(rows: MockRows) {
|
||||
return {
|
||||
query: vi.fn(async (sql: string, params?: unknown[]) => {
|
||||
clientCalls.push({ sql, params: params ?? [] });
|
||||
|
||||
if (sql.includes('requester_contact_id, company_id, created_at')) {
|
||||
return { rows: rows.ownReport ?? [], rowCount: rows.ownReport?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('FROM messages') && sql.includes('WHERE report_id = $1')) {
|
||||
return { rows: rows.ownMessage ?? [], rowCount: rows.ownMessage?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('WHERE m.message_id = $1')) {
|
||||
return { rows: rows.tier1 ?? [], rowCount: rows.tier1?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('FROM indicators') && sql.includes('WHERE message_id = $1')) {
|
||||
return { rows: rows.ownIndicators ?? [], rowCount: rows.ownIndicators?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('BETWEEN $2::timestamptz')) {
|
||||
return { rows: rows.tier2Candidates ?? [], rowCount: rows.tier2Candidates?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('message_id = ANY')) {
|
||||
return { rows: rows.candidateIndicators ?? [], rowCount: rows.candidateIndicators?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('BETWEEN $4::timestamptz')) {
|
||||
return { rows: rows.tier3 ?? [], rowCount: rows.tier3?.length ?? 0 };
|
||||
}
|
||||
if (sql.includes('UPDATE campaigns')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes('UPDATE reports SET campaign_id')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes('INSERT INTO campaigns')) {
|
||||
return { rows: rows.insertCampaign ?? [{ id: 'unstaged-campaign-id' }], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unstaged query in test mock: ${sql}`);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function callsContaining(needle: string) {
|
||||
return clientCalls.filter((c) => c.sql.includes(needle));
|
||||
}
|
||||
|
||||
describe('groupReportIntoCampaign', () => {
|
||||
beforeEach(() => {
|
||||
queryMock.mockReset();
|
||||
transactionMock.mockReset();
|
||||
clientCalls = [];
|
||||
});
|
||||
|
||||
function stage(rows: MockRows) {
|
||||
transactionMock.mockImplementation(async (callback: (client: unknown) => Promise<unknown>) =>
|
||||
callback(makeClient(rows))
|
||||
);
|
||||
}
|
||||
|
||||
const REPORT_ROW = {
|
||||
title: 'Re: Invoice Alert',
|
||||
requester_contact_id: 5,
|
||||
company_id: 10,
|
||||
created_at: '2026-07-15T10:00:00Z',
|
||||
};
|
||||
|
||||
it('Tier 3 match increments existing campaign report_count, updates last_seen_at, links campaign_id, and never creates a second campaign', async () => {
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [],
|
||||
tier3: [{ campaign_id: 'campaign-1', title: 'Invoice Alert' }],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('report-2');
|
||||
|
||||
expect(result).toEqual({
|
||||
campaignId: 'campaign-1',
|
||||
groupMethod: 'sender_subject_client',
|
||||
created: false,
|
||||
});
|
||||
|
||||
const updateCampaignCalls = callsContaining('UPDATE campaigns');
|
||||
expect(updateCampaignCalls).toHaveLength(1);
|
||||
expect(updateCampaignCalls[0].sql).toContain('report_count = report_count + 1');
|
||||
expect(updateCampaignCalls[0].sql).toContain('last_seen_at = NOW()');
|
||||
expect(updateCampaignCalls[0].params).toEqual(['campaign-1']);
|
||||
|
||||
const updateReportCalls = callsContaining('UPDATE reports SET campaign_id');
|
||||
expect(updateReportCalls).toHaveLength(1);
|
||||
expect(updateReportCalls[0].params).toEqual(['campaign-1', 'report-2']);
|
||||
|
||||
expect(callsContaining('INSERT INTO campaigns')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('creates exactly one new campaign when no tier matches anything', async () => {
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [],
|
||||
tier3: [],
|
||||
insertCampaign: [{ id: 'new-campaign-id' }],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('report-3');
|
||||
|
||||
expect(result).toEqual({
|
||||
campaignId: 'new-campaign-id',
|
||||
groupMethod: 'sender_subject_client',
|
||||
created: true,
|
||||
});
|
||||
|
||||
const insertCalls = callsContaining('INSERT INTO campaigns');
|
||||
expect(insertCalls).toHaveLength(1);
|
||||
expect(insertCalls[0].params).toEqual([
|
||||
'sender_subject_client:5:invoice alert:10',
|
||||
'sender_subject_client',
|
||||
]);
|
||||
expect(callsContaining('UPDATE campaigns')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('skipIfAlreadyGrouped:true returns null when the pre-check SELECT reports a non-null campaign_id (D-08)', async () => {
|
||||
queryMock.mockResolvedValueOnce({ rows: [{ campaign_id: 'already-grouped' }], rowCount: 1 });
|
||||
|
||||
const result = await groupReportIntoCampaign('report-4', { skipIfAlreadyGrouped: true });
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(transactionMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skipIfAlreadyGrouped:true proceeds to full matching when campaign_id is null', async () => {
|
||||
queryMock.mockResolvedValueOnce({ rows: [{ campaign_id: null }], rowCount: 1 });
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [],
|
||||
tier3: [{ campaign_id: 'campaign-9', title: 'Invoice Alert' }],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('report-5', { skipIfAlreadyGrouped: true });
|
||||
|
||||
expect(result).toEqual({
|
||||
campaignId: 'campaign-9',
|
||||
groupMethod: 'sender_subject_client',
|
||||
created: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('Tier 1 (Message-ID) matches before Tier 2/3 are ever queried', async () => {
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [{ id: 'own-msg-id', message_id: 'shared-message-id-123' }],
|
||||
tier1: [{ campaign_id: 'campaign-t1' }],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('report-6');
|
||||
|
||||
expect(result).toEqual({
|
||||
campaignId: 'campaign-t1',
|
||||
groupMethod: 'message_id',
|
||||
created: false,
|
||||
});
|
||||
expect(callsContaining('FROM indicators')).toHaveLength(0);
|
||||
expect(callsContaining('JOIN contacts c')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('Tier 2 (attachment-hash/URL-domain + subject + sender + 24h) matches when Tier 1 finds nothing', async () => {
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [{ id: 'own-msg-id', message_id: 'unique-message-id-no-other-match' }],
|
||||
tier1: [], // own message_id matches no other already-grouped report
|
||||
ownIndicators: [
|
||||
{ indicator_type: 'attachment_hash', value: 'hash123' },
|
||||
{ indicator_type: 'sender', value: 'attacker@evil.example.com' },
|
||||
],
|
||||
tier2Candidates: [
|
||||
{
|
||||
report_id: 'other-report',
|
||||
campaign_id: 'campaign-t2',
|
||||
title: 'Invoice Alert',
|
||||
message_id: 'other-msg-id',
|
||||
},
|
||||
],
|
||||
candidateIndicators: [
|
||||
{ message_id: 'other-msg-id', indicator_type: 'attachment_hash', value: 'hash123' },
|
||||
{ message_id: 'other-msg-id', indicator_type: 'sender', value: 'attacker@evil.example.com' },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('report-7');
|
||||
|
||||
expect(result).toEqual({
|
||||
campaignId: 'campaign-t2',
|
||||
groupMethod: 'attachment_or_url',
|
||||
created: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('extracts URL-domain (not raw URL string) when matching Tier 2 via url indicators', async () => {
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [{ id: 'own-msg-id', message_id: 'unique-message-id-no-other-match-2' }],
|
||||
tier1: [],
|
||||
ownIndicators: [
|
||||
{ indicator_type: 'url', value: 'https://evil.example.com/phish?x=1' },
|
||||
{ indicator_type: 'sender', value: 'attacker@evil.example.com' },
|
||||
],
|
||||
tier2Candidates: [
|
||||
{
|
||||
report_id: 'other-report',
|
||||
campaign_id: 'campaign-t2-url',
|
||||
title: 'Invoice Alert',
|
||||
message_id: 'other-msg-id-2',
|
||||
},
|
||||
],
|
||||
candidateIndicators: [
|
||||
// Different raw URL string, same domain — must match via extractUrlDomain, not exact string equality.
|
||||
{ message_id: 'other-msg-id-2', indicator_type: 'url', value: 'https://evil.example.com/different-path' },
|
||||
{ message_id: 'other-msg-id-2', indicator_type: 'sender', value: 'attacker@evil.example.com' },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('report-8');
|
||||
|
||||
expect(result).toEqual({
|
||||
campaignId: 'campaign-t2-url',
|
||||
groupMethod: 'attachment_or_url',
|
||||
created: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('re-running on an already-grouped report whose own messages/indicators would otherwise self-match does not increment report_count a second time (self-exclusion)', async () => {
|
||||
// Own message_id and own indicators exist (this report has already been
|
||||
// through parseAndStoreMessage), but NO other report shares them — the
|
||||
// only thing that *could* match is this report's own row, which every
|
||||
// tier query excludes via `r.id != <this report's id>`.
|
||||
stage({
|
||||
ownReport: [REPORT_ROW],
|
||||
ownMessage: [{ id: 'own-msg-id', message_id: 'self-message-id' }],
|
||||
tier1: [], // self-exclusion means the report's own linked campaign never surfaces here
|
||||
ownIndicators: [
|
||||
{ indicator_type: 'attachment_hash', value: 'self-hash' },
|
||||
{ indicator_type: 'sender', value: 'attacker@evil.example.com' },
|
||||
],
|
||||
tier2Candidates: [], // no other report in the 24h window
|
||||
tier3: [], // no other report matches sender+subject+client either
|
||||
insertCampaign: [{ id: 'fresh-campaign-id' }],
|
||||
});
|
||||
|
||||
const result = await groupReportIntoCampaign('already-grouped-report');
|
||||
|
||||
// Tier 1's own message_id exists, so the new campaign is created with
|
||||
// the strongest available key (message_id) — see computeTier1Key.
|
||||
expect(result).toEqual({
|
||||
campaignId: 'fresh-campaign-id',
|
||||
groupMethod: 'message_id',
|
||||
created: true,
|
||||
});
|
||||
// The critical assertion: no existing campaign's report_count is bumped
|
||||
// a second time for this report re-run.
|
||||
expect(callsContaining('UPDATE campaigns')).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
389
lib/services/campaign-grouping-service.ts
Normal file
389
lib/services/campaign-grouping-service.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
/**
|
||||
* Campaign Grouping Service
|
||||
*
|
||||
* Shared grouping core called by all 3 trigger sites: the webhook path
|
||||
* (`webhook-service.ts`'s `triggerPhishingDetection()`), the cron sweep
|
||||
* (`phishing-sweep-service.ts`), and the on-demand
|
||||
* `POST /api/phishing/tickets/{id}/analyze` route. One deterministic,
|
||||
* testable function computes a tiered campaign key (CAMP-01) and
|
||||
* finds-or-creates a `campaigns` row (CAMP-02) — no duplicated matching
|
||||
* logic between callers, mirroring `phishing-detector.ts`'s shared-core
|
||||
* architecture.
|
||||
*
|
||||
* D-07 limitation (load-bearing, stated explicitly): `parseAndStoreMessage`
|
||||
* (the only writer of `messages`/`indicators` rows — Phase 16) is not wired
|
||||
* into the automatic webhook/cron path this phase. That means the automatic
|
||||
* path only ever has `reports`/`contacts` data available, so Tier 1
|
||||
* (Message-ID) and Tier 2 (attachment-hash/URL-domain) can only ever match
|
||||
* for a report that has already been through an explicit `/analyze` call at
|
||||
* least once. Until then, automatic grouping effectively only reaches
|
||||
* Tier 3 (sender + normalized subject + client + 24h window).
|
||||
*/
|
||||
|
||||
import { postgresClient } from './postgres-client';
|
||||
|
||||
// =============================================================================
|
||||
// Pure logic — tier-key helpers (mirrors matchesPhishingPatterns / computePhishingContentHash)
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* D-03: strip leading Re:/Fwd:/Fw: (repeated, case-insensitive), lowercase, trim.
|
||||
*/
|
||||
export function normalizeSubject(subject: string | null): string {
|
||||
let s = (subject ?? '').trim();
|
||||
const prefixRe = /^(re|fwd|fw):\s*/i;
|
||||
while (prefixRe.test(s)) {
|
||||
s = s.replace(prefixRe, '').trim();
|
||||
}
|
||||
return s.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Pitfall 4: indicators.value for indicator_type='url' is a bare URL string,
|
||||
* not a domain — extract at read time, guarded (malformed/relative URLs are
|
||||
* possible in real-world phishing emails).
|
||||
*/
|
||||
export function extractUrlDomain(url: string): string | null {
|
||||
try {
|
||||
return new URL(url).hostname || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Orchestration — groupReportIntoCampaign (mirrors detectPhishingTicket)
|
||||
// =============================================================================
|
||||
|
||||
export interface GroupReportResult {
|
||||
campaignId: string;
|
||||
groupMethod: 'message_id' | 'attachment_or_url' | 'sender_subject_client';
|
||||
created: boolean;
|
||||
}
|
||||
|
||||
interface OwnReportRow {
|
||||
title: string | null;
|
||||
requester_contact_id: number | null;
|
||||
company_id: number | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
interface OwnMessageRow {
|
||||
id: string;
|
||||
message_id: string | null;
|
||||
}
|
||||
|
||||
interface IndicatorRow {
|
||||
indicator_type: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface CandidateIndicatorRow extends IndicatorRow {
|
||||
message_id: string;
|
||||
}
|
||||
|
||||
interface Tier2CandidateRow {
|
||||
report_id: string;
|
||||
campaign_id: string;
|
||||
title: string | null;
|
||||
message_id: string;
|
||||
}
|
||||
|
||||
/** Builds a `message_id:...` key — the strongest available signal (Tier 1). */
|
||||
function computeTier1Key(messageId: string | null): string | null {
|
||||
return messageId ? `message_id:${messageId}` : null;
|
||||
}
|
||||
|
||||
/** Builds an `attachment_or_url:...` key from own indicators + subject + sender (Tier 2). */
|
||||
function computeTier2Key(
|
||||
attachmentHashes: string[],
|
||||
urlDomains: string[],
|
||||
normalizedSubject: string,
|
||||
senderValue: string | null
|
||||
): string | null {
|
||||
if (!senderValue || !normalizedSubject) return null;
|
||||
if (attachmentHashes.length === 0 && urlDomains.length === 0) return null;
|
||||
const keyParts = [...attachmentHashes, ...urlDomains].sort();
|
||||
return `attachment_or_url:${keyParts.join(',')}:${normalizedSubject}:${senderValue}`;
|
||||
}
|
||||
|
||||
/** Builds a `sender_subject_client:...` key from sender + subject + company (Tier 3). */
|
||||
function computeTier3Key(
|
||||
requesterContactId: number | null,
|
||||
normalizedSubject: string,
|
||||
companyId: number | null
|
||||
): string | null {
|
||||
if (!requesterContactId || !normalizedSubject || !companyId) return null;
|
||||
return `sender_subject_client:${requesterContactId}:${normalizedSubject}:${companyId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared entry point called by the webhook path, the cron sweep, and the
|
||||
* on-demand `/analyze` route. Looks up the report's own signal (message,
|
||||
* indicators, requester/company), tries Tier 1 -> Tier 2 -> Tier 3 matching
|
||||
* queries in order inside a single transaction (Pitfall 2 — `campaigns.
|
||||
* campaign_key` has no UNIQUE constraint), and either links the report to an
|
||||
* existing campaign (bumping report_count/last_seen_at, CAMP-02) or inserts a
|
||||
* new campaigns row.
|
||||
*
|
||||
* D-08: `opts.skipIfAlreadyGrouped` short-circuits (returns null) when the
|
||||
* report is already linked — used by the automatic webhook/cron path to
|
||||
* avoid redundant work on every re-fire/sweep pass. The `/analyze` route
|
||||
* omits this option so it always re-runs full tiered matching (it may
|
||||
* upgrade a Tier-3-only report to Tier 1 after `parseAndStoreMessage` has
|
||||
* just populated `messages`/`indicators` for the first time).
|
||||
*
|
||||
* D-04: never merges campaigns — attaches to the first/best tiered match
|
||||
* found, or creates a new campaign. Multi-campaign-merge logic is explicitly
|
||||
* out of scope this phase.
|
||||
*/
|
||||
export async function groupReportIntoCampaign(
|
||||
reportId: string,
|
||||
opts?: { skipIfAlreadyGrouped?: boolean }
|
||||
): Promise<GroupReportResult | null> {
|
||||
try {
|
||||
if (opts?.skipIfAlreadyGrouped) {
|
||||
const existing = await postgresClient.query<{ campaign_id: string | null }>(
|
||||
`SELECT campaign_id::text AS campaign_id FROM reports WHERE id = $1`,
|
||||
[reportId]
|
||||
);
|
||||
if (existing.rows[0]?.campaign_id) {
|
||||
return null; // already grouped, short-circuit (D-08)
|
||||
}
|
||||
}
|
||||
|
||||
return await postgresClient.transaction(async (client) => {
|
||||
const ownReportRes = await client.query<OwnReportRow>(
|
||||
`SELECT title, requester_contact_id, company_id, created_at
|
||||
FROM reports
|
||||
WHERE id = $1`,
|
||||
[reportId]
|
||||
);
|
||||
const ownReport = ownReportRes.rows[0];
|
||||
if (!ownReport) {
|
||||
throw new Error(`groupReportIntoCampaign: report ${reportId} not found`);
|
||||
}
|
||||
|
||||
const normalizedSubject = normalizeSubject(ownReport.title);
|
||||
|
||||
// Own messages row, if `parseAndStoreMessage` has already run for this
|
||||
// report (D-07 — not the case for the automatic webhook/cron path
|
||||
// until an explicit /analyze call has happened at least once).
|
||||
const ownMessageRes = await client.query<OwnMessageRow>(
|
||||
`SELECT id::text AS id, message_id
|
||||
FROM messages
|
||||
WHERE report_id = $1
|
||||
LIMIT 1`,
|
||||
[reportId]
|
||||
);
|
||||
const ownMessage = ownMessageRes.rows[0] ?? null;
|
||||
|
||||
let matchCampaignId: string | null = null;
|
||||
let matchGroupMethod: GroupReportResult['groupMethod'] | null = null;
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tier 1: Message-ID. Self-exclusion (`r.id != $2`) is REQUIRED — without
|
||||
// it this query would trivially match the report's own messages row
|
||||
// against itself, double-incrementing its own already-linked campaign
|
||||
// on every re-run (plan-checker finding).
|
||||
// ---------------------------------------------------------------------
|
||||
if (ownMessage?.message_id) {
|
||||
const tier1 = await client.query<{ campaign_id: string }>(
|
||||
`SELECT r.campaign_id::text AS campaign_id
|
||||
FROM messages m
|
||||
JOIN reports r ON r.id = m.report_id
|
||||
WHERE m.message_id = $1
|
||||
AND r.campaign_id IS NOT NULL
|
||||
AND r.id != $2
|
||||
ORDER BY r.created_at ASC
|
||||
LIMIT 1`,
|
||||
[ownMessage.message_id, reportId]
|
||||
);
|
||||
if (tier1.rows[0]?.campaign_id) {
|
||||
matchCampaignId = tier1.rows[0].campaign_id;
|
||||
matchGroupMethod = 'message_id';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tier 2: attachment-hash / URL-domain + normalized subject + sender,
|
||||
// within a 24h window (D-02). Only reachable if the own report has its
|
||||
// own indicators (i.e. parseAndStoreMessage already ran for it).
|
||||
// Self-exclusion (`r.id != $1`) required for the same reason as Tier 1.
|
||||
// ---------------------------------------------------------------------
|
||||
let ownAttachmentHashes: string[] = [];
|
||||
let ownUrlDomains: string[] = [];
|
||||
let ownSenderValue: string | null = null;
|
||||
|
||||
if (!matchCampaignId && ownMessage) {
|
||||
const ownIndicatorsRes = await client.query<IndicatorRow>(
|
||||
`SELECT indicator_type, value
|
||||
FROM indicators
|
||||
WHERE message_id = $1`,
|
||||
[ownMessage.id]
|
||||
);
|
||||
ownAttachmentHashes = ownIndicatorsRes.rows
|
||||
.filter((i) => i.indicator_type === 'attachment_hash')
|
||||
.map((i) => i.value);
|
||||
ownUrlDomains = ownIndicatorsRes.rows
|
||||
.filter((i) => i.indicator_type === 'url')
|
||||
.map((i) => extractUrlDomain(i.value))
|
||||
.filter((d): d is string => d !== null);
|
||||
ownSenderValue =
|
||||
ownIndicatorsRes.rows.find((i) => i.indicator_type === 'sender')?.value ?? null;
|
||||
|
||||
if (
|
||||
(ownAttachmentHashes.length > 0 || ownUrlDomains.length > 0) &&
|
||||
ownSenderValue &&
|
||||
normalizedSubject
|
||||
) {
|
||||
const tier2Candidates = await client.query<Tier2CandidateRow>(
|
||||
`SELECT r.id::text AS report_id, r.campaign_id::text AS campaign_id, r.title,
|
||||
m.id::text AS message_id
|
||||
FROM messages m
|
||||
JOIN reports r ON r.id = m.report_id
|
||||
WHERE r.campaign_id IS NOT NULL
|
||||
AND r.id != $1
|
||||
AND r.created_at BETWEEN $2::timestamptz - INTERVAL '24 hours'
|
||||
AND $2::timestamptz + INTERVAL '24 hours'`,
|
||||
[reportId, ownReport.created_at]
|
||||
);
|
||||
|
||||
if (tier2Candidates.rows.length > 0) {
|
||||
const candidateMessageIds = tier2Candidates.rows.map((c) => c.message_id);
|
||||
const candidateIndicatorsRes = await client.query<CandidateIndicatorRow>(
|
||||
`SELECT message_id::text AS message_id, indicator_type, value
|
||||
FROM indicators
|
||||
WHERE message_id = ANY($1::uuid[])`,
|
||||
[candidateMessageIds]
|
||||
);
|
||||
const indicatorsByMessage = new Map<string, CandidateIndicatorRow[]>();
|
||||
for (const ind of candidateIndicatorsRes.rows) {
|
||||
const arr = indicatorsByMessage.get(ind.message_id) ?? [];
|
||||
arr.push(ind);
|
||||
indicatorsByMessage.set(ind.message_id, arr);
|
||||
}
|
||||
|
||||
for (const candidate of tier2Candidates.rows) {
|
||||
if (normalizeSubject(candidate.title) !== normalizedSubject) continue;
|
||||
const candidateIndicators = indicatorsByMessage.get(candidate.message_id) ?? [];
|
||||
const candidateSender = candidateIndicators.find(
|
||||
(i) => i.indicator_type === 'sender'
|
||||
)?.value;
|
||||
if (candidateSender !== ownSenderValue) continue;
|
||||
const candidateHashes = candidateIndicators
|
||||
.filter((i) => i.indicator_type === 'attachment_hash')
|
||||
.map((i) => i.value);
|
||||
const candidateDomains = candidateIndicators
|
||||
.filter((i) => i.indicator_type === 'url')
|
||||
.map((i) => extractUrlDomain(i.value))
|
||||
.filter((d): d is string => d !== null);
|
||||
const hashMatch = ownAttachmentHashes.some((h) => candidateHashes.includes(h));
|
||||
const domainMatch = ownUrlDomains.some((d) => candidateDomains.includes(d));
|
||||
if (hashMatch || domainMatch) {
|
||||
matchCampaignId = candidate.campaign_id;
|
||||
matchGroupMethod = 'attachment_or_url';
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tier 3: sender + normalizeSubject(title) + client + 24h window
|
||||
// (D-02). Joins reports.requester_contact_id -> contacts (Pitfall 5 —
|
||||
// NOT `contact_id`). Self-exclusion (`r.id != $3`) required for the
|
||||
// same reason as Tiers 1-2.
|
||||
// ---------------------------------------------------------------------
|
||||
if (!matchCampaignId && ownReport.requester_contact_id && ownReport.company_id && normalizedSubject) {
|
||||
const tier3 = await client.query<{ campaign_id: string; title: string | null }>(
|
||||
`SELECT r.campaign_id::text AS campaign_id, r.title
|
||||
FROM reports r
|
||||
JOIN contacts c ON c.id = r.requester_contact_id
|
||||
WHERE r.requester_contact_id = $1
|
||||
AND r.company_id = $2
|
||||
AND r.campaign_id IS NOT NULL
|
||||
AND r.id != $3
|
||||
AND r.created_at BETWEEN $4::timestamptz - INTERVAL '24 hours'
|
||||
AND $4::timestamptz + INTERVAL '24 hours'
|
||||
ORDER BY r.created_at ASC`,
|
||||
[ownReport.requester_contact_id, ownReport.company_id, reportId, ownReport.created_at]
|
||||
);
|
||||
const match = tier3.rows.find((r) => normalizeSubject(r.title) === normalizedSubject);
|
||||
if (match) {
|
||||
matchCampaignId = match.campaign_id;
|
||||
matchGroupMethod = 'sender_subject_client';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Find-or-create against `campaigns` (CAMP-02).
|
||||
// ---------------------------------------------------------------------
|
||||
if (matchCampaignId && matchGroupMethod) {
|
||||
await client.query(
|
||||
`UPDATE campaigns
|
||||
SET report_count = report_count + 1, last_seen_at = NOW(), updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[matchCampaignId]
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE reports SET campaign_id = $1, updated_at = NOW() WHERE id = $2`,
|
||||
[matchCampaignId, reportId]
|
||||
);
|
||||
return { campaignId: matchCampaignId, groupMethod: matchGroupMethod, created: false };
|
||||
}
|
||||
|
||||
// No match on any tier — create a new campaign. Prefer the strongest
|
||||
// available key (Tier 1 > Tier 2 > Tier 3) so future duplicates of THIS
|
||||
// report can match it; do NOT set/transition campaigns.status (leave
|
||||
// migration default 'open' — Phase 19/20's concern).
|
||||
const tier1Key = computeTier1Key(ownMessage?.message_id ?? null);
|
||||
const tier2Key = computeTier2Key(
|
||||
ownAttachmentHashes,
|
||||
ownUrlDomains,
|
||||
normalizedSubject,
|
||||
ownSenderValue
|
||||
);
|
||||
const tier3Key = computeTier3Key(
|
||||
ownReport.requester_contact_id,
|
||||
normalizedSubject,
|
||||
ownReport.company_id
|
||||
);
|
||||
|
||||
let newCampaignKey: string;
|
||||
let newGroupMethod: GroupReportResult['groupMethod'];
|
||||
if (tier1Key) {
|
||||
newCampaignKey = tier1Key;
|
||||
newGroupMethod = 'message_id';
|
||||
} else if (tier2Key) {
|
||||
newCampaignKey = tier2Key;
|
||||
newGroupMethod = 'attachment_or_url';
|
||||
} else if (tier3Key) {
|
||||
newCampaignKey = tier3Key;
|
||||
newGroupMethod = 'sender_subject_client';
|
||||
} else {
|
||||
newCampaignKey = `report:${reportId}`;
|
||||
newGroupMethod = 'sender_subject_client';
|
||||
}
|
||||
|
||||
const newCampaign = await client.query<{ id: string }>(
|
||||
`INSERT INTO campaigns (campaign_key, group_method, first_seen_at, last_seen_at, report_count)
|
||||
VALUES ($1, $2, NOW(), NOW(), 1)
|
||||
RETURNING id::text AS id`,
|
||||
[newCampaignKey, newGroupMethod]
|
||||
);
|
||||
const newCampaignId = newCampaign.rows[0].id;
|
||||
|
||||
await client.query(
|
||||
`UPDATE reports SET campaign_id = $1, updated_at = NOW() WHERE id = $2`,
|
||||
[newCampaignId, reportId]
|
||||
);
|
||||
|
||||
return { campaignId: newCampaignId, groupMethod: newGroupMethod, created: true };
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[CAMPAIGN-GROUPING] Failed to group report into campaign', reportId, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue