diff --git a/.planning/phases/15-data-model-detection-ticket-evidence/15-02-SUMMARY.md b/.planning/phases/15-data-model-detection-ticket-evidence/15-02-SUMMARY.md new file mode 100644 index 0000000..914607b --- /dev/null +++ b/.planning/phases/15-data-model-detection-ticket-evidence/15-02-SUMMARY.md @@ -0,0 +1,108 @@ +--- +phase: 15-data-model-detection-ticket-evidence +plan: 02 +subsystem: services +tags: [detection, phishing-triage, evidence, postgres, autotask, sha256] + +# Dependency graph +requires: + - phase: 15-01 + provides: "reports table (ticket_id UNIQUE FK, content_hash, matched_patterns JSONB, evidence JSONB)" +provides: + - "lib/services/phishing-detector.ts — KNOWN_PHISHING_PATTERNS, matchesPhishingPatterns, computePhishingContentHash, gatherTicketEvidence, detectPhishingTicket" + - "One shared detection entry point (detectPhishingTicket) for both the webhook path and cron sweep in Plan 03" +affects: [15-03-webhook-and-sweep, 16-message-parsing] + +# Tech tracking +tech-stack: + added: [] + patterns: + - "Case-insensitive substring matcher via .toLowerCase()+.includes() only (no RegExp/eval) — mirrors robotic-classifier.evaluateContains" + - "sha256 content-hash over only the fields that define reprocessing eligibility (title+description), excluding bump-prone fields — mirrors analyzer/preprocessor.computeContentHash" + - "Check-before-write idempotency: SELECT existing content_hash, skip evidence-gathering and write entirely when unchanged — mirrors analyzer/persistence.findExistingAnalysisByContentHash" + +key-files: + created: [lib/services/phishing-detector.ts, lib/services/phishing-detector.test.ts] + modified: [] + +key-decisions: + - "Split Task 1 pure-logic implementation from Task 2 evidence/orchestration into two separate commits (test -> feat -> feat) even though both live in the same file, so the TDD RED/GREEN gate sequence is unambiguous in git history" + - "Autotask client instantiated as a lazy module-level singleton with env-var config, mirroring ticket-reconciliation-service.ts's getClient() pattern, rather than introducing a shared factory (out of scope for this plan)" + - "Idempotency guard compares stored reports.content_hash to the freshly computed hash BEFORE gathering evidence, so an unchanged ticket never re-queries ticket_notes/time_entries/Autotask attachments" + +patterns-established: + - "Pattern: detector modules expose pure matching/hashing functions separately from the async DB/API orchestration function, so vitest can cover the pure logic without mocking postgresClient or AutotaskClient" + +requirements-completed: [DETECT-01, DETECT-02, EVID-01] + +# Metrics +duration: 13min +completed: 2026-07-15 +--- + +# Phase 15 Plan 02: Phishing Detector Summary + +**`lib/services/phishing-detector.ts` — a single deterministic detection core matching 8 locked DETECT-01 patterns, sha256 content-hashing for D-04 idempotent reprocessing, and EVID-01 evidence capture (company/notes/time-entries/attachment-metadata) upserted into the Plan 01 `reports` table via `ON CONFLICT (ticket_id)`.** + +## Performance + +- **Duration:** 13 min +- **Started:** 2026-07-15T11:41:00Z +- **Completed:** 2026-07-15T11:47:59Z +- **Tasks:** 2 completed +- **Files modified:** 2 + +## Accomplishments +- Implemented `KNOWN_PHISHING_PATTERNS` (the 8 locked DETECT-01 strings verbatim) and `matchesPhishingPatterns` — case-insensitive substring matching (`.toLowerCase()` + `.includes()` only, no `RegExp`/`eval`), returning both a `flagged` boolean and the exact subset of patterns present. +- Implemented `computePhishingContentHash` — sha256 over `{ title, description }` only, stable for identical input, changes on either field, and normalizes `null` description to `''`. +- Wrote and ran a 17-assertion vitest suite (`phishing-detector.test.ts`) covering all 8 patterns individually, the negative case, case-insensitivity, matched[] exactness, and hash stability/change/null-normalization — RED confirmed (module didn't exist) before GREEN implementation. +- Implemented `gatherTicketEvidence` — parameterized `$1` queries against `companies`, `ticket_notes`, and `time_entries`, plus Autotask attachment metadata (`fullPath`/`title`/`contentType` only, never base64 `data`), with the Autotask call wrapped in try/catch so an API failure degrades to an empty attachments array instead of throwing. +- Implemented `detectPhishingTicket` — the shared orchestration entry point: matches, hashes, checks the D-04 idempotency guard (SELECT existing `content_hash`, skip gathering/writing when unchanged), then upserts one `reports` row via `ON CONFLICT (ticket_id) DO UPDATE ... RETURNING id`, binding `requester_contact_id` from `ticket.contact_id` and `created_by_contact_id` from `ticket.created_by_contact_id`. + +## Task Commits + +Each task was committed atomically, with Task 1 following the full TDD RED/GREEN gate sequence: + +1. **Task 1 (RED): add failing tests for pattern matcher + content hash** - `0e7daf9` (test) +2. **Task 1 (GREEN): implement phishing pattern matcher + content hash** - `aabf532` (feat) +3. **Task 2: add evidence capture + detectPhishingTicket orchestration** - `15d0caa` (feat) + +**Plan metadata:** (this SUMMARY.md commit) + +_Note: Task 1 is TDD — test → feat. No REFACTOR commit was needed; the GREEN implementation was already clean._ + +## Files Created/Modified +- `lib/services/phishing-detector.ts` - Pure matcher/hash functions (`KNOWN_PHISHING_PATTERNS`, `matchesPhishingPatterns`, `computePhishingContentHash`) plus evidence capture and orchestration (`gatherTicketEvidence`, `detectPhishingTicket`) — the shared detection core for Plan 03's webhook and cron-sweep callers +- `lib/services/phishing-detector.test.ts` - 17 vitest assertions covering all 8 locked patterns individually, negative case, case-insensitivity, matched[] exactness, and content-hash stability/change/null-normalization + +## Decisions Made +- Split Task 1's pure-logic commit from Task 2's evidence/orchestration commit even though both extend the same file, so the RED (`test(...)`) → GREEN (`feat(...)`) gate sequence required by the TDD workflow is unambiguous in `git log`, and Task 2's orchestration work is its own reviewable `feat(...)` commit. +- Reused the `ticket-reconciliation-service.ts` lazy-singleton pattern for the AutotaskClient (env-var config, module-level cache) rather than introducing a new factory — no existing `getAutotaskClient()`/`isAutotaskConfigured()` factory was present to reuse, and adding one was out of scope for this plan. +- Idempotency check queries only `id, content_hash` from `reports` (not the full row) and returns immediately on a hash match, before any evidence-gathering queries run — this is what makes the D-04 guarantee ("reprocessing only when content hash changed") cheap for the common case of an unchanged ticket being re-scanned by the cron sweep. + +## Deviations from Plan + +None - plan executed exactly as written. The plan's own Task 1 instruction ("Do not add DB access in this task — pure functions only") was honored by writing only the pure matcher/hash functions in the first commit, then extending the same file with DB/API-touching code in Task 2's commit, exactly as the plan's two-task structure specifies. + +## Issues Encountered +None. + +## User Setup Required +None - no external service configuration required. The detector uses the existing `AUTOTASK_*` env vars already configured elsewhere in the codebase (no new credentials introduced). + +## Next Phase Readiness +- `detectPhishingTicket(ticket)` is ready to be called from both the Autotask webhook handler and a cron sweep in Plan 03 — same underlying logic, no duplicated matching/hashing/idempotency code between the two callers. +- The reports upsert path is fully wired against the Plan 01 schema (`ON CONFLICT (ticket_id)`, `content_hash`, `matched_patterns` JSONB, `evidence` JSONB) — verified via `npx tsc --noEmit --pretty` and the passing vitest suite; no live DB write was exercised in this plan (that happens when Plan 03 wires a real ticket through the detector against the dev Postgres instance). +- No blockers for Plan 03. + +--- +*Phase: 15-data-model-detection-ticket-evidence* +*Completed: 2026-07-15* + +## Self-Check: PASSED +- FOUND: lib/services/phishing-detector.ts +- FOUND: lib/services/phishing-detector.test.ts +- FOUND: .planning/phases/15-data-model-detection-ticket-evidence/15-02-SUMMARY.md +- FOUND: commit 0e7daf9 +- FOUND: commit aabf532 +- FOUND: commit 15d0caa diff --git a/lib/services/phishing-detector.test.ts b/lib/services/phishing-detector.test.ts new file mode 100644 index 0000000..aa7bb8a --- /dev/null +++ b/lib/services/phishing-detector.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect } from 'vitest'; +import { + KNOWN_PHISHING_PATTERNS, + matchesPhishingPatterns, + computePhishingContentHash, +} from './phishing-detector'; + +describe('KNOWN_PHISHING_PATTERNS', () => { + it('has exactly 8 locked patterns', () => { + expect(KNOWN_PHISHING_PATTERNS).toHaveLength(8); + }); +}); + +describe('matchesPhishingPatterns', () => { + it('flags a title containing "Phishing Report"', () => { + const result = matchesPhishingPatterns('Fwd: Phishing Report', null); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Phishing Report'); + }); + + it('flags a title containing "Spam Alert"', () => { + const result = matchesPhishingPatterns('Spam Alert - user reported', null); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Spam Alert'); + }); + + it('flags a title containing "Phishing Alert - Email Security Report"', () => { + const result = matchesPhishingPatterns('Phishing Alert - Email Security Report', null); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Phishing Alert - Email Security Report'); + }); + + it('flags a description containing "KnowBe4 Phish Alert Report"', () => { + const result = matchesPhishingPatterns(null, 'This is a KnowBe4 Phish Alert Report for review'); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('KnowBe4 Phish Alert Report'); + }); + + it('flags a description containing "Source: KnowBe4 Phish Alert Button"', () => { + const result = matchesPhishingPatterns(null, 'Source: KnowBe4 Phish Alert Button'); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Source: KnowBe4 Phish Alert Button'); + }); + + it('flags a description containing "userSubmissionsReportMessage"', () => { + const result = matchesPhishingPatterns(null, 'Generated by userSubmissionsReportMessage flow'); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('userSubmissionsReportMessage'); + }); + + it('flags a description containing "reported message destinations"', () => { + const result = matchesPhishingPatterns(null, 'See reported message destinations below'); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('reported message destinations'); + }); + + it('flags a description containing "Microsoft directly"', () => { + const result = matchesPhishingPatterns(null, 'This message was reported to Microsoft directly'); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Microsoft directly'); + }); + + it('does not flag a ticket with none of the patterns', () => { + const result = matchesPhishingPatterns('Re: order confirmation', 'please review invoice'); + expect(result.flagged).toBe(false); + expect(result.matched).toEqual([]); + }); + + it('matches case-insensitively', () => { + const result = matchesPhishingPatterns('SPAM ALERT from user', null); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Spam Alert'); + }); + + it('returns only the patterns actually present in matched[]', () => { + const result = matchesPhishingPatterns('Phishing Report', 'unrelated body text'); + expect(result.matched).toEqual(['Phishing Report']); + }); + + it('checks both title and description for a match', () => { + const result = matchesPhishingPatterns('Unrelated subject', 'Spam Alert triggered'); + expect(result.flagged).toBe(true); + expect(result.matched).toContain('Spam Alert'); + }); +}); + +describe('computePhishingContentHash', () => { + it('is stable for identical title+description', () => { + expect(computePhishingContentHash('t', 'd')).toBe(computePhishingContentHash('t', 'd')); + }); + + it('changes when the title changes', () => { + expect(computePhishingContentHash('t', 'd')).not.toBe(computePhishingContentHash('t2', 'd')); + }); + + it('changes when the description changes', () => { + expect(computePhishingContentHash('t', 'd')).not.toBe(computePhishingContentHash('t', 'd2')); + }); + + it('is defined and stable when description is null', () => { + const hash1 = computePhishingContentHash('t', null); + const hash2 = computePhishingContentHash('t', null); + expect(hash1).toBeTruthy(); + expect(hash1).toBe(hash2); + }); +}); diff --git a/lib/services/phishing-detector.ts b/lib/services/phishing-detector.ts new file mode 100644 index 0000000..a88de0f --- /dev/null +++ b/lib/services/phishing-detector.ts @@ -0,0 +1,248 @@ +/** + * Phishing Detector + * + * Shared detection core called by both the webhook path (Plan 03) and the + * cron sweep (Plan 03). Matches a ticket's title+description against the 8 + * locked DETECT-01 patterns, computes a content hash for D-04 idempotency, + * gathers EVID-01 evidence, and upserts a single `reports` row per candidate + * ticket — reprocessing only when the content hash changed. + * + * One deterministic, testable detector with no duplicated matching logic + * between callers (CONTEXT.md discretion: "both call the same underlying + * logic"). + */ + +import { createHash } from 'crypto'; +import { postgresClient } from './postgres-client'; +import { AutotaskClient } from './autotask-client'; + +// ============================================================================= +// Pure detection logic — pattern matcher + content hash +// ============================================================================= + +/** + * The 8 locked DETECT-01 patterns (case-insensitive substring match). + */ +export const KNOWN_PHISHING_PATTERNS: readonly string[] = [ + 'Phishing Report', + 'Spam Alert', + 'Phishing Alert - Email Security Report', + 'KnowBe4 Phish Alert Report', + 'Source: KnowBe4 Phish Alert Button', + 'userSubmissionsReportMessage', + 'reported message destinations', + 'Microsoft directly', +]; + +/** + * Case-insensitive substring match against the locked pattern list — mirrors + * robotic-classifier.evaluateContains (.toLowerCase() + .includes() only, + * NO regex, NO eval). + */ +export function matchesPhishingPatterns( + title: string | null, + description: string | null +): { flagged: boolean; matched: string[] } { + const haystack = `${title ?? ''} ${description ?? ''}`.toLowerCase(); + const matched = KNOWN_PHISHING_PATTERNS.filter((pattern) => + haystack.includes(pattern.toLowerCase()) + ); + return { flagged: matched.length > 0, matched }; +} + +/** + * sha256 over title+description only (D-04) — does NOT include + * last_activity_date, status, or any bump-prone field, so status/assignee + * churn never forces reprocessing. + */ +export function computePhishingContentHash( + title: string | null, + description: string | null +): string { + return createHash('sha256') + .update(JSON.stringify({ title: title ?? '', description: description ?? '' })) + .digest('hex'); +} + +// ============================================================================= +// Evidence capture + orchestration +// ============================================================================= + +export interface DetectableTicket { + id: number; + ticket_number: string | null; + title: string | null; + description: string | null; + company_id: number | null; + contact_id?: number | null; + created_by_contact_id?: number | null; +} + +interface EvidenceNote { + id: number; + title: string | null; + description: string | null; + note_type: number | null; + creator_resource_id: number | null; + created_at: string; +} + +interface EvidenceTimeEntry { + id: number; + resource_id: number | null; + entry_date: string | null; + hours_worked: number | null; + start_date_time: string | null; + end_date_time: string | null; +} + +interface EvidenceAttachment { + fullPath: string; + title: string; + contentType?: string; +} + +export interface EvidencePayload { + company_name: string | null; + notes: EvidenceNote[]; + time_entries: EvidenceTimeEntry[]; + attachments: EvidenceAttachment[]; +} + +let _autotaskClient: AutotaskClient | null = null; +function getAutotaskClient(): AutotaskClient { + if (!_autotaskClient) { + _autotaskClient = new AutotaskClient({ + apiUrl: process.env.AUTOTASK_API_URL || '', + username: process.env.AUTOTASK_USERNAME || '', + password: process.env.AUTOTASK_SECRET || '', + apiIntegrationCode: process.env.AUTOTASK_API_INTEGRATION_CODE || '', + }); + } + return _autotaskClient; +} + +/** + * Gather EVID-01 evidence for a candidate ticket: company name, ticket notes, + * time entries, and attachment metadata (never base64 `data` — content fetch + * is Phase 16). + */ +export async function gatherTicketEvidence( + ticket: DetectableTicket +): Promise { + const companyResult = await postgresClient.query<{ company_name: string | null }>( + `SELECT company_name FROM companies WHERE id = $1`, + [ticket.company_id] + ); + const company_name = companyResult.rows[0]?.company_name ?? null; + + const notesResult = await postgresClient.query( + `SELECT id, title, description, note_type, creator_resource_id, created_at + FROM ticket_notes + WHERE ticket_id = $1 + ORDER BY created_at`, + [ticket.id] + ); + + const timeEntriesResult = await postgresClient.query( + `SELECT id, resource_id, entry_date, hours_worked, start_date_time, end_date_time + FROM time_entries + WHERE ticket_id = $1 + ORDER BY entry_date`, + [ticket.id] + ); + + let attachments: EvidenceAttachment[] = []; + try { + const rawAttachments = await getAutotaskClient().getAttachments('Tickets', ticket.id); + attachments = rawAttachments.map((attachment) => ({ + fullPath: attachment.fullPath, + title: attachment.title, + contentType: attachment.contentType, + })); + } catch (error) { + console.error('[PHISHING-DETECT] Failed to fetch attachments for ticket', ticket.id, error); + attachments = []; + } + + return { + company_name, + notes: notesResult.rows, + time_entries: timeEntriesResult.rows, + attachments, + }; +} + +export interface DetectPhishingResult { + flagged: boolean; + reportId?: string; + skippedUnchanged?: boolean; +} + +/** + * Shared entry point called by both the webhook path and the cron sweep. + * Matches, hashes, checks the D-04 idempotency guard, gathers EVID-01 + * evidence, and upserts one `reports` row per candidate ticket. + */ +export async function detectPhishingTicket( + ticket: DetectableTicket +): Promise { + const { flagged, matched } = matchesPhishingPatterns(ticket.title, ticket.description); + if (!flagged) { + return { flagged: false }; + } + + const contentHash = computePhishingContentHash(ticket.title, ticket.description); + + try { + const existing = await postgresClient.query<{ id: string; content_hash: string }>( + `SELECT id::text AS id, content_hash FROM reports WHERE ticket_id = $1`, + [ticket.id] + ); + + if (existing.rowCount && existing.rowCount > 0 && existing.rows[0].content_hash === contentHash) { + return { flagged: true, skippedUnchanged: true }; + } + + const evidence = await gatherTicketEvidence(ticket); + + const upsertResult = await postgresClient.query<{ id: string }>( + `INSERT INTO reports ( + ticket_id, ticket_number, company_id, company_name, requester_contact_id, + created_by_contact_id, title, description, matched_patterns, content_hash, evidence + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11::jsonb) + ON CONFLICT (ticket_id) DO UPDATE SET + ticket_number = EXCLUDED.ticket_number, + company_id = EXCLUDED.company_id, + company_name = EXCLUDED.company_name, + requester_contact_id = EXCLUDED.requester_contact_id, + created_by_contact_id = EXCLUDED.created_by_contact_id, + title = EXCLUDED.title, + description = EXCLUDED.description, + matched_patterns = EXCLUDED.matched_patterns, + content_hash = EXCLUDED.content_hash, + evidence = EXCLUDED.evidence, + updated_at = NOW() + RETURNING id::text AS id`, + [ + ticket.id, + ticket.ticket_number, + ticket.company_id, + evidence.company_name, + ticket.contact_id ?? null, + ticket.created_by_contact_id ?? null, + ticket.title, + ticket.description, + JSON.stringify(matched), + contentHash, + JSON.stringify(evidence), + ] + ); + + return { flagged: true, reportId: upsertResult.rows[0].id }; + } catch (error) { + console.error('[PHISHING-DETECT] Failed to detect/persist report for ticket', ticket.id, error); + throw error; + } +}