import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { readFileSync } from 'fs'; import { resolve } from 'path'; import { runPipeline, _PIPELINE_INTERNALS } from './pipeline'; import * as persistence from './persistence'; import type { RawTicketBundle } from './preprocessor'; import type Anthropic from '@anthropic-ai/sdk'; import type { ITGlueSearchResult } from './itglue-search'; const FIXTURE = JSON.parse( readFileSync( resolve(__dirname, 'fixtures', 'T20260424.0045.input.json'), 'utf8' ) ) as RawTicketBundle; let findExistingSpy: ReturnType; beforeEach(() => { findExistingSpy = vi .spyOn(persistence, 'findExistingAnalysisByContentHash') .mockResolvedValue(null); }); afterEach(() => { findExistingSpy.mockRestore(); vi.restoreAllMocks(); }); interface Reply { text: string; usage?: Partial; } /** * Sequential mock: each stage is a separate `messages.create` call. Queue the * replies in order: Haiku → Sonnet → (Opus, optional). */ function makeFakeAnthropic(queue: Reply[]): { fake: Anthropic; bodies: any[]; } { const bodies: any[] = []; let i = 0; const create = vi.fn(async (body: any) => { bodies.push(body); const next = queue[i++]; if (!next) throw new Error('No more queued LLM replies'); return { id: `msg_${i}`, type: 'message', role: 'assistant', content: [{ type: 'text', text: next.text }], model: body.model, stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: next.usage?.input_tokens ?? 5_000, output_tokens: next.usage?.output_tokens ?? 500, cache_creation_input_tokens: next.usage?.cache_creation_input_tokens ?? 0, cache_read_input_tokens: next.usage?.cache_read_input_tokens ?? 0, }, } as Anthropic.Message; }); return { fake: { messages: { create } } as unknown as Anthropic, bodies, }; } const validTriage = (overrides: Record = {}) => JSON.stringify({ ticket_type: 'service_request', category: 'Vendor Integration', entities: { client_name: 'Seubert and Associates', site_name: null, devices: [], users: ['Lorentz Hinrichsen'], applications: ['AMS360'], vendors: ['Vertafore'], }, is_resolved: false, status_matches_reality: false, complexity_tier: 'medium', complexity_reasons: ['vendor case opened after requestor pivoted'], itglue_lookup_needed: false, itglue_search_hints: [], ...overrides, }); const validSonnet = (overrides: Record = {}) => JSON.stringify({ summary: 'Lorentz pivoted; tech opened a Vertafore case anyway.', timeline: [ { timestamp: '2026-04-24T14:34:46.583Z', actor: 'Lorentz Hinrichsen', actor_type: 'wulf_tech', source: 'ticket_note', visibility: 'customer_facing', action: 'Said no further outreach to Vertafore was needed.', }, ], what_was_done: ['Opened a case with Vertafore on 04/24'], what_should_have_been_done: ['Confirmed with requestor before opening case'], gaps: [ { description: 'Work continued after requestor said to stop.', severity: 'high', evidence_timestamps: ['2026-04-27T00:00:00.000Z'], }, ], next_step: 'Confirm with requestor whether Vertafore endpoint info is still useful.', next_step_rationale: 'They effectively closed the loop on 04/24.', post_resolution_analysis: null, confidence_score: 0.7, needs_human_review: false, human_review_reasons: [], ambiguities_for_opus: [], itglue_docs_referenced: [], ...overrides, }); const validOpus = (overrides: Record = {}) => JSON.stringify({ opus_notes: 'Reviewed Sonnet analysis; agree on the high-severity gap.', updates: {}, ...overrides, }); describe('runPipeline', () => { it('short-circuits when an existing analysis with the same content hash exists', async () => { findExistingSpy.mockResolvedValueOnce({ id: 'existing-uuid', analysis_version: 3, }); const { fake } = makeFakeAnthropic([]); const result = await runPipeline( { bundle: FIXTURE }, { anthropic: fake } ); expect(result.outcome).toBe('idempotent_short_circuit'); if (result.outcome === 'idempotent_short_circuit') { expect(result.existing_analysis_id).toBe('existing-uuid'); expect(result.existing_analysis_version).toBe(3); } }); it('does NOT short-circuit when force=true', async () => { findExistingSpy.mockResolvedValueOnce({ id: 'existing-uuid', analysis_version: 3, }); const { fake } = makeFakeAnthropic([ { text: validTriage({ status_matches_reality: true }) }, { text: validSonnet() }, ]); const result = await runPipeline( { bundle: FIXTURE, force: true }, { anthropic: fake } ); expect(result.outcome).toBe('complete'); }); it('runs Stage 1 → Stage 3 happy path with no Opus when nothing triggers it', async () => { const { fake, bodies } = makeFakeAnthropic([ { text: validTriage({ status_matches_reality: true, complexity_tier: 'low' }) }, { text: validSonnet({ confidence_score: 0.85, ambiguities_for_opus: [], }), }, ]); const result = await runPipeline( { bundle: FIXTURE }, { anthropic: fake } ); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; expect(bodies).toHaveLength(2); expect(bodies[0].model).toBe('claude-haiku-4-5'); expect(bodies[1].model).toBe('claude-sonnet-4-6'); expect(result.meta.haiku_used).toBe(true); expect(result.meta.sonnet_used).toBe(true); expect(result.meta.opus_used).toBe(false); expect(result.analysis.confidence_score).toBe(0.85); }); it('runs Opus when triage.status_matches_reality=false', async () => { const { fake, bodies } = makeFakeAnthropic([ { text: validTriage({ status_matches_reality: false }) }, { text: validSonnet({ confidence_score: 0.9 }) }, { text: validOpus() }, ]); const result = await runPipeline( { bundle: FIXTURE }, { anthropic: fake } ); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; expect(bodies.map((b) => b.model)).toEqual([ 'claude-haiku-4-5', 'claude-sonnet-4-6', 'claude-opus-4-7', ]); expect(result.meta.opus_used).toBe(true); }); it('applies Opus updates over the Sonnet result', async () => { const { fake } = makeFakeAnthropic([ { text: validTriage({ complexity_tier: 'high' }) }, { text: validSonnet({ next_step: 'sonnet step', confidence_score: 0.4, }), }, { text: validOpus({ updates: { next_step: 'opus step', confidence_score: 0.85, needs_human_review: true, human_review_reasons: ['opus flagged for review'], }, }), }, ]); const result = await runPipeline({ bundle: FIXTURE }, { anthropic: fake }); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; expect(result.analysis.next_step).toBe('opus step'); expect(result.analysis.confidence_score).toBe(0.85); expect(result.analysis.needs_human_review).toBe(true); expect(result.analysis.human_review_reasons).toEqual(['opus flagged for review']); }); it('respects forceSkipOpus even when triggers fire', async () => { const { fake, bodies } = makeFakeAnthropic([ { text: validTriage({ complexity_tier: 'high' }) }, { text: validSonnet({ ambiguities_for_opus: ['why?'] }) }, ]); const result = await runPipeline( { bundle: FIXTURE, forceSkipOpus: true }, { anthropic: fake } ); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; expect(result.meta.opus_used).toBe(false); expect(bodies.map((b) => b.model)).toEqual([ 'claude-haiku-4-5', 'claude-sonnet-4-6', ]); }); it('trips the cost circuit breaker before Opus when running cost ≥ ceiling', async () => { const ceiling = _PIPELINE_INTERNALS.COST_CEILING_USD; // Force the running cost above the ceiling by inflating Sonnet input/output. // Sonnet pricing: $3/1M input, $15/1M output. Spend 700M tokens on output // — well over $2 — to deterministically trip the breaker. const { fake, bodies } = makeFakeAnthropic([ { text: validTriage({ complexity_tier: 'high' }) }, { text: validSonnet({ ambiguities_for_opus: ['why?'] }), usage: { input_tokens: 1, output_tokens: 700_000_000 }, }, ]); const result = await runPipeline({ bundle: FIXTURE }, { anthropic: fake }); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; expect(result.meta.opus_used).toBe(false); expect(result.meta.cost_circuit_breaker_tripped).toBe(true); expect(result.analysis.needs_human_review).toBe(true); expect(result.analysis.human_review_reasons.some((r) => r.includes('cost ceiling'))).toBe(true); expect(bodies.map((b) => b.model)).toEqual([ 'claude-haiku-4-5', 'claude-sonnet-4-6', ]); expect(result.meta.estimated_cost_usd).toBeGreaterThanOrEqual(ceiling); }); it('runs IT Glue search when triage requests it AND succeeds', async () => { const { fake } = makeFakeAnthropic([ { text: validTriage({ itglue_lookup_needed: true, itglue_search_hints: ['AMS360', 'VSSO'], status_matches_reality: true, }), }, { text: validSonnet({ confidence_score: 0.85 }) }, ]); const fakeItglue = vi.fn().mockResolvedValue({ org_id: '42', org_name: 'Seubert and Associates', docs: [ { id: 'doc1', name: 'AMS360 Server', doc_type: 'configuration', organization_id: '42', organization_name: 'Seubert and Associates', snippet: 'sanitized snippet', url: null, updated_at: '2026-04-01T00:00:00.000Z', }, ], alias_used: false, } satisfies ITGlueSearchResult); const result = await runPipeline( { bundle: FIXTURE }, { anthropic: fake, itglueSearch: fakeItglue } ); expect(fakeItglue).toHaveBeenCalledTimes(1); expect(fakeItglue).toHaveBeenCalledWith({ org_name: 'Seubert and Associates', hints: ['AMS360', 'VSSO'], }); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; expect(result.itglue_search_used).toBe(true); expect(result.itglue_org_resolved).toBe(true); expect(result.model_traces.itglue?.doc_count).toBe(1); }); it('continues without IT Glue context when search throws', async () => { const { fake } = makeFakeAnthropic([ { text: validTriage({ itglue_lookup_needed: true, itglue_search_hints: ['AMS360'], status_matches_reality: true, }), }, { text: validSonnet({ confidence_score: 0.85 }) }, ]); const fakeItglue = vi.fn().mockRejectedValue(new Error('IT Glue 502')); const result = await runPipeline( { bundle: FIXTURE }, { anthropic: fake, itglueSearch: fakeItglue } ); expect(result.outcome).toBe('complete'); }); it('reports filtered_noise_count from preprocessor', async () => { const { fake } = makeFakeAnthropic([ { text: validTriage({ status_matches_reality: true, complexity_tier: 'low' }) }, { text: validSonnet({ confidence_score: 0.85 }) }, ]); const result = await runPipeline({ bundle: FIXTURE }, { anthropic: fake }); expect(result.outcome).toBe('complete'); if (result.outcome !== 'complete') return; // T20260424.0045 has 4 workflow-rule firings + 4 service-desk-notification rows = 8 noise. expect(result.filtered_noise_count).toBe(8); }); it('drives onStage callbacks in order', async () => { const stages: string[] = []; const { fake } = makeFakeAnthropic([ { text: validTriage({ status_matches_reality: true, complexity_tier: 'low' }) }, { text: validSonnet({ confidence_score: 0.85 }) }, ]); await runPipeline( { bundle: FIXTURE }, { anthropic: fake }, { onStage: (stage) => void stages.push(stage) } ); expect(stages).toEqual(['fetching', 'triaging', 'analyzing']); }); it('drives onStage callbacks including itglue and deep_review when applicable', async () => { const stages: string[] = []; const { fake } = makeFakeAnthropic([ { text: validTriage({ itglue_lookup_needed: true, itglue_search_hints: ['x'], complexity_tier: 'high', status_matches_reality: false, }), }, { text: validSonnet({ ambiguities_for_opus: ['?'] }) }, { text: validOpus() }, ]); const fakeItglue = vi.fn().mockResolvedValue({ org_id: '1', org_name: 'X', docs: [], alias_used: false, } satisfies ITGlueSearchResult); await runPipeline( { bundle: FIXTURE }, { anthropic: fake, itglueSearch: fakeItglue }, { onStage: (stage) => void stages.push(stage) } ); expect(stages).toEqual([ 'fetching', 'triaging', 'itglue', 'analyzing', 'deep_review', ]); }); });