wulf-pulse/lib/services/analyzer/pipeline.test.ts

import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { readFileSync } from 'fs';
import { resolve } from 'path';
import { runPipeline, _PIPELINE_INTERNALS } from './pipeline';
import * as persistence from './persistence';
import type { RawTicketBundle } from './preprocessor';
import type Anthropic from '@anthropic-ai/sdk';
import type { ITGlueSearchResult } from './itglue-search';

const FIXTURE = JSON.parse(
  readFileSync(
    resolve(__dirname, 'fixtures', 'T20260424.0045.input.json'),
    'utf8'
  )
) as RawTicketBundle;

let findExistingSpy: ReturnType<typeof vi.spyOn>;

beforeEach(() => {
  findExistingSpy = vi
    .spyOn(persistence, 'findExistingAnalysisByContentHash')
    .mockResolvedValue(null);
});

afterEach(() => {
  findExistingSpy.mockRestore();
  vi.restoreAllMocks();
});

interface Reply {
  text: string;
  usage?: Partial<Anthropic.Usage>;
}

/**
 * Sequential mock: each stage is a separate `messages.create` call. Queue the
 * replies in order: Haiku → Sonnet → (Opus, optional).
 */
function makeFakeAnthropic(queue: Reply[]): {
  fake: Anthropic;
  bodies: any[];
} {
  const bodies: any[] = [];
  let i = 0;
  const create = vi.fn(async (body: any) => {
    bodies.push(body);
    const next = queue[i++];
    if (!next) throw new Error('No more queued LLM replies');
    return {
      id: `msg_${i}`,
      type: 'message',
      role: 'assistant',
      content: [{ type: 'text', text: next.text }],
      model: body.model,
      stop_reason: 'end_turn',
      stop_sequence: null,
      usage: {
        input_tokens: next.usage?.input_tokens ?? 5_000,
        output_tokens: next.usage?.output_tokens ?? 500,
        cache_creation_input_tokens: next.usage?.cache_creation_input_tokens ?? 0,
        cache_read_input_tokens: next.usage?.cache_read_input_tokens ?? 0,
      },
    } as Anthropic.Message;
  });
  return {
    fake: { messages: { create } } as unknown as Anthropic,
    bodies,
  };
}

const validTriage = (overrides: Record<string, unknown> = {}) =>
  JSON.stringify({
    ticket_type: 'service_request',
    category: 'Vendor Integration',
    entities: {
      client_name: 'Seubert and Associates',
      site_name: null,
      devices: [],
      users: ['Lorentz Hinrichsen'],
      applications: ['AMS360'],
      vendors: ['Vertafore'],
    },
    is_resolved: false,
    status_matches_reality: false,
    complexity_tier: 'medium',
    complexity_reasons: ['vendor case opened after requestor pivoted'],
    itglue_lookup_needed: false,
    itglue_search_hints: [],
    ...overrides,
  });

const validSonnet = (overrides: Record<string, unknown> = {}) =>
  JSON.stringify({
    summary: 'Lorentz pivoted; tech opened a Vertafore case anyway.',
    timeline: [
      {
        timestamp: '2026-04-24T14:34:46.583Z',
        actor: 'Lorentz Hinrichsen',
        actor_type: 'wulf_tech',
        source: 'ticket_note',
        visibility: 'customer_facing',
        action: 'Said no further outreach to Vertafore was needed.',
      },
    ],
    what_was_done: ['Opened a case with Vertafore on 04/24'],
    what_should_have_been_done: ['Confirmed with requestor before opening case'],
    gaps: [
      {
        description: 'Work continued after requestor said to stop.',
        severity: 'high',
        evidence_timestamps: ['2026-04-27T00:00:00.000Z'],
      },
    ],
    next_step: 'Confirm with requestor whether Vertafore endpoint info is still useful.',
    next_step_rationale: 'They effectively closed the loop on 04/24.',
    post_resolution_analysis: null,
    confidence_score: 0.7,
    needs_human_review: false,
    human_review_reasons: [],
    ambiguities_for_opus: [],
    itglue_docs_referenced: [],
    ...overrides,
  });

const validOpus = (overrides: Record<string, unknown> = {}) =>
  JSON.stringify({
    opus_notes: 'Reviewed Sonnet analysis; agree on the high-severity gap.',
    updates: {},
    ...overrides,
  });

describe('runPipeline', () => {
  it('short-circuits when an existing analysis with the same content hash exists', async () => {
    findExistingSpy.mockResolvedValueOnce({
      id: 'existing-uuid',
      analysis_version: 3,
    });
    const { fake } = makeFakeAnthropic([]);
    const result = await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake }
    );
    expect(result.outcome).toBe('idempotent_short_circuit');
    if (result.outcome === 'idempotent_short_circuit') {
      expect(result.existing_analysis_id).toBe('existing-uuid');
      expect(result.existing_analysis_version).toBe(3);
    }
  });

  it('does NOT short-circuit when force=true', async () => {
    findExistingSpy.mockResolvedValueOnce({
      id: 'existing-uuid',
      analysis_version: 3,
    });
    const { fake } = makeFakeAnthropic([
      { text: validTriage({ status_matches_reality: true }) },
      { text: validSonnet() },
    ]);
    const result = await runPipeline(
      { bundle: FIXTURE, force: true },
      { anthropic: fake }
    );
    expect(result.outcome).toBe('complete');
  });

  it('runs Stage 1 → Stage 3 happy path with no Opus when nothing triggers it', async () => {
    const { fake, bodies } = makeFakeAnthropic([
      { text: validTriage({ status_matches_reality: true, complexity_tier: 'low' }) },
      {
        text: validSonnet({
          confidence_score: 0.85,
          ambiguities_for_opus: [],
        }),
      },
    ]);

    const result = await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake }
    );

    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;
    expect(bodies).toHaveLength(2);
    expect(bodies[0].model).toBe('claude-haiku-4-5');
    expect(bodies[1].model).toBe('claude-sonnet-4-6');
    expect(result.meta.haiku_used).toBe(true);
    expect(result.meta.sonnet_used).toBe(true);
    expect(result.meta.opus_used).toBe(false);
    expect(result.analysis.confidence_score).toBe(0.85);
  });

  it('runs Opus when triage.status_matches_reality=false', async () => {
    const { fake, bodies } = makeFakeAnthropic([
      { text: validTriage({ status_matches_reality: false }) },
      { text: validSonnet({ confidence_score: 0.9 }) },
      { text: validOpus() },
    ]);

    const result = await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake }
    );

    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;
    expect(bodies.map((b) => b.model)).toEqual([
      'claude-haiku-4-5',
      'claude-sonnet-4-6',
      'claude-opus-4-7',
    ]);
    expect(result.meta.opus_used).toBe(true);
  });

  it('applies Opus updates over the Sonnet result', async () => {
    const { fake } = makeFakeAnthropic([
      { text: validTriage({ complexity_tier: 'high' }) },
      {
        text: validSonnet({
          next_step: 'sonnet step',
          confidence_score: 0.4,
        }),
      },
      {
        text: validOpus({
          updates: {
            next_step: 'opus step',
            confidence_score: 0.85,
            needs_human_review: true,
            human_review_reasons: ['opus flagged for review'],
          },
        }),
      },
    ]);

    const result = await runPipeline({ bundle: FIXTURE }, { anthropic: fake });
    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;
    expect(result.analysis.next_step).toBe('opus step');
    expect(result.analysis.confidence_score).toBe(0.85);
    expect(result.analysis.needs_human_review).toBe(true);
    expect(result.analysis.human_review_reasons).toEqual(['opus flagged for review']);
  });

  it('respects forceSkipOpus even when triggers fire', async () => {
    const { fake, bodies } = makeFakeAnthropic([
      { text: validTriage({ complexity_tier: 'high' }) },
      { text: validSonnet({ ambiguities_for_opus: ['why?'] }) },
    ]);
    const result = await runPipeline(
      { bundle: FIXTURE, forceSkipOpus: true },
      { anthropic: fake }
    );
    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;
    expect(result.meta.opus_used).toBe(false);
    expect(bodies.map((b) => b.model)).toEqual([
      'claude-haiku-4-5',
      'claude-sonnet-4-6',
    ]);
  });

  it('trips the cost circuit breaker before Opus when running cost ≥ ceiling', async () => {
    const ceiling = _PIPELINE_INTERNALS.COST_CEILING_USD;
    // Force the running cost above the ceiling by inflating Sonnet input/output.
    // Sonnet pricing: $3/1M input, $15/1M output. Spend 700M tokens on output
    // — well over $2 — to deterministically trip the breaker.
    const { fake, bodies } = makeFakeAnthropic([
      { text: validTriage({ complexity_tier: 'high' }) },
      {
        text: validSonnet({ ambiguities_for_opus: ['why?'] }),
        usage: { input_tokens: 1, output_tokens: 700_000_000 },
      },
    ]);

    const result = await runPipeline({ bundle: FIXTURE }, { anthropic: fake });
    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;

    expect(result.meta.opus_used).toBe(false);
    expect(result.meta.cost_circuit_breaker_tripped).toBe(true);
    expect(result.analysis.needs_human_review).toBe(true);
    expect(result.analysis.human_review_reasons.some((r) => r.includes('cost ceiling'))).toBe(true);
    expect(bodies.map((b) => b.model)).toEqual([
      'claude-haiku-4-5',
      'claude-sonnet-4-6',
    ]);
    expect(result.meta.estimated_cost_usd).toBeGreaterThanOrEqual(ceiling);
  });

  it('runs IT Glue search when triage requests it AND succeeds', async () => {
    const { fake } = makeFakeAnthropic([
      {
        text: validTriage({
          itglue_lookup_needed: true,
          itglue_search_hints: ['AMS360', 'VSSO'],
          status_matches_reality: true,
        }),
      },
      { text: validSonnet({ confidence_score: 0.85 }) },
    ]);

    const fakeItglue = vi.fn().mockResolvedValue({
      org_id: '42',
      org_name: 'Seubert and Associates',
      docs: [
        {
          id: 'doc1',
          name: 'AMS360 Server',
          doc_type: 'configuration',
          organization_id: '42',
          organization_name: 'Seubert and Associates',
          snippet: 'sanitized snippet',
          url: null,
          updated_at: '2026-04-01T00:00:00.000Z',
        },
      ],
      alias_used: false,
    } satisfies ITGlueSearchResult);

    const result = await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake, itglueSearch: fakeItglue }
    );

    expect(fakeItglue).toHaveBeenCalledTimes(1);
    expect(fakeItglue).toHaveBeenCalledWith({
      org_name: 'Seubert and Associates',
      hints: ['AMS360', 'VSSO'],
    });

    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;
    expect(result.itglue_search_used).toBe(true);
    expect(result.itglue_org_resolved).toBe(true);
    expect(result.model_traces.itglue?.doc_count).toBe(1);
  });

  it('continues without IT Glue context when search throws', async () => {
    const { fake } = makeFakeAnthropic([
      {
        text: validTriage({
          itglue_lookup_needed: true,
          itglue_search_hints: ['AMS360'],
          status_matches_reality: true,
        }),
      },
      { text: validSonnet({ confidence_score: 0.85 }) },
    ]);

    const fakeItglue = vi.fn().mockRejectedValue(new Error('IT Glue 502'));
    const result = await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake, itglueSearch: fakeItglue }
    );
    expect(result.outcome).toBe('complete');
  });

  it('reports filtered_noise_count from preprocessor', async () => {
    const { fake } = makeFakeAnthropic([
      { text: validTriage({ status_matches_reality: true, complexity_tier: 'low' }) },
      { text: validSonnet({ confidence_score: 0.85 }) },
    ]);
    const result = await runPipeline({ bundle: FIXTURE }, { anthropic: fake });
    expect(result.outcome).toBe('complete');
    if (result.outcome !== 'complete') return;
    // T20260424.0045 has 4 workflow-rule firings + 4 service-desk-notification rows = 8 noise.
    expect(result.filtered_noise_count).toBe(8);
  });

  it('drives onStage callbacks in order', async () => {
    const stages: string[] = [];
    const { fake } = makeFakeAnthropic([
      { text: validTriage({ status_matches_reality: true, complexity_tier: 'low' }) },
      { text: validSonnet({ confidence_score: 0.85 }) },
    ]);
    await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake },
      { onStage: (stage) => void stages.push(stage) }
    );
    expect(stages).toEqual(['fetching', 'triaging', 'analyzing']);
  });

  it('drives onStage callbacks including itglue and deep_review when applicable', async () => {
    const stages: string[] = [];
    const { fake } = makeFakeAnthropic([
      {
        text: validTriage({
          itglue_lookup_needed: true,
          itglue_search_hints: ['x'],
          complexity_tier: 'high',
          status_matches_reality: false,
        }),
      },
      { text: validSonnet({ ambiguities_for_opus: ['?'] }) },
      { text: validOpus() },
    ]);
    const fakeItglue = vi.fn().mockResolvedValue({
      org_id: '1',
      org_name: 'X',
      docs: [],
      alias_used: false,
    } satisfies ITGlueSearchResult);

    await runPipeline(
      { bundle: FIXTURE },
      { anthropic: fake, itglueSearch: fakeItglue },
      { onStage: (stage) => void stages.push(stage) }
    );
    expect(stages).toEqual([
      'fetching',
      'triaging',
      'itglue',
      'analyzing',
      'deep_review',
    ]);
  });
});