Eight sub-phases per docs/ticket-analyzer-phase2-spec.md:
2.1 Schema (migration 070): analyzer_stage_executions table; source_snapshot,
aggregate_fingerprint, fingerprint_generated_at columns on analyzer_analyses.
model_traces marked LEGACY (kept for back-compat).
2.2 Every pipeline stage records a row to analyzer_stage_executions, success
or failure. Worker persists a status='failed' analyzer_analyses row when
the pipeline throws so partial stage records have a parent. Pipeline
exposes raw triage/sonnet/opus responses for downstream stages.
2.3 Stage 3 prompt updated with markdown formatting rules + banned filler
phrases. Added react-markdown + remark-gfm + @tailwindcss/typography.
New <AnalysisMarkdown> component replaces <ProseText>; coerces stray
headers to bold paragraphs.
2.4 Stage 6 fingerprint (Haiku) runs after persistence, failure-tolerant.
scripts/backfill-fingerprints.ts reconstructs Stage 6 input from the
legacy model_traces blob.
2.5 Browse UI rebuild at /analyzer/tickets: multi-select for client/issue/
queue/status/priority/assignee, sticky filter bar, active-filter chips,
bulk selection persisted via localStorage, "Analyze N selected" +
"Generate aggregate report" actions. New <MultiSelect> primitive.
Staleness uses last_activity_date > completed_at heuristic per spec C.1.
2.6 Aggregate reports (migration 071): runner is fire-and-forget, persists
SQL distributions immediately so UI shows partial state during the
Sonnet reduce call. Three endpoints, three pages (/analyzer/reports[/new
/:id]). IT Glue context fetcher capped at 200 doc titles.
2.7 Cost guards (migration 072): per-request $5 confirmation, soft-warn at
$20/day, hard-block at $50/day with ANALYZER_DAILY_COST_OVERRIDE_USERS
override. Every gating decision audited.
2.8 Runbook + build notes updated.
128 vitest tests passing, tsc clean. Migrations 070/071/072 idempotent
(IF NOT EXISTS). model_traces double-write retained — drop in a future
migration once aggregate reports have soaked.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
287 lines
9.3 KiB
TypeScript
287 lines
9.3 KiB
TypeScript
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
import { readFileSync } from 'fs';
|
|
import { resolve } from 'path';
|
|
import { analyzerWorker } from './worker';
|
|
import * as dataAccess from './data-access';
|
|
import * as persistence from './persistence';
|
|
import * as pipelineModule from './pipeline';
|
|
import * as stage6Module from './stages/stage6-fingerprint';
|
|
import type { RawTicketBundle } from './preprocessor';
|
|
|
|
const FIXTURE = JSON.parse(
|
|
readFileSync(
|
|
resolve(__dirname, 'fixtures', 'T20260424.0045.input.json'),
|
|
'utf8'
|
|
)
|
|
) as RawTicketBundle;
|
|
|
|
let loadSpy: ReturnType<typeof vi.spyOn>;
|
|
let runPipelineSpy: ReturnType<typeof vi.spyOn>;
|
|
let insertSpy: ReturnType<typeof vi.spyOn>;
|
|
let completeSpy: ReturnType<typeof vi.spyOn>;
|
|
let failSpy: ReturnType<typeof vi.spyOn>;
|
|
let updateStatusSpy: ReturnType<typeof vi.spyOn>;
|
|
let bulkStageSpy: ReturnType<typeof vi.spyOn>;
|
|
let insertFailedSpy: ReturnType<typeof vi.spyOn>;
|
|
|
|
beforeEach(() => {
|
|
loadSpy = vi.spyOn(dataAccess, 'loadTicketBundle');
|
|
runPipelineSpy = vi.spyOn(pipelineModule, 'runPipeline');
|
|
insertSpy = vi
|
|
.spyOn(persistence, 'insertAnalysis')
|
|
.mockResolvedValue({ id: 'an_uuid', analysis_version: 1 });
|
|
completeSpy = vi.spyOn(persistence, 'completeJob').mockResolvedValue();
|
|
failSpy = vi.spyOn(persistence, 'failJob').mockResolvedValue();
|
|
updateStatusSpy = vi.spyOn(persistence, 'updateJobStatus').mockResolvedValue();
|
|
bulkStageSpy = vi
|
|
.spyOn(persistence, 'bulkInsertStageExecutions')
|
|
.mockResolvedValue();
|
|
insertFailedSpy = vi
|
|
.spyOn(persistence, 'insertFailedAnalysis')
|
|
.mockResolvedValue({ id: 'failed_uuid', analysis_version: 1 });
|
|
vi.spyOn(persistence, 'updateAnalysisFingerprint').mockResolvedValue();
|
|
// Default Stage 6 to a no-op success in tests; worker treats failures as
|
|
// non-fatal anyway, so we just need it not to make real HTTP calls.
|
|
vi.spyOn(stage6Module, 'runFingerprintStage').mockRejectedValue(
|
|
new Error('fingerprint stub: tests do not exercise stage 6')
|
|
);
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
describe('analyzerWorker.runJob', () => {
|
|
it('writes the analysis row and marks the job complete on a happy-path run', async () => {
|
|
loadSpy.mockResolvedValueOnce(FIXTURE);
|
|
runPipelineSpy.mockResolvedValueOnce({
|
|
outcome: 'complete',
|
|
analysis: {
|
|
summary: 's',
|
|
timeline: [],
|
|
what_was_done: [],
|
|
what_should_have_been_done: [],
|
|
gaps: [],
|
|
next_step: 'next',
|
|
next_step_rationale: 'why',
|
|
post_resolution_analysis: null,
|
|
confidence_score: 0.8,
|
|
needs_human_review: false,
|
|
human_review_reasons: [],
|
|
ambiguities_for_opus: [],
|
|
itglue_docs_referenced: [],
|
|
},
|
|
pre: {
|
|
header: {
|
|
ticket_number: 'T20260424.0045',
|
|
autotask_ticket_id: 680282,
|
|
title: 't',
|
|
status_label: 'Waiting Customer',
|
|
priority_label: 'Minor Service',
|
|
queue: 'Level 2 Support',
|
|
account_name: 'Seubert and Associates',
|
|
contact_name: 'Tyler Lyster',
|
|
contact_email: 'tlyster@seubert.com',
|
|
created_at: '2026-04-24T12:53:50.163Z',
|
|
resolved_at: null,
|
|
},
|
|
events: [],
|
|
counts: {
|
|
total_events: 0,
|
|
customer_facing: 0,
|
|
internal_only: 0,
|
|
mixed: 0,
|
|
filtered_noise: 8,
|
|
},
|
|
content_hash: 'a'.repeat(64),
|
|
},
|
|
meta: {
|
|
haiku_used: true,
|
|
sonnet_used: true,
|
|
opus_used: false,
|
|
total_input_tokens: 6_000,
|
|
total_output_tokens: 800,
|
|
total_cache_creation_tokens: 0,
|
|
total_cache_read_tokens: 0,
|
|
estimated_cost_usd: 0.05,
|
|
cost_circuit_breaker_tripped: false,
|
|
},
|
|
filtered_noise_count: 8,
|
|
itglue_search_used: false,
|
|
itglue_org_resolved: false,
|
|
triage_response: {} as never,
|
|
sonnet_response: {} as never,
|
|
opus_response: null,
|
|
model_traces: {},
|
|
});
|
|
|
|
const result = await analyzerWorker.runJob(
|
|
'job_1',
|
|
'T20260424.0045',
|
|
'user_abc'
|
|
);
|
|
|
|
expect(result.outcome).toBe('complete');
|
|
expect(result.analysis_id).toBe('an_uuid');
|
|
expect(insertSpy).toHaveBeenCalledTimes(1);
|
|
expect(insertSpy.mock.calls[0][0]).toMatchObject({
|
|
ticket_number: 'T20260424.0045',
|
|
autotask_ticket_id: 680282,
|
|
triggered_by_user_id: 'user_abc',
|
|
status: 'complete',
|
|
haiku_used: true,
|
|
sonnet_used: true,
|
|
opus_used: false,
|
|
filtered_noise_count: 8,
|
|
});
|
|
expect(completeSpy).toHaveBeenCalledWith('job_1', 'an_uuid');
|
|
expect(failSpy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('completes the job pointing at the existing analysis on idempotent short-circuit', async () => {
|
|
loadSpy.mockResolvedValueOnce(FIXTURE);
|
|
runPipelineSpy.mockResolvedValueOnce({
|
|
outcome: 'idempotent_short_circuit',
|
|
existing_analysis_id: 'existing_id',
|
|
existing_analysis_version: 4,
|
|
pre: {
|
|
header: {
|
|
ticket_number: 'T20260424.0045',
|
|
autotask_ticket_id: 680282,
|
|
title: 't',
|
|
status_label: 'x',
|
|
priority_label: null,
|
|
queue: null,
|
|
account_name: null,
|
|
contact_name: null,
|
|
contact_email: null,
|
|
created_at: '2026-04-24T12:53:50.163Z',
|
|
resolved_at: null,
|
|
},
|
|
events: [],
|
|
counts: {
|
|
total_events: 0,
|
|
customer_facing: 0,
|
|
internal_only: 0,
|
|
mixed: 0,
|
|
filtered_noise: 0,
|
|
},
|
|
content_hash: 'b'.repeat(64),
|
|
},
|
|
});
|
|
|
|
const result = await analyzerWorker.runJob('job_1', 'T20260424.0045', null);
|
|
expect(result.outcome).toBe('idempotent_short_circuit');
|
|
expect(result.analysis_id).toBe('existing_id');
|
|
expect(insertSpy).not.toHaveBeenCalled();
|
|
expect(completeSpy).toHaveBeenCalledWith('job_1', 'existing_id');
|
|
});
|
|
|
|
it('marks the job failed with a clear message when the ticket is missing', async () => {
|
|
loadSpy.mockRejectedValueOnce(
|
|
new dataAccess.TicketNotFoundError('T20260424.0045')
|
|
);
|
|
|
|
const result = await analyzerWorker.runJob(
|
|
'job_1',
|
|
'T20260424.0045',
|
|
null
|
|
);
|
|
expect(result.outcome).toBe('failed');
|
|
expect(result.analysis_id).toBeNull();
|
|
expect(failSpy).toHaveBeenCalledTimes(1);
|
|
expect(failSpy.mock.calls[0][1]).toMatch(/not found in local mirror/);
|
|
expect(insertSpy).not.toHaveBeenCalled();
|
|
expect(completeSpy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('fails the job and persists the error message when the pipeline throws', async () => {
|
|
loadSpy.mockResolvedValueOnce(FIXTURE);
|
|
runPipelineSpy.mockRejectedValueOnce(new Error('LLM stage on claude-sonnet-4-6 failed twice'));
|
|
|
|
const result = await analyzerWorker.runJob('job_1', 'T20260424.0045', null);
|
|
expect(result.outcome).toBe('failed');
|
|
expect(failSpy).toHaveBeenCalledTimes(1);
|
|
expect(failSpy.mock.calls[0][1]).toMatch(/failed twice/);
|
|
});
|
|
|
|
it('drives updateJobStatus through the pipeline progress callbacks', async () => {
|
|
loadSpy.mockResolvedValueOnce(FIXTURE);
|
|
runPipelineSpy.mockImplementationOnce((async (
|
|
_input: unknown,
|
|
_deps: unknown,
|
|
callbacks: { onStage?: (stage: string) => Promise<void> | void } | undefined
|
|
) => {
|
|
await callbacks?.onStage?.('fetching');
|
|
await callbacks?.onStage?.('triaging');
|
|
await callbacks?.onStage?.('analyzing');
|
|
return {
|
|
outcome: 'complete',
|
|
analysis: {
|
|
summary: 's',
|
|
timeline: [],
|
|
what_was_done: [],
|
|
what_should_have_been_done: [],
|
|
gaps: [],
|
|
next_step: 'n',
|
|
next_step_rationale: 'r',
|
|
post_resolution_analysis: null,
|
|
confidence_score: 0.8,
|
|
needs_human_review: false,
|
|
human_review_reasons: [],
|
|
ambiguities_for_opus: [],
|
|
itglue_docs_referenced: [],
|
|
},
|
|
pre: {
|
|
header: {
|
|
ticket_number: 'T20260424.0045',
|
|
autotask_ticket_id: 680282,
|
|
title: 't',
|
|
status_label: null,
|
|
priority_label: null,
|
|
queue: null,
|
|
account_name: null,
|
|
contact_name: null,
|
|
contact_email: null,
|
|
created_at: '2026-04-24T12:53:50.163Z',
|
|
resolved_at: null,
|
|
},
|
|
events: [],
|
|
counts: {
|
|
total_events: 0,
|
|
customer_facing: 0,
|
|
internal_only: 0,
|
|
mixed: 0,
|
|
filtered_noise: 0,
|
|
},
|
|
content_hash: 'c'.repeat(64),
|
|
},
|
|
meta: {
|
|
haiku_used: true,
|
|
sonnet_used: true,
|
|
opus_used: false,
|
|
total_input_tokens: 0,
|
|
total_output_tokens: 0,
|
|
total_cache_creation_tokens: 0,
|
|
total_cache_read_tokens: 0,
|
|
estimated_cost_usd: 0,
|
|
cost_circuit_breaker_tripped: false,
|
|
},
|
|
filtered_noise_count: 0,
|
|
itglue_search_used: false,
|
|
itglue_org_resolved: false,
|
|
triage_response: {} as never,
|
|
sonnet_response: {} as never,
|
|
opus_response: null,
|
|
model_traces: {},
|
|
};
|
|
}) as never);
|
|
|
|
await analyzerWorker.runJob('job_1', 'T20260424.0045', null);
|
|
|
|
const stagesPassedToUpdate = (updateStatusSpy.mock.calls as Array<[string, string]>).map(
|
|
(c) => c[1]
|
|
);
|
|
expect(stagesPassedToUpdate).toEqual(['fetching', 'triaging', 'analyzing']);
|
|
});
|
|
});
|