/** * Analyzer job worker. * * Polls `analyzer_jobs` for queued rows, runs the full pipeline, persists the * result, and updates the job's status / result_analysis_id throughout. * * Concurrency model: a single in-process polling loop. Multiple Next.js * workers will all import this module, but `claimQueuedJob()` uses * `FOR UPDATE SKIP LOCKED` so they cooperate at the row level — each job is * processed exactly once. * * Auto-start: the worker self-initializes on first server-side import in * production. In dev / tests, set ANALYZER_WORKER_AUTOSTART=1 to opt in. */ import { bulkInsertStageExecutions, claimQueuedJob, completeJob, failJob, insertAnalysis, insertFailedAnalysis, resetStaleJobsToQueued, updateAnalysisFingerprint, updateJobStatus, } from './persistence'; import { loadTicketBundle, TicketNotFoundError } from './data-access'; import { runPipeline, type PipelineResult } from './pipeline'; import { runFingerprintStage } from './stages/stage6-fingerprint'; import { chainTriggerForCompletedAnalysis, runAggregateReport, } from './aggregate-persistence'; import { insertReferencedXrefsFromAnalysis } from './asset-audit/xrefs'; import { stageModelsFor, type Provider } from '@/lib/services/llm/models'; import type { PreprocessedTicket, StageExecutionRecord, } from '@/lib/types/analyzer'; const POLL_INTERVAL_MS = 2_000; const STALE_JOB_RESET_MINUTES = 10; class AnalyzerWorker { private timer: NodeJS.Timeout | null = null; private running = false; private inFlight = false; async start(): Promise { if (this.running) return; this.running = true; console.log('[ANALYZER-WORKER] starting; polling every 2s'); // Reclaim jobs orphaned by a previous restart. Any active-state job // whose started_at is older than the pipeline ceiling is presumed // orphaned and gets reset to 'queued' so this worker can re-claim it. try { const reset = await resetStaleJobsToQueued(STALE_JOB_RESET_MINUTES); if (reset > 0) { console.log( `[ANALYZER-WORKER] reset ${reset} stale in-flight job(s) to queued (older than ${STALE_JOB_RESET_MINUTES}min)` ); } } catch (err) { console.error('[ANALYZER-WORKER] stale-job reset failed:', err); } this.scheduleNextPoll(0); } async stop(): Promise { this.running = false; if (this.timer) { clearTimeout(this.timer); this.timer = null; } } private scheduleNextPoll(delay: number): void { if (!this.running) return; this.timer = setTimeout(() => { void this.poll(); }, delay); } private async poll(): Promise { if (!this.running) return; if (this.inFlight) { // Avoid overlap if a previous poll is still running. this.scheduleNextPoll(POLL_INTERVAL_MS); return; } this.inFlight = true; try { const claimed = await claimQueuedJob(); if (claimed) { await this.runJob( claimed.id, claimed.ticket_number, claimed.queued_by_user_id, claimed.provider ); } } catch (err) { console.error('[ANALYZER-WORKER] poll error:', err); } finally { this.inFlight = false; this.scheduleNextPoll(POLL_INTERVAL_MS); } } /** * Process a single job end-to-end. Public so tests + manual triggers can * call it directly without going through the poll loop. */ async runJob( jobId: string, ticketNumber: string, triggeredByUserId: string | null, provider: Provider = 'anthropic' ): Promise<{ analysis_id: string | null; outcome: PipelineResult['outcome'] | 'failed' }> { // Phase 2: collect per-stage records as the pipeline runs, plus the // preprocessed bundle, so we can persist a failed analyzer_analyses row // (with source_snapshot) when a stage throws. const stageRecords: StageExecutionRecord[] = []; let capturedPre: PreprocessedTicket | null = null; try { const bundle = await loadTicketBundle(ticketNumber); const result = await runPipeline( { bundle, force: false, provider }, {}, { onStage: (stage) => updateJobStatus(jobId, stage), onStageRecord: (rec) => { stageRecords.push(rec); }, onPreprocessed: (pre) => { capturedPre = pre; }, } ); if (result.outcome === 'idempotent_short_circuit') { // Point the job at the existing analysis so the UI can navigate to it. await completeJob(jobId, result.existing_analysis_id); // Chain-trigger may need to run here too: the bundle endpoint queues // jobs for tickets whose content hash didn't match a complete row, but // a parallel analysis may have completed between then and now. try { const chain = await chainTriggerForCompletedAnalysis( ticketNumber, result.existing_analysis_id ); for (const reportId of chain.readyReportIds) { void runAggregateReport(reportId).catch((err) => { console.error( `[ANALYZER-WORKER] aggregate report ${reportId} runner threw:`, err ); }); } } catch (chainErr) { console.error( '[ANALYZER-WORKER] chain-trigger (short-circuit) failed:', chainErr ); } return { analysis_id: result.existing_analysis_id, outcome: 'idempotent_short_circuit', }; } const inserted = await insertAnalysis({ ticket_number: result.pre.header.ticket_number, autotask_ticket_id: result.pre.header.autotask_ticket_id, content_hash: result.pre.content_hash, triggered_by_user_id: triggeredByUserId, status: 'complete', provider, haiku_used: result.meta.haiku_used, sonnet_used: result.meta.sonnet_used, opus_used: result.meta.opus_used, total_input_tokens: result.meta.total_input_tokens, total_output_tokens: result.meta.total_output_tokens, estimated_cost_usd: result.meta.estimated_cost_usd, analysis: result.analysis, filtered_noise_count: result.filtered_noise_count, model_traces: result.model_traces, source_snapshot: result.pre.events, }); // Stage 6 — fingerprint. Failure-tolerant: log and continue. const fingerprintModel = stageModelsFor(provider).fingerprint; const fpStart = new Date(); let fpInputTokens: number | null = null; let fpOutputTokens: number | null = null; let fpOutput: unknown = {}; let fpErr: Error | null = null; try { const fp = await runFingerprintStage( { triage: result.triage_response, sonnet: result.sonnet_response, opus: result.opus_response, }, undefined, fingerprintModel ); fpInputTokens = fp.usage.input_tokens; fpOutputTokens = fp.usage.output_tokens; fpOutput = fp.data; await updateAnalysisFingerprint(inserted.id, fp.data); } catch (err) { fpErr = err instanceof Error ? err : new Error(String(err)); console.warn( `[ANALYZER-WORKER] fingerprint failed for analysis ${inserted.id}: ${fpErr.message}` ); } const fpEnd = new Date(); stageRecords.push({ stage: 'fingerprint', stage_order: 6, model_id: fingerprintModel, input_payload: { triage_category: result.triage_response.category, ticket_number: result.pre.header.ticket_number, opus_used: result.opus_response !== null, }, output_payload: fpErr ? {} : fpOutput, input_tokens: fpInputTokens, output_tokens: fpOutputTokens, latency_ms: fpEnd.getTime() - fpStart.getTime(), started_at: fpStart, completed_at: fpEnd, error_message: fpErr ? fpErr.message : null, }); await bulkInsertStageExecutions(inserted.id, stageRecords); await completeJob(jobId, inserted.id); // Phase 4.1: ingest xref rows for every IT Glue doc the analyzer cited. // Best-effort; never fail the job on xref failure. try { const refs = result.analysis.itglue_docs_referenced ?? []; if (refs.length > 0) { await insertReferencedXrefsFromAnalysis({ ticketNumber: result.pre.header.ticket_number, analysisId: inserted.id, references: refs.map((r) => ({ id: r.id, name: r.name, url: r.url, doc_type: r.doc_type, relevance_reason: r.relevance_reason, })), }); } } catch (xrefErr) { console.warn( `[ANALYZER-WORKER] xref ingestion failed for analysis ${inserted.id}:`, xrefErr instanceof Error ? xrefErr.message : xrefErr ); } // Chain-trigger any pending_analyses bundles waiting on this ticket. // Best-effort: a failure here must not fail the job. try { const chain = await chainTriggerForCompletedAnalysis( result.pre.header.ticket_number, inserted.id ); for (const reportId of chain.readyReportIds) { void runAggregateReport(reportId).catch((err) => { console.error( `[ANALYZER-WORKER] aggregate report ${reportId} runner threw:`, err ); }); } } catch (chainErr) { console.error( '[ANALYZER-WORKER] chain-trigger failed (job already complete):', chainErr ); } return { analysis_id: inserted.id, outcome: 'complete' }; } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error( `[ANALYZER-WORKER] job ${jobId} (${ticketNumber}) failed:`, message ); const reason = err instanceof TicketNotFoundError ? `Ticket ${ticketNumber} not found in local mirror — confirm sync is current.` : message; // Phase 2: when we have a preprocessed bundle, persist a 'failed' // analyzer_analyses row with source_snapshot + accumulated stage rows. // Best-effort — if this fails we still fail the job below. if (capturedPre !== null) { const pre: PreprocessedTicket = capturedPre; try { const failedAnalysis = await insertFailedAnalysis({ ticket_number: pre.header.ticket_number, autotask_ticket_id: pre.header.autotask_ticket_id, content_hash: pre.content_hash, triggered_by_user_id: triggeredByUserId, source_snapshot: pre.events, filtered_noise_count: pre.counts.filtered_noise, error_message: reason, partial_input_tokens: 0, partial_output_tokens: 0, partial_cost_usd: 0, haiku_used: stageRecords.some((r) => r.stage === 'triage'), sonnet_used: stageRecords.some((r) => r.stage === 'analyze'), opus_used: stageRecords.some((r) => r.stage === 'deep_review'), provider, }); await bulkInsertStageExecutions(failedAnalysis.id, stageRecords); } catch (persistErr) { console.error( `[ANALYZER-WORKER] failed to persist failed-analysis row for job ${jobId}:`, persistErr ); } } await failJob(jobId, reason); return { analysis_id: null, outcome: 'failed' }; } } } export const analyzerWorker = new AnalyzerWorker(); /** * Should we auto-start the worker on import? * - Skip in browsers (typeof window check). * - Skip during vitest. * - Auto-start in production by default. * - Otherwise opt in via ANALYZER_WORKER_AUTOSTART=1. */ function shouldAutoStart(): boolean { if (typeof window !== 'undefined') return false; if (process.env.VITEST === 'true') return false; if (process.env.NODE_ENV === 'production') return true; return process.env.ANALYZER_WORKER_AUTOSTART === '1'; } if (shouldAutoStart()) { analyzerWorker.start().catch((err) => { console.error('[ANALYZER-WORKER] failed to auto-start:', err); }); } export const _WORKER_INTERNALS = { POLL_INTERVAL_MS };