/** * Persistence + runner for aggregate reports. * * Spec: docs/ticket-analyzer-phase2-spec.md → Sections D.4–D.6 */ import postgresClient from '@/lib/services/postgres-client'; import { type AggregateFingerprint, type AggregateReduceResponse, type AggregateReportStatus, type StageExecutionRecord, } from '@/lib/types/analyzer'; import { getITGlueClient } from '@/lib/services/itglue-client'; import { runAggregateReduceStage } from './stages/aggregate-reduce'; interface AggregateReportRow { id: string; generated_by_user_id: string | null; generated_at: Date; filter_criteria: unknown; analysis_ids: string[]; ticket_count: number; include_itglue_context: boolean; report_title: string | null; category_distribution: Record | null; client_distribution: Record | null; resolution_path_distribution: Record | null; root_cause_distribution: Record | null; date_range_actual: { earliest: string | null; latest: string | null } | null; documentation_gaps: unknown; process_gaps: unknown; client_patterns: unknown; recurrence_clusters: unknown; systemic_observations: unknown; recommended_actions: unknown; narrative_summary: string | null; executive_summary: string | null; total_input_tokens: number | null; total_output_tokens: number | null; estimated_cost_usd: string | null; model_used: string | null; itglue_context_included: boolean | null; status: AggregateReportStatus; error_message: string | null; expected_ticket_numbers: string[] | null; triggered_by_ticket_number: string | null; } export interface AggregateReportSummary { id: string; generatedByUserId: string | null; generatedAt: string; filterCriteria: unknown; analysisIds: string[]; ticketCount: number; includeItglueContext: boolean; reportTitle: string | null; status: AggregateReportStatus; errorMessage: string | null; // SQL outputs categoryDistribution: Record | null; clientDistribution: Record | null; resolutionPathDistribution: Record | null; rootCauseDistribution: Record | null; dateRangeActual: { earliest: string | null; latest: string | null } | null; // LLM outputs documentationGaps: unknown; processGaps: unknown; clientPatterns: unknown; recurrenceClusters: unknown; systemicObservations: unknown; recommendedActions: unknown; narrativeSummary: string | null; executiveSummary: string | null; // Cost totalInputTokens: number | null; totalOutputTokens: number | null; estimatedCostUsd: number | null; modelUsed: string | null; // Bundle (Phase 3) expectedTicketNumbers: string[] | null; triggeredByTicketNumber: string | null; } function rowToSummary(r: AggregateReportRow): AggregateReportSummary { return { id: r.id, generatedByUserId: r.generated_by_user_id, generatedAt: r.generated_at.toISOString(), filterCriteria: r.filter_criteria, analysisIds: r.analysis_ids, ticketCount: r.ticket_count, includeItglueContext: r.include_itglue_context, reportTitle: r.report_title, status: r.status, errorMessage: r.error_message, categoryDistribution: r.category_distribution, clientDistribution: r.client_distribution, resolutionPathDistribution: r.resolution_path_distribution, rootCauseDistribution: r.root_cause_distribution, dateRangeActual: r.date_range_actual, documentationGaps: r.documentation_gaps, processGaps: r.process_gaps, clientPatterns: r.client_patterns, recurrenceClusters: r.recurrence_clusters, systemicObservations: r.systemic_observations, recommendedActions: r.recommended_actions, narrativeSummary: r.narrative_summary, executiveSummary: r.executive_summary, totalInputTokens: r.total_input_tokens, totalOutputTokens: r.total_output_tokens, estimatedCostUsd: r.estimated_cost_usd === null ? null : Number(r.estimated_cost_usd), modelUsed: r.model_used, expectedTicketNumbers: r.expected_ticket_numbers, triggeredByTicketNumber: r.triggered_by_ticket_number, }; } const REPORT_SELECT = ` id::text AS id, generated_by_user_id, generated_at, filter_criteria, analysis_ids::text[] AS analysis_ids, ticket_count, include_itglue_context, report_title, category_distribution, client_distribution, resolution_path_distribution, root_cause_distribution, date_range_actual, documentation_gaps, process_gaps, client_patterns, recurrence_clusters, systemic_observations, recommended_actions, narrative_summary, executive_summary, total_input_tokens, total_output_tokens, estimated_cost_usd::text AS estimated_cost_usd, model_used, itglue_context_included, status, error_message, expected_ticket_numbers, triggered_by_ticket_number `; export interface CreateAggregateReportInput { generatedByUserId: string | null; filterCriteria: unknown; analysisIds: string[]; ticketCount: number; includeItglueContext: boolean; reportTitle: string | null; /** * Bundle mode (Phase 3): when set, the report is created in the * 'pending_analyses' state and the worker will transition it to 'pending' * once every expected ticket has a complete analysis. Leave undefined for * the legacy manual-multi-select flow. */ expectedTicketNumbers?: string[]; triggeredByTicketNumber?: string; } export async function createAggregateReport( input: CreateAggregateReportInput ): Promise<{ id: string }> { const isBundle = Array.isArray(input.expectedTicketNumbers) && input.expectedTicketNumbers.length > 0; const initialStatus = isBundle ? 'pending_analyses' : 'pending'; const res = await postgresClient.query<{ id: string }>( `INSERT INTO analyzer_aggregate_reports (generated_by_user_id, filter_criteria, analysis_ids, ticket_count, include_itglue_context, report_title, status, expected_ticket_numbers, triggered_by_ticket_number) VALUES ($1, $2::jsonb, $3::uuid[], $4, $5, $6, $7, $8::text[], $9) RETURNING id::text AS id`, [ input.generatedByUserId, JSON.stringify(input.filterCriteria), input.analysisIds, input.ticketCount, input.includeItglueContext, input.reportTitle, initialStatus, input.expectedTicketNumbers ?? null, input.triggeredByTicketNumber ?? null, ] ); return { id: res.rows[0].id }; } /** * Worker chain-trigger. * * Called after a single-ticket analysis completes successfully. For each * pending_analyses report waiting on this ticket: append the analysis_id (if * not already present), and if all expected tickets now have a complete * analysis, transition status='pending' and fire runAggregateReport. * * Idempotent: safe to invoke multiple times for the same analysis (the * deduplicating UPDATE skips no-ops; the status transition is gated on the * full set being present so the second call is a no-op). */ export async function chainTriggerForCompletedAnalysis( ticketNumber: string, analysisId: string ): Promise<{ readyReportIds: string[]; touchedReportIds: string[] }> { const res = await postgresClient.query<{ id: string; expected_ticket_numbers: string[]; analysis_ids: string[]; }>( `SELECT id::text AS id, expected_ticket_numbers, analysis_ids::text[] AS analysis_ids FROM analyzer_aggregate_reports WHERE status = 'pending_analyses' AND expected_ticket_numbers @> ARRAY[$1]::text[]`, [ticketNumber] ); const touched: string[] = []; const ready: string[] = []; for (const r of res.rows) { if (!r.analysis_ids.includes(analysisId)) { await postgresClient.query( `UPDATE analyzer_aggregate_reports SET analysis_ids = analysis_ids || $2::uuid WHERE id = $1 AND NOT (analysis_ids @> ARRAY[$2::uuid])`, [r.id, analysisId] ); touched.push(r.id); } // Re-check whether the full set is now satisfied: every expected ticket // must have at least one complete analysis whose id is in analysis_ids. // Reads the latest analysis_ids (the UPDATE above isn't reflected in the // copy we loaded earlier). const ready_check = await postgresClient.query<{ satisfied: boolean }>( `SELECT ( (SELECT COUNT(DISTINCT aa.ticket_number) FROM analyzer_analyses aa JOIN analyzer_aggregate_reports r ON r.id = $1 WHERE aa.id = ANY(r.analysis_ids) AND aa.status = 'complete' AND aa.ticket_number = ANY(r.expected_ticket_numbers)) = (SELECT array_length(expected_ticket_numbers, 1) FROM analyzer_aggregate_reports WHERE id = $1) ) AS satisfied`, [r.id] ); if (ready_check.rows[0]?.satisfied) { const transition = await postgresClient.query<{ id: string }>( `UPDATE analyzer_aggregate_reports SET status = 'pending' WHERE id = $1 AND status = 'pending_analyses' RETURNING id::text AS id`, [r.id] ); if (transition.rowCount && transition.rowCount > 0) { ready.push(r.id); } } } return { readyReportIds: ready, touchedReportIds: touched }; } export async function getAggregateReport( id: string ): Promise { const res = await postgresClient.query( `SELECT ${REPORT_SELECT} FROM analyzer_aggregate_reports WHERE id = $1`, [id] ); if (res.rowCount === 0) return null; return rowToSummary(res.rows[0]); } export async function listAggregateReports(opts: { limit?: number; offset?: number; generatedByUserId?: string; }): Promise { const limit = Math.min(opts.limit ?? 50, 200); const offset = opts.offset ?? 0; const params: unknown[] = [limit, offset]; let userClause = ''; if (opts.generatedByUserId) { params.push(opts.generatedByUserId); userClause = `WHERE generated_by_user_id = $${params.length}`; } const res = await postgresClient.query( `SELECT ${REPORT_SELECT} FROM analyzer_aggregate_reports ${userClause} ORDER BY generated_at DESC LIMIT $1 OFFSET $2`, params ); return res.rows.map(rowToSummary); } interface FingerprintRow { id: string; ticket_number: string; aggregate_fingerprint: AggregateFingerprint; triggered_at: Date; } async function loadFingerprints( analysisIds: string[] ): Promise<{ ticket_number: string; fingerprint: AggregateFingerprint; triggered_at: Date }[]> { if (analysisIds.length === 0) return []; const res = await postgresClient.query( `SELECT id::text AS id, ticket_number, aggregate_fingerprint, triggered_at FROM analyzer_analyses WHERE id = ANY($1::uuid[]) AND aggregate_fingerprint IS NOT NULL ORDER BY ticket_number, analysis_version DESC`, [analysisIds] ); return res.rows.map((r) => ({ ticket_number: r.ticket_number, fingerprint: r.aggregate_fingerprint, triggered_at: r.triggered_at, })); } function bucketCount(items: string[]): Record { const out: Record = {}; for (const i of items) out[i] = (out[i] ?? 0) + 1; return out; } async function fetchITGlueDocTitles( clientNames: string[] ): Promise<{ client_name: string; doc_titles: string[] }[]> { let client; try { client = getITGlueClient(); } catch { return []; // not configured — caller should fall back gracefully } const result: { client_name: string; doc_titles: string[] }[] = []; for (const name of clientNames) { try { const org = await client.findOrganizationByName(name); if (!org) continue; const docs = await client.getFlexibleAssetsForOrganization(org.id); const titles = docs .map((d) => (d as { name?: string }).name) .filter((t): t is string => typeof t === 'string') .slice(0, 50); result.push({ client_name: name, doc_titles: titles }); } catch { // Tolerate per-client failures. } } return result; } const ITGLUE_DOC_TITLE_CAP = 200; export async function bulkInsertReportStageExecutions( reportId: string, records: StageExecutionRecord[] ): Promise { if (records.length === 0) return; const values: unknown[] = [reportId]; const tuples: string[] = []; for (const r of records) { const base = values.length; values.push( r.stage, r.stage_order, r.model_id, JSON.stringify(r.input_payload ?? {}), JSON.stringify(r.output_payload ?? {}), r.input_tokens, r.output_tokens, r.latency_ms, r.started_at, r.completed_at, r.error_message ); tuples.push( `($1, $${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}::jsonb, ` + `$${base + 5}::jsonb, $${base + 6}, $${base + 7}, $${base + 8}, ` + `$${base + 9}, $${base + 10}, $${base + 11})` ); } await postgresClient.query( `INSERT INTO analyzer_stage_executions (aggregate_report_id, stage, stage_order, model_id, input_payload, output_payload, input_tokens, output_tokens, latency_ms, started_at, completed_at, error_message) VALUES ${tuples.join(', ')}`, values ); } /** * Fire-and-forget runner. Intended to be invoked from the POST endpoint with * `void runAggregateReport(id)` — the route returns immediately, this updates * the row when work completes (or fails). */ export async function runAggregateReport(reportId: string): Promise { const stageRecords: StageExecutionRecord[] = []; try { await postgresClient.query( `UPDATE analyzer_aggregate_reports SET status = 'running' WHERE id = $1`, [reportId] ); const report = await getAggregateReport(reportId); if (!report) throw new Error('report row vanished'); // ── Step 1: load fingerprints + compute SQL distributions ── const sqlStart = new Date(); const fingerprints = await loadFingerprints(report.analysisIds); if (fingerprints.length === 0) { throw new Error('no analyses with fingerprints found for the given IDs'); } const categories = bucketCount(fingerprints.map((f) => f.fingerprint.category)); const clients = bucketCount(fingerprints.map((f) => f.fingerprint.client_name)); const resolutionPaths = bucketCount( fingerprints.map((f) => f.fingerprint.resolution_path) ); const rootCauses = bucketCount( fingerprints.map((f) => f.fingerprint.root_cause_class) ); const dates = fingerprints.map((f) => f.triggered_at.getTime()); const dateRange = { earliest: new Date(Math.min(...dates)).toISOString(), latest: new Date(Math.max(...dates)).toISOString(), }; const sqlEnd = new Date(); stageRecords.push({ stage: 'analyze', // Reusing 'analyze' since CHECK constraint enumerates only stage names; a future migration could add 'aggregate_sql' / 'aggregate_reduce'. stage_order: 1, model_id: null, input_payload: { analysis_ids: report.analysisIds }, output_payload: { category_distribution: categories, client_distribution: clients, resolution_path_distribution: resolutionPaths, root_cause_distribution: rootCauses, date_range_actual: dateRange, fingerprint_count: fingerprints.length, }, input_tokens: null, output_tokens: null, latency_ms: sqlEnd.getTime() - sqlStart.getTime(), started_at: sqlStart, completed_at: sqlEnd, error_message: null, }); // Persist partial results immediately so UI can show distributions. await postgresClient.query( `UPDATE analyzer_aggregate_reports SET category_distribution = $2::jsonb, client_distribution = $3::jsonb, resolution_path_distribution = $4::jsonb, root_cause_distribution = $5::jsonb, date_range_actual = $6::jsonb WHERE id = $1`, [ reportId, JSON.stringify(categories), JSON.stringify(clients), JSON.stringify(resolutionPaths), JSON.stringify(rootCauses), JSON.stringify(dateRange), ] ); // ── Step 2: IT Glue context (optional) ── let itglueDocTitles: { client_name: string; doc_titles: string[] }[] | undefined; let itglueIncluded = false; if (report.includeItglueContext) { const uniqueClients = Array.from( new Set(fingerprints.map((f) => f.fingerprint.client_name)) ); const itglueStart = new Date(); itglueDocTitles = await fetchITGlueDocTitles(uniqueClients); // Cap to spec total (200 doc titles across all clients). let remaining = ITGLUE_DOC_TITLE_CAP; itglueDocTitles = itglueDocTitles.map((c) => { if (remaining <= 0) return { client_name: c.client_name, doc_titles: [] }; const titles = c.doc_titles.slice(0, remaining); remaining -= titles.length; return { client_name: c.client_name, doc_titles: titles }; }); itglueIncluded = itglueDocTitles.some((c) => c.doc_titles.length > 0); const itglueEnd = new Date(); stageRecords.push({ stage: 'itglue', stage_order: 2, model_id: null, input_payload: { client_count: uniqueClients.length }, output_payload: { doc_count: itglueDocTitles.reduce((a, c) => a + c.doc_titles.length, 0) }, input_tokens: null, output_tokens: null, latency_ms: itglueEnd.getTime() - itglueStart.getTime(), started_at: itglueStart, completed_at: itglueEnd, error_message: null, }); } // ── Step 3: reduce LLM call ── // Honour the provider the bundle was created with. Manual aggregate reports // (no provider in filter_criteria) default to anthropic. const reduceProvider: 'anthropic' | 'openrouter' = (report.filterCriteria as { provider?: string } | null)?.provider === 'openrouter' ? 'openrouter' : 'anthropic'; const reduceStart = new Date(); let reduceResult; try { reduceResult = await runAggregateReduceStage( { distributions: { category_distribution: categories, client_distribution: clients, resolution_path_distribution: resolutionPaths, root_cause_distribution: rootCauses, date_range_actual: dateRange, }, fingerprints: fingerprints.map((f) => ({ ticket_number: f.ticket_number, fingerprint: f.fingerprint, })), itglue_doc_titles: itglueDocTitles, }, { provider: reduceProvider } ); } catch (err) { const reduceEnd = new Date(); stageRecords.push({ stage: 'analyze', stage_order: 3, model_id: null, input_payload: { fingerprint_count: fingerprints.length }, output_payload: {}, input_tokens: null, output_tokens: null, latency_ms: reduceEnd.getTime() - reduceStart.getTime(), started_at: reduceStart, completed_at: reduceEnd, error_message: err instanceof Error ? err.message : String(err), }); throw err; } const reduceEnd = new Date(); stageRecords.push({ stage: 'analyze', stage_order: 3, model_id: reduceResult.model_used, input_payload: { fingerprint_count: fingerprints.length }, output_payload: reduceResult.data, input_tokens: reduceResult.usage.input_tokens, output_tokens: reduceResult.usage.output_tokens, latency_ms: reduceEnd.getTime() - reduceStart.getTime(), started_at: reduceStart, completed_at: reduceEnd, error_message: null, }); // ── Step 4: persist outputs ── await postgresClient.query( `UPDATE analyzer_aggregate_reports SET documentation_gaps = $2::jsonb, process_gaps = $3::jsonb, client_patterns = $4::jsonb, recurrence_clusters = $5::jsonb, systemic_observations = $6::jsonb, recommended_actions = $7::jsonb, narrative_summary = $8, executive_summary = $9, total_input_tokens = $10, total_output_tokens = $11, estimated_cost_usd = $12, model_used = $13, itglue_context_included = $14, status = 'complete' WHERE id = $1`, [ reportId, JSON.stringify(reduceResult.data.documentation_gaps), JSON.stringify(reduceResult.data.process_gaps), JSON.stringify(reduceResult.data.client_patterns), JSON.stringify(reduceResult.data.recurrence_clusters), JSON.stringify(reduceResult.data.systemic_observations), JSON.stringify(reduceResult.data.recommended_actions), reduceResult.data.narrative_summary, reduceResult.data.executive_summary, reduceResult.usage.input_tokens, reduceResult.usage.output_tokens, reduceResult.estimated_cost_usd, reduceResult.model_used, itglueIncluded, ] ); await bulkInsertReportStageExecutions(reportId, stageRecords); } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(`[ANALYZER-REPORT] runAggregateReport ${reportId} failed:`, message); await postgresClient .query( `UPDATE analyzer_aggregate_reports SET status = 'failed', error_message = $2 WHERE id = $1`, [reportId, message] ) .catch(() => {}); if (stageRecords.length > 0) { await bulkInsertReportStageExecutions(reportId, stageRecords).catch(() => {}); } } }