fix(analyzer): reset stale in-flight jobs on worker boot

A container restart leaves analyzer_jobs rows stuck in
fetching/triaging/itglue/analyzing/deep_review forever — the worker's
claimQueuedJob only picks up status='queued', so a job mid-pipeline
when the process died gets orphaned.

resetStaleJobsToQueued() reverts any active-state row whose started_at
is older than 10 min back to 'queued' with started_at=NULL. The worker
calls it once on start() before scheduling the first poll. 10 min is
3x the realistic pipeline ceiling — well past Sonnet+Opus combined.

Logs the count when nonzero so restarts that recover work are visible.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
lorentz 2026-04-29 14:56:00 -04:00
parent a0a6e7f192
commit 378e68ad8a
2 changed files with 39 additions and 0 deletions

View file

@ -329,6 +329,28 @@ export async function completeJob(
);
}
/**
* Reclaim orphaned in-flight jobs whose started_at is older than the threshold.
* Called once on worker boot any job in an active state (fetching/triaging/
* itglue/analyzing/deep_review) that's been "running" longer than the expected
* pipeline ceiling is almost certainly orphaned by a container restart and
* needs to be re-queued. Returns the number of rows reset.
*/
export async function resetStaleJobsToQueued(
thresholdMinutes = 10
): Promise<number> {
const res = await postgresClient.query(
`UPDATE analyzer_jobs
SET status = 'queued',
started_at = NULL
WHERE status IN ('fetching','triaging','itglue','analyzing','deep_review')
AND started_at IS NOT NULL
AND started_at < NOW() - ($1::int || ' minutes')::interval`,
[thresholdMinutes]
);
return res.rowCount ?? 0;
}
export async function failJob(jobId: string, errorMessage: string): Promise<void> {
await postgresClient.query(
`UPDATE analyzer_jobs

View file

@ -20,6 +20,7 @@ import {
failJob,
insertAnalysis,
insertFailedAnalysis,
resetStaleJobsToQueued,
updateAnalysisFingerprint,
updateJobStatus,
} from './persistence';
@ -33,6 +34,7 @@ import type {
} from '@/lib/types/analyzer';
const POLL_INTERVAL_MS = 2_000;
const STALE_JOB_RESET_MINUTES = 10;
class AnalyzerWorker {
private timer: NodeJS.Timeout | null = null;
@ -43,6 +45,21 @@ class AnalyzerWorker {
if (this.running) return;
this.running = true;
console.log('[ANALYZER-WORKER] starting; polling every 2s');
// Reclaim jobs orphaned by a previous restart. Any active-state job
// whose started_at is older than the pipeline ceiling is presumed
// orphaned and gets reset to 'queued' so this worker can re-claim it.
try {
const reset = await resetStaleJobsToQueued(STALE_JOB_RESET_MINUTES);
if (reset > 0) {
console.log(
`[ANALYZER-WORKER] reset ${reset} stale in-flight job(s) to queued (older than ${STALE_JOB_RESET_MINUTES}min)`
);
}
} catch (err) {
console.error('[ANALYZER-WORKER] stale-job reset failed:', err);
}
this.scheduleNextPoll(0);
}