chore: merge executor worktree (worktree-agent-a30d26dba3410e0da) — plan 24-04

.planning/phases/24-aws-route-53-dns-sync-track-changes-crud-operations-full-aud/24-04-SUMMARY.md

@@ -0,0 +1,182 @@
+---
+phase: 24-aws-route-53-dns-sync-track-changes-crud-operations-full-aud
+plan: 04
+subsystem: aws-route53
+tags: [route53, integration-health, dns, node-dns, delegation-check]
+dependency-graph:
+  requires:
+    - "lib/services/route53-factory.ts (isRoute53Configured / getRoute53Client, plan 24-01)"
+    - "lib/types/route53.ts (plan 24-01)"
+    - "migrations/102_route53_tables.sql (route53_zones, plan 24-01)"
+  provides:
+    - "lib/services/route53-dns-delegation.ts (normalizeNsList / compareNsDelegation / resolveLiveNs / checkAllZoneDelegations)"
+    - "checkRoute53() registered in lib/services/integration-health.ts"
+    - "HealthStatus union gained 'degraded' member"
+    - "IntegrationHealth gained nsDelegationMismatches? / nsDelegationErrors? fields"
+  affects:
+    - "app/api/dashboard/integration-health/route.ts (now returns a route53 entry)"
+    - "/admin/integrations and /status pages (consume the health list; both already fall back safely on an unrecognized status string)"
+tech-stack:
+  added: []
+  patterns:
+    - "dedicated dns.Resolver() instance pinned to public resolvers, never the process-global dns module (D-12, T-24-13)"
+    - "bounded-concurrency worker-pool batch check (checkAllZoneDelegations, T-24-14)"
+key-files:
+  created:
+    - lib/services/route53-dns-delegation.ts
+    - lib/services/route53-dns-delegation.test.ts
+  modified:
+    - lib/services/integration-health.ts
+decisions:
+  - "Used Node's dns module directly (Resolver + setServers(['1.1.1.1','8.8.8.8'])), not the DoH-over-HTTPS fallback — plan 24-01's checkpoint confirmed EGRESS-OK from inside the pulse-app container."
+  - "Chosen degraded-state HealthStatus member: added a new 'degraded' member to the union (no existing member represented 'reachable, authenticated, but a secondary check found a problem') — every existing member was either a full pass ('ok'/'unknown'), a hard failure ('auth_failed'/'unreachable'/'not_configured'), or operator-suppressed ('disabled')."
+  - "Implemented a local, private sanitizeAwsError()/isAwsAuthError() inside integration-health.ts instead of importing from lib/services/route53-record-validation.ts — that file is scoped to plan 24-03, which runs in a sibling parallel worktree and was not available in this isolated worktree (24-04's depends_on only lists 24-01). See Deviations below."
+metrics:
+  duration: "~45 min, 2 tasks, TDD RED/GREEN on Task 1"
+  completed: "2026-08-05"
+---
+
+# Phase 24 Plan 4: Route 53 Integration Health + D-12 DNS Delegation Check Summary
+
+Added `checkRoute53()` to the existing integration-health aggregator (auth probe via
+`ListHostedZonesCommand`) plus a D-12-specific extension: every synced hosted zone's
+Route-53-authoritative name servers are compared against a **live public DNS lookup**
+(dedicated `dns.Resolver()` pinned to `1.1.1.1`/`8.8.8.8`, never the process-global
+resolver) and a mismatch degrades the reported health to a new `'degraded'`
+`HealthStatus` member — no manually-maintained "expected NS" field anywhere.
+
+## What Was Built
+
+**Task 1 — `lib/services/route53-dns-delegation.ts` (TDD RED/GREEN):**
+- Pure half, fully unit-tested (13/13 assertions, no network I/O in tests):
+  - `normalizeNsList(input: unknown): string[]` — lowercase, strip trailing dot,
+    de-dupe, sort; `[]` for `null`/`undefined`/non-array input.
+  - `compareNsDelegation(authoritative, live)` — set-diff mismatch detection.
+    Empty authoritative list is treated as unjudgeable (`mismatch: false`, no false
+    alarm); a non-empty authoritative list with an empty live answer is a real
+    delegation problem (`mismatch: true`).
+- I/O half (not unit-tested per plan instruction — network calls are flaky in CI;
+  covered by manual verification in 24-VALIDATION.md):
+  - `resolveLiveNs(domain, timeoutMs=5000)` — constructs `new Resolver()` (callback
+    API from `'dns'`) and calls `.setServers(['1.1.1.1','8.8.8.8'])` **on that
+    instance only**. The process-global `dns.setServers()` is never called anywhere
+    in this file (T-24-13) — verified by grep in the acceptance criteria.
+  - `checkAllZoneDelegations(zones, opts?)` — bounded-concurrency (default 5) worker
+    pool over the zone list; a lookup error yields `{ mismatch: false, error }` (an
+    unreachable resolver is an infra problem, not delegation drift — T-24-14); zones
+    whose `authoritativeNameServers` normalizes to `[]` are skipped.
+
+**Task 2 — `checkRoute53()` registered in `lib/services/integration-health.ts`:**
+- `IntegrationHealth` gained two optional fields: `nsDelegationMismatches?: string[] | null`
+  and `nsDelegationErrors?: string[] | null` (kept separate from `error`, per 24-PATTERNS.md).
+- `checkRoute53()` placed next to `checkDattoRmm()`: config gate via `isRoute53Configured()`
+  → `getRoute53Client().send(new ListHostedZonesCommand({ MaxItems: 1 }))` auth probe,
+  timed for `latencyMs`. Auth errors (`InvalidClientTokenId`, `SignatureDoesNotMatch`,
+  `AccessDenied`, `UnrecognizedClientException`, or HTTP 401/403 via `$metadata.httpStatusCode`)
+  map to `status: 'auth_failed'`; any other error maps to `'unreachable'`. Both branches
+  redact the error through a local `sanitizeAwsError()` before it reaches `IntegrationHealth.error`.
+- D-12 delegation step: `SELECT id, name, authoritative_name_servers FROM route53_zones
+  WHERE is_deleted = false ORDER BY name LIMIT 50`, fed to `checkAllZoneDelegations()`.
+  A non-empty mismatch list downgrades `status` from `'ok'` to the new `'degraded'`
+  member and sets a summary `error` string (`'NS delegation mismatch for N zone(s): ...'`,
+  noting truncation if the 50-zone cap was hit). The entire delegation step is wrapped
+  in its own try/catch — a Postgres failure or blocked resolver degrades to
+  `nsDelegationErrors` and leaves the auth-probe status untouched, never throwing out
+  of `checkIntegrationHealth()`'s `Promise.all` (T-24-16).
+- Registered `checkRoute53(),` as a bare (unwrapped) entry in the `Promise.all` array
+  alongside `checkAutotask()` / `checkDattoRmm()` / `checkItglue()` / `checkS1()`.
+- No changes to `applyDisableOverlay()` — it already keys off `item.key` generically,
+  so `'route53'` is covered automatically (D-10, display-only; `integration_settings`
+  grep count unchanged from before this task).
+- **Bug fix (Rule 1):** `summarize()`'s status-bucketing `if/else if` chain didn't
+  account for the new `'degraded'` status — it would have silently fallen through
+  uncounted (not `ok`, not `failed`, not `notConfigured`), breaking the invariant that
+  bucket counts sum to `total`. Added `'degraded'` to the `failed` bucket (and thus
+  `hasIssues`) alongside `'auth_failed'`/`'unreachable'`.
+- **Type fix (Rule 3):** the plan's example passed `MaxItems: '1'` (string) to
+  `ListHostedZonesCommand`; this SDK version (`@aws-sdk/client-route-53` ^3.1104.0)
+  types `MaxItems` as `number`. Changed to `MaxItems: 1`.
+
+## Deviations from Plan
+
+### Auto-fixed Issues
+
+**1. [Rule 1 - Bug] `summarize()` didn't bucket the new `'degraded'` status**
+- Found during: Task 2, after adding `'degraded'` to `HealthStatus`.
+- Fix: added `'degraded'` to the existing `failed`-bucket branch.
+- Files modified: `lib/services/integration-health.ts`
+- Commit: ea04672
+
+**2. [Rule 3 - Blocking issue] `ListHostedZonesCommand({ MaxItems: '1' })` failed `tsc`**
+- Found during: Task 2 verification (`npx tsc --noEmit`).
+- Issue: this SDK version types `MaxItems` as `number`, not `string` as the plan's
+  action text described.
+- Fix: `MaxItems: 1`.
+- Files modified: `lib/services/integration-health.ts`
+- Commit: ea04672
+
+**3. [Rule 3 - Blocking issue] `lib/services/route53-record-validation.ts` (plan 24-03)
+   does not exist in this worktree**
+- Found during: Task 2, reading `<read_first>` / `<action>` which reference
+  `sanitizeAwsError` from that file.
+- Why: this plan's `depends_on` frontmatter lists only `24-01`; plan 24-03 (which owns
+  `sanitizeAwsError`) runs concurrently in a sibling parallel worktree in this same wave
+  and is not merged into this worktree's history.
+- Fix: implemented a local, private `sanitizeAwsError()` + `isAwsAuthError()` pair
+  inside `lib/services/integration-health.ts`, using the **identical redaction rules**
+  spec'd in 24-03-PLAN.md (`AKIA[0-9A-Z]{16}` → `[redacted-key-id]`, `arn:aws:[^\s"']+`
+  → `[redacted-arn]`, 12-digit account ids → `[redacted-account-id]`, truncate to 500
+  chars) so behavior is consistent regardless of which implementation ships. Documented
+  inline with a NOTE comment pointing at this deviation.
+- **Follow-up for a human/future plan:** once 24-03 lands on `master`, the local copy in
+  `integration-health.ts` should be replaced with an import from
+  `@/lib/services/route53-record-validation` to keep a single source of truth — flagging
+  this explicitly since it is a cross-plan duplication introduced by parallel worktree
+  execution, not by design.
+- Files modified: `lib/services/integration-health.ts`
+- Commit: ea04672
+
+### Verification Note (not a deviation)
+
+The plan's Task 2 acceptance criteria includes an optional curl check against the running
+`pulse-app` container's `/api/dashboard/integration-health` route. That container
+(confirmed via `docker inspect`) has no source-code volume mount — it runs a pre-built
+standalone image from before this plan's commits, and the route additionally redirects
+unauthenticated requests (307) per `middleware.ts`. Live end-to-end verification against
+the running container was therefore not performed in this worktree; `npx tsc --noEmit`,
+`npx vitest run lib/services/route53-dns-delegation.test.ts` (13/13 passing), and the full
+`npm test` suite (480/482 passing — the 2 failures are pre-existing/unrelated, see below)
+are the verifications actually run. A container rebuild + authenticated curl is left to
+the orchestrator/human at merge time if desired.
+
+### Out-of-Scope Discovery (logged, not fixed)
+
+`npm test` (full suite) surfaced the same 2 pre-existing failures in
+`lib/services/analyzer/itglue-search.test.ts` already logged in this phase's
+`deferred-items.md` by plan 24-01. Neither that file nor `itglue-search.ts` were touched
+by this plan.
+
+## Self-Check: PASSED
+
+All created/modified files confirmed present:
+- FOUND: lib/services/route53-dns-delegation.ts
+- FOUND: lib/services/route53-dns-delegation.test.ts
+- FOUND: lib/services/integration-health.ts (modified)
+
+All commits confirmed present in `git log`:
+- 7396f07 test(24-04): add failing test for NS normalization and delegation comparison
+- 06ebae5 feat(24-04): implement NS normalization and delegation-comparison module
+- ea04672 feat(24-04): register checkRoute53() in the integration health aggregator
+
+## TDD Gate Compliance
+
+Task 1 followed RED → GREEN: `test(24-04)` commit (7396f07) precedes the `feat(24-04)`
+implementation commit (06ebae5); no REFACTOR commit was needed (implementation matched
+the test contract on first pass). Task 2 is `type="auto"` without `tdd="true"` per the
+plan, so no RED/GREEN gate applied there — verified with `tsc` + full `npm test` instead.
+
+## Threat Flags
+
+None beyond what's already covered by this plan's own `<threat_model>` (T-24-13, T-24-14,
+T-24-03, T-24-15, T-24-16 — all addressed as designed, see "What Was Built" above). No new
+network endpoints, auth paths, or schema changes were introduced outside that register.

lib/services/integration-health.ts

@@ -11,12 +11,19 @@
  * Usage:
  *   const results = await checkIntegrationHealth();
  *
- * Tools covered live: S1, Datto RMM, IT Glue, Autotask. Others report
- * configured / not_configured only — extending to live checks is mechanical.
+ * Tools covered live: S1, Datto RMM, IT Glue, Autotask, AWS Route 53. Others
+ * report configured / not_configured only — extending to live checks is
+ * mechanical.
  */
 
+import { ListHostedZonesCommand } from '@aws-sdk/client-route-53';
+import { isRoute53Configured, getRoute53Client } from '@/lib/services/route53-factory';
+import { checkAllZoneDelegations } from '@/lib/services/route53-dns-delegation';
+
 export type HealthStatus =
   | 'ok'              // configured, auth succeeded
+  | 'degraded'        // configured, auth succeeded, but a secondary check found a problem
+                      // (e.g. Route 53 D-12 NS-delegation mismatch) — reachable, not fully healthy
   | 'auth_failed'     // configured, server returned 401/403
   | 'unreachable'     // configured, network/DNS/TLS error
   | 'not_configured'  // env vars missing
@@ -40,6 +47,10 @@ export interface IntegrationHealth {
   error?: string | null;
   tokenExpiry?: TokenExpiry | null;
   checkedAt: string;
+  /** D-12: zone names whose live NS answer mismatches Route 53's authoritative NS list. */
+  nsDelegationMismatches?: string[] | null;
+  /** D-12: zone names whose live NS lookup failed (infrastructure problem, not a mismatch). */
+  nsDelegationErrors?: string[] | null;
 }
 
 interface CacheEntry {
@@ -190,6 +201,115 @@ async function checkDattoRmm(): Promise<IntegrationHealth> {
   }
 }
 
+/**
+ * Redact an AWS SDK error down to a message safe to surface in the health
+ * API response and (eventually) route53_audit_log.error_message.
+ *
+ * NOTE: this duplicates the `sanitizeAwsError` spec'd for
+ * `lib/services/route53-record-validation.ts` in plan 24-03 (T-24-03). That
+ * file did not exist in this plan's isolated worktree at execution time
+ * (24-03 runs in a sibling parallel worktree and depends_on for this plan
+ * only lists 24-01) — see this plan's SUMMARY "Deviations" section. This
+ * local copy uses the identical redaction rules so behavior is consistent
+ * regardless of which implementation ships; if 24-03 lands first in a
+ * future merge, this local copy should be replaced with an import from
+ * `@/lib/services/route53-record-validation` for a single source of truth.
+ */
+function sanitizeAwsError(err: unknown): string {
+  const message = err instanceof Error ? err.message : String(err);
+  return message
+    .replace(/AKIA[0-9A-Z]{16}/g, '[redacted-key-id]')
+    .replace(/arn:aws:[^\s"']+/g, '[redacted-arn]')
+    .replace(/\b[0-9]{12}\b/g, '[redacted-account-id]')
+    .slice(0, 500);
+}
+
+const AWS_AUTH_ERROR_NAMES = new Set([
+  'InvalidClientTokenId',
+  'SignatureDoesNotMatch',
+  'AccessDenied',
+  'UnrecognizedClientException',
+]);
+
+function isAwsAuthError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  const name = (err as Error & { name?: string }).name;
+  if (name && AWS_AUTH_ERROR_NAMES.has(name)) return true;
+  const httpStatusCode = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
+  return httpStatusCode === 401 || httpStatusCode === 403;
+}
+
+const ROUTE53_ZONE_CHECK_LIMIT = 50;
+
+async function checkRoute53(): Promise<IntegrationHealth> {
+  const checkedAt = new Date().toISOString();
+  if (!isRoute53Configured()) {
+    return {
+      key: 'route53', name: 'AWS Route 53', category: 'network',
+      status: 'not_configured', configured: false, checkedAt,
+    };
+  }
+
+  const start = Date.now();
+  let status: HealthStatus;
+  let error: string | null = null;
+  try {
+    await getRoute53Client().send(new ListHostedZonesCommand({ MaxItems: 1 }));
+    status = 'ok';
+  } catch (err) {
+    status = isAwsAuthError(err) ? 'auth_failed' : 'unreachable';
+    error = sanitizeAwsError(err);
+  }
+  const latencyMs = Date.now() - start;
+
+  // D-12: live NS-delegation check. Wrapped in its own try/catch — a
+  // Postgres failure or a blocked resolver must degrade to
+  // nsDelegationErrors, never throw out of checkIntegrationHealth()'s
+  // Promise.all (T-24-16).
+  let nsDelegationMismatches: string[] | null = null;
+  let nsDelegationErrors: string[] | null = null;
+  try {
+    // Lazy import to avoid pulling postgres-client into edge runtimes.
+    const { default: postgresClient } = await import('@/lib/services/postgres-client');
+    const res = await postgresClient.query<{ id: string; name: string; authoritative_name_servers: unknown }>(
+      `SELECT id, name, authoritative_name_servers
+       FROM route53_zones
+       WHERE is_deleted = false
+       ORDER BY name
+       LIMIT ${ROUTE53_ZONE_CHECK_LIMIT}`,
+    );
+    const zones = res.rows.map((r) => ({
+      id: r.id,
+      name: r.name,
+      authoritativeNameServers: r.authoritative_name_servers,
+    }));
+    const delegationResults = await checkAllZoneDelegations(zones);
+    const mismatches = delegationResults.filter((r) => r.mismatch).map((r) => r.zoneName);
+    const errors = delegationResults.filter((r) => r.error).map((r) => r.zoneName);
+    if (mismatches.length > 0) nsDelegationMismatches = mismatches;
+    if (errors.length > 0) nsDelegationErrors = errors;
+
+    if (mismatches.length > 0 && status === 'ok') {
+      status = 'degraded';
+      const truncationNote = res.rowCount === ROUTE53_ZONE_CHECK_LIMIT
+        ? ` (checked first ${ROUTE53_ZONE_CHECK_LIMIT} zones by name)`
+        : '';
+      error = `NS delegation mismatch for ${mismatches.length} zone(s): ${mismatches.join(', ')}${truncationNote}`;
+    }
+  } catch (err) {
+    // Postgres unreachable, migration not applied yet, or resolver blocked —
+    // an infrastructure problem, not evidence of delegation drift. Preserve
+    // the auth-probe status; just note the delegation check itself failed.
+    nsDelegationErrors = [err instanceof Error ? err.message : String(err)];
+  }
+
+  return {
+    key: 'route53', name: 'AWS Route 53', category: 'network',
+    status, configured: true, latencyMs, error, checkedAt,
+    nsDelegationMismatches, nsDelegationErrors,
+  };
+}
+
 async function checkItglue(): Promise<IntegrationHealth> {
   const apiKey = process.env.ITGLUE_API_KEY;
   const checkedAt = new Date().toISOString();
@@ -327,6 +447,7 @@ export async function checkIntegrationHealth(opts?: { skipCache?: boolean }): Pr
     checkDattoRmm(),
     checkItglue(),
     checkS1(),
+    checkRoute53(),
     Promise.resolve(checkConfigOnly('veeam', 'Veeam VSPC', 'backup',
       ['VEEAM_VSPC_URL', 'VEEAM_VSPC_API_KEY'])),
     Promise.resolve(checkConfigOnly('msgraph', 'Microsoft Graph', 'productivity',
@@ -379,7 +500,7 @@ export function summarize(items: IntegrationHealth[]): HealthSummary {
       continue;
     }
     if (i.status === 'ok' || i.status === 'unknown') ok += 1;
-    else if (i.status === 'auth_failed' || i.status === 'unreachable') failed += 1;
+    else if (i.status === 'auth_failed' || i.status === 'unreachable' || i.status === 'degraded') failed += 1;
     else if (i.status === 'not_configured') notConfigured += 1;
     if (i.tokenExpiry) {
       if (i.tokenExpiry.daysRemaining <= 0) expired += 1;

lib/services/route53-dns-delegation.test.ts

@@ -0,0 +1,87 @@
+import { describe, it, expect } from 'vitest';
+import { normalizeNsList, compareNsDelegation } from './route53-dns-delegation';
+
+describe('normalizeNsList', () => {
+  it('lowercases and strips trailing dots', () => {
+    expect(normalizeNsList(['NS-123.AWSDNS-45.com.', 'ns-999.awsdns-01.org'])).toEqual([
+      'ns-123.awsdns-45.com',
+      'ns-999.awsdns-01.org',
+    ]);
+  });
+
+  it('returns [] for null', () => {
+    expect(normalizeNsList(null)).toEqual([]);
+  });
+
+  it('returns [] for undefined', () => {
+    expect(normalizeNsList(undefined)).toEqual([]);
+  });
+
+  it('returns [] for a non-array input', () => {
+    expect(normalizeNsList('not-an-array')).toEqual([]);
+    expect(normalizeNsList(42)).toEqual([]);
+    expect(normalizeNsList({})).toEqual([]);
+  });
+
+  it('de-duplicates and sorts so ordering differences never register as a mismatch', () => {
+    expect(normalizeNsList(['ns-2.awsdns.com.', 'ns-1.awsdns.com', 'ns-1.awsdns.com.'])).toEqual([
+      'ns-1.awsdns.com',
+      'ns-2.awsdns.com',
+    ]);
+  });
+
+  it('drops empty strings after trimming', () => {
+    expect(normalizeNsList(['  ', 'ns-1.awsdns.com', ''])).toEqual(['ns-1.awsdns.com']);
+  });
+});
+
+describe('compareNsDelegation', () => {
+  it('returns mismatch: false with empty diffs when sets are identical', () => {
+    const result = compareNsDelegation(
+      ['ns-1.awsdns.com', 'ns-2.awsdns.com'],
+      ['NS-1.AWSDNS.com.', 'ns-2.awsdns.com.'],
+    );
+    expect(result.mismatch).toBe(false);
+    expect(result.missingFromLive).toEqual([]);
+    expect(result.extraInLive).toEqual([]);
+  });
+
+  it('returns mismatch: true with populated missingFromLive when an authoritative NS is absent from the live answer', () => {
+    const result = compareNsDelegation(
+      ['ns-1.awsdns.com', 'ns-2.awsdns.com'],
+      ['ns-1.awsdns.com'],
+    );
+    expect(result.mismatch).toBe(true);
+    expect(result.missingFromLive).toEqual(['ns-2.awsdns.com']);
+    expect(result.extraInLive).toEqual([]);
+  });
+
+  it('returns mismatch: true with populated extraInLive when the live answer contains an NS Route 53 does not consider authoritative', () => {
+    const result = compareNsDelegation(
+      ['ns-1.awsdns.com'],
+      ['ns-1.awsdns.com', 'ns-rogue.example.com'],
+    );
+    expect(result.mismatch).toBe(true);
+    expect(result.extraInLive).toEqual(['ns-rogue.example.com']);
+    expect(result.missingFromLive).toEqual([]);
+  });
+
+  it('returns mismatch: true when live has no answer but authoritative is non-empty (delegation problem, not a pass)', () => {
+    const result = compareNsDelegation(['ns1.example.com'], []);
+    expect(result.mismatch).toBe(true);
+  });
+
+  it('returns mismatch: false when authoritative is empty (unjudgeable, must not false-alarm)', () => {
+    const result = compareNsDelegation([], ['ns1.example.com']);
+    expect(result.mismatch).toBe(false);
+  });
+
+  it('is case-insensitive and trailing-dot-insensitive on both sides', () => {
+    const result = compareNsDelegation(['NS-1.AWSDNS.COM.'], ['ns-1.awsdns.com']);
+    expect(result.mismatch).toBe(false);
+  });
+
+  it('handles null/undefined inputs on both sides without throwing', () => {
+    expect(compareNsDelegation(null, undefined)).toMatchObject({ mismatch: false });
+  });
+});

lib/services/route53-dns-delegation.ts

@@ -0,0 +1,195 @@
+/**
+ * D-12 DNS-delegation health check helpers.
+ *
+ * Split into a pure half (normalization + set comparison, fully unit-testable)
+ * and an I/O half (live public DNS lookup via a dedicated `dns.Resolver()`
+ * instance). The pure half never touches the network; the I/O half never
+ * touches the process-global resolver — see the CRITICAL comment on
+ * `resolveLiveNs` below (24-RESEARCH.md Pitfall 5 / threat T-24-13).
+ */
+
+import { Resolver } from 'dns';
+
+// ============================================================================
+// Pure half — NS normalization + comparison (unit-testable, no I/O)
+// ============================================================================
+
+/**
+ * Normalize an unknown input (expected to be a JSONB-sourced string array,
+ * e.g. route53_zones.authoritative_name_servers or a raw DNS answer) into a
+ * lowercase, trailing-dot-stripped, de-duplicated, sorted list of hostnames.
+ *
+ * Returns [] for null, undefined, or any non-array input rather than
+ * throwing — callers treat an empty list as "unjudgeable", not an error.
+ */
+export function normalizeNsList(input: unknown): string[] {
+  if (!Array.isArray(input)) return [];
+  const set = new Set<string>();
+  for (const entry of input) {
+    const normalized = String(entry).trim().toLowerCase().replace(/\.$/, '');
+    if (normalized.length > 0) set.add(normalized);
+  }
+  return Array.from(set).sort();
+}
+
+export interface NsDelegationComparison {
+  mismatch: boolean;
+  authoritative: string[];
+  live: string[];
+  missingFromLive: string[];
+  extraInLive: string[];
+}
+
+/**
+ * Compare Route 53's authoritative NS list against a live public DNS answer.
+ *
+ * - An empty (normalized) authoritative list is unjudgeable — a zone Pulse
+ *   has no recorded NS data for cannot be flagged as drifted. Returns
+ *   `mismatch: false`.
+ * - A non-empty authoritative list with an empty live answer IS a delegation
+ *   problem (the domain resolved to nothing) — returns `mismatch: true`.
+ * - Otherwise mismatch is true iff either side has an entry the other lacks.
+ */
+export function compareNsDelegation(authoritative: unknown, live: unknown): NsDelegationComparison {
+  const authNorm = normalizeNsList(authoritative);
+  const liveNorm = normalizeNsList(live);
+
+  if (authNorm.length === 0) {
+    return { mismatch: false, authoritative: authNorm, live: liveNorm, missingFromLive: [], extraInLive: [] };
+  }
+  if (liveNorm.length === 0) {
+    return {
+      mismatch: true,
+      authoritative: authNorm,
+      live: liveNorm,
+      missingFromLive: [...authNorm],
+      extraInLive: [],
+    };
+  }
+
+  const authSet = new Set(authNorm);
+  const liveSet = new Set(liveNorm);
+  const missingFromLive = authNorm.filter((ns) => !liveSet.has(ns));
+  const extraInLive = liveNorm.filter((ns) => !authSet.has(ns));
+
+  return {
+    mismatch: missingFromLive.length > 0 || extraInLive.length > 0,
+    authoritative: authNorm,
+    live: liveNorm,
+    missingFromLive,
+    extraInLive,
+  };
+}
+
+// ============================================================================
+// I/O half — live public DNS lookup (dedicated resolver instance)
+// ============================================================================
+
+export type ResolveLiveNsResult =
+  | { ok: true; nameServers: string[] }
+  | { ok: false; error: string };
+
+/**
+ * Resolve a domain's live NS records against a dedicated public resolver
+ * (Cloudflare 1.1.1.1 + Google 8.8.8.8), NOT the container's default
+ * resolver (24-RESEARCH.md Pitfall 5).
+ *
+ * CRITICAL: this constructs its own `new Resolver()` instance and calls
+ * `setServers()` on THAT instance only. The process-global `dns.setServers()`
+ * is never called anywhere in this file — doing so would repoint DNS
+ * resolution for the entire Node process, including Postgres/Redis/AWS
+ * hostname resolution (T-24-13).
+ */
+export function resolveLiveNs(domain: string, timeoutMs = 5000): Promise<ResolveLiveNsResult> {
+  const target = domain.trim().replace(/\.$/, '');
+  const resolver = new Resolver();
+  resolver.setServers(['1.1.1.1', '8.8.8.8']);
+
+  return new Promise((resolve) => {
+    let settled = false;
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      resolver.cancel();
+      resolve({ ok: false, error: `DNS lookup timed out after ${timeoutMs}ms` });
+    }, timeoutMs);
+
+    resolver.resolveNs(target, (err, addresses) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (err) {
+        resolve({ ok: false, error: err.message });
+        return;
+      }
+      resolve({ ok: true, nameServers: addresses });
+    });
+  });
+}
+
+export interface ZoneDelegationInput {
+  id: string;
+  name: string;
+  authoritativeNameServers: unknown;
+}
+
+export interface ZoneDelegationResult {
+  zoneId: string;
+  zoneName: string;
+  mismatch: boolean;
+  error?: string;
+  missingFromLive: string[];
+  extraInLive: string[];
+}
+
+/**
+ * Check live NS delegation for a batch of zones, bounded to at most
+ * `opts.concurrency` (default 5) concurrent lookups so a large zone list
+ * does not open hundreds of concurrent UDP sockets (T-24-14).
+ *
+ * Zones whose authoritativeNameServers normalizes to an empty list are
+ * skipped (unjudgeable). A lookup failure yields `{ mismatch: false, error }`
+ * — an unreachable resolver is an infrastructure problem, not evidence of
+ * delegation drift, and must never be reported as a mismatch.
+ */
+export async function checkAllZoneDelegations(
+  zones: ZoneDelegationInput[],
+  opts?: { concurrency?: number },
+): Promise<ZoneDelegationResult[]> {
+  const concurrency = opts?.concurrency ?? 5;
+  const judgeable = zones.filter((z) => normalizeNsList(z.authoritativeNameServers).length > 0);
+  const results: ZoneDelegationResult[] = [];
+
+  let cursor = 0;
+  async function worker(): Promise<void> {
+    while (cursor < judgeable.length) {
+      const idx = cursor;
+      cursor += 1;
+      const zone = judgeable[idx];
+      const live = await resolveLiveNs(zone.name);
+      if (!live.ok) {
+        results.push({
+          zoneId: zone.id,
+          zoneName: zone.name,
+          mismatch: false,
+          error: live.error,
+          missingFromLive: [],
+          extraInLive: [],
+        });
+        continue;
+      }
+      const comparison = compareNsDelegation(zone.authoritativeNameServers, live.nameServers);
+      results.push({
+        zoneId: zone.id,
+        zoneName: zone.name,
+        mismatch: comparison.mismatch,
+        missingFromLive: comparison.missingFromLive,
+        extraInLive: comparison.extraInLive,
+      });
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(concurrency, judgeable.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}