chore: merge executor worktree (worktree-agent-a30d26dba3410e0da) — plan 24-04

lib/services/integration-health.ts

@@ -11,12 +11,19 @@
  * Usage:
  *   const results = await checkIntegrationHealth();
  *
- * Tools covered live: S1, Datto RMM, IT Glue, Autotask. Others report
- * configured / not_configured only — extending to live checks is mechanical.
+ * Tools covered live: S1, Datto RMM, IT Glue, Autotask, AWS Route 53. Others
+ * report configured / not_configured only — extending to live checks is
+ * mechanical.
  */
 
+import { ListHostedZonesCommand } from '@aws-sdk/client-route-53';
+import { isRoute53Configured, getRoute53Client } from '@/lib/services/route53-factory';
+import { checkAllZoneDelegations } from '@/lib/services/route53-dns-delegation';
+
 export type HealthStatus =
   | 'ok'              // configured, auth succeeded
+  | 'degraded'        // configured, auth succeeded, but a secondary check found a problem
+                      // (e.g. Route 53 D-12 NS-delegation mismatch) — reachable, not fully healthy
   | 'auth_failed'     // configured, server returned 401/403
   | 'unreachable'     // configured, network/DNS/TLS error
   | 'not_configured'  // env vars missing
@@ -40,6 +47,10 @@ export interface IntegrationHealth {
   error?: string | null;
   tokenExpiry?: TokenExpiry | null;
   checkedAt: string;
+  /** D-12: zone names whose live NS answer mismatches Route 53's authoritative NS list. */
+  nsDelegationMismatches?: string[] | null;
+  /** D-12: zone names whose live NS lookup failed (infrastructure problem, not a mismatch). */
+  nsDelegationErrors?: string[] | null;
 }
 
 interface CacheEntry {
@@ -190,6 +201,115 @@ async function checkDattoRmm(): Promise<IntegrationHealth> {
   }
 }
 
+/**
+ * Redact an AWS SDK error down to a message safe to surface in the health
+ * API response and (eventually) route53_audit_log.error_message.
+ *
+ * NOTE: this duplicates the `sanitizeAwsError` spec'd for
+ * `lib/services/route53-record-validation.ts` in plan 24-03 (T-24-03). That
+ * file did not exist in this plan's isolated worktree at execution time
+ * (24-03 runs in a sibling parallel worktree and depends_on for this plan
+ * only lists 24-01) — see this plan's SUMMARY "Deviations" section. This
+ * local copy uses the identical redaction rules so behavior is consistent
+ * regardless of which implementation ships; if 24-03 lands first in a
+ * future merge, this local copy should be replaced with an import from
+ * `@/lib/services/route53-record-validation` for a single source of truth.
+ */
+function sanitizeAwsError(err: unknown): string {
+  const message = err instanceof Error ? err.message : String(err);
+  return message
+    .replace(/AKIA[0-9A-Z]{16}/g, '[redacted-key-id]')
+    .replace(/arn:aws:[^\s"']+/g, '[redacted-arn]')
+    .replace(/\b[0-9]{12}\b/g, '[redacted-account-id]')
+    .slice(0, 500);
+}
+
+const AWS_AUTH_ERROR_NAMES = new Set([
+  'InvalidClientTokenId',
+  'SignatureDoesNotMatch',
+  'AccessDenied',
+  'UnrecognizedClientException',
+]);
+
+function isAwsAuthError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  const name = (err as Error & { name?: string }).name;
+  if (name && AWS_AUTH_ERROR_NAMES.has(name)) return true;
+  const httpStatusCode = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
+  return httpStatusCode === 401 || httpStatusCode === 403;
+}
+
+const ROUTE53_ZONE_CHECK_LIMIT = 50;
+
+async function checkRoute53(): Promise<IntegrationHealth> {
+  const checkedAt = new Date().toISOString();
+  if (!isRoute53Configured()) {
+    return {
+      key: 'route53', name: 'AWS Route 53', category: 'network',
+      status: 'not_configured', configured: false, checkedAt,
+    };
+  }
+
+  const start = Date.now();
+  let status: HealthStatus;
+  let error: string | null = null;
+  try {
+    await getRoute53Client().send(new ListHostedZonesCommand({ MaxItems: 1 }));
+    status = 'ok';
+  } catch (err) {
+    status = isAwsAuthError(err) ? 'auth_failed' : 'unreachable';
+    error = sanitizeAwsError(err);
+  }
+  const latencyMs = Date.now() - start;
+
+  // D-12: live NS-delegation check. Wrapped in its own try/catch — a
+  // Postgres failure or a blocked resolver must degrade to
+  // nsDelegationErrors, never throw out of checkIntegrationHealth()'s
+  // Promise.all (T-24-16).
+  let nsDelegationMismatches: string[] | null = null;
+  let nsDelegationErrors: string[] | null = null;
+  try {
+    // Lazy import to avoid pulling postgres-client into edge runtimes.
+    const { default: postgresClient } = await import('@/lib/services/postgres-client');
+    const res = await postgresClient.query<{ id: string; name: string; authoritative_name_servers: unknown }>(
+      `SELECT id, name, authoritative_name_servers
+       FROM route53_zones
+       WHERE is_deleted = false
+       ORDER BY name
+       LIMIT ${ROUTE53_ZONE_CHECK_LIMIT}`,
+    );
+    const zones = res.rows.map((r) => ({
+      id: r.id,
+      name: r.name,
+      authoritativeNameServers: r.authoritative_name_servers,
+    }));
+    const delegationResults = await checkAllZoneDelegations(zones);
+    const mismatches = delegationResults.filter((r) => r.mismatch).map((r) => r.zoneName);
+    const errors = delegationResults.filter((r) => r.error).map((r) => r.zoneName);
+    if (mismatches.length > 0) nsDelegationMismatches = mismatches;
+    if (errors.length > 0) nsDelegationErrors = errors;
+
+    if (mismatches.length > 0 && status === 'ok') {
+      status = 'degraded';
+      const truncationNote = res.rowCount === ROUTE53_ZONE_CHECK_LIMIT
+        ? ` (checked first ${ROUTE53_ZONE_CHECK_LIMIT} zones by name)`
+        : '';
+      error = `NS delegation mismatch for ${mismatches.length} zone(s): ${mismatches.join(', ')}${truncationNote}`;
+    }
+  } catch (err) {
+    // Postgres unreachable, migration not applied yet, or resolver blocked —
+    // an infrastructure problem, not evidence of delegation drift. Preserve
+    // the auth-probe status; just note the delegation check itself failed.
+    nsDelegationErrors = [err instanceof Error ? err.message : String(err)];
+  }
+
+  return {
+    key: 'route53', name: 'AWS Route 53', category: 'network',
+    status, configured: true, latencyMs, error, checkedAt,
+    nsDelegationMismatches, nsDelegationErrors,
+  };
+}
+
 async function checkItglue(): Promise<IntegrationHealth> {
   const apiKey = process.env.ITGLUE_API_KEY;
   const checkedAt = new Date().toISOString();
@@ -327,6 +447,7 @@ export async function checkIntegrationHealth(opts?: { skipCache?: boolean }): Pr
     checkDattoRmm(),
     checkItglue(),
     checkS1(),
+    checkRoute53(),
     Promise.resolve(checkConfigOnly('veeam', 'Veeam VSPC', 'backup',
       ['VEEAM_VSPC_URL', 'VEEAM_VSPC_API_KEY'])),
     Promise.resolve(checkConfigOnly('msgraph', 'Microsoft Graph', 'productivity',
@@ -379,7 +500,7 @@ export function summarize(items: IntegrationHealth[]): HealthSummary {
       continue;
     }
     if (i.status === 'ok' || i.status === 'unknown') ok += 1;
-    else if (i.status === 'auth_failed' || i.status === 'unreachable') failed += 1;
+    else if (i.status === 'auth_failed' || i.status === 'unreachable' || i.status === 'degraded') failed += 1;
     else if (i.status === 'not_configured') notConfigured += 1;
     if (i.tokenExpiry) {
       if (i.tokenExpiry.daysRemaining <= 0) expired += 1;

lib/services/route53-dns-delegation.test.ts

@@ -0,0 +1,87 @@
+import { describe, it, expect } from 'vitest';
+import { normalizeNsList, compareNsDelegation } from './route53-dns-delegation';
+
+describe('normalizeNsList', () => {
+  it('lowercases and strips trailing dots', () => {
+    expect(normalizeNsList(['NS-123.AWSDNS-45.com.', 'ns-999.awsdns-01.org'])).toEqual([
+      'ns-123.awsdns-45.com',
+      'ns-999.awsdns-01.org',
+    ]);
+  });
+
+  it('returns [] for null', () => {
+    expect(normalizeNsList(null)).toEqual([]);
+  });
+
+  it('returns [] for undefined', () => {
+    expect(normalizeNsList(undefined)).toEqual([]);
+  });
+
+  it('returns [] for a non-array input', () => {
+    expect(normalizeNsList('not-an-array')).toEqual([]);
+    expect(normalizeNsList(42)).toEqual([]);
+    expect(normalizeNsList({})).toEqual([]);
+  });
+
+  it('de-duplicates and sorts so ordering differences never register as a mismatch', () => {
+    expect(normalizeNsList(['ns-2.awsdns.com.', 'ns-1.awsdns.com', 'ns-1.awsdns.com.'])).toEqual([
+      'ns-1.awsdns.com',
+      'ns-2.awsdns.com',
+    ]);
+  });
+
+  it('drops empty strings after trimming', () => {
+    expect(normalizeNsList(['  ', 'ns-1.awsdns.com', ''])).toEqual(['ns-1.awsdns.com']);
+  });
+});
+
+describe('compareNsDelegation', () => {
+  it('returns mismatch: false with empty diffs when sets are identical', () => {
+    const result = compareNsDelegation(
+      ['ns-1.awsdns.com', 'ns-2.awsdns.com'],
+      ['NS-1.AWSDNS.com.', 'ns-2.awsdns.com.'],
+    );
+    expect(result.mismatch).toBe(false);
+    expect(result.missingFromLive).toEqual([]);
+    expect(result.extraInLive).toEqual([]);
+  });
+
+  it('returns mismatch: true with populated missingFromLive when an authoritative NS is absent from the live answer', () => {
+    const result = compareNsDelegation(
+      ['ns-1.awsdns.com', 'ns-2.awsdns.com'],
+      ['ns-1.awsdns.com'],
+    );
+    expect(result.mismatch).toBe(true);
+    expect(result.missingFromLive).toEqual(['ns-2.awsdns.com']);
+    expect(result.extraInLive).toEqual([]);
+  });
+
+  it('returns mismatch: true with populated extraInLive when the live answer contains an NS Route 53 does not consider authoritative', () => {
+    const result = compareNsDelegation(
+      ['ns-1.awsdns.com'],
+      ['ns-1.awsdns.com', 'ns-rogue.example.com'],
+    );
+    expect(result.mismatch).toBe(true);
+    expect(result.extraInLive).toEqual(['ns-rogue.example.com']);
+    expect(result.missingFromLive).toEqual([]);
+  });
+
+  it('returns mismatch: true when live has no answer but authoritative is non-empty (delegation problem, not a pass)', () => {
+    const result = compareNsDelegation(['ns1.example.com'], []);
+    expect(result.mismatch).toBe(true);
+  });
+
+  it('returns mismatch: false when authoritative is empty (unjudgeable, must not false-alarm)', () => {
+    const result = compareNsDelegation([], ['ns1.example.com']);
+    expect(result.mismatch).toBe(false);
+  });
+
+  it('is case-insensitive and trailing-dot-insensitive on both sides', () => {
+    const result = compareNsDelegation(['NS-1.AWSDNS.COM.'], ['ns-1.awsdns.com']);
+    expect(result.mismatch).toBe(false);
+  });
+
+  it('handles null/undefined inputs on both sides without throwing', () => {
+    expect(compareNsDelegation(null, undefined)).toMatchObject({ mismatch: false });
+  });
+});

lib/services/route53-dns-delegation.ts

@@ -0,0 +1,195 @@
+/**
+ * D-12 DNS-delegation health check helpers.
+ *
+ * Split into a pure half (normalization + set comparison, fully unit-testable)
+ * and an I/O half (live public DNS lookup via a dedicated `dns.Resolver()`
+ * instance). The pure half never touches the network; the I/O half never
+ * touches the process-global resolver — see the CRITICAL comment on
+ * `resolveLiveNs` below (24-RESEARCH.md Pitfall 5 / threat T-24-13).
+ */
+
+import { Resolver } from 'dns';
+
+// ============================================================================
+// Pure half — NS normalization + comparison (unit-testable, no I/O)
+// ============================================================================
+
+/**
+ * Normalize an unknown input (expected to be a JSONB-sourced string array,
+ * e.g. route53_zones.authoritative_name_servers or a raw DNS answer) into a
+ * lowercase, trailing-dot-stripped, de-duplicated, sorted list of hostnames.
+ *
+ * Returns [] for null, undefined, or any non-array input rather than
+ * throwing — callers treat an empty list as "unjudgeable", not an error.
+ */
+export function normalizeNsList(input: unknown): string[] {
+  if (!Array.isArray(input)) return [];
+  const set = new Set<string>();
+  for (const entry of input) {
+    const normalized = String(entry).trim().toLowerCase().replace(/\.$/, '');
+    if (normalized.length > 0) set.add(normalized);
+  }
+  return Array.from(set).sort();
+}
+
+export interface NsDelegationComparison {
+  mismatch: boolean;
+  authoritative: string[];
+  live: string[];
+  missingFromLive: string[];
+  extraInLive: string[];
+}
+
+/**
+ * Compare Route 53's authoritative NS list against a live public DNS answer.
+ *
+ * - An empty (normalized) authoritative list is unjudgeable — a zone Pulse
+ *   has no recorded NS data for cannot be flagged as drifted. Returns
+ *   `mismatch: false`.
+ * - A non-empty authoritative list with an empty live answer IS a delegation
+ *   problem (the domain resolved to nothing) — returns `mismatch: true`.
+ * - Otherwise mismatch is true iff either side has an entry the other lacks.
+ */
+export function compareNsDelegation(authoritative: unknown, live: unknown): NsDelegationComparison {
+  const authNorm = normalizeNsList(authoritative);
+  const liveNorm = normalizeNsList(live);
+
+  if (authNorm.length === 0) {
+    return { mismatch: false, authoritative: authNorm, live: liveNorm, missingFromLive: [], extraInLive: [] };
+  }
+  if (liveNorm.length === 0) {
+    return {
+      mismatch: true,
+      authoritative: authNorm,
+      live: liveNorm,
+      missingFromLive: [...authNorm],
+      extraInLive: [],
+    };
+  }
+
+  const authSet = new Set(authNorm);
+  const liveSet = new Set(liveNorm);
+  const missingFromLive = authNorm.filter((ns) => !liveSet.has(ns));
+  const extraInLive = liveNorm.filter((ns) => !authSet.has(ns));
+
+  return {
+    mismatch: missingFromLive.length > 0 || extraInLive.length > 0,
+    authoritative: authNorm,
+    live: liveNorm,
+    missingFromLive,
+    extraInLive,
+  };
+}
+
+// ============================================================================
+// I/O half — live public DNS lookup (dedicated resolver instance)
+// ============================================================================
+
+export type ResolveLiveNsResult =
+  | { ok: true; nameServers: string[] }
+  | { ok: false; error: string };
+
+/**
+ * Resolve a domain's live NS records against a dedicated public resolver
+ * (Cloudflare 1.1.1.1 + Google 8.8.8.8), NOT the container's default
+ * resolver (24-RESEARCH.md Pitfall 5).
+ *
+ * CRITICAL: this constructs its own `new Resolver()` instance and calls
+ * `setServers()` on THAT instance only. The process-global `dns.setServers()`
+ * is never called anywhere in this file — doing so would repoint DNS
+ * resolution for the entire Node process, including Postgres/Redis/AWS
+ * hostname resolution (T-24-13).
+ */
+export function resolveLiveNs(domain: string, timeoutMs = 5000): Promise<ResolveLiveNsResult> {
+  const target = domain.trim().replace(/\.$/, '');
+  const resolver = new Resolver();
+  resolver.setServers(['1.1.1.1', '8.8.8.8']);
+
+  return new Promise((resolve) => {
+    let settled = false;
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      resolver.cancel();
+      resolve({ ok: false, error: `DNS lookup timed out after ${timeoutMs}ms` });
+    }, timeoutMs);
+
+    resolver.resolveNs(target, (err, addresses) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (err) {
+        resolve({ ok: false, error: err.message });
+        return;
+      }
+      resolve({ ok: true, nameServers: addresses });
+    });
+  });
+}
+
+export interface ZoneDelegationInput {
+  id: string;
+  name: string;
+  authoritativeNameServers: unknown;
+}
+
+export interface ZoneDelegationResult {
+  zoneId: string;
+  zoneName: string;
+  mismatch: boolean;
+  error?: string;
+  missingFromLive: string[];
+  extraInLive: string[];
+}
+
+/**
+ * Check live NS delegation for a batch of zones, bounded to at most
+ * `opts.concurrency` (default 5) concurrent lookups so a large zone list
+ * does not open hundreds of concurrent UDP sockets (T-24-14).
+ *
+ * Zones whose authoritativeNameServers normalizes to an empty list are
+ * skipped (unjudgeable). A lookup failure yields `{ mismatch: false, error }`
+ * — an unreachable resolver is an infrastructure problem, not evidence of
+ * delegation drift, and must never be reported as a mismatch.
+ */
+export async function checkAllZoneDelegations(
+  zones: ZoneDelegationInput[],
+  opts?: { concurrency?: number },
+): Promise<ZoneDelegationResult[]> {
+  const concurrency = opts?.concurrency ?? 5;
+  const judgeable = zones.filter((z) => normalizeNsList(z.authoritativeNameServers).length > 0);
+  const results: ZoneDelegationResult[] = [];
+
+  let cursor = 0;
+  async function worker(): Promise<void> {
+    while (cursor < judgeable.length) {
+      const idx = cursor;
+      cursor += 1;
+      const zone = judgeable[idx];
+      const live = await resolveLiveNs(zone.name);
+      if (!live.ok) {
+        results.push({
+          zoneId: zone.id,
+          zoneName: zone.name,
+          mismatch: false,
+          error: live.error,
+          missingFromLive: [],
+          extraInLive: [],
+        });
+        continue;
+      }
+      const comparison = compareNsDelegation(zone.authoritativeNameServers, live.nameServers);
+      results.push({
+        zoneId: zone.id,
+        zoneName: zone.name,
+        mismatch: comparison.mismatch,
+        missingFromLive: comparison.missingFromLive,
+        extraInLive: comparison.extraInLive,
+      });
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(concurrency, judgeable.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}