feat(24-03): add Route 53 audit lifecycle and pulse_crud history persistence

- createPendingAuditLog/markAuditCommitted/markAuditFailed implement the pending -> committed/failed lifecycle (D-07, SC-3); markAuditFailed always sanitizes via sanitizeAwsError
- insertPulseCrudHistory writes 'pulse_crud' history rows, documented as callable only after a committed write
- upsertMirrorRecord/softDeleteMirrorRecord/loadMirrorRecord manage the route53_records mirror; mirror writes are best-effort and soft-delete only (D-08), audit/history writes are not best-effort
- log same pre-existing itglue-search.test.ts failures (unrelated, out of scope) in deferred-items.md

.planning/phases/24-aws-route-53-dns-sync-track-changes-crud-operations-full-aud/deferred-items.md

@@ -14,3 +14,11 @@ changes).
   confirms zero changes to either path from Task 1 or Task 2). Unrelated to
   the Route 53 factory/schema work in this plan — not fixed, logged here for
   visibility.
+
+## Plan 24-03
+
+- Same 2 pre-existing `lib/services/analyzer/itglue-search.test.ts` failures
+  re-surfaced by `npm test` (full suite) while verifying Task 2's
+  `route53-write-persistence.ts`. Neither `itglue-search.ts` nor its test
+  file were touched by this plan. Out of scope per the scope boundary rule —
+  not fixed.

lib/services/route53-write-persistence.test.ts

@@ -0,0 +1,180 @@
+/**
+ * lib/services/route53-write-persistence.ts unit tests — SQL contract checks
+ * against a mocked postgresClient. No real Postgres connection is made.
+ * Follows the mocking discipline in lib/services/pax8-sync-service.test.ts.
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// Mock postgresClient BEFORE importing the module under test.
+const queryMock = vi.fn();
+vi.mock('@/lib/services/postgres-client', () => ({
+  default: {
+    query: (...args: unknown[]) => queryMock(...args),
+  },
+}));
+
+// Import AFTER the mock is declared so vi.mock hoisting takes effect.
+import {
+  createPendingAuditLog,
+  markAuditCommitted,
+  markAuditFailed,
+  insertPulseCrudHistory,
+  upsertMirrorRecord,
+  softDeleteMirrorRecord,
+  loadMirrorRecord,
+} from './route53-write-persistence';
+
+interface MockCall {
+  sql: string;
+  params: unknown[];
+}
+
+function calls(): MockCall[] {
+  return queryMock.mock.calls.map(([sql, params]) => ({
+    sql: String(sql),
+    params: (params as unknown[]) ?? [],
+  }));
+}
+
+beforeEach(() => {
+  queryMock.mockReset();
+});
+
+describe('createPendingAuditLog', () => {
+  it("issues an INSERT into route53_audit_log whose SQL contains 'pending'", async () => {
+    queryMock.mockResolvedValueOnce({ rows: [{ id: 'audit-1' }] });
+    const result = await createPendingAuditLog({
+      operation: 'create',
+      zoneId: 'Z123',
+      recordKey: 'Z123:www.example.com.:A:',
+      recordName: 'www.example.com.',
+      recordType: 'A',
+      beforeValue: null,
+      afterValue: { ttl: 300, resourceRecords: [{ value: '1.2.3.4' }] },
+      performedByUserId: 'user-1',
+      performedByEmail: 'lorentz@wulfconsulting.com',
+    });
+    expect(result).toEqual({ id: 'audit-1' });
+    expect(calls()).toHaveLength(1);
+    expect(calls()[0].sql).toContain('INSERT INTO route53_audit_log');
+    expect(calls()[0].sql).toContain("'pending'");
+  });
+});
+
+describe('markAuditCommitted', () => {
+  it("issues an UPDATE setting status = 'committed'", async () => {
+    queryMock.mockResolvedValueOnce({ rows: [] });
+    await markAuditCommitted('audit-1', 'change-1', 'INSYNC', { ResponseMetadata: {} });
+    expect(calls()[0].sql).toContain('UPDATE route53_audit_log');
+    expect(calls()[0].sql).toContain("'committed'");
+  });
+});
+
+describe('markAuditFailed', () => {
+  it('passes a sanitized error string — a raw AWS key id never reaches the bound parameters', async () => {
+    queryMock.mockResolvedValueOnce({ rows: [] });
+    const rawError = new Error('AccessDenied for AKIAIOSFODNN7EXAMPLE');
+    await markAuditFailed('audit-1', rawError);
+
+    expect(calls()).toHaveLength(1);
+    expect(calls()[0].sql).toContain("'failed'");
+    const boundParams = calls()[0].params;
+    expect(boundParams).not.toContain(rawError);
+    for (const param of boundParams) {
+      if (typeof param === 'string') {
+        expect(param).not.toContain('AKIAIOSFODNN7EXAMPLE');
+      }
+    }
+  });
+});
+
+describe('insertPulseCrudHistory', () => {
+  it("binds the literal 'pulse_crud' as the history source", async () => {
+    queryMock.mockResolvedValueOnce({ rows: [] });
+    await insertPulseCrudHistory({
+      zoneId: 'Z123',
+      recordKey: 'Z123:www.example.com.:A:',
+      recordName: 'www.example.com.',
+      recordType: 'A',
+      changeAction: 'create',
+      beforeValue: null,
+      afterValue: { ttl: 300 },
+      changedByUserId: 'user-1',
+      changedByEmail: 'lorentz@wulfconsulting.com',
+      auditLogId: 'audit-1',
+    });
+    expect(calls()[0].sql).toContain('INSERT INTO route53_record_history');
+    expect(calls()[0].sql).toContain("'pulse_crud'");
+  });
+});
+
+describe('upsertMirrorRecord', () => {
+  it('issues an INSERT ... ON CONFLICT upsert and never throws even if the query rejects', async () => {
+    queryMock.mockRejectedValueOnce(new Error('connection reset'));
+    await expect(
+      upsertMirrorRecord({
+        recordKey: 'Z123:www.example.com.:A:',
+        zoneId: 'Z123',
+        name: 'www.example.com.',
+        type: 'A',
+        setIdentifier: null,
+        ttl: 300,
+        resourceRecords: [{ value: '1.2.3.4' }],
+        aliasTarget: null,
+        rawPayload: null,
+      })
+    ).resolves.toBeUndefined();
+  });
+});
+
+describe('softDeleteMirrorRecord', () => {
+  it('issues an UPDATE and never a DELETE FROM', async () => {
+    queryMock.mockResolvedValueOnce({ rows: [] });
+    await softDeleteMirrorRecord('Z123:www.example.com.:A:');
+    expect(calls()[0].sql).toContain('UPDATE route53_records');
+    expect(calls()[0].sql).not.toContain('DELETE FROM');
+  });
+
+  it('never throws even if the query rejects (best-effort)', async () => {
+    queryMock.mockRejectedValueOnce(new Error('connection reset'));
+    await expect(softDeleteMirrorRecord('Z123:www.example.com.:A:')).resolves.toBeUndefined();
+  });
+});
+
+describe('loadMirrorRecord', () => {
+  it('returns a camelCase object when found', async () => {
+    queryMock.mockResolvedValueOnce({
+      rowCount: 1,
+      rows: [
+        {
+          record_key: 'Z123:www.example.com.:A:',
+          zone_id: 'Z123',
+          name: 'www.example.com.',
+          type: 'A',
+          set_identifier: null,
+          ttl: 300,
+          resource_records: [{ value: '1.2.3.4' }],
+          alias_target: null,
+        },
+      ],
+    });
+    const result = await loadMirrorRecord('Z123:www.example.com.:A:');
+    expect(result).toEqual({
+      recordKey: 'Z123:www.example.com.:A:',
+      zoneId: 'Z123',
+      name: 'www.example.com.',
+      type: 'A',
+      setIdentifier: null,
+      ttl: 300,
+      resourceRecords: [{ value: '1.2.3.4' }],
+      aliasTarget: null,
+    });
+  });
+
+  it('returns null when not found', async () => {
+    queryMock.mockResolvedValueOnce({ rowCount: 0, rows: [] });
+    const result = await loadMirrorRecord('missing-key');
+    expect(result).toBeNull();
+  });
+});

lib/services/route53-write-persistence.ts

@@ -0,0 +1,297 @@
+/**
+ * AWS Route 53 DNS sync — CRUD write-back persistence.
+ *
+ * Implements the pending -> committed/failed audit lifecycle, the single
+ * most important pattern in this phase (lifted from
+ * lib/services/analyzer/asset-audit/persistence.ts's createPendingWrite /
+ * markWriteCommitted / markWriteFailed shape, itself following
+ * migrations/075_itglue_audit.sql's itglue_writes precedent).
+ *
+ * Discipline (24-RESEARCH.md Pattern 3, D-07): an audit row is created with
+ * status='pending' BEFORE any AWS command is constructed. It is then
+ * transitioned to 'committed' or 'failed' after the AWS call resolves.
+ * Callers must never construct a ChangeResourceRecordSetsCommand without an
+ * audit row already in flight.
+ */
+
+import postgresClient from '@/lib/services/postgres-client';
+import { sanitizeAwsError } from '@/lib/services/route53-record-validation';
+import type { Route53RecordValue } from '@/lib/types/route53';
+
+const LOG_PREFIX = '[ROUTE53-WRITE]';
+
+// ============================================================================
+// Audit lifecycle (route53_audit_log)
+// ============================================================================
+
+export interface CreatePendingAuditLogInput {
+  operation: 'create' | 'update' | 'delete';
+  zoneId: string;
+  recordKey: string;
+  recordName: string;
+  recordType: string;
+  beforeValue: unknown;
+  afterValue: unknown;
+  performedByUserId: string | null;
+  performedByEmail: string | null;
+}
+
+/**
+ * Insert a 'pending' audit row. Must be called and awaited BEFORE any AWS
+ * ChangeResourceRecordSetsCommand is constructed — that ordering is the
+ * whole point of this pattern: no write to Route 53 can occur without an
+ * audit row already in flight (D-07, SC-3). This write is not best-effort —
+ * let it throw so a DB failure fails the request rather than silently
+ * allowing an unlogged DNS mutation (T-24-07).
+ */
+export async function createPendingAuditLog(
+  input: CreatePendingAuditLogInput
+): Promise<{ id: string }> {
+  const res = await postgresClient.query<{ id: string }>(
+    `INSERT INTO route53_audit_log
+       (operation, zone_id, record_key, record_name, record_type,
+        before_value, after_value,
+        performed_by_user_id, performed_by_email, status)
+     VALUES ($1, $2, $3, $4, $5,
+             $6::jsonb, $7::jsonb,
+             $8, $9, 'pending')
+     RETURNING id::text AS id`,
+    [
+      input.operation,
+      input.zoneId,
+      input.recordKey,
+      input.recordName,
+      input.recordType,
+      JSON.stringify(input.beforeValue ?? null),
+      JSON.stringify(input.afterValue ?? null),
+      input.performedByUserId,
+      input.performedByEmail,
+    ]
+  );
+  return { id: res.rows[0].id };
+}
+
+/**
+ * Transition an audit row to 'committed' after a successful AWS call. Not
+ * best-effort — let it throw.
+ */
+export async function markAuditCommitted(
+  id: string,
+  awsChangeId: string | null,
+  awsChangeStatus: string | null,
+  awsResponse: unknown
+): Promise<void> {
+  await postgresClient.query(
+    `UPDATE route53_audit_log
+        SET status = 'committed',
+            completed_at = NOW(),
+            aws_change_id = $2,
+            aws_change_status = $3,
+            aws_response = $4::jsonb
+      WHERE id = $1`,
+    [id, awsChangeId, awsChangeStatus, JSON.stringify(awsResponse ?? null)]
+  );
+}
+
+/**
+ * Transition an audit row to 'failed' after an unsuccessful AWS call.
+ * D-07 + T-24-03: the error is always passed through sanitizeAwsError first
+ * — never pass a raw error object or JSON.stringify(err) into error_message.
+ * Not best-effort — let it throw.
+ */
+export async function markAuditFailed(id: string, err: unknown): Promise<void> {
+  await postgresClient.query(
+    `UPDATE route53_audit_log
+        SET status = 'failed',
+            completed_at = NOW(),
+            error_message = $2
+      WHERE id = $1`,
+    [id, sanitizeAwsError(err)]
+  );
+}
+
+// ============================================================================
+// pulse_crud history (route53_record_history)
+// ============================================================================
+
+export interface InsertPulseCrudHistoryInput {
+  zoneId: string;
+  recordKey: string;
+  recordName: string;
+  recordType: string;
+  changeAction: 'create' | 'update' | 'delete';
+  beforeValue: unknown;
+  afterValue: unknown;
+  changedByUserId: string | null;
+  changedByEmail: string | null;
+  auditLogId: string;
+}
+
+/**
+ * Insert a 'pulse_crud' history row.
+ *
+ * IMPORTANT: callers must invoke this ONLY after markAuditCommitted. A
+ * failed AWS call changed nothing on AWS's side, so it gets an audit row
+ * (for traceability of the attempt) but must NOT get a history row (which is
+ * a timeline of resolved, actually-applied changes). See 24-RESEARCH.md
+ * Pattern 3.
+ */
+export async function insertPulseCrudHistory(
+  input: InsertPulseCrudHistoryInput
+): Promise<void> {
+  await postgresClient.query(
+    `INSERT INTO route53_record_history
+       (zone_id, record_key, record_name, record_type,
+        change_action, before_value, after_value,
+        source, changed_by_user_id, changed_by_email, audit_log_id)
+     VALUES ($1, $2, $3, $4,
+             $5, $6::jsonb, $7::jsonb,
+             'pulse_crud', $8, $9, $10)`,
+    [
+      input.zoneId,
+      input.recordKey,
+      input.recordName,
+      input.recordType,
+      input.changeAction,
+      JSON.stringify(input.beforeValue ?? null),
+      JSON.stringify(input.afterValue ?? null),
+      input.changedByUserId,
+      input.changedByEmail,
+      input.auditLogId,
+    ]
+  );
+}
+
+// ============================================================================
+// Mirror refresh (route53_records) — best-effort
+// ============================================================================
+
+export interface UpsertMirrorRecordInput {
+  recordKey: string;
+  zoneId: string;
+  name: string;
+  type: string;
+  setIdentifier: string | null;
+  ttl: number | null;
+  resourceRecords: Route53RecordValue[] | null;
+  aliasTarget: Record<string, unknown> | null;
+  rawPayload: unknown;
+}
+
+/**
+ * Best-effort refresh of route53_records after a committed write so the
+ * admin UI reflects the change before the next scheduled sync. A failure
+ * here is logged but never thrown — the AWS write already succeeded and the
+ * next incremental sync reconciles the mirror regardless.
+ */
+export async function upsertMirrorRecord(input: UpsertMirrorRecordInput): Promise<void> {
+  try {
+    await postgresClient.query(
+      `INSERT INTO route53_records
+         (record_key, zone_id, name, type, set_identifier, ttl,
+          resource_records, alias_target, raw_payload,
+          synced_at, updated_at, is_deleted, deleted_at)
+       VALUES ($1, $2, $3, $4, $5, $6,
+               $7::jsonb, $8::jsonb, $9::jsonb,
+               NOW(), NOW(), false, NULL)
+       ON CONFLICT (record_key) DO UPDATE SET
+         zone_id = EXCLUDED.zone_id,
+         name = EXCLUDED.name,
+         type = EXCLUDED.type,
+         set_identifier = EXCLUDED.set_identifier,
+         ttl = EXCLUDED.ttl,
+         resource_records = EXCLUDED.resource_records,
+         alias_target = EXCLUDED.alias_target,
+         raw_payload = EXCLUDED.raw_payload,
+         synced_at = NOW(),
+         updated_at = NOW(),
+         is_deleted = false,
+         deleted_at = NULL`,
+      [
+        input.recordKey,
+        input.zoneId,
+        input.name,
+        input.type,
+        input.setIdentifier,
+        input.ttl,
+        JSON.stringify(input.resourceRecords ?? null),
+        JSON.stringify(input.aliasTarget ?? null),
+        JSON.stringify(input.rawPayload ?? null),
+      ]
+    );
+  } catch (err) {
+    console.error(`${LOG_PREFIX} upsertMirrorRecord failed (best-effort):`, sanitizeAwsError(err));
+  }
+}
+
+/**
+ * Soft-delete a mirror record. Never hard-delete: route53_record_history
+ * references record_key and the ledger is unbounded by design (D-08). Best
+ * effort — logged, never thrown.
+ */
+export async function softDeleteMirrorRecord(recordKey: string): Promise<void> {
+  try {
+    await postgresClient.query(
+      `UPDATE route53_records
+          SET is_deleted = true, deleted_at = NOW(), updated_at = NOW()
+        WHERE record_key = $1`,
+      [recordKey]
+    );
+  } catch (err) {
+    console.error(`${LOG_PREFIX} softDeleteMirrorRecord failed (best-effort):`, sanitizeAwsError(err));
+  }
+}
+
+// ============================================================================
+// Mirror read (route53_records)
+// ============================================================================
+
+export interface MirrorRecordRow {
+  recordKey: string;
+  zoneId: string;
+  name: string;
+  type: string;
+  setIdentifier: string | null;
+  ttl: number | null;
+  resourceRecords: Route53RecordValue[] | null;
+  aliasTarget: Record<string, unknown> | null;
+}
+
+interface RawMirrorRecordRow {
+  record_key: string;
+  zone_id: string;
+  name: string;
+  type: string;
+  set_identifier: string | null;
+  ttl: number | null;
+  resource_records: Route53RecordValue[] | null;
+  alias_target: Record<string, unknown> | null;
+}
+
+/**
+ * Load the current mirror row for a record key, if present and not
+ * soft-deleted. Supplies the before_value for updates/deletes and,
+ * critically, the exact TTL and value set a Route 53 DELETE action requires
+ * to match — a DELETE with a mismatched TTL or value set fails or targets
+ * the wrong recordset (24-RESEARCH.md Pitfall 3).
+ */
+export async function loadMirrorRecord(recordKey: string): Promise<MirrorRecordRow | null> {
+  const res = await postgresClient.query<RawMirrorRecordRow>(
+    `SELECT record_key, zone_id, name, type, set_identifier, ttl, resource_records, alias_target
+       FROM route53_records
+      WHERE record_key = $1 AND is_deleted = false`,
+    [recordKey]
+  );
+  if (res.rowCount === 0) return null;
+  const row = res.rows[0];
+  return {
+    recordKey: row.record_key,
+    zoneId: row.zone_id,
+    name: row.name,
+    type: row.type,
+    setIdentifier: row.set_identifier,
+    ttl: row.ttl,
+    resourceRecords: row.resource_records,
+    aliasTarget: row.alias_target,
+  };
+}