From 51f0b32cb32b52d1d12ed75b01cf302238f61a63 Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 21 May 2026 11:08:24 -0400 Subject: [PATCH 001/463] feat(260521-fci-01): add ticket reconciliation service + API route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New lib/services/ticket-reconciliation-service.ts: reconcileStaleTickets() scans tickets where is_deleted=false AND status<>5 AND synced_at older than 7 days (capped at 500), re-fetches each from Autotask, and either upserts via the webhook SQL pattern or soft-deletes when Autotask returns null. - Returns { scanned, updated, statusFlippedToComplete, softDeleted, errors }. - New POST /api/sync/reconcile-tickets — fire-and-forget trigger mirroring /api/sync/incremental (public per existing middleware allowlist). --- app/api/sync/reconcile-tickets/route.ts | 31 ++++ lib/services/ticket-reconciliation-service.ts | 151 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 app/api/sync/reconcile-tickets/route.ts create mode 100644 lib/services/ticket-reconciliation-service.ts diff --git a/app/api/sync/reconcile-tickets/route.ts b/app/api/sync/reconcile-tickets/route.ts new file mode 100644 index 0000000..bffdc5a --- /dev/null +++ b/app/api/sync/reconcile-tickets/route.ts @@ -0,0 +1,31 @@ +/** + * Ticket Reconciliation API Endpoint + * POST /api/sync/reconcile-tickets - Trigger a stale-ticket reconciliation pass. + * Public per middleware.ts (matches /api/sync/incremental). Fire-and-forget. + */ + +import { NextRequest, NextResponse } from 'next/server'; +import { reconcileStaleTickets } from '@/lib/services/ticket-reconciliation-service'; + +export async function POST(request: NextRequest) { + try { + const body = await request.json().catch(() => ({})); + const triggeredBy = body.triggeredBy || 'api'; + + // Non-blocking — same pattern as /api/sync/incremental. + reconcileStaleTickets().catch((error) => { + console.error('[RECONCILE] Ticket reconciliation failed:', error); + }); + + return NextResponse.json({ + message: 'Ticket reconciliation started', + triggeredBy, + }); + } catch (error) { + console.error('Failed to start ticket reconciliation:', error); + return NextResponse.json( + { error: 'Failed to start ticket reconciliation' }, + { status: 500 } + ); + } +} diff --git a/lib/services/ticket-reconciliation-service.ts b/lib/services/ticket-reconciliation-service.ts new file mode 100644 index 0000000..c859df6 --- /dev/null +++ b/lib/services/ticket-reconciliation-service.ts @@ -0,0 +1,151 @@ +/** + * Ticket Reconciliation Service (STOPGAP) + * + * Nightly backstop for drift between the postgres `tickets` mirror and Autotask. + * Webhooks can drop events and the incremental sync's high-water-mark filter + * can miss tickets whose lastActivityDate never advances after a status change. + * This job bounds local staleness at 7 days by re-fetching each stale-open + * ticket from Autotask and either re-upserting or soft-deleting. + * + * NOTE: This is a stopgap. The underlying incremental-filter issue is tracked + * separately and should fix the root cause. + */ + +import { postgresClient } from './postgres-client'; +import { AutotaskClient } from './autotask-client'; +import { autotaskRateLimiter } from './rate-limiter'; +import { mapAutotaskToDatabase } from '../utils/entity-mapper'; +import { getTableName } from '../utils/sync-helpers'; +import { EntityType } from '../types/sync'; +import { createSyncLogger } from '../utils/sync-logger'; + +export interface ReconcileResult { + scanned: number; + updated: number; + statusFlippedToComplete: number; + softDeleted: number; + errors: number; +} + +const STALE_DAYS = 7; +const SCAN_LIMIT = 500; +const COMPLETE_STATUS = 5; // Autotask status id for Complete + +let _client: AutotaskClient | null = null; +function getClient(): AutotaskClient { + if (!_client) { + _client = new AutotaskClient({ + apiUrl: process.env.AUTOTASK_API_URL || '', + username: process.env.AUTOTASK_USERNAME || '', + password: process.env.AUTOTASK_SECRET || '', + apiIntegrationCode: process.env.AUTOTASK_API_INTEGRATION_CODE || '', + }); + } + return _client; +} + +/** + * Re-fetch up to SCAN_LIMIT stale-open tickets from Autotask and reconcile + * the local mirror. Idempotent and safe to run alongside the regular sync. + */ +export async function reconcileStaleTickets(): Promise { + const logger = createSyncLogger({ component: 'TicketReconciliation' }); + const startedAt = Date.now(); + const result: ReconcileResult = { + scanned: 0, + updated: 0, + statusFlippedToComplete: 0, + softDeleted: 0, + errors: 0, + }; + + // 1. Find stale-open tickets in the postgres mirror. + const staleQuery = ` + SELECT id, status + FROM tickets + WHERE is_deleted = false + AND status <> $1 + AND synced_at < NOW() - INTERVAL '${STALE_DAYS} days' + ORDER BY synced_at ASC + LIMIT $2 + `; + const stale = await postgresClient.query<{ id: string; status: number | null }>( + staleQuery, + [COMPLETE_STATUS, SCAN_LIMIT] + ); + result.scanned = stale.rows.length; + + logger.info(`Scanning ${result.scanned} stale-open tickets (>${STALE_DAYS}d, limit=${SCAN_LIMIT})`); + + if (result.scanned === 0) { + logger.info('No stale tickets found', { duration: Date.now() - startedAt }); + return result; + } + + const client = getClient(); + const tableName = getTableName(EntityType.TICKETS); // 'tickets' + + // 2. Walk each id. Use the singleton rate limiter as an extra guard on top of + // AutotaskClient's internal limiter (defense in depth for parallel callers). + for (const row of stale.rows) { + const ticketId = Number(row.id); + const priorStatus = row.status; + try { + const entity = await autotaskRateLimiter.throttle(() => + client.getEntityById>('Tickets', ticketId) + ); + + if (!entity) { + // Not in Autotask anymore -> soft-delete locally. + await postgresClient.query( + `UPDATE ${tableName} + SET is_deleted = true, deleted_at = NOW(), updated_at = NOW() + WHERE id = $1`, + [ticketId] + ); + result.softDeleted += 1; + continue; + } + + // 3. Upsert using the same SQL shape the webhook handler uses. + const mapped = mapAutotaskToDatabase(EntityType.TICKETS, entity); + if (!mapped) { + throw new Error(`mapAutotaskToDatabase returned null for ticket ${ticketId}`); + } + const keys = Object.keys(mapped); + const values = Object.values(mapped); + const placeholders = keys.map((_, i) => `$${i + 1}`).join(', '); + const updateClause = keys + .filter(k => k !== 'id') + .map(k => `${k} = EXCLUDED.${k}`) + .join(', '); + const upsertSql = ` + INSERT INTO ${tableName} (${keys.join(', ')}) + VALUES (${placeholders}) + ON CONFLICT (id) + DO UPDATE SET ${updateClause}, updated_at = NOW() + `; + await postgresClient.query(upsertSql, values); + result.updated += 1; + + const newStatus = (mapped.status ?? null) as number | null; + if (priorStatus !== COMPLETE_STATUS && newStatus === COMPLETE_STATUS) { + result.statusFlippedToComplete += 1; + } + } catch (err) { + result.errors += 1; + logger.warn( + `Reconcile failed for ticket ${ticketId}`, + { ticketId }, + err instanceof Error ? err : new Error(String(err)) + ); + } + } + + logger.info( + `Reconciliation complete: scanned=${result.scanned} updated=${result.updated} flippedComplete=${result.statusFlippedToComplete} softDeleted=${result.softDeleted} errors=${result.errors}`, + { duration: Date.now() - startedAt } + ); + + return result; +} From badd7181946cb51dabcfabc63dce3742be770dce Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 21 May 2026 11:09:36 -0400 Subject: [PATCH 002/463] feat(260521-fci-02): wire tickets-reconcile schedule + migration 090 - sync-scheduler.ts: extend sync_type union with 'tickets-reconcile', add a default schedule entry (disabled, 30 4 * * *), and a dispatch case using the device-link-reconcile / integration-health dynamic-import pattern. - migrations/090_ticket_reconcile_schedule.sql: idempotent INSERT (ON CONFLICT DO NOTHING) so existing installs pick up the row without disturbing the fresh-DB default-seed path. --- lib/services/sync-scheduler.ts | 29 +++++++++++++++++++- migrations/090_ticket_reconcile_schedule.sql | 24 ++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 migrations/090_ticket_reconcile_schedule.sql diff --git a/lib/services/sync-scheduler.ts b/lib/services/sync-scheduler.ts index cae7f18..09b9382 100644 --- a/lib/services/sync-scheduler.ts +++ b/lib/services/sync-scheduler.ts @@ -22,7 +22,7 @@ export interface ScheduleConfig { name: string; description: string; cron_expression: string; - sync_type: 'incremental' | 'full' | 'veeam-incremental' | 'veeam-full' | 'veeam-rpo-check' | 'contract-services' | 'engagement-daily' | 'zoom-daily' | 'morning-summary' | 'ticket-digest-daily' | 'ticket-digest-weekly' | 'ticket-digest-monthly' | 'device-link-reconcile' | 'integration-health'; + sync_type: 'incremental' | 'full' | 'veeam-incremental' | 'veeam-full' | 'veeam-rpo-check' | 'contract-services' | 'engagement-daily' | 'zoom-daily' | 'morning-summary' | 'ticket-digest-daily' | 'ticket-digest-weekly' | 'ticket-digest-monthly' | 'device-link-reconcile' | 'integration-health' | 'qbo' | 'appgate-sessions' | 'appgate-daily' | 'tickets-reconcile'; years_back?: number; is_enabled: boolean; last_run?: Date; @@ -291,6 +291,14 @@ class SyncScheduler { sync_type: 'integration-health', is_enabled: false, }, + { + id: 'tickets-reconcile', + name: 'Tickets Reconciliation', + description: 'STOPGAP backstop: re-fetches stale-open tickets (>7d since last sync) from Autotask and reconciles status / soft-deletes missing rows. Runs daily at 4:30 AM. Capped at 500 tickets per run.', + cron_expression: '30 4 * * *', + sync_type: 'tickets-reconcile', + is_enabled: false, + }, ]; for (const schedule of defaultSchedules) { @@ -434,6 +442,25 @@ class SyncScheduler { console.log( `[SCHEDULER] integration-health: failed=${result.summary.failed} expired=${result.summary.expired} expiringSoon=${result.summary.expiringWithin14Days} alertSent=${result.alertSent}` ); + } else if (config.sync_type === 'appgate-sessions' || config.sync_type === 'appgate-daily') { + const { isAppgateConfigured } = await import('@/lib/services/appgate-factory'); + if (!isAppgateConfigured()) { + console.log(`[SCHEDULER] Skipping ${config.sync_type} — AppGate not configured`); + } else { + const { getAppgateSyncService } = await import('@/lib/services/appgate-sync-service'); + const svc = getAppgateSyncService(); + if (config.sync_type === 'appgate-daily') { + await svc.dailySync('scheduled'); + } else { + await svc.sessionsSync('scheduled'); + } + } + } else if (config.sync_type === 'tickets-reconcile') { + const { reconcileStaleTickets } = await import('@/lib/services/ticket-reconciliation-service'); + const result = await reconcileStaleTickets(); + console.log( + `[SCHEDULER] tickets-reconcile: scanned=${result.scanned} updated=${result.updated} flippedComplete=${result.statusFlippedToComplete} softDeleted=${result.softDeleted} errors=${result.errors}` + ); } else if (config.sync_type === 'incremental') { await this.syncService.incrementalSync('scheduled'); } else { diff --git a/migrations/090_ticket_reconcile_schedule.sql b/migrations/090_ticket_reconcile_schedule.sql new file mode 100644 index 0000000..728ed80 --- /dev/null +++ b/migrations/090_ticket_reconcile_schedule.sql @@ -0,0 +1,24 @@ +-- Migration 090: Seed the tickets-reconcile sync schedule (STOPGAP). +-- +-- The sync_scheduler.createDefaultSchedules() path only seeds defaults on a +-- virgin sync_schedules table; this migration covers existing installs. +-- Idempotent via ON CONFLICT (id) DO NOTHING. + +INSERT INTO sync_schedules ( + id, + name, + description, + cron_expression, + sync_type, + years_back, + is_enabled +) VALUES ( + 'tickets-reconcile', + 'Tickets Reconciliation', + 'STOPGAP backstop: re-fetches stale-open tickets (>7d since last sync) from Autotask and reconciles status / soft-deletes missing rows. Runs daily at 4:30 AM. Capped at 500 tickets per run.', + '30 4 * * *', + 'tickets-reconcile', + NULL, + false +) +ON CONFLICT (id) DO NOTHING; From d02796e8639a80e146287edba6546439b920a7ee Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 21 May 2026 11:11:33 -0400 Subject: [PATCH 003/463] docs(quick-260521-fci): Stopgap nightly reconciliation for stale open tickets in postgres mirror --- .planning/STATE.md | 3 +- .../260521-fci-PLAN.md | 525 ++++++++++++++++++ .../260521-fci-SUMMARY.md | 122 ++++ 3 files changed, 649 insertions(+), 1 deletion(-) create mode 100644 .planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-PLAN.md create mode 100644 .planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-SUMMARY.md diff --git a/.planning/STATE.md b/.planning/STATE.md index 2d55ac6..1d07cf4 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -28,7 +28,7 @@ See: .planning/PROJECT.md (updated 2026-05-03) Phase: 09.1 (ntfy-backend-fix) — EXECUTING Plan: 1 of 1 Status: Executing Phase 09.1 -Last activity: 2026-05-19 - Completed quick task 260519-0oz: Add QBO createPayment + createDeposit + .FH reconciliation script +Last activity: 2026-05-21 - Completed quick task 260521-fci: Stopgap nightly reconciliation for stale open tickets in postgres mirror Progress: [░░░░░░░░░░] 0% @@ -94,6 +94,7 @@ None yet. | # | Description | Date | Commit | Directory | |---|-------------|------|--------|-----------| | 260519-0oz | Add QBO createPayment + createDeposit + .FH reconciliation script | 2026-05-19 | 5497458 | [260519-0oz-add-qbo-createpayment-createdeposit-fh-r](./quick/260519-0oz-add-qbo-createpayment-createdeposit-fh-r/) | +| 260521-fci | Stopgap nightly reconciliation for stale open tickets in postgres mirror | 2026-05-21 | badd718 | [260521-fci-stopgap-nightly-reconciliation-for-stale](./quick/260521-fci-stopgap-nightly-reconciliation-for-stale/) | ## Session Continuity diff --git a/.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-PLAN.md b/.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-PLAN.md new file mode 100644 index 0000000..4d4753d --- /dev/null +++ b/.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-PLAN.md @@ -0,0 +1,525 @@ +--- +phase: quick-260521-fci +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - lib/services/ticket-reconciliation-service.ts + - app/api/sync/reconcile-tickets/route.ts + - lib/services/sync-scheduler.ts + - migrations/090_ticket_reconcile_schedule.sql +autonomous: true +requirements: + - QUICK-260521-FCI-01 +must_haves: + truths: + - "A nightly job scans the postgres tickets mirror for rows where is_deleted=false AND status<>5 AND synced_at < NOW() - INTERVAL '7 days' (capped at LIMIT 500)" + - "For each stale ticket id, the job fetches the current Autotask record via AutotaskClient.getEntityById('Tickets', id) and either upserts (using the existing mapAutotaskToDatabase + webhook upsert SQL) or soft-deletes (is_deleted=true, deleted_at=NOW()) when Autotask returns null" + - "The job returns { scanned, updated, statusFlippedToComplete, softDeleted, errors } and writes one summary log line per run" + - "POST /api/sync/reconcile-tickets triggers the job non-blocking (fire-and-forget), mirroring /api/sync/incremental" + - "A 'tickets-reconcile' default schedule (id=tickets-reconcile, cron='30 4 * * *') exists in sync_schedules and is dispatched via the scheduler's switch (disabled by default, like other defaults)" + - "Migration 090 idempotently seeds the schedule row with ON CONFLICT (id) DO NOTHING so it lands on a fresh DB without disturbing existing installs" + - "`npx tsc --noEmit --pretty` passes — type-check is the safety net (no tests for sync services)" + artifacts: + - path: "lib/services/ticket-reconciliation-service.ts" + provides: "reconcileStaleTickets() function returning ReconcileResult" + exports: ["reconcileStaleTickets", "ReconcileResult"] + - path: "app/api/sync/reconcile-tickets/route.ts" + provides: "POST handler — fire-and-forget trigger for reconcileStaleTickets()" + exports: ["POST"] + - path: "lib/services/sync-scheduler.ts" + provides: "Updated sync_type union + dispatch case + default schedule entry for tickets-reconcile" + contains: "tickets-reconcile" + - path: "migrations/090_ticket_reconcile_schedule.sql" + provides: "Idempotent INSERT for tickets-reconcile schedule row" + contains: "ON CONFLICT (id) DO NOTHING" + key_links: + - from: "lib/services/sync-scheduler.ts" + to: "lib/services/ticket-reconciliation-service.ts" + via: "dynamic import + direct call in executeScheduledSync switch (matches device-link-reconcile / integration-health pattern)" + pattern: "reconcileStaleTickets\\(" + - from: "app/api/sync/reconcile-tickets/route.ts" + to: "lib/services/ticket-reconciliation-service.ts" + via: "fire-and-forget call with .catch() — mirrors /api/sync/incremental" + pattern: "reconcileStaleTickets\\(\\)\\.catch" + - from: "lib/services/ticket-reconciliation-service.ts" + to: "lib/utils/entity-mapper.ts" + via: "mapAutotaskToDatabase(EntityType.TICKETS, entity) — reuse, do not parallel-implement" + pattern: "mapAutotaskToDatabase\\(EntityType\\.TICKETS" +--- + + +Build a STOPGAP nightly reconciliation job that detects "drift" between the +postgres `tickets` mirror and Autotask: tickets that have not been re-synced in +>7 days and are still marked open locally. For each, fetch the canonical +Autotask record by id and either re-upsert (status may have flipped to Complete +out-of-band, e.g. via a missed webhook) or soft-delete (Autotask no longer +returns the row). Wire it into the existing scheduler as a nightly cron. + +Purpose: The webhook can drop events (Autotask retries are limited, network +hiccups happen), and the incremental sync uses a high-water-mark filter that +will miss a ticket whose `lastActivityDate` never advances after a status +change. This job is a backstop — not a replacement for fixing the incremental +filter, but a way to bound staleness at 7 days while that fix is scoped +separately. + +Output: +- `lib/services/ticket-reconciliation-service.ts` (new) +- `app/api/sync/reconcile-tickets/route.ts` (new) +- Edits to `lib/services/sync-scheduler.ts` (union + default + dispatch) +- `migrations/090_ticket_reconcile_schedule.sql` (new) + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/STATE.md +@CLAUDE.md + + +@lib/services/webhook-service.ts +@lib/services/sync-scheduler.ts +@app/api/sync/incremental/route.ts +@lib/utils/entity-mapper.ts +@lib/utils/sync-helpers.ts +@lib/utils/sync-logger.ts +@lib/services/autotask-client.ts +@lib/services/rate-limiter.ts +@lib/services/postgres-client.ts +@lib/types/sync.ts +@middleware.ts + + + + +From `lib/services/autotask-client.ts:164`: +```typescript +async getEntityById(entityName: string, id: number): Promise +// Internally calls this.rateLimiter.throttle() (10 req/sec). For Tickets, pass 'Tickets'. +``` + +From `lib/utils/entity-mapper.ts`: +```typescript +export function mapAutotaskToDatabase( + entity: EntityType, + data: any +): Record +// For EntityType.TICKETS, runs the ticket-specific mapper and appends synced_at=new Date() +``` + +From `lib/utils/sync-helpers.ts`: +```typescript +export function getTableName(entity: EntityType): string +// getTableName(EntityType.TICKETS) -> 'tickets' +``` + +From `lib/utils/sync-logger.ts:250`: +```typescript +export function createSyncLogger(context?: LogContext): SyncLogger +// Use: const logger = createSyncLogger({ component: 'TicketReconciliation' }) +// Methods: logger.info(msg, ctx?), logger.warn(msg, ctx?, err?), logger.error(msg, ctx?, err?) +``` + +From `lib/services/rate-limiter.ts:114`: +```typescript +export const autotaskRateLimiter: RateLimiter +// Use: await autotaskRateLimiter.throttle(async () => { ... }) +``` + +From `lib/types/sync.ts`: +```typescript +export enum EntityType { TICKETS = 'tickets', ... } +``` + +From `lib/services/postgres-client.ts`: +```typescript +postgresClient.query(sql: string, params?: any[]): Promise> +// .rows: T[] +``` + +**Webhook upsert pattern to mirror exactly** (`lib/services/webhook-service.ts:201-234`): +```typescript +const mappedData = mapAutotaskToDatabase(internalEntityType, entityData); +const tableName = getTableName(internalEntityType); +const keys = Object.keys(mappedData); +const values = Object.values(mappedData); +const placeholders = keys.map((_, i) => `$${i + 1}`).join(', '); +const updateClause = keys + .filter(k => k !== 'id') + .map(k => `${k} = EXCLUDED.${k}`) + .join(', '); +const query = ` + INSERT INTO ${tableName} (${keys.join(', ')}) + VALUES (${placeholders}) + ON CONFLICT (id) + DO UPDATE SET ${updateClause}, updated_at = NOW() +`; +await postgresClient.query(query, values); +``` + +**Tickets table schema reminders:** +- `id` is `BIGINT` (Autotask id) +- Audit cols: `created_at`, `updated_at`, `synced_at`, `is_deleted`, `deleted_at` +- Status 5 = Complete in Autotask convention (see `ticket-digest-service.ts:193` precedent: `status NOT IN (5)`) + +**Existing dispatch precedent** (`sync-scheduler.ts:425-449`): +The `device-link-reconcile` and `integration-health` cases use dynamic `await import()` +and call the service directly (not via HTTP). The dispatch logs a one-line summary. +Mirror that pattern — do NOT POST to the API route from the scheduler. (The API +route exists for manual/admin triggering, the scheduler calls the service directly.) + +**Middleware note:** `/api/sync` is already in `publicRoutes` (middleware.ts:30), +so the new `/api/sync/reconcile-tickets` route is auto-allowed. No middleware +edit needed. + +**Default-schedule seeding caveat:** `createDefaultSchedules()` in +`sync-scheduler.ts` only inserts when the table is empty (line 173 check). +For existing DBs the in-code default won't fire — that's why migration 090 +seeds the row explicitly with `ON CONFLICT (id) DO NOTHING`. On a fresh DB +both paths are idempotent. + + + + + + + Task 1: Build reconciliation service + API route + + lib/services/ticket-reconciliation-service.ts, + app/api/sync/reconcile-tickets/route.ts + + +**Create `lib/services/ticket-reconciliation-service.ts`:** + +```typescript +/** + * Ticket Reconciliation Service (STOPGAP) + * + * Nightly backstop for drift between the postgres `tickets` mirror and Autotask. + * Webhooks can drop events and the incremental sync's high-water-mark filter + * can miss tickets whose lastActivityDate never advances after a status change. + * This job bounds local staleness at 7 days by re-fetching each stale-open + * ticket from Autotask and either re-upserting or soft-deleting. + * + * NOTE: This is a stopgap. The underlying incremental-filter issue is tracked + * separately and should fix the root cause. + */ + +import { postgresClient } from './postgres-client'; +import { AutotaskClient } from './autotask-client'; +import { autotaskRateLimiter } from './rate-limiter'; +import { mapAutotaskToDatabase } from '../utils/entity-mapper'; +import { getTableName } from '../utils/sync-helpers'; +import { EntityType } from '../types/sync'; +import { createSyncLogger } from '../utils/sync-logger'; + +export interface ReconcileResult { + scanned: number; + updated: number; + statusFlippedToComplete: number; + softDeleted: number; + errors: number; +} + +const STALE_DAYS = 7; +const SCAN_LIMIT = 500; +const COMPLETE_STATUS = 5; // Autotask status id for Complete + +let _client: AutotaskClient | null = null; +function getClient(): AutotaskClient { + if (!_client) { + _client = new AutotaskClient({ + apiUrl: process.env.AUTOTASK_API_URL || '', + username: process.env.AUTOTASK_USERNAME || '', + password: process.env.AUTOTASK_SECRET || '', + apiIntegrationCode: process.env.AUTOTASK_API_INTEGRATION_CODE || '', + }); + } + return _client; +} + +/** + * Re-fetch up to SCAN_LIMIT stale-open tickets from Autotask and reconcile + * the local mirror. Idempotent and safe to run alongside the regular sync. + */ +export async function reconcileStaleTickets(): Promise { + const logger = createSyncLogger({ component: 'TicketReconciliation' }); + const startedAt = Date.now(); + const result: ReconcileResult = { + scanned: 0, + updated: 0, + statusFlippedToComplete: 0, + softDeleted: 0, + errors: 0, + }; + + // 1. Find stale-open tickets in the postgres mirror. + const staleQuery = ` + SELECT id, status + FROM tickets + WHERE is_deleted = false + AND status <> $1 + AND synced_at < NOW() - INTERVAL '${STALE_DAYS} days' + ORDER BY synced_at ASC + LIMIT $2 + `; + const stale = await postgresClient.query<{ id: string; status: number | null }>( + staleQuery, + [COMPLETE_STATUS, SCAN_LIMIT] + ); + result.scanned = stale.rows.length; + + logger.info(`Scanning ${result.scanned} stale-open tickets (>${STALE_DAYS}d, limit=${SCAN_LIMIT})`); + + if (result.scanned === 0) { + logger.info('No stale tickets found', { duration: Date.now() - startedAt }); + return result; + } + + const client = getClient(); + const tableName = getTableName(EntityType.TICKETS); // 'tickets' + + // 2. Walk each id. Use the singleton rate limiter as an extra guard on top of + // AutotaskClient's internal limiter (defense in depth for parallel callers). + for (const row of stale.rows) { + const ticketId = Number(row.id); + const priorStatus = row.status; + try { + const entity = await autotaskRateLimiter.throttle(() => + client.getEntityById>('Tickets', ticketId) + ); + + if (!entity) { + // Not in Autotask anymore -> soft-delete locally. + await postgresClient.query( + `UPDATE ${tableName} + SET is_deleted = true, deleted_at = NOW(), updated_at = NOW() + WHERE id = $1`, + [ticketId] + ); + result.softDeleted += 1; + continue; + } + + // 3. Upsert using the same SQL shape the webhook handler uses. + const mapped = mapAutotaskToDatabase(EntityType.TICKETS, entity); + if (!mapped) { + throw new Error(`mapAutotaskToDatabase returned null for ticket ${ticketId}`); + } + const keys = Object.keys(mapped); + const values = Object.values(mapped); + const placeholders = keys.map((_, i) => `$${i + 1}`).join(', '); + const updateClause = keys + .filter(k => k !== 'id') + .map(k => `${k} = EXCLUDED.${k}`) + .join(', '); + const upsertSql = ` + INSERT INTO ${tableName} (${keys.join(', ')}) + VALUES (${placeholders}) + ON CONFLICT (id) + DO UPDATE SET ${updateClause}, updated_at = NOW() + `; + await postgresClient.query(upsertSql, values); + result.updated += 1; + + const newStatus = (mapped.status ?? null) as number | null; + if (priorStatus !== COMPLETE_STATUS && newStatus === COMPLETE_STATUS) { + result.statusFlippedToComplete += 1; + } + } catch (err) { + result.errors += 1; + logger.warn( + `Reconcile failed for ticket ${ticketId}`, + { ticketId }, + err instanceof Error ? err : new Error(String(err)) + ); + } + } + + logger.info( + `Reconciliation complete: scanned=${result.scanned} updated=${result.updated} flippedComplete=${result.statusFlippedToComplete} softDeleted=${result.softDeleted} errors=${result.errors}`, + { duration: Date.now() - startedAt } + ); + + return result; +} +``` + +**Create `app/api/sync/reconcile-tickets/route.ts`:** + +```typescript +/** + * Ticket Reconciliation API Endpoint + * POST /api/sync/reconcile-tickets - Trigger a stale-ticket reconciliation pass. + * Public per middleware.ts (matches /api/sync/incremental). Fire-and-forget. + */ + +import { NextRequest, NextResponse } from 'next/server'; +import { reconcileStaleTickets } from '@/lib/services/ticket-reconciliation-service'; + +export async function POST(request: NextRequest) { + try { + const body = await request.json().catch(() => ({})); + const triggeredBy = body.triggeredBy || 'api'; + + // Non-blocking — same pattern as /api/sync/incremental. + reconcileStaleTickets().catch((error) => { + console.error('[RECONCILE] Ticket reconciliation failed:', error); + }); + + return NextResponse.json({ + message: 'Ticket reconciliation started', + triggeredBy, + }); + } catch (error) { + console.error('Failed to start ticket reconciliation:', error); + return NextResponse.json( + { error: 'Failed to start ticket reconciliation' }, + { status: 500 } + ); + } +} +``` + +**Constraints reminders (CLAUDE.md):** +- No Zod validation in route. +- No `requireAuth()` — `/api/sync` is in middleware's `publicRoutes`; matches `incremental/route.ts`. +- `postgresClient.query()`, manual snake_case ↔ camelCase via `mapAutotaskToDatabase`. +- Errors: `try/catch` + `NextResponse.json({ error, message }, { status })`. 500 for runtime. +- kebab-case filenames (both are). Use `@/lib/...` and `@/lib/services/...` imports — no relative `../../`. +- Do NOT introduce a parallel mapping path. Do NOT touch `sync-helpers.ts`. + + + npx tsc --noEmit --pretty 2>&1 | grep -E "(ticket-reconciliation|reconcile-tickets)" ; echo "exit=$?" + + +- Both files exist at the listed paths. +- `npx tsc --noEmit --pretty` is clean for the two new files (no errors referencing + `ticket-reconciliation-service.ts` or `app/api/sync/reconcile-tickets/route.ts`). +- `reconcileStaleTickets()` exports a `ReconcileResult` shape matching + `{ scanned, updated, statusFlippedToComplete, softDeleted, errors }`. +- The route is structured exactly like `/api/sync/incremental` (no auth, body parse + with `.catch(() => ({}))`, fire-and-forget call with `.catch()`). +- Upsert SQL is the webhook-service pattern, not a new shape. + + + + + Task 2: Wire scheduler + migration + + lib/services/sync-scheduler.ts, + migrations/090_ticket_reconcile_schedule.sql + + +**Edit `lib/services/sync-scheduler.ts`:** + +1. **Extend the `sync_type` union** on the `ScheduleConfig` interface (line ~25). Append `| 'tickets-reconcile'` to the existing union literal: + +```typescript + sync_type: 'incremental' | 'full' | 'veeam-incremental' | 'veeam-full' | 'veeam-rpo-check' | 'contract-services' | 'engagement-daily' | 'zoom-daily' | 'morning-summary' | 'ticket-digest-daily' | 'ticket-digest-weekly' | 'ticket-digest-monthly' | 'device-link-reconcile' | 'integration-health' | 'qbo' | 'appgate-sessions' | 'appgate-daily' | 'tickets-reconcile'; +``` + +2. **Add a default schedule entry** inside `createDefaultSchedules()` `defaultSchedules` array (alongside the `integration-health` entry around line 286-293). Append: + +```typescript + { + id: 'tickets-reconcile', + name: 'Tickets Reconciliation', + description: 'STOPGAP backstop: re-fetches stale-open tickets (>7d since last sync) from Autotask and reconciles status / soft-deletes missing rows. Runs daily at 4:30 AM. Capped at 500 tickets per run.', + cron_expression: '30 4 * * *', + sync_type: 'tickets-reconcile', + is_enabled: false, + }, +``` + +(Match the other defaults: `is_enabled: false` so this seeds disabled and an admin enables it from `/admin`.) + +3. **Add a dispatch case** in `executeScheduledSync()`'s switch (the long `else if` chain starting at line ~397). Insert it next to `device-link-reconcile` / `integration-health` (anywhere in the chain before the trailing `incremental` / fallthrough cases). Use the dynamic-import pattern those two cases use: + +```typescript + } else if (config.sync_type === 'tickets-reconcile') { + const { reconcileStaleTickets } = await import('@/lib/services/ticket-reconciliation-service'); + const result = await reconcileStaleTickets(); + console.log( + `[SCHEDULER] tickets-reconcile: scanned=${result.scanned} updated=${result.updated} flippedComplete=${result.statusFlippedToComplete} softDeleted=${result.softDeleted} errors=${result.errors}` + ); +``` + +**Do NOT** change anything about `weekly-full`, the morning summary, or any other existing schedule. Do NOT modify `sync-helpers.ts`. + +**Create `migrations/090_ticket_reconcile_schedule.sql`:** + +```sql +-- Migration 090: Seed the tickets-reconcile sync schedule (STOPGAP). +-- +-- The sync_scheduler.createDefaultSchedules() path only seeds defaults on a +-- virgin sync_schedules table; this migration covers existing installs. +-- Idempotent via ON CONFLICT (id) DO NOTHING. + +INSERT INTO sync_schedules ( + id, + name, + description, + cron_expression, + sync_type, + years_back, + is_enabled +) VALUES ( + 'tickets-reconcile', + 'Tickets Reconciliation', + 'STOPGAP backstop: re-fetches stale-open tickets (>7d since last sync) from Autotask and reconciles status / soft-deletes missing rows. Runs daily at 4:30 AM. Capped at 500 tickets per run.', + '30 4 * * *', + 'tickets-reconcile', + NULL, + false +) +ON CONFLICT (id) DO NOTHING; +``` + +**Notes:** +- Schedule starts **disabled** (`is_enabled = false`) — admin enables from `/admin/sync-schedules` (or equivalent UI) when ready. Matches the convention of nearly every other default. +- Cron `30 4 * * *` = 4:30 AM daily, deliberately staggered from the 4:00 AM contract-services job to avoid Autotask rate-limit contention. +- The migration is idempotent: on a fresh DB the default-schedule seeder may insert first; on an existing DB this migration is the only insert path. Either way `ON CONFLICT DO NOTHING` makes the order irrelevant. +- No new migration numbers conflict — `ls migrations/` shows highest is `089_appgate_tables.sql`. + + + npx tsc --noEmit --pretty 2>&1 | grep -E "sync-scheduler" ; echo "exit=$?" + + +- `lib/services/sync-scheduler.ts` includes `'tickets-reconcile'` in the `sync_type` union, a new `defaultSchedules` entry with id `tickets-reconcile`, and a dispatch case calling `reconcileStaleTickets()` via dynamic import. +- `migrations/090_ticket_reconcile_schedule.sql` exists with `ON CONFLICT (id) DO NOTHING` and the same id/cron/description as the default entry. +- `npx tsc --noEmit --pretty` is clean for `sync-scheduler.ts` (no errors mentioning the file). +- Manual smoke check (optional, not required for done): grep confirms no leftover edits to `weekly-full` or `sync-helpers.ts`. + + + + + + +- `npx tsc --noEmit --pretty` exits clean across the whole repo (the project's only safety net for sync code). +- `grep -n "tickets-reconcile" lib/services/sync-scheduler.ts` shows three hits: union, default entry, dispatch case. +- `grep -n "reconcileStaleTickets" lib/services/` shows the export in the new service file and the import in the scheduler. +- `ls migrations/090_*.sql` returns the new file. +- Sanity-only manual check (do NOT execute as part of automated verify): a one-shot trigger via `curl -X POST http://localhost:3100/api/sync/reconcile-tickets -H 'Content-Type: application/json' -d '{"triggeredBy":"manual-smoke"}'` should return `{ "message": "Ticket reconciliation started", ... }` and the next container log line should be `[TicketReconciliation] Scanning N stale-open tickets...`. Document this in the SUMMARY, don't gate on it. + + + +- All four artifacts (service, route, scheduler edits, migration) exist and type-check. +- The job's behavioral contract holds: stale-open tickets are either updated or soft-deleted, never silently left behind. +- No CLAUDE.md violations: no ORM, no Zod-in-route, no parallel mapper, no auth on `/api/sync/*`, kebab-case filenames, `@/...` imports. +- Migration 090 is idempotent. The schedule lands disabled by default. +- Scope discipline: zero changes to `weekly-full`, `sync-helpers.ts`, or any non-listed file. + + + +After completion, create `.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-SUMMARY.md` with: +- Files created/modified (paths + 1-line role each) +- The `ReconcileResult` shape and where it's logged +- Cron + default-disabled note (for the eventual ops handoff) +- Any deviations from this plan and why +- Pointer to the underlying incremental-filter issue this stopgaps (link to existing tracking, or note "no separate tracker yet") + diff --git a/.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-SUMMARY.md b/.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-SUMMARY.md new file mode 100644 index 0000000..cb2c54c --- /dev/null +++ b/.planning/quick/260521-fci-stopgap-nightly-reconciliation-for-stale/260521-fci-SUMMARY.md @@ -0,0 +1,122 @@ +--- +phase: quick-260521-fci +plan: 01 +subsystem: sync +tags: [stopgap, autotask, reconciliation, scheduler] +requires: + - existing AutotaskClient.getEntityById + - existing mapAutotaskToDatabase / getTableName / autotaskRateLimiter + - existing sync_schedules table + sync-scheduler dispatch chain +provides: + - reconcileStaleTickets() — re-fetches stale-open tickets and reconciles + - POST /api/sync/reconcile-tickets — fire-and-forget trigger + - tickets-reconcile cron schedule (disabled by default, 30 4 * * *) +affects: + - lib/services/sync-scheduler.ts (sync_type union + default schedule + dispatch case) +tech_stack: + added: [] + patterns: + - dynamic-import-in-scheduler-dispatch + - webhook-style ON CONFLICT (id) DO UPDATE upsert + - rate-limited per-row Autotask getEntityById walk +key_files: + created: + - lib/services/ticket-reconciliation-service.ts + - app/api/sync/reconcile-tickets/route.ts + - migrations/090_ticket_reconcile_schedule.sql + modified: + - lib/services/sync-scheduler.ts +decisions: + - STOPGAP only — does NOT fix the underlying incremental-filter root cause + - LIMIT 500 per run + 10 req/sec rate limit caps worst-case API cost + - 4:30 AM cron deliberately staggered from 4:00 AM contract-services job + - Soft-delete (not hard-delete) when Autotask returns null — preserves audit + - Schedule lands DISABLED on both fresh + existing DBs; admin enables via UI +metrics: + duration_seconds: 129 + completed_at: 2026-05-21T15:09:45Z + tasks_completed: 2 + files_created: 3 + files_modified: 1 +--- + +# Quick Task 260521-fci: STOPGAP nightly reconciliation for stale tickets — Summary + +One-liner: A nightly cron walks postgres tickets that are still open locally but +haven't synced in >7 days, re-fetches each from Autotask via getEntityById, and +either upserts the canonical row or soft-deletes it — backstopping missed +webhooks and the incremental sync's high-water-mark blind spot. + +## Files Created / Modified + +| File | Role | +| ---- | ---- | +| `lib/services/ticket-reconciliation-service.ts` (created) | `reconcileStaleTickets()` — finds stale-open tickets, re-fetches from Autotask, upserts or soft-deletes. Returns `ReconcileResult`. | +| `app/api/sync/reconcile-tickets/route.ts` (created) | POST endpoint, fire-and-forget trigger mirroring `/api/sync/incremental` (no auth — public per existing `/api/sync` middleware allowlist). | +| `lib/services/sync-scheduler.ts` (modified) | Extended `sync_type` union with `'tickets-reconcile'`, added a `defaultSchedules` entry (disabled), and a dispatch case using the dynamic-import pattern matching `device-link-reconcile` / `integration-health`. | +| `migrations/090_ticket_reconcile_schedule.sql` (created) | Idempotent `INSERT … ON CONFLICT (id) DO NOTHING` so existing installs pick up the schedule row without disturbing the fresh-DB default-seed path. | + +## Result Shape + +`ReconcileResult` is logged once per run in two places: + +```ts +interface ReconcileResult { + scanned: number; // rows pulled from postgres for processing + updated: number; // successfully upserted from Autotask + statusFlippedToComplete: number; // priorStatus != 5, newStatus == 5 + softDeleted: number; // Autotask returned null -> is_deleted=true + errors: number; // per-row failures (logged, do not abort run) +} +``` + +Log lines: +1. **Service:** `[TicketReconciliation] Reconciliation complete: scanned=N updated=N flippedComplete=N softDeleted=N errors=N` + (via `createSyncLogger({ component: 'TicketReconciliation' })`) +2. **Scheduler:** `[SCHEDULER] tickets-reconcile: scanned=N updated=N flippedComplete=N softDeleted=N errors=N` + +## Operational Notes (handoff) + +- **Cron:** `30 4 * * *` (4:30 AM daily). Staggered 30 min after the 4:00 AM `contract-services` job to avoid rate-limit contention. +- **Default state:** `is_enabled = false`. Enable via the existing `/admin` sync-schedules UI when ready. +- **Capacity:** Hard-capped at `LIMIT 500` per run. With the 10 req/sec Autotask limiter that's ~50s minimum walltime if every ticket is stale; usually far less. +- **Manual trigger:** `curl -X POST http://localhost:3100/api/sync/reconcile-tickets -H 'Content-Type: application/json' -d '{"triggeredBy":"manual-smoke"}'` — returns `{ "message": "Ticket reconciliation started", "triggeredBy": "manual-smoke" }` immediately; result lines appear in container logs. +- **Idempotency:** Safe to run alongside the regular incremental sync — the upsert path uses the same SQL the webhook uses, and the soft-delete is column-level (no row deletion). +- **Failure mode:** Per-row exceptions increment `errors` and continue; only postgres infra failures abort the run. + +## Pointer to Underlying Issue + +This is a **STOPGAP**. The root cause is that Autotask's incremental sync uses +`lastTrackedModificationDateTime` (a.k.a. `lastActivityDate`-style high-water-mark) +which can fail to advance after some status transitions, so a status change to +Complete may never trigger an incremental row in our pipeline. The webhook path +also drops events on rare network/timeout edges. + +**No separate tracker exists yet** for the root cause. When that work is scoped +(likely as a "fix incremental filter for ticket status transitions" plan), this +job can either be retired or downgraded to a weekly safety net. + +## Deviations from Plan + +None — the plan was followed exactly as written. The only nuance was that the +union already contained `qbo | appgate-sessions | appgate-daily` from earlier +work in this worktree (pre-existing uncommitted changes), so the edit appended +`| 'tickets-reconcile'` to that already-extended union rather than the one +literally shown in the plan. Behavior is identical. + +## Verification + +- `npx tsc --noEmit --pretty` — clean (no output, full repo passes) +- `grep -n "tickets-reconcile" lib/services/sync-scheduler.ts` — three hits (union L25, default L295/L299, dispatch L458) +- `grep -rn "reconcileStaleTickets" lib/services/ app/api/sync/reconcile-tickets/` — one export, two imports (scheduler + route) +- `ls migrations/090_*.sql` — present +- Manual smoke (not run as part of automated verify, per plan): the curl POST above is the suggested ops handoff smoke test. + +## Self-Check: PASSED + +- `lib/services/ticket-reconciliation-service.ts` — FOUND +- `app/api/sync/reconcile-tickets/route.ts` — FOUND +- `migrations/090_ticket_reconcile_schedule.sql` — FOUND +- Commit `51f0b32` (Task 1: service + route) — FOUND +- Commit `badd718` (Task 2: scheduler + migration) — FOUND +- Full `npx tsc --noEmit --pretty` — clean From 1ecaefe85a63e41f8e979cd536cb7d505a57ee86 Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 21 May 2026 11:21:48 -0400 Subject: [PATCH 004/463] fix(quick-260521-foj-01): widen Companies full sync to fetch all companies - Add buildCompaniesFilter() returning id > 0 in lib/utils/sync-helpers.ts - Route COMPANIES through buildCompaniesFilter on full sync instead of the generic buildActiveFilter (which applied isActive=true and missed inactive companies with tickets, causing tickets_company_id_fkey on weekly-full and full syncs since 2026-05-15) - hasAppliedFilters stays true (filter is non-empty), so soft-delete of companies is not triggered --- lib/services/entity-sync.ts | 14 +++++++++----- lib/utils/sync-helpers.ts | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 1103cb6..bb57f3b 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -10,18 +10,19 @@ import { EntityType } from '../types/sync'; import { mapAutotaskToDatabase, mapAutotaskBatch } from '../utils/entity-mapper'; import { bulkUpsertRecords, getLastSyncTime, softDeleteMissingRecords } from '../utils/db-helpers'; import { syncProgressTracker } from './sync-progress-tracker'; -import { - getAutotaskEntityName, +import { + getAutotaskEntityName, buildIncrementalFilter, buildActiveFilter, buildDateRangeFilter, + buildCompaniesFilter, buildContractsFilter, buildContractServicesFilter, buildProjectsFilter, buildProjectPhasesFilter, buildTimeEntriesFilter, buildBillingItemsFilter, - getTableName + getTableName } from '../utils/sync-helpers'; import { createSyncLogger, SyncPhase, categorizeError } from '../utils/sync-logger'; @@ -128,9 +129,12 @@ export class EntitySyncService { } const filters: Array<{ field: string; op: string; value: any }> = []; - + // Special handling for entities that require filters - if (entity === EntityType.CONTRACTS) { + if (entity === EntityType.COMPANIES) { + filters.push(...buildCompaniesFilter()); + entityLogger.info('Full sync of all companies (active + inactive)'); + } else if (entity === EntityType.CONTRACTS) { filters.push(...buildContractsFilter()); entityLogger.info('Full sync with status filter for active contracts'); } else if (entity === EntityType.CONTRACT_SERVICES) { diff --git a/lib/utils/sync-helpers.ts b/lib/utils/sync-helpers.ts index 25274d9..0d0b836 100644 --- a/lib/utils/sync-helpers.ts +++ b/lib/utils/sync-helpers.ts @@ -91,6 +91,22 @@ export function buildContractServicesFilter(): Array<{ field: string; op: string ]; } +/** + * Build filter for companies (Companies endpoint requires a filter — use id > 0 to fetch all + * companies regardless of isActive. Active-only filtering misses inactive companies that + * have tickets, causing tickets_company_id_fkey violations on full sync.) + * @returns Query filter array for all companies + */ +export function buildCompaniesFilter(): Array<{ field: string; op: string; value: any }> { + return [ + { + field: 'id', + op: 'gt', + value: 0, + }, + ]; +} + /** * Get table name for entity type * @param entity Entity type From 62c529fb91ca92adc6d55fccef1b524ac3846d93 Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 21 May 2026 11:22:33 -0400 Subject: [PATCH 005/463] fix(quick-260521-foj-02): defensively nullify ticket company_id for missing companies - Add getValidCompanyIds() helper mirroring getValidResourceIds() - Add a new TICKETS validation block that nullifies ticket.company_id when the referenced company is not present (is_deleted=false) in the Pulse mirror, instead of letting tickets_company_id_fkey roll back the bulkUpsert transaction - Block runs after the existing recordsWithoutCompany filter and before the existing resource-FK nullification block (correct ordering) - Belt-and-suspenders on top of Task 1: covers hard-deleted-in-Autotask companies that Task 1's widening still won't fetch --- lib/services/entity-sync.ts | 38 ++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index bb57f3b..744d0a7 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -255,10 +255,35 @@ export class EntitySyncService { } } + // Validate company_id foreign keys for tickets (defensive — Task 1 widening + // the Companies filter should make this a near-zero count, but catches the + // truly hard-deleted Autotask company case). + if (entity === EntityType.TICKETS) { + const validCompanyIds = await this.getValidCompanyIds(); + let nullifiedCompanyCount = 0; + const nullifiedCompanySamples: number[] = []; + mappedRecords = mappedRecords.map(ticket => { + if (ticket.company_id && !validCompanyIds.has(ticket.company_id)) { + if (nullifiedCompanySamples.length < 5) { + nullifiedCompanySamples.push(ticket.id); + } + ticket.company_id = null; + nullifiedCompanyCount++; + } + return ticket; + }); + if (nullifiedCompanyCount > 0) { + entityLogger.warn('Nullified ticket company_id for companies missing from mirror', { + nullifiedCount: nullifiedCompanyCount, + sampleTicketIds: nullifiedCompanySamples, + }); + } + } + // Validate resource foreign keys for tickets if (entity === EntityType.TICKETS) { const initialCount = mappedRecords.length; - + // Get all valid resource IDs from database const validResourceIds = await this.getValidResourceIds(); @@ -721,6 +746,17 @@ export class EntitySyncService { return new Set(result.rows.map(row => Number(row.id))); } + /** + * Get all valid company IDs from the database + * Used to validate foreign key references before insert + * @returns Set of valid company IDs + */ + private async getValidCompanyIds(): Promise> { + const query = 'SELECT id FROM companies WHERE is_deleted = false'; + const result = await postgresClient.query<{ id: number }>(query); + return new Set(result.rows.map(row => Number(row.id))); + } + /** * Used to validate foreign key references before insert * @returns Set of valid contact IDs From 078e087d5fa024c62f152eac0fda10ee2be4cb8b Mon Sep 17 00:00:00 2001 From: lorentz Date: Thu, 21 May 2026 11:24:34 -0400 Subject: [PATCH 006/463] docs(quick-260521-foj): Fix weekly-full FK error - widen Companies filter + defensive ticket company_id validation --- .planning/STATE.md | 3 +- .../260521-foj-PLAN.md | 373 ++++++++++++++++++ .../260521-foj-SUMMARY.md | 122 ++++++ 3 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 .planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-PLAN.md create mode 100644 .planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-SUMMARY.md diff --git a/.planning/STATE.md b/.planning/STATE.md index 1d07cf4..88285c8 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -28,7 +28,7 @@ See: .planning/PROJECT.md (updated 2026-05-03) Phase: 09.1 (ntfy-backend-fix) — EXECUTING Plan: 1 of 1 Status: Executing Phase 09.1 -Last activity: 2026-05-21 - Completed quick task 260521-fci: Stopgap nightly reconciliation for stale open tickets in postgres mirror +Last activity: 2026-05-21 - Completed quick task 260521-foj: Fix weekly-full FK error: widen Companies filter + defensive ticket company_id validation Progress: [░░░░░░░░░░] 0% @@ -95,6 +95,7 @@ None yet. |---|-------------|------|--------|-----------| | 260519-0oz | Add QBO createPayment + createDeposit + .FH reconciliation script | 2026-05-19 | 5497458 | [260519-0oz-add-qbo-createpayment-createdeposit-fh-r](./quick/260519-0oz-add-qbo-createpayment-createdeposit-fh-r/) | | 260521-fci | Stopgap nightly reconciliation for stale open tickets in postgres mirror | 2026-05-21 | badd718 | [260521-fci-stopgap-nightly-reconciliation-for-stale](./quick/260521-fci-stopgap-nightly-reconciliation-for-stale/) | +| 260521-foj | Fix weekly-full FK error: widen Companies filter + defensive ticket company_id validation | 2026-05-21 | 62c529f | [260521-foj-fix-weekly-full-fk-error-widen-companies](./quick/260521-foj-fix-weekly-full-fk-error-widen-companies/) | ## Session Continuity diff --git a/.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-PLAN.md b/.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-PLAN.md new file mode 100644 index 0000000..846a760 --- /dev/null +++ b/.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-PLAN.md @@ -0,0 +1,373 @@ +--- +phase: quick-260521-foj +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - lib/utils/sync-helpers.ts + - lib/services/entity-sync.ts +autonomous: true +requirements: + - QUICK-260521-FOJ +must_haves: + truths: + - "Full sync of Companies entity fetches all companies (active + inactive) — no isActive=true filter applied" + - "Tickets sync with a company_id pointing at a company missing from the Pulse mirror does not raise tickets_company_id_fkey; the company_id is nullified and the ticket is preserved" + - "`hasAppliedFilters` remains true on Companies full sync, so soft-delete of companies is NOT triggered" + - "Existing behavior unchanged for all other entities — getActiveField/buildActiveFilter still works as before for resources, contacts, statuses, etc." + - "`npx tsc --noEmit --pretty` passes after both edits" + artifacts: + - path: "lib/utils/sync-helpers.ts" + provides: "New exported `buildCompaniesFilter()` returning `[{ field: 'id', op: 'gt', value: 0 }]`" + contains: "export function buildCompaniesFilter" + - path: "lib/services/entity-sync.ts" + provides: "New `getValidCompanyIds()` private method + COMPANIES branch in full-sync filter chain + ticket company_id nullification block" + contains: "buildCompaniesFilter" + key_links: + - from: "lib/services/entity-sync.ts (full-sync filter chain ~line 133-165)" + to: "buildCompaniesFilter (lib/utils/sync-helpers.ts)" + via: "import + branch above the generic `else` that calls buildActiveFilter" + pattern: "entity === EntityType.COMPANIES.*buildCompaniesFilter" + - from: "lib/services/entity-sync.ts (ticket validation block)" + to: "companies table (is_deleted = false)" + via: "getValidCompanyIds() called inside the `if (entity === EntityType.TICKETS)` block" + pattern: "getValidCompanyIds" +--- + + +Fix the weekly-full and full ticket sync FK error (`tickets_company_id_fkey`) caused by Companies sync filtering out inactive companies that nonetheless have tickets in the same sync window. + +Purpose: Every full and weekly-full sync since 2026-05-15 fails because Companies full sync applies `isActive=true` (via `buildActiveFilter`), which misses inactive companies that have tickets. The ticket transaction rolls back on the FK violation. Live evidence in `sync_history`: every recent full sync shows `[DATABASE_CONSTRAINT_ERROR] ... tickets_company_id_fkey`. + +Output: +- A new `buildCompaniesFilter()` in `lib/utils/sync-helpers.ts` mirroring the `buildProjectPhasesFilter` template (`id > 0` to fetch all). +- A new branch in `entity-sync.ts` full-sync filter chain that routes COMPANIES through `buildCompaniesFilter()` instead of the generic `buildActiveFilter`. +- A defensive ticket `company_id` nullification block in `entity-sync.ts` (belt-and-suspenders for the rare case of truly hard-deleted Autotask companies). + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@./CLAUDE.md +@lib/utils/sync-helpers.ts +@lib/services/entity-sync.ts +@lib/services/postgres-client.ts + + + + +From `lib/utils/sync-helpers.ts` — existing filter-builder templates this task must mirror: +```ts +/** + * Build filter for contract services (requires contractID filter — fetch all via contractIDs) + * @returns Query filter array for active contract services + */ +export function buildContractServicesFilter(): Array<{ field: string; op: string; value: any }> { + return [ + { + field: 'contractID', + op: 'gt', + value: 0, + }, + ]; +} + +/** + * Build filter for project phases (Phases endpoint requires a filter) + * @returns Query filter array for all phases + */ +export function buildProjectPhasesFilter(): Array<{ field: string; op: string; value: any }> { + return [ + { + field: 'id', + op: 'gt', + value: 0, + }, + ]; +} + +/** + * Get active status field name for entity + * Returns 'isActive' for COMPANIES — DO NOT change this mapping. It is still + * referenced by `buildActiveFilter` for other entities (resources, contacts, etc.). + */ +export function getActiveField(entity: EntityType): string | null { /* ... */ } +``` + +From `lib/services/entity-sync.ts` — the full-sync filter chain (~lines 130-170) is where the new COMPANIES branch goes: +```ts +const filters: Array<{ field: string; op: string; value: any }> = []; + +// Special handling for entities that require filters +if (entity === EntityType.CONTRACTS) { + filters.push(...buildContractsFilter()); + entityLogger.info('Full sync with status filter for active contracts'); +} else if (entity === EntityType.CONTRACT_SERVICES) { + filters.push(...buildContractServicesFilter()); + entityLogger.info('Full sync of all contract services'); +} else if (entity === EntityType.PROJECTS) { + filters.push(...buildProjectsFilter()); + entityLogger.info('Full sync with status filter for non-completed projects'); +} else if (entity === EntityType.PROJECT_PHASES) { + filters.push(...buildProjectPhasesFilter()); + entityLogger.info('Full sync of all project phases'); +} else if (entity === EntityType.TIME_ENTRIES) { /* ... */ +} else if (entity === EntityType.BILLING_ITEMS) { /* ... */ +} else { + // Add active filter if applicable + const activeFilter = buildActiveFilter(entity); + if (activeFilter) { + filters.push(...activeFilter); + entityLogger.info('Full sync with active filter'); + } + // ... date range filter ... +} +``` + +From `lib/services/entity-sync.ts` — the existing `getValidResourceIds` template the new method must mirror (lines 710-718): +```ts +/** + * Get all valid resource IDs from the database + * Used to validate foreign key references before insert + * @returns Set of valid resource IDs + */ +private async getValidResourceIds(): Promise> { + const query = 'SELECT id FROM resources WHERE is_deleted = false'; + const result = await postgresClient.query<{ id: number }>(query); + return new Set(result.rows.map(row => Number(row.id))); +} +``` + +From `lib/services/entity-sync.ts` — the existing tickets validation block where the new `company_id` validation slots in (lines 235-285): +- Lines 238-252: filters out tickets without `company_id` (`recordsWithoutCompany`). KEEP AS-IS. +- Lines 255-285: nullifies invalid `assigned_resource_id` / `first_response_*` resource FKs against `getValidResourceIds`. KEEP AS-IS. +- NEW VALIDATION goes immediately after the `recordsWithoutCompany` filter, before or alongside the resource validation block. Must use the same `mappedRecords = mappedRecords.map(...)` mutation pattern and the same `entityLogger.warn` shape. + +From `lib/services/postgres-client.ts` — singleton DB interface: +```ts +postgresClient.query<{ id: number }>(sql, params?: any[]): Promise> +``` + + + + + + + Task 1: Add buildCompaniesFilter and wire COMPANIES into the full-sync filter chain + lib/utils/sync-helpers.ts, lib/services/entity-sync.ts + +**Step 1 — `lib/utils/sync-helpers.ts`:** + +Add a new exported function `buildCompaniesFilter()` adjacent to `buildContractServicesFilter` / `buildProjectPhasesFilter` (around line 92, right after `buildContractServicesFilter`). Mirror the JSDoc style of `buildContractServicesFilter` exactly: + +```ts +/** + * Build filter for companies (Companies endpoint requires a filter — use id > 0 to fetch all + * companies regardless of isActive. Active-only filtering misses inactive companies that + * have tickets, causing tickets_company_id_fkey violations on full sync.) + * @returns Query filter array for all companies + */ +export function buildCompaniesFilter(): Array<{ field: string; op: string; value: any }> { + return [ + { + field: 'id', + op: 'gt', + value: 0, + }, + ]; +} +``` + +DO NOT touch `getActiveField(COMPANIES)` — leave it returning `'isActive'`. It's still referenced by `buildActiveFilter` for other entities and by call sites we are not changing. + +**Step 2 — `lib/services/entity-sync.ts`:** + +1. Update the sync-helpers import at the top of the file (currently lines 13-25). Add `buildCompaniesFilter` to the named imports — alphabetize alongside the other `build*` helpers. The import block becomes: + +```ts +import { + getAutotaskEntityName, + buildIncrementalFilter, + buildActiveFilter, + buildDateRangeFilter, + buildCompaniesFilter, + buildContractsFilter, + buildContractServicesFilter, + buildProjectsFilter, + buildProjectPhasesFilter, + buildTimeEntriesFilter, + buildBillingItemsFilter, + getTableName +} from '../utils/sync-helpers'; +``` + +2. In the full-sync filter chain (around lines 133-165), add a new branch for COMPANIES. Place it as the FIRST branch in the chain (just after the `const filters: Array<...> = [];` declaration and before the existing `if (entity === EntityType.CONTRACTS)` branch). The chain after edit: + +```ts +const filters: Array<{ field: string; op: string; value: any }> = []; + +// Special handling for entities that require filters +if (entity === EntityType.COMPANIES) { + filters.push(...buildCompaniesFilter()); + entityLogger.info('Full sync of all companies (active + inactive)'); +} else if (entity === EntityType.CONTRACTS) { + filters.push(...buildContractsFilter()); + entityLogger.info('Full sync with status filter for active contracts'); +} else if (entity === EntityType.CONTRACT_SERVICES) { + // ... existing branches unchanged ... +``` + +Critical: the new COMPANIES branch MUST execute BEFORE the generic `else` block that calls `buildActiveFilter`. Result: the active filter is no longer applied to Companies on full sync, so inactive companies are fetched too. + +**Why this is safe (state this in your verification reasoning, do not add it as a code comment):** +- `hasAppliedFilters` (entity-sync.ts:173) stays true because the new filter array is non-empty (`id > 0`). So `softDeleteMissingRecords` is NOT triggered for companies — no accidental mass soft-delete. +- The existing DB already has 145 `is_active=false` companies referenced by tickets; downstream UI/API code tolerates inactive companies. +- Companies don't support incremental sync (entity-sync.ts:106 `supportsIncremental` excludes COMPANIES), so this code path runs on every Companies sync — full, weekly-full, and the incremental→full fallback path. + +DO NOT touch: +- `getActiveField`. +- The `lastActivityDate` filter on tickets (out of scope). +- Any other branch in the filter chain. +- Adjacent formatting / unrelated code. + + + cd /opt/stacks/pulse && npx tsc --noEmit --pretty 2>&1 | tail -40 && echo "---" && grep -n "buildCompaniesFilter" lib/utils/sync-helpers.ts lib/services/entity-sync.ts && echo "---" && grep -n "entity === EntityType.COMPANIES" lib/services/entity-sync.ts + + +- `buildCompaniesFilter` exported from `lib/utils/sync-helpers.ts` with the exact JSDoc and body shown above. +- `buildCompaniesFilter` imported in `lib/services/entity-sync.ts` named-imports block. +- Exactly one new COMPANIES branch in the full-sync filter chain, placed before all other `if/else if` branches, with the `entityLogger.info('Full sync of all companies (active + inactive)')` line. +- `getActiveField` and `buildActiveFilter` definitions UNCHANGED. +- `npx tsc --noEmit --pretty` passes with zero errors. +- `grep -n "buildCompaniesFilter" lib/services/entity-sync.ts` shows exactly two matches (one import, one usage). + + + + + Task 2: Defensive company_id nullification in tickets sync (belt-and-suspenders) + lib/services/entity-sync.ts + +This task adds a second layer of defense after Task 1: even if a ticket somehow references a company that doesn't exist in the Pulse mirror (e.g. hard-deleted from Autotask), nullify the `company_id` rather than letting the FK violation roll back the whole ticket transaction. + +**Step 1 — Add new private method `getValidCompanyIds`:** + +Find the existing `getValidResourceIds` method (currently lines ~710-718, just after `calculateMonthlyChunks` is defined further down — search for `private async getValidResourceIds`). Add the new method immediately AFTER it, before `getValidContactIds`. Match the exact pattern: + +```ts + /** + * Get all valid company IDs from the database + * Used to validate foreign key references before insert + * @returns Set of valid company IDs + */ + private async getValidCompanyIds(): Promise> { + const query = 'SELECT id FROM companies WHERE is_deleted = false'; + const result = await postgresClient.query<{ id: number }>(query); + return new Set(result.rows.map(row => Number(row.id))); + } +``` + +**Step 2 — Add the company_id validation block in the tickets validation pipeline:** + +Find the existing `if (entity === EntityType.TICKETS) {` block that nullifies invalid resource IDs (currently lines ~254-285, the block that uses `getValidResourceIds`). Add a NEW `if (entity === EntityType.TICKETS) { ... }` block IMMEDIATELY ABOVE it (between the `recordsWithoutCompany` filter block at lines ~238-252 and the resource validation block at lines ~254-285). The new block: + +```ts + // Validate company_id foreign keys for tickets (defensive — Task 1 widening + // the Companies filter should make this a near-zero count, but catches the + // truly hard-deleted Autotask company case). + if (entity === EntityType.TICKETS) { + const validCompanyIds = await this.getValidCompanyIds(); + let nullifiedCompanyCount = 0; + const nullifiedCompanySamples: number[] = []; + mappedRecords = mappedRecords.map(ticket => { + if (ticket.company_id && !validCompanyIds.has(ticket.company_id)) { + if (nullifiedCompanySamples.length < 5) { + nullifiedCompanySamples.push(ticket.id); + } + ticket.company_id = null; + nullifiedCompanyCount++; + } + return ticket; + }); + if (nullifiedCompanyCount > 0) { + entityLogger.warn('Nullified ticket company_id for companies missing from mirror', { + nullifiedCount: nullifiedCompanyCount, + sampleTicketIds: nullifiedCompanySamples, + }); + } + } +``` + +Ordering rationale (must be respected): +1. `recordsWithoutCompany` filter (existing) — drops tickets without any `company_id`. Runs first so we only validate tickets that have a `company_id`. +2. NEW company_id nullification block — nullifies `company_id` that doesn't exist in mirror. Runs second. +3. Resource validation block (existing) — nullifies invalid resource refs. Runs third. +4. Everything else (`bulkUpsert` at ~line 372) runs after. + +The `tickets.company_id` column is nullable (confirmed via the recent stopgap reconciliation work and existing handling that filters rows without `company_id`). Nullifying it is non-destructive — it preserves the ticket row and just severs the company link, matching the resource-FK handling pattern already used in the same function. + +DO NOT touch: +- The existing `recordsWithoutCompany` filter block. +- The existing resource validation block. +- The chunked tickets sync (`syncTicketsChunked`) — out of scope. That path has its own (separate) flow; this fix is for `syncEntity`. +- `cachedValidResourceIds` or `cachedValidContactIds` fields. +- Adjacent code / formatting. + +DO NOT manually trigger a sync from this task. Note in SUMMARY.md only: +``` +Manual verify (user runs after merge): + curl -X POST http://localhost:3100/api/sync/full \ + -H 'content-type: application/json' \ + -d '{"triggeredBy":"manual-verify"}' +Then check `sync_history` for the next ticket sync row — it should complete without `tickets_company_id_fkey` in `error_message`. +``` + + + cd /opt/stacks/pulse && npx tsc --noEmit --pretty 2>&1 | tail -40 && echo "---" && grep -cn "getValidCompanyIds" lib/services/entity-sync.ts && echo "---" && grep -n "Nullified ticket company_id for companies missing from mirror" lib/services/entity-sync.ts && echo "---" && grep -n "if (entity === EntityType.TICKETS)" lib/services/entity-sync.ts + + +- `getValidCompanyIds` method exists exactly once in `lib/services/entity-sync.ts`, mirroring `getValidResourceIds`. +- A new `if (entity === EntityType.TICKETS)` block exists between the `recordsWithoutCompany` filter and the resource validation block. +- The new block calls `await this.getValidCompanyIds()` and logs via `entityLogger.warn('Nullified ticket company_id for companies missing from mirror', ...)` when count > 0. +- `grep -c "getValidCompanyIds" lib/services/entity-sync.ts` returns 2 (one definition, one call site). +- `grep -n "if (entity === EntityType.TICKETS)" lib/services/entity-sync.ts` returns at least 2 matches (the new block + the existing resource validation block); ordering must put the new block earlier in the file (lower line number) than the resource block. +- `npx tsc --noEmit --pretty` passes with zero errors. +- No edits to `syncTicketsChunked`, `getValidResourceIds`, `getValidContactIds`, or `getValidProjectIds`. + + + + + + +1. Type check passes: `npx tsc --noEmit --pretty` from repo root — zero errors. +2. Targeted greps: + - `grep -n "buildCompaniesFilter" lib/utils/sync-helpers.ts lib/services/entity-sync.ts` → 3 matches (one definition + import + usage). + - `grep -c "getValidCompanyIds" lib/services/entity-sync.ts` → 2. + - `grep -n "entity === EntityType.COMPANIES" lib/services/entity-sync.ts` → 1 new match in the filter chain. +3. Negative greps (proving we didn't touch out-of-scope code): + - `git diff lib/utils/sync-helpers.ts -- ':!**/buildCompaniesFilter*'` should show only the new function added. + - `git diff lib/services/entity-sync.ts` should show: import addition, COMPANIES filter branch, new `getValidCompanyIds` method, and new ticket company_id validation block. NOTHING else. +4. Read-back sanity: + - `entity-sync.ts` line ~152 (was the start of the generic `else` calling `buildActiveFilter`) is now NOT reached for COMPANIES because the new branch fires first. + - `hasAppliedFilters` (line ~173) is still true for COMPANIES full sync because `buildCompaniesFilter()` returns a non-empty filter. + + + +- `lib/utils/sync-helpers.ts` exports `buildCompaniesFilter()` returning `[{ field: 'id', op: 'gt', value: 0 }]`. +- `lib/services/entity-sync.ts` routes COMPANIES through `buildCompaniesFilter` on full sync; the generic `buildActiveFilter` branch is no longer hit for COMPANIES. +- `lib/services/entity-sync.ts` defensively nullifies ticket `company_id` for any company missing from the Pulse mirror, mirroring the existing resource-FK nullification pattern. +- `npx tsc --noEmit --pretty` passes. +- No changes to `getActiveField`, `buildActiveFilter`, `syncTicketsChunked`, the reconciliation service from quick-260521-fci, or the sync-progress-lock code. +- SUMMARY.md notes the manual `curl -X POST http://localhost:3100/api/sync/full` verification step but does NOT execute it. + + + +After completion, create `.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-SUMMARY.md` with: +- What changed in each file (1-2 lines per file). +- Why this fixes the FK error (1 paragraph: Companies sync was filtering on `isActive=true` via `buildActiveFilter`, missing inactive companies referenced by tickets; widening to `id > 0` closes the gap; the defensive nullification handles the residual hard-deleted-in-Autotask edge case). +- Manual verify step for the user: `curl -X POST http://localhost:3100/api/sync/full -H 'content-type: application/json' -d '{"triggeredBy":"manual-verify"}'`, then check `sync_history` for the next full ticket sync. +- Explicit note: did NOT modify `getActiveField`, `syncTicketsChunked`, or the reconciliation service. + diff --git a/.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-SUMMARY.md b/.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-SUMMARY.md new file mode 100644 index 0000000..7a95dfb --- /dev/null +++ b/.planning/quick/260521-foj-fix-weekly-full-fk-error-widen-companies/260521-foj-SUMMARY.md @@ -0,0 +1,122 @@ +--- +phase: quick-260521-foj +plan: 01 +subsystem: sync +tags: + - autotask-sync + - companies + - tickets + - foreign-key + - bugfix +requirements: + - QUICK-260521-FOJ +dependency_graph: + requires: + - lib/utils/sync-helpers.ts (existing build*Filter pattern) + - lib/services/entity-sync.ts (existing full-sync filter chain + tickets validation pipeline) + - lib/services/postgres-client.ts (singleton query interface) + provides: + - buildCompaniesFilter() (lib/utils/sync-helpers.ts) + - EntitySyncService.getValidCompanyIds() (private) + - Defensive ticket company_id nullification in syncEntity() + affects: + - Companies full sync (now fetches active + inactive) + - Tickets full + weekly-full sync (no longer rolls back on tickets_company_id_fkey) +tech_stack: + added: [] + patterns: + - Mirror existing buildProjectPhasesFilter / buildContractServicesFilter "id > 0" template for any Autotask entity that requires a filter but where we want everything + - Mirror existing getValidResourceIds + null-on-miss pattern for FK validation in syncEntity() +key_files: + created: [] + modified: + - lib/utils/sync-helpers.ts + - lib/services/entity-sync.ts +decisions: + - Widen Companies full sync to id > 0 (fetch all) rather than carrying isActive=true via buildActiveFilter, so inactive Autotask companies referenced by tickets are mirrored. + - Add a belt-and-suspenders ticket company_id nullification block for the truly hard-deleted Autotask company case. Non-destructive — preserves the ticket row. + - Did NOT touch getActiveField — still returns 'isActive' for COMPANIES because other entities and call sites use that mapping. + - Did NOT touch syncTicketsChunked — out of scope; that path has its own flow and is not on the failing weekly-full path. +metrics: + duration: 1m 49s + completed_date: 2026-05-21 +--- + +# Quick 260521-foj: Fix weekly-full FK error — widen Companies full sync Summary + +One-liner: Widen Companies full sync to fetch active+inactive companies (id > 0) and defensively nullify ticket.company_id for hard-deleted-in-Autotask companies — eliminates the `tickets_company_id_fkey` rollback that has been failing every full and weekly-full sync since 2026-05-15. + +## What changed + +**`lib/utils/sync-helpers.ts`** +- Added new exported `buildCompaniesFilter()` returning `[{ field: 'id', op: 'gt', value: 0 }]`, mirroring the `buildProjectPhasesFilter` template. JSDoc explains the rationale (Companies endpoint requires a filter; active-only misses inactive companies referenced by tickets). + +**`lib/services/entity-sync.ts`** +- Added `buildCompaniesFilter` to the named-imports block from `../utils/sync-helpers`. +- Inserted a new `COMPANIES` branch as the first branch in the full-sync filter chain (placed before `CONTRACTS`). It pushes the `id > 0` filter and logs `Full sync of all companies (active + inactive)`. This routes COMPANIES around the generic `else` block that would otherwise apply `buildActiveFilter` (`isActive=true`). +- Added a new private `getValidCompanyIds()` method that selects all `companies.id WHERE is_deleted = false`, mirroring `getValidResourceIds`. +- Added a new `if (entity === EntityType.TICKETS)` block between the existing `recordsWithoutCompany` filter and the existing resource-FK nullification block. It iterates `mappedRecords`, nullifies `ticket.company_id` when the referenced company is missing from the Pulse mirror, tracks up to 5 sample ticket IDs, and emits a `warn` log when the count is non-zero. Pattern matches the surrounding resource nullification block exactly. + +## Why this fixes the FK error + +Every full and weekly-full sync since 2026-05-15 was failing with `[DATABASE_CONSTRAINT_ERROR] ... tickets_company_id_fkey`, rolling back the tickets bulkUpsert transaction. Root cause: Companies full sync was applying `buildActiveFilter`, which sends `isActive=true` to the Autotask API. Companies marked inactive in Autotask but still referenced by open tickets were therefore not mirrored into Pulse. When the tickets sync ran next, the FK constraint on `tickets.company_id → companies.id` fired and rolled the whole batch back. Widening the Companies filter to `id > 0` closes the primary gap by fetching active+inactive companies. The defensive nullification covers the residual edge case where a company was hard-deleted from Autotask entirely — instead of rolling back, the ticket survives with `company_id = null` (the column is nullable, and the existing `recordsWithoutCompany` filter only drops tickets that have no `company_id` from the start, so the bulkUpsert path handles `null` cleanly via the resource-nullification precedent). + +`hasAppliedFilters` stays true on Companies full sync because the new filter array is non-empty (`id > 0`), so `softDeleteMissingRecords` is NOT triggered — no accidental mass soft-delete of companies. Companies do not support incremental sync (per `supportsIncremental` at entity-sync.ts line 106), so this code path runs on every Companies sync (full, weekly-full, and the incremental→full fallback). + +## Commits + +| Task | Description | Commit | Files | +| ---- | ------------------------------------------------------------------------------------ | ------- | ------------------------------------------------ | +| 1 | Add buildCompaniesFilter + wire COMPANIES into the full-sync filter chain | 1ecaefe | lib/utils/sync-helpers.ts, lib/services/entity-sync.ts | +| 2 | Add getValidCompanyIds + defensive ticket company_id nullification | 62c529f | lib/services/entity-sync.ts | + +## Verification + +Type check passed cleanly: + +``` +cd /opt/stacks/pulse && npx tsc --noEmit --pretty +# (zero errors) +``` + +Targeted greps confirm shape: +- `grep -n "buildCompaniesFilter" lib/utils/sync-helpers.ts lib/services/entity-sync.ts` → 3 matches (definition + import + usage) +- `grep -c "getValidCompanyIds" lib/services/entity-sync.ts` → 2 (definition + call site) +- `grep -n "entity === EntityType.COMPANIES" lib/services/entity-sync.ts` → 1 new match (line 134, in the filter chain) +- `grep -n "if (entity === EntityType.TICKETS)" lib/services/entity-sync.ts` → 2 matches: line 261 (new company-validation block) BEFORE line 284 (existing resource-validation block), ordering correct. + +## Manual verify step (user runs after merge) + +```bash +curl -X POST http://localhost:3100/api/sync/full \ + -H 'content-type: application/json' \ + -d '{"triggeredBy":"manual-verify"}' +``` + +Then check `sync_history` for the next ticket sync row — `error_message` should no longer contain `tickets_company_id_fkey`, and `status` should be `completed`. Also sanity-check `SELECT count(*) FROM companies WHERE is_active = false AND is_deleted = false;` — the count should be ≥ the previous ~145 (or whatever the prior baseline was) once Companies full sync picks up inactive companies that weren't being fetched before. + +## Out-of-scope (explicit non-changes) + +- `getActiveField` in `lib/utils/sync-helpers.ts` — unchanged. Still returns `'isActive'` for COMPANIES (used by other call sites and for other entities like RESOURCES, CONTACTS, etc.). +- `buildActiveFilter` — unchanged. +- `syncTicketsChunked` — unchanged. The chunked path has its own (separate) flow and was not on the failing weekly-full path. Touching it was explicitly out of scope per the plan. +- The reconciliation service from `quick-260521-fci` — unchanged. +- `cachedValidResourceIds` / `cachedValidContactIds` — unchanged. +- The `lastActivityDate` filter on tickets — unchanged. +- Other branches in the full-sync filter chain (CONTRACTS, CONTRACT_SERVICES, PROJECTS, PROJECT_PHASES, TIME_ENTRIES, BILLING_ITEMS) — unchanged. + +## Deviations from Plan + +None — plan executed exactly as written. + +## Known Stubs + +None — both functions are wired end-to-end into the live sync path. + +## Self-Check: PASSED + +- `lib/utils/sync-helpers.ts` exists and contains `export function buildCompaniesFilter` at line 100. +- `lib/services/entity-sync.ts` exists; import (line 18), usage (line 135), COMPANIES branch (line 134), `getValidCompanyIds` method, and the new TICKETS company-validation block (line 261) all present. +- Commit `1ecaefe` exists in `git log` (Task 1). +- Commit `62c529f` exists in `git log` (Task 2). +- `npx tsc --noEmit --pretty` exits 0. From 758b7e7f15a6b5d1fa801b1a23ab928884841537 Mon Sep 17 00:00:00 2001 From: lorentz Date: Tue, 2 Jun 2026 20:25:14 -0400 Subject: [PATCH 007/463] feat(engagement): replace Graph email counts with real-time mimecast data - Broaden mimecast retention from 30 days to 18 months rolling - Re-enable mimecast-sync schedule (was disabled since March 17) - Full sync triggered: 35,559 messages loaded for last 30 days - Users list API: LATERAL join on mimecast_messages for emails_sent/received - User detail API: add emails{d7,d30,d90} field from mimecast - Engagement page: prefer mimecast email counts in detail panel sub-label Graph API has 48-72hr reporting lag; mimecast is same-day --- app/api/engagement/user/[userId]/route.ts | 21 +++++++++++++++++++++ app/api/engagement/users/route.ts | 12 ++++++++++-- app/engagement/page.tsx | 3 ++- lib/services/mimecast-sync-service.ts | 9 +++++---- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/app/api/engagement/user/[userId]/route.ts b/app/api/engagement/user/[userId]/route.ts index e424169..2b86848 100644 --- a/app/api/engagement/user/[userId]/route.ts +++ b/app/api/engagement/user/[userId]/route.ts @@ -207,6 +207,22 @@ export async function GET( [user.email, periodDays] ).catch(() => null); + // Mimecast real-time email counts + const mimecastEmailsResult = await postgresClient.query( + `SELECT + COUNT(*) FILTER (WHERE LOWER(sender_address) = LOWER($1) AND direction IN ('outbound','internal') AND sent_datetime >= NOW() - INTERVAL '7 days') AS sent_d7, + COUNT(*) FILTER (WHERE LOWER(sender_address) = LOWER($1) AND direction IN ('outbound','internal') AND sent_datetime >= NOW() - INTERVAL '30 days') AS sent_d30, + COUNT(*) FILTER (WHERE LOWER(sender_address) = LOWER($1) AND direction IN ('outbound','internal') AND sent_datetime >= NOW() - INTERVAL '90 days') AS sent_d90, + COUNT(*) FILTER (WHERE LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted') AND sent_datetime >= NOW() - INTERVAL '7 days') AS received_d7, + COUNT(*) FILTER (WHERE LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted') AND sent_datetime >= NOW() - INTERVAL '30 days') AS received_d30, + COUNT(*) FILTER (WHERE LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted') AND sent_datetime >= NOW() - INTERVAL '90 days') AS received_d90 + FROM mimecast_messages + WHERE LOWER(sender_address) = LOWER($1) + OR (LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted'))`, + [user.email] + ).catch(() => null); + const me = mimecastEmailsResult?.rows[0]; + // After-hours meetings (5:30 PM – 7:00 AM America/New_York) const afterHoursMeetingsResult = await postgresClient.query( `SELECT COUNT(*) as count @@ -496,6 +512,11 @@ export async function GET( messagesPct: totalMessages > 0 ? Math.round((afterHoursMessages / totalMessages) * 100) : 0, meetingsPct: totalMeetings > 0 ? Math.round((afterHoursMeetings / totalMeetings) * 100) : 0, }, + emails: me ? { + d7: { sent: parseInt(me.sent_d7 ?? 0), received: parseInt(me.received_d7 ?? 0) }, + d30: { sent: parseInt(me.sent_d30 ?? 0), received: parseInt(me.received_d30 ?? 0) }, + d90: { sent: parseInt(me.sent_d90 ?? 0), received: parseInt(me.received_d90 ?? 0) }, + } : null, snapshots: snapshotsResult.rows, hours: hours ? { diff --git a/app/api/engagement/users/route.ts b/app/api/engagement/users/route.ts index f86ffb2..1f6df95 100644 --- a/app/api/engagement/users/route.ts +++ b/app/api/engagement/users/route.ts @@ -82,8 +82,8 @@ export async function GET(request: NextRequest) { COALESCE(es.teams_calls, 0) as teams_calls, COALESCE(es.teams_meetings_attended, 0) as teams_meetings_attended, COALESCE(es.teams_meetings_organized, 0) as teams_meetings_organized, - COALESCE(es.emails_sent, 0) as emails_sent, - COALESCE(es.emails_received, 0) as emails_received, + COALESCE(mc.emails_sent, 0) as emails_sent, + COALESCE(mc.emails_received, 0) as emails_received, COALESCE(es.emails_read, 0) as emails_read, COALESCE(es.audio_duration_seconds, 0) as audio_duration_seconds, COALESCE(es.meeting_duration_seconds, 0) as meeting_duration_seconds, @@ -139,6 +139,14 @@ export async function GET(request: NextRequest) { WHERE start_time >= NOW() - INTERVAL '${interval}' GROUP BY host_email ) zm ON LOWER(r.email) = LOWER(zm.host_email) + LEFT JOIN LATERAL ( + SELECT + COUNT(*) FILTER (WHERE LOWER(mm.sender_address) = LOWER(gu.email) AND mm.direction IN ('outbound', 'internal')) AS emails_sent, + COUNT(*) FILTER (WHERE LOWER(mm.recipient_address) = LOWER(gu.email) AND mm.direction IN ('inbound', 'internal') AND mm.status IN ('archived', 'accepted')) AS emails_received + FROM mimecast_messages mm + WHERE (LOWER(mm.sender_address) = LOWER(gu.email) OR LOWER(mm.recipient_address) = LOWER(gu.email)) + AND mm.sent_datetime >= NOW() - INTERVAL '${interval}' + ) mc ON true WHERE gu.account_enabled = true AND LOWER(gu.email) LIKE '%@wulfconsulting.%' AND LOWER(gu.email) NOT LIKE '%#ext#%' diff --git a/app/engagement/page.tsx b/app/engagement/page.tsx index 86af00a..091565a 100644 --- a/app/engagement/page.tsx +++ b/app/engagement/page.tsx @@ -146,6 +146,7 @@ interface UserDetail { end_date_time: string | null; }>; }>; + emails: Record<'d7' | 'd30' | 'd90', { sent: number; received: number }> | null; meetingCounts: { total: number; withClients: number }; zoom: { calls: Record<'d7' | 'd30' | 'd90', { total: number; client: number; outbound: number; inbound: number; durationSeconds: number }>; @@ -989,7 +990,7 @@ export default function EngagementPage() { }, ...(snap ? [{ label: 'Messages', - sub: `${snap.emails_sent} emails`, + sub: userDetail.emails?.[pKey] ? `${userDetail.emails[pKey].sent} emails sent` : `${snap.emails_sent} emails`, value: snap.teams_chat_messages + snap.teams_private_messages, peerMax: userDetail.peerMax?.messages || (period === 'D7' ? 200 : period === 'D30' ? 800 : 2400), prevValue: null, diff --git a/lib/services/mimecast-sync-service.ts b/lib/services/mimecast-sync-service.ts index 02dd2fe..ecc1043 100644 --- a/lib/services/mimecast-sync-service.ts +++ b/lib/services/mimecast-sync-service.ts @@ -1,6 +1,6 @@ /** * Mimecast Sync Service - * Full sync (120 days back), incremental (since last sync), body fetch, 120-day purge + * Full sync (18 months back), incremental (since last sync), body fetch, 18-month rolling purge */ import { getMimecastClient, MimecastMessage, MimecastThreatEvent } from './mimecast-client'; @@ -15,9 +15,10 @@ export interface MimecastSyncResult { durationMs: number; } -const RETENTION_DAYS = 30; // API max lookback is ~30 days +const RETENTION_DAYS = 548; // 18-month rolling retention in DB +const FULL_SYNC_DAYS = 548; // How far back a full sync reaches const BODY_FETCH_LIMIT = 500; -const CHUNK_DAYS = 7; // Pages the 30-day window in 7-day chunks to avoid 5000-result cap +const CHUNK_DAYS = 7; // Pages the window in 7-day chunks to avoid 5000-result cap per request // ── Upsert helpers ──────────────────────────────────────────────────────────── @@ -283,7 +284,7 @@ export async function runMimecastFullSync(): Promise { const toDate = new Date(); const fromDate = new Date(); - fromDate.setDate(fromDate.getDate() - RETENTION_DAYS); + fromDate.setDate(fromDate.getDate() - FULL_SYNC_DAYS); const messagesUpserted = await syncMimecastMessages(fromDate, toDate, errors); const threatsUpserted = await syncMimecastThreats(errors); From 4f7e9be0596d8023cd18824299766dd4f83ff31a Mon Sep 17 00:00:00 2001 From: lorentz Date: Fri, 10 Jul 2026 11:45:11 -0400 Subject: [PATCH 008/463] =?UTF-8?q?docs:=20capture=20exploration=20?= =?UTF-8?q?=E2=80=94=20PAX8=20integration=20scope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plant two seeds (PAX8 sync + Autotask mapping, general Pulse data assistant) and one research question (PAX8 company identifier alternatives to fuzzy name matching), surfaced while scoping a future PAX8 integration. --- {.windsurf => .devin}/workflows/kiosk.md | 0 {.windsurf => .devin}/workflows/plan.md | 0 .planning/research/questions.md | 20 ++++ .planning/seeds/SEED-002-pax8-integration.md | 92 +++++++++++++++++++ .../SEED-003-general-pulse-data-assistant.md | 85 +++++++++++++++++ 5 files changed, 197 insertions(+) rename {.windsurf => .devin}/workflows/kiosk.md (100%) rename {.windsurf => .devin}/workflows/plan.md (100%) create mode 100644 .planning/research/questions.md create mode 100644 .planning/seeds/SEED-002-pax8-integration.md create mode 100644 .planning/seeds/SEED-003-general-pulse-data-assistant.md diff --git a/.windsurf/workflows/kiosk.md b/.devin/workflows/kiosk.md similarity index 100% rename from .windsurf/workflows/kiosk.md rename to .devin/workflows/kiosk.md diff --git a/.windsurf/workflows/plan.md b/.devin/workflows/plan.md similarity index 100% rename from .windsurf/workflows/plan.md rename to .devin/workflows/plan.md diff --git a/.planning/research/questions.md b/.planning/research/questions.md new file mode 100644 index 0000000..c73fd75 --- /dev/null +++ b/.planning/research/questions.md @@ -0,0 +1,20 @@ +# Research Questions + +Open questions surfaced during exploration, not yet investigated. Pull into the +relevant phase's research step when picked up. + +## RESEARCH-pax8-company-identifiers + +**Raised:** 2026-07-10, during exploration of [[SEED-002]] (PAX8 Integration) + +**Question:** Does the PAX8 API expose a stable identifier for companies beyond +name — a domain, external/tenant ID, or similar — that could serve as a more +reliable join key to Autotask companies than fuzzy name matching? + +**Why it matters:** The PAX8 integration scope defaulted to "fuzzy name match at +sync time, flag unmatched/ambiguous for manual review" for mapping PAX8 +companies to Autotask companies. That's a reasonable starting point, but if the +API exposes something like a primary domain, it would be materially more +reliable and worth using instead (or as a tiebreaker for ambiguous name +matches). Check the PAX8 API docs (`companies` endpoint) before committing to +fuzzy-name as the permanent strategy. diff --git a/.planning/seeds/SEED-002-pax8-integration.md b/.planning/seeds/SEED-002-pax8-integration.md new file mode 100644 index 0000000..b71bbed --- /dev/null +++ b/.planning/seeds/SEED-002-pax8-integration.md @@ -0,0 +1,92 @@ +--- +id: SEED-002 +status: dormant +planted: 2026-07-10 +planted_during: v1.0 milestone (Pulse Mobile Shell Redesign), Phase 9.1 in flight +trigger_when: Whenever the developer is ready to start this as its own milestone — surface during the next /gsd-new-milestone scan +scope: Medium +--- + +# SEED-002: PAX8 Integration + +A read-only integration syncing PAX8 (vendor licensing/distribution marketplace) +data into Pulse — companies, subscriptions, product catalog, and orders/invoices — +mapped to existing Autotask companies, surfaced on a new `/pax8` page. + +## Why This Matters + +Three overlapping needs, all satisfied by the same underlying data sync: +1. **Cost reconciliation** — compare what PAX8 bills per client/subscription + against what's actually provisioned or invoiced. +2. **License visibility** — see subscription/seat counts per company alongside + other Pulse company data. +3. **Autotask cost mapping** — tie PAX8 companies and subscription costs to + Autotask companies/contracts for margin/profitability reporting. + +The developer already has a PAX8 client ID + client secret provisioned, so auth +setup is not a blocker — this seed is purely about scope/sequencing, not +credential access. + +## When to Surface + +**Trigger:** Whenever the developer wants to start this — not gated on any other +milestone closing, but naturally lands after the current Mobile Shell Redesign +wraps. Also surface if a future milestone's scope mentions: "PAX8", "license +reconciliation", "vendor subscription costs", "seat count audit", or "Autotask +contract margin". + +## Scope Estimate + +**Medium** — roughly 2–4 phases. New external client integration (auth, 4 +entities), a sync service, a new migration, a company-matching pass with manual +review for exceptions, and one new UI page. Follows an existing, well-worn +pattern in this codebase (autotask-factory, veeam-sync-service, etc.) — no new +architectural concepts, just new surface area. + +## Breadcrumbs + +- Auth: PAX8 REST API at `https://api.pax8.com/v1`, OAuth2 client-credentials. + Client ID + secret already provisioned by the developer. +- Pattern to follow: `lib/services/-client.ts` + `-factory.ts` + (`isConfigured()`) — see `lib/services/veeam-client.ts` / + `veeam-factory.ts` or `lib/services/msgraph-client.ts` / `msgraph-factory.ts` + for the shape. +- Sync pattern: `lib/services/entity-sync.ts` (Autotask) and + `lib/services/engagement-sync-service.ts` — incremental-if-supported, else + full upsert via `postgresClient.bulkUpsert()`. +- Env vars: `PAX8_*` (add to the integrations table in `CLAUDE.md` / + `INTEGRATIONS.md` once built). +- Scheduler: new `pax8-daily` schedule via `lib/services/sync-scheduler.ts`, + matching cadence of other integrations (see `engagement-daily`). +- Admin disable: wire into `/admin/integrations` (`integration_settings` table, + migration 081) like other integrations, so it can be toggled without a + container restart. +- UI: new top-level page (like `/engagement`) — company list, subscriptions, + cost breakdown. Not folded into the existing company detail modal. +- This is explicitly the first planned data source for [[SEED-003]] (General + Pulse Data Assistant) — build the Postgres schema with that eventual consumer + in mind (clean, well-typed tables; avoid PAX8-API-shaped blobs). + +## Notes + +### Entities in scope (v1) +- Companies (join key to Autotask companies) +- Subscriptions (product, seat count, billing term — current state) +- Products/Catalog (SKUs, categories — needed to make subscriptions readable) +- Orders/Invoices (historical line items — needed for actual cost reconciliation, + not just current-state seats) + +### Key design forks (decided during exploration) +- **Read-only** — no write-back to PAX8 (no seat adjustments, no orders) in v1. +- **Company matching: fuzzy name match at sync time** — auto-match by name + similarity, flag unmatched/ambiguous companies for manual review. See + [[RESEARCH-pax8-company-identifiers]] — worth checking whether PAX8 exposes a + more stable identifier (domain, external ID) before committing to fuzzy + matching as the permanent strategy. +- **New dedicated `/pax8` page**, not folded into company detail — daily sync + via the scheduler, not on-demand-only. + +### Anti-goals +- Not building write access (seat changes, placing orders) in v1. +- Not building the chatbot/NL query layer here — that's [[SEED-003]], a + separate initiative that consumes this data once it exists. diff --git a/.planning/seeds/SEED-003-general-pulse-data-assistant.md b/.planning/seeds/SEED-003-general-pulse-data-assistant.md new file mode 100644 index 0000000..3b8d4d4 --- /dev/null +++ b/.planning/seeds/SEED-003-general-pulse-data-assistant.md @@ -0,0 +1,85 @@ +--- +id: SEED-003 +status: dormant +planted: 2026-07-10 +planted_during: v1.0 milestone (Pulse Mobile Shell Redesign), Phase 9.1 in flight +trigger_when: After PAX8 (SEED-002) and ideally other core data sources are mirrored into Postgres — surface during the next /gsd-new-milestone scan +scope: Large +--- + +# SEED-003: General Pulse Data Assistant + +A natural-language chatbot interface over Pulse's own data generally — tickets, +companies, engagement, finance, and PAX8 once it exists — not scoped to any +single integration. + +## Why This Matters + +Surfaced while scoping [[SEED-002]] (PAX8 integration): the developer wants +"the ability to ask questions through a chatbot style interface" as one of the +motivations for pulling PAX8 data in, but on reflection the real want is broader +than PAX8 — a general assistant that happens to need PAX8 (and other sources) +as data it can draw on, not a PAX8-specific bot. + +This is a meaningfully different initiative from any single data integration: +it needs a query/tool-calling layer that can span multiple Postgres tables +safely, decide what's answerable, and present results — not just sync data +into a table. + +## When to Surface + +**Trigger:** After PAX8 data (SEED-002) — and ideally other core sources +(tickets, engagement, finance) — are mirrored into Postgres and stable. Also +surface if a future milestone's scope mentions: "chatbot", "natural language +query", "ask Pulse", "data assistant", or "LLM interface over dashboard data". + +## Scope Estimate + +**Large** — likely its own multi-phase milestone. Needs: query/tool-calling +architecture (what can the LLM call — raw SQL? scoped read helpers per domain?), +guardrails against unsafe/expensive queries, a chat UI surface (new page, or a +persistent widget across pages?), conversation state/history, and an answer +format that handles tabular results well (not just prose). + +## Breadcrumbs + +- Existing LLM plumbing to reuse rather than rebuild: `@anthropic-ai/sdk` + already wired for the AI Ticket Analyzer pipeline + (`lib/services/analyzer/pipeline.ts`, `lib/services/analyzer/stages/*.ts`), + plus `ai-triage-service.ts`. Provider is per-request (`anthropic` | + `openrouter`) — same pattern likely applies here. +- Analyzer's cost-ceiling guard (Stage 4 skipped above $2.00 estimated cost) is + a pattern worth mirroring for a chat interface that could otherwise run + unbounded queries/tokens per question. +- IT Glue data has a hard redaction requirement before hitting an LLM + (`lib/services/analyzer/itglue-search.ts`) — any assistant that can see IT + Glue-sourced data must route through the same redaction, not raw client + output. +- `postgresClient` singleton (`lib/services/postgres-client.ts`) is the only + DB access path — any tool-calling layer built for this assistant should call + through it, not open a second connection path. +- Data sources this assistant should eventually reach: tickets (Autotask sync), + companies, engagement snapshots (`engagement_snapshots`), finance/QBO data, + and PAX8 once [[SEED-002]] lands. + +## Notes + +### Key design forks (decide during brainstorming, not now) +- **Tool-calling over scoped read helpers vs. raw SQL generation** — raw + text-to-SQL is riskier (injection, runaway queries, schema drift); scoped + helpers per domain (e.g. `getCompanySubscriptionCosts(companyId)`) are safer + but need maintenance as new questions come up. Current lean: scoped helpers, + mirroring the analyzer's structured-stage-output philosophy. +- **Where it surfaces** — dedicated page vs. a persistent chat widget + available across desktop and mobile. Not decided. +- **Read-only vs. can it trigger actions** — e.g. could it kick off a sync, or + strictly answer questions? Current lean: read-only, at least for v1. +- **Conversation memory** — single-turn Q&A vs. multi-turn session with + history. Affects whether this needs new schema (a `chat_sessions` / + `chat_messages` table) beyond the query layer itself. + +### Anti-goals +- Not a replacement for the existing dashboards/pages — a supplement for + ad-hoc questions those views don't answer directly. +- Not scoped to any single integration (started life as a "PAX8 chatbot" idea, + deliberately broadened). From bb737bdf65b27ec3beb96b1653531eacecb67092 Mon Sep 17 00:00:00 2001 From: lorentz Date: Fri, 10 Jul 2026 11:54:06 -0400 Subject: [PATCH 009/463] docs: start milestone v2.0 PAX8 Integration --- .planning/PROJECT.md | 35 ++++++++++++++++++++++++++++++++--- .planning/STATE.md | 31 ++++++++++++++----------------- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md index 104a3db..5732f1a 100644 --- a/.planning/PROJECT.md +++ b/.planning/PROJECT.md @@ -15,6 +15,29 @@ A manager can open Pulse on their phone and, in under 30 seconds, see the state of the business and triage tickets — without ever needing to switch to desktop for read-only awareness. +## Current Milestone: v2.0 PAX8 Integration + +**Goal:** Sync PAX8 licensing/subscription data into Pulse, read-only, mapped +to Autotask companies, so managers can see subscription costs and seat counts +alongside existing company data. + +**Target features:** +- PAX8 client + factory (OAuth2 client-credentials; credentials already + provisioned) +- Read-only sync of companies, subscriptions, product catalog, and + orders/invoices into Postgres +- Fuzzy-name matching of PAX8 companies to Autotask companies, with a + manual-review path for unmatched/ambiguous cases +- New `/pax8` page (company list, subscriptions, cost breakdown) +- Daily sync via the existing scheduler pattern + +Unrelated in domain to the v1.0 Mobile Shell Redesign above — this milestone +adds a new backend integration and admin-facing surface, not a mobile change. +See `.planning/seeds/SEED-002-pax8-integration.md` for the exploration that +scoped this milestone. `.planning/seeds/SEED-003-general-pulse-data-assistant.md` +(NL chatbot over Pulse data) is explicitly out of scope here — a separate +future milestone that will consume this data once it exists. + ## Requirements ### Validated @@ -112,14 +135,20 @@ desktop for read-only awareness. ### Active - + -_No active hypotheses — all planned milestone phases validated._ +- PAX8 client + factory, read-only sync of companies/subscriptions/catalog/ + orders into Postgres, fuzzy-name Autotask company matching, new `/pax8` + page, daily sync cadence — v2.0 PAX8 Integration milestone (see above) ### Out of Scope +- PAX8 write access (seat adjustments, placing orders) — read-only in v2.0; + revisit only if reconciliation surfaces a concrete need to act, not just view +- General natural-language data assistant / chatbot over Pulse data — separate + future milestone (SEED-003), deliberately not bundled with the PAX8 data sync - Service worker / offline cache / push notifications — deferred until a clear offline use-case lands - Tablet breakpoint (`md:max-w-2xl`) — noted as follow-up, keep `max-w-lg` @@ -198,4 +227,4 @@ This document evolves at phase transitions and milestone boundaries. 4. Update Context with current state --- -*Last updated: 2026-05-11 — Phase 9 complete (User Profile & Preferences)* +*Last updated: 2026-07-10 — v2.0 PAX8 Integration milestone started* diff --git a/.planning/STATE.md b/.planning/STATE.md index 88285c8..aacb014 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -1,17 +1,16 @@ --- gsd_state_version: 1.0 -milestone: v1.0 -milestone_name: milestone -status: executing -stopped_at: Phase 9 UI-SPEC approved -last_updated: "2026-05-11T10:38:01.535Z" -last_activity: 2026-05-11 -- Phase 09.1 execution started +milestone: v2.0 +milestone_name: PAX8 Integration +status: planning +last_updated: "2026-07-10T15:53:23.273Z" +last_activity: 2026-07-10 progress: - total_phases: 11 - completed_phases: 10 - total_plans: 31 - completed_plans: 30 - percent: 97 + total_phases: 0 + completed_phases: 0 + total_plans: 0 + completed_plans: 0 + percent: 0 --- # Project State @@ -25,12 +24,10 @@ See: .planning/PROJECT.md (updated 2026-05-03) ## Current Position -Phase: 09.1 (ntfy-backend-fix) — EXECUTING -Plan: 1 of 1 -Status: Executing Phase 09.1 -Last activity: 2026-05-21 - Completed quick task 260521-foj: Fix weekly-full FK error: widen Companies filter + defensive ticket company_id validation - -Progress: [░░░░░░░░░░] 0% +Phase: Not started (defining requirements) +Plan: — +Status: Defining requirements +Last activity: 2026-07-10 — Milestone v2.0 started ## Performance Metrics From 46a775e81f632ed39c4c8d76a6f2b407ba2526cb Mon Sep 17 00:00:00 2001 From: lorentz Date: Fri, 10 Jul 2026 11:54:22 -0400 Subject: [PATCH 010/463] chore: clear v1.0 phase artifacts for v2.0 milestone start Phase directories (01-09.1) are fully preserved in git history at prior commits; this just clears the working tree so v2.0 roadmapping starts clean, per gsd-new-milestone's phases.clear step. --- .../phases/01-pwa-scaffolding/01-01-PLAN.md | 374 ----- .../01-pwa-scaffolding/01-01-SUMMARY.md | 142 -- .../phases/01-pwa-scaffolding/01-02-PLAN.md | 286 ---- .../01-pwa-scaffolding/01-02-SUMMARY.md | 174 -- .../phases/01-pwa-scaffolding/01-HUMAN-UAT.md | 36 - .../01-pwa-scaffolding/01-VERIFICATION.md | 152 -- .../02-mobile-shell-more-drawer/02-01-PLAN.md | 754 --------- .../02-01-SUMMARY.md | 129 -- .../02-mobile-shell-more-drawer/02-02-PLAN.md | 406 ----- .../02-02-SUMMARY.md | 155 -- .../02-mobile-shell-more-drawer/02-CONTEXT.md | 142 -- .../02-HUMAN-UAT.md | 52 - .../02-VERIFICATION.md | 184 --- .../phases/03-dashboard-restyle/03-01-PLAN.md | 578 ------- .../03-dashboard-restyle/03-01-SUMMARY.md | 140 -- .../phases/03-dashboard-restyle/03-02-PLAN.md | 426 ----- .../03-dashboard-restyle/03-02-SUMMARY.md | 121 -- .../03-dashboard-restyle/03-HUMAN-UAT.md | 45 - .../03-dashboard-restyle/03-VERIFICATION.md | 134 -- .../phases/04-tickets-restyle/04-01-PLAN.md | 583 ------- .../04-tickets-restyle/04-01-SUMMARY.md | 188 --- .../phases/04-tickets-restyle/04-02-PLAN.md | 590 ------- .../04-tickets-restyle/04-02-SUMMARY.md | 136 -- .../phases/04-tickets-restyle/04-03-PLAN.md | 218 --- .../04-tickets-restyle/04-03-SUMMARY.md | 110 -- .../phases/04-tickets-restyle/04-CONTEXT.md | 142 -- .../04-tickets-restyle/04-DISCUSSION-LOG.md | 127 -- .../phases/04-tickets-restyle/04-HUMAN-UAT.md | 74 - .../phases/04-tickets-restyle/04-UI-SPEC.md | 367 ----- .../04-tickets-restyle/04-VERIFICATION.md | 206 --- .../phases/05-finance-restyle/05-01-PLAN.md | 309 ---- .../05-finance-restyle/05-01-SUMMARY.md | 109 -- .../phases/05-finance-restyle/05-02-PLAN.md | 676 -------- .../05-finance-restyle/05-02-SUMMARY.md | 143 -- .../phases/05-finance-restyle/05-CONTEXT.md | 211 --- .../05-finance-restyle/05-DISCUSSION-LOG.md | 28 - .../phases/05-finance-restyle/05-HUMAN-UAT.md | 74 - .../phases/05-finance-restyle/05-UI-SPEC.md | 425 ----- .../05-finance-restyle/05-VERIFICATION.md | 207 --- .../phases/06-analyzer-feed-new/06-01-PLAN.md | 458 ------ .../06-analyzer-feed-new/06-01-SUMMARY.md | 138 -- .../phases/06-analyzer-feed-new/06-02-PLAN.md | 696 -------- .../06-analyzer-feed-new/06-02-SUMMARY.md | 140 -- .../phases/06-analyzer-feed-new/06-03-PLAN.md | 473 ------ .../06-analyzer-feed-new/06-03-SUMMARY.md | 116 -- .../phases/06-analyzer-feed-new/06-CONTEXT.md | 373 ----- .../06-analyzer-feed-new/06-DISCUSSION-LOG.md | 167 -- .../06-analyzer-feed-new/06-HUMAN-UAT.md | 44 - .../phases/06-analyzer-feed-new/06-UI-SPEC.md | 480 ------ .../06-analyzer-feed-new/06-VERIFICATION.md | 171 -- .../07-engagement-overview-new/07-01-PLAN.md | 558 ------- .../07-01-SUMMARY.md | 131 -- .../07-engagement-overview-new/07-02-PLAN.md | 782 --------- .../07-02-SUMMARY.md | 161 -- .../07-engagement-overview-new/07-03-PLAN.md | 703 -------- .../07-03-SUMMARY.md | 165 -- .../07-engagement-overview-new/07-CONTEXT.md | 430 ----- .../07-DISCUSSION-LOG.md | 152 -- .../07-HUMAN-UAT.md | 52 - .../07-engagement-overview-new/07-UI-SPEC.md | 613 ------- .../07-VERIFICATION.md | 202 --- .../07.1-01-PLAN.md | 296 ---- .../07.1-01-SUMMARY.md | 130 -- .../07.1-02-PLAN.md | 342 ---- .../07.1-02-SUMMARY.md | 162 -- .../07.1-03-PLAN.md | 669 -------- .../07.1-03-SUMMARY.md | 305 ---- .../07.1-04-AUDIT.md | 183 --- .../07.1-04-PLAN.md | 531 ------ .../07.1-04-SUMMARY.md | 230 --- .../07.1-05-MANIFEST.md | 550 ------- .../07.1-05-PLAN.md | 424 ----- .../07.1-05-SUMMARY.md | 270 --- .../07.1-HUMAN-UAT.md | 102 -- .../07.1-VERIFICATION.md | 162 -- .../08-01-PLAN.md | 412 ----- .../08-01-SUMMARY.md | 99 -- .../08-02-PLAN.md | 1446 ----------------- .../08-02-SUMMARY.md | 99 -- .../08-CONTEXT.md | 167 -- .../08-DISCUSSION-LOG.md | 152 -- .../08-HUMAN-UAT.md | 29 - .../08-UI-SPEC.md | 286 ---- .../08-VERIFICATION.md | 113 -- .../09-01-PLAN.md | 421 ----- .../09-01-SUMMARY.md | 126 -- .../09-02-PLAN.md | 617 ------- .../09-02-SUMMARY.md | 144 -- .../09-03-PLAN.md | 709 -------- .../09-03-SUMMARY.md | 186 --- .../09-04-PLAN.md | 466 ------ .../09-04-SUMMARY.md | 172 -- .../09-05-PLAN.md | 436 ----- .../09-05-SUMMARY.md | 143 -- .../09-06-PLAN.md | 523 ------ .../09-06-SUMMARY.md | 177 -- .../09-CONTEXT.md | 459 ------ .../09-DISCUSSION-LOG.md | 188 --- .../09-HUMAN-UAT.md | 93 -- .../09-UI-SPEC.md | 386 ----- .../09-VERIFICATION.md | 257 --- .../09.1-ntfy-backend-fix/09.1-01-PLAN.md | 437 ----- .../09.1-ntfy-backend-fix/09.1-01-SUMMARY.md | 139 -- .../09.1-ntfy-backend-fix/09.1-HUMAN-UAT.md | 32 - .../09.1-VERIFICATION.md | 117 -- 105 files changed, 29945 deletions(-) delete mode 100644 .planning/phases/01-pwa-scaffolding/01-01-PLAN.md delete mode 100644 .planning/phases/01-pwa-scaffolding/01-01-SUMMARY.md delete mode 100644 .planning/phases/01-pwa-scaffolding/01-02-PLAN.md delete mode 100644 .planning/phases/01-pwa-scaffolding/01-02-SUMMARY.md delete mode 100644 .planning/phases/01-pwa-scaffolding/01-HUMAN-UAT.md delete mode 100644 .planning/phases/01-pwa-scaffolding/01-VERIFICATION.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-01-PLAN.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-01-SUMMARY.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-02-PLAN.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-02-SUMMARY.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-CONTEXT.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-HUMAN-UAT.md delete mode 100644 .planning/phases/02-mobile-shell-more-drawer/02-VERIFICATION.md delete mode 100644 .planning/phases/03-dashboard-restyle/03-01-PLAN.md delete mode 100644 .planning/phases/03-dashboard-restyle/03-01-SUMMARY.md delete mode 100644 .planning/phases/03-dashboard-restyle/03-02-PLAN.md delete mode 100644 .planning/phases/03-dashboard-restyle/03-02-SUMMARY.md delete mode 100644 .planning/phases/03-dashboard-restyle/03-HUMAN-UAT.md delete mode 100644 .planning/phases/03-dashboard-restyle/03-VERIFICATION.md delete mode 100644 .planning/phases/04-tickets-restyle/04-01-PLAN.md delete mode 100644 .planning/phases/04-tickets-restyle/04-01-SUMMARY.md delete mode 100644 .planning/phases/04-tickets-restyle/04-02-PLAN.md delete mode 100644 .planning/phases/04-tickets-restyle/04-02-SUMMARY.md delete mode 100644 .planning/phases/04-tickets-restyle/04-03-PLAN.md delete mode 100644 .planning/phases/04-tickets-restyle/04-03-SUMMARY.md delete mode 100644 .planning/phases/04-tickets-restyle/04-CONTEXT.md delete mode 100644 .planning/phases/04-tickets-restyle/04-DISCUSSION-LOG.md delete mode 100644 .planning/phases/04-tickets-restyle/04-HUMAN-UAT.md delete mode 100644 .planning/phases/04-tickets-restyle/04-UI-SPEC.md delete mode 100644 .planning/phases/04-tickets-restyle/04-VERIFICATION.md delete mode 100644 .planning/phases/05-finance-restyle/05-01-PLAN.md delete mode 100644 .planning/phases/05-finance-restyle/05-01-SUMMARY.md delete mode 100644 .planning/phases/05-finance-restyle/05-02-PLAN.md delete mode 100644 .planning/phases/05-finance-restyle/05-02-SUMMARY.md delete mode 100644 .planning/phases/05-finance-restyle/05-CONTEXT.md delete mode 100644 .planning/phases/05-finance-restyle/05-DISCUSSION-LOG.md delete mode 100644 .planning/phases/05-finance-restyle/05-HUMAN-UAT.md delete mode 100644 .planning/phases/05-finance-restyle/05-UI-SPEC.md delete mode 100644 .planning/phases/05-finance-restyle/05-VERIFICATION.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-01-PLAN.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-01-SUMMARY.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-02-PLAN.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-02-SUMMARY.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-03-PLAN.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-03-SUMMARY.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-CONTEXT.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-DISCUSSION-LOG.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-HUMAN-UAT.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-UI-SPEC.md delete mode 100644 .planning/phases/06-analyzer-feed-new/06-VERIFICATION.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-01-PLAN.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-01-SUMMARY.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-02-PLAN.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-02-SUMMARY.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-03-PLAN.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-03-SUMMARY.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-CONTEXT.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-DISCUSSION-LOG.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-HUMAN-UAT.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-UI-SPEC.md delete mode 100644 .planning/phases/07-engagement-overview-new/07-VERIFICATION.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-01-PLAN.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-01-SUMMARY.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-02-PLAN.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-02-SUMMARY.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-03-PLAN.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-03-SUMMARY.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-04-AUDIT.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-04-PLAN.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-04-SUMMARY.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-05-MANIFEST.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-05-PLAN.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-05-SUMMARY.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-HUMAN-UAT.md delete mode 100644 .planning/phases/07.1-user-timezone-fix-inserted-urgent/07.1-VERIFICATION.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-01-PLAN.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-01-SUMMARY.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-02-PLAN.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-02-SUMMARY.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-CONTEXT.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-DISCUSSION-LOG.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-HUMAN-UAT.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-UI-SPEC.md delete mode 100644 .planning/phases/08-engagement-user-profile-new/08-VERIFICATION.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-01-PLAN.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-01-SUMMARY.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-02-PLAN.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-02-SUMMARY.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-03-PLAN.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-03-SUMMARY.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-04-PLAN.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-04-SUMMARY.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-05-PLAN.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-05-SUMMARY.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-06-PLAN.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-06-SUMMARY.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-CONTEXT.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-DISCUSSION-LOG.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-HUMAN-UAT.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-UI-SPEC.md delete mode 100644 .planning/phases/09-user-profile-preferences-new/09-VERIFICATION.md delete mode 100644 .planning/phases/09.1-ntfy-backend-fix/09.1-01-PLAN.md delete mode 100644 .planning/phases/09.1-ntfy-backend-fix/09.1-01-SUMMARY.md delete mode 100644 .planning/phases/09.1-ntfy-backend-fix/09.1-HUMAN-UAT.md delete mode 100644 .planning/phases/09.1-ntfy-backend-fix/09.1-VERIFICATION.md diff --git a/.planning/phases/01-pwa-scaffolding/01-01-PLAN.md b/.planning/phases/01-pwa-scaffolding/01-01-PLAN.md deleted file mode 100644 index 6a48d35..0000000 --- a/.planning/phases/01-pwa-scaffolding/01-01-PLAN.md +++ /dev/null @@ -1,374 +0,0 @@ ---- -phase: 01-pwa-scaffolding -plan: 01 -type: execute -wave: 1 -depends_on: [] -files_modified: - - public/manifest.json - - app/layout.tsx -autonomous: true -requirements: - - PWA-01 - - PWA-02 - - PWA-03 - -must_haves: - truths: - - "Visiting /manifest.json returns valid JSON with name 'Pulse', short_name 'Pulse', display 'standalone', start_url '/mobile'" - - "The manifest theme_color matches the Wulf brand blue (#0075AD) and background_color matches the light shell" - - "app/layout.tsx references the manifest so Chrome/Safari pick it up automatically (via metadata.manifest or )" - - "app/layout.tsx exports a viewport object whose viewportFit is 'cover' so the rendered contains 'viewport-fit=cover'" - - "Installing Pulse to a phone home screen launches a chromeless app that opens to /mobile" - artifacts: - - path: "public/manifest.json" - provides: "Web App Manifest — name, short_name, display, start_url, theme/background, icons" - contains: '"display": "standalone"' - - path: "app/layout.tsx" - provides: "Root layout exporting metadata (with manifest) and viewport (with viewportFit:'cover')" - contains: "viewportFit" - key_links: - - from: "app/layout.tsx" - to: "public/manifest.json" - via: "metadata.manifest or " - pattern: "manifest" - - from: "public/manifest.json" - to: "/mobile" - via: "start_url field" - pattern: '"start_url"\s*:\s*"/mobile"' - - from: "app/layout.tsx (viewport export)" - to: "rendered " - via: "Next.js viewport export → viewport-fit=cover in DOM" - pattern: "viewportFit" ---- - - -Add the PWA install surface: a valid Web App Manifest at `/manifest.json`, a manifest reference from the root layout, and a viewport export with `viewportFit: 'cover'` so future shell phases can paint behind the device home indicator. - -Purpose: PWA-01, PWA-02, PWA-03 — make Pulse installable to a phone home screen and have the install land on `/mobile` in standalone (chromeless) mode. No service worker, no offline. - -Output: `public/manifest.json` (new file) and an updated `app/layout.tsx` that adds the manifest reference and a Next 16 viewport export. Verifiable by `curl http://localhost:3100/manifest.json` and grep on `app/layout.tsx`. - - - -@$HOME/.claude/get-shit-done/workflows/execute-plan.md -@$HOME/.claude/get-shit-done/templates/summary.md - - - -@.planning/PROJECT.md -@.planning/ROADMAP.md -@.planning/STATE.md -@.planning/REQUIREMENTS.md -@docs/superpowers/specs/2026-05-03-mobile-shell-design.md -@CLAUDE.md -@app/layout.tsx -@app/globals.css -@app/styles/brand.css - - - - -**Existing `app/layout.tsx` shape (what is currently there):** -- Imports `Metadata` from `next` (already imported). -- Exports `const metadata: Metadata = { title, description, icons: { icon: [...], shortcut: '/favicon.png', apple: '/wulff-logo.png' } }`. -- Does NOT currently export `viewport`. Next 16 expects a separate `viewport` export of type `Viewport` from `next`. -- The metadata object already has an `icons` field. Do NOT remove it; the PWA `manifest` field is added alongside `icons`. - -**Brand colors (already defined in `app/styles/brand.css`):** -- Wulf primary blue: `#0075AD` (oklch `0.540 0.136 233.3`). This is the `theme_color`. -- Light shell background: white (`#FFFFFF`). This is the `background_color` (the manifest only allows one; the light shell is the standard splash background). - -**Existing icon assets (in `/public`):** -- `/public/wulff-logo.png` — square PNG, used today as `metadata.icons.apple` (apple-touch-icon). -- `/public/favicon.png` — square PNG. -- `/public/branding/wulf-mark.png` — Wulf "W" mark, square PNG. -- `/public/branding/wulf-wordmark.png` — Wulf "Pulse" wordmark. -None of these have explicit pixel sizes verified, but they're used today and PWA install tools accept them with `"sizes": "any"`. - -**Next.js 16 metadata API for manifest:** -- The recommended way to reference a manifest is `metadata.manifest = '/manifest.json'` in the metadata export. Next emits `` automatically. This satisfies the spec wording (``) without hand-rolling the link tag. -- Alternative: hand-roll `` inside ``. Either approach is acceptable per the spec; prefer `metadata.manifest` because the file already uses the metadata API. - -**Next.js 16 viewport API:** -- Import: `import type { Viewport } from 'next'`. -- Export: `export const viewport: Viewport = { ... }` (separate from `metadata`; Next 16 deprecated `metadata.viewport`). -- The `viewportFit` field is camelCase in TS; Next emits `viewport-fit=cover` in the rendered `` tag. -- Reasonable default fields: `width: 'device-width'`, `initialScale: 1`, `viewportFit: 'cover'`. Do NOT add `maximumScale` or `userScalable: false` (accessibility). - -**Theme color / dark mode caveat:** -- The manifest only allows one `theme_color`. Use the Wulf blue `#0075AD` so the system UI tint matches the brand in both light and dark modes. -- Optionally also add a viewport `themeColor` array with `media: '(prefers-color-scheme: dark)'` variants in the viewport export. This is a Next.js helper that emits `` per-scheme. NOT required for PWA-01..03; only add if it falls out naturally. - -**Verification commands the executor will use:** -- `curl -sf http://localhost:3100/manifest.json | jq .` (dev server must be running) -- `grep -E "viewportFit|viewport-fit" app/layout.tsx` -- `grep -E 'manifest:|rel="manifest"' app/layout.tsx` -- `npx tsc --noEmit --pretty` (must pass) - - - - - - - Task 1: Create public/manifest.json - public/manifest.json - - - docs/superpowers/specs/2026-05-03-mobile-shell-design.md (§4 — exact field requirements) - - .planning/REQUIREMENTS.md (PWA-01) - - app/styles/brand.css (line 28: `--wulf-blue` is `#0075AD` — this is theme_color) - - app/globals.css (lines 47-48: light theme `--background` is white; lines 82-83: dark theme background) - - public/ directory listing — confirm `wulff-logo.png`, `favicon.png`, `branding/wulf-mark.png` exist - - -Create `public/manifest.json` (new file) with exactly this JSON shape. Hand-write the file; do not use a generator. - -```json -{ - "name": "Pulse", - "short_name": "Pulse", - "description": "Wulf Consulting operations console — tickets, RMM, backups, and analytics on the go.", - "start_url": "/mobile", - "scope": "/", - "display": "standalone", - "orientation": "portrait", - "theme_color": "#0075AD", - "background_color": "#FFFFFF", - "icons": [ - { - "src": "/wulff-logo.png", - "sizes": "any", - "type": "image/png", - "purpose": "any" - }, - { - "src": "/branding/wulf-mark.png", - "sizes": "any", - "type": "image/png", - "purpose": "any" - }, - { - "src": "/favicon.png", - "sizes": "any", - "type": "image/png", - "purpose": "any" - } - ] -} -``` - -Notes on the choices (so a reviewer doesn't have to ask): -- `name` and `short_name` both "Pulse" — matches spec §4 verbatim. -- `start_url: "/mobile"` — spec §4 verbatim. The phone install lands on the mobile shell, not the desktop dashboard. -- `scope: "/"` — allow the standalone window to navigate anywhere in the app without falling out to the browser. (Spec doesn't specify; root scope is the safe default for an installed PSA console.) -- `display: "standalone"` — spec §4 verbatim. Chromeless app surface. -- `orientation: "portrait"` — phone-first per the spec's overall framing (§1, §2). Tablet landscape is explicit out-of-scope (§7). -- `theme_color: "#0075AD"` — Wulf brand blue from `app/styles/brand.css` line 28 (`--wulf-blue`). Matches the `--primary` token in both light and dark modes (oklch values resolve to this brand blue, slightly lifted for dark). -- `background_color: "#FFFFFF"` — light shell background. Manifest only allows one value; the iOS/Android splash uses this. White matches Pulse's default theme on light devices and is acceptable on dark devices (brief flash, not a regression). -- `icons` — three entries reusing existing assets in `/public`. Using `"sizes": "any"` because the assets are not explicitly sized — install tools accept this for PNGs and pick the largest. Do NOT generate new icon PNGs in this task; reuse what's there. (A future polish phase can add density-specific 192/512 icons if install warns.) - -Do NOT: -- Add a `serviceworker` field (no SW in v1, spec §4 explicit). -- Add `display_override` or `prefer_related_applications` (not needed; not in spec). -- Add `categories` or `lang` (cosmetic; not in spec scope). -- Reference `next-pwa` or any plugin (forbidden by spec §4 and CLAUDE.md). -- Edit any existing migration, lib/, or component file. - -The file must be served directly by Next.js as a static asset — placing it at `public/manifest.json` makes it available at `http://localhost:3100/manifest.json`. - - - test -f public/manifest.json && jq -e '.name == "Pulse" and .short_name == "Pulse" and .display == "standalone" and .start_url == "/mobile" and .theme_color == "#0075AD" and .background_color == "#FFFFFF" and (.icons | length) >= 1' public/manifest.json - - - - File `public/manifest.json` exists. - - `jq -r .name public/manifest.json` outputs `Pulse`. - - `jq -r .short_name public/manifest.json` outputs `Pulse`. - - `jq -r .display public/manifest.json` outputs `standalone`. - - `jq -r .start_url public/manifest.json` outputs `/mobile`. - - `jq -r .theme_color public/manifest.json` outputs `#0075AD`. - - `jq -r .background_color public/manifest.json` outputs `#FFFFFF`. - - `jq -e '.icons | length >= 1' public/manifest.json` exits 0. - - `jq -e '.icons[0].src' public/manifest.json` outputs a path beginning with `/` (e.g., `/wulff-logo.png`). - - File is valid JSON: `jq empty public/manifest.json` exits 0. - - No `serviceworker` field present: `jq -e '.serviceworker == null' public/manifest.json` exits 0. - - When dev server is running on port 3100: `curl -sf http://localhost:3100/manifest.json` exits 0 and the body equals the file contents. - - - `public/manifest.json` exists, is valid JSON, contains the spec-mandated fields with the values above, references at least one icon from `/public`, and is reachable at `http://localhost:3100/manifest.json` when the dev server is running. - - - - - Task 2: Add manifest reference and viewport export to app/layout.tsx - app/layout.tsx - - - app/layout.tsx (current file — already exports `metadata: Metadata`, no `viewport` export yet) - - docs/superpowers/specs/2026-05-03-mobile-shell-design.md (§4 — viewport-fit=cover wording) - - .planning/REQUIREMENTS.md (PWA-02, PWA-03) - - public/manifest.json (created in Task 1 — must exist before this task ships) - - -Edit `app/layout.tsx` (do NOT create a new file). Two changes, both at the top of the file alongside the existing `metadata` export. The body of `RootLayout` is unchanged. - -**Change 1 — add `manifest: '/manifest.json'` to the existing `metadata` object.** - -The current export looks like: - -```ts -export const metadata: Metadata = { - title: "Pulse · Operations console", - description: "Wulf Consulting operations console — tickets, RMM, IT Glue, backups, and analytics in one place.", - icons: { - icon: [ - { url: "/favicon.png", sizes: "any" }, - { url: "/wulff-logo.png", sizes: "32x32", type: "image/png" }, - ], - shortcut: "/favicon.png", - apple: "/wulff-logo.png", - }, -}; -``` - -Add a `manifest` field alongside `icons`. The result should be: - -```ts -export const metadata: Metadata = { - title: "Pulse · Operations console", - description: "Wulf Consulting operations console — tickets, RMM, IT Glue, backups, and analytics in one place.", - manifest: "/manifest.json", - icons: { - icon: [ - { url: "/favicon.png", sizes: "any" }, - { url: "/wulff-logo.png", sizes: "32x32", type: "image/png" }, - ], - shortcut: "/favicon.png", - apple: "/wulff-logo.png", - }, -}; -``` - -Next.js 16 emits `` automatically from this field — this satisfies the spec wording (`` from §4) without hand-rolling the tag. - -**Change 2 — add a `Viewport` import and a separate `viewport` export.** - -Update the `next` type import on line 1. The current import is: - -```ts -import type { Metadata } from "next"; -``` - -Change it to: - -```ts -import type { Metadata, Viewport } from "next"; -``` - -Then, immediately after the `metadata` export (and before `export default function RootLayout(...)`), add: - -```ts -export const viewport: Viewport = { - width: "device-width", - initialScale: 1, - viewportFit: "cover", - themeColor: [ - { media: "(prefers-color-scheme: light)", color: "#FFFFFF" }, - { media: "(prefers-color-scheme: dark)", color: "#0A0A0A" }, - ], -}; -``` - -Notes on the choices: -- `viewportFit: 'cover'` — the only field PWA-03 strictly requires. Emits `viewport-fit=cover` in the rendered `` tag. With this set, Phase 2's safe-area-inset utilities can paint behind the home indicator. -- `width: 'device-width'` and `initialScale: 1` — standard mobile viewport defaults; they were absent before and Next 16 would warn without them. Adding them here removes the warning and makes the viewport explicit. -- `themeColor` — paired light/dark values for the system browser chrome (status bar tint). Light = white (matches manifest `background_color`); dark = `#0A0A0A` (close to the existing `--background` oklch `0.145 0 0` in `app/globals.css` line 83). This is OPTIONAL for PWA-03 (the manifest's `theme_color` already covers the install chrome), but it's a one-line improvement that ships better dark-mode rendering and costs nothing. Keep it; remove if it ever conflicts with a future per-page override. -- Do NOT add `maximumScale`, `userScalable: false`, or `minimumScale` — accessibility regression. - -Do NOT: -- Touch the `RootLayout` function body. -- Touch the `ThemeProvider`, `AppNavigation`, `CommandPalette`, `TaglineFooter`, `Toaster`, or `AuthProvider` imports. -- Add any `` JSX (no hand-rolled `` tag — let Next emit it from `metadata.manifest`). -- Touch the `IBM_Plex_Sans` / `IBM_Plex_Mono` font setup. -- Add `'use client'` — root layout is a server component. - - - grep -q 'manifest: "/manifest.json"' app/layout.tsx && grep -q 'viewportFit: "cover"' app/layout.tsx && grep -q 'import type { Metadata, Viewport } from "next"' app/layout.tsx && grep -q 'export const viewport: Viewport' app/layout.tsx && npx tsc --noEmit --pretty 2>&1 | tee /tmp/tsc-out && ! grep -E "app/layout\\.tsx.*error" /tmp/tsc-out - - - - `grep -E '^import type \{ Metadata, Viewport \} from "next"' app/layout.tsx` matches one line (or `Metadata` and `Viewport` both appear in a single named-import line from `next`). - - `grep -E 'manifest:\s*"/manifest\.json"' app/layout.tsx` matches one line inside the `metadata` object. - - `grep -E '^export const viewport: Viewport = \{' app/layout.tsx` matches exactly one line. - - `grep -E 'viewportFit:\s*"cover"' app/layout.tsx` matches one line inside the `viewport` export. - - `grep -E 'width:\s*"device-width"' app/layout.tsx` matches one line. - - `grep -E 'initialScale:\s*1' app/layout.tsx` matches one line. - - The `metadata.icons` object is unchanged (still contains `apple: "/wulff-logo.png"`): `grep -E 'apple:\s*"/wulff-logo\.png"' app/layout.tsx` matches. - - The `RootLayout` default export is unchanged: `grep -E 'export default function RootLayout' app/layout.tsx` matches. - - No `'use client'` pragma added: `! grep -E "^'use client'" app/layout.tsx`. - - Type check passes: `npx tsc --noEmit --pretty` exits 0 (or, if other files have unrelated pre-existing errors, no error rows mention `app/layout.tsx`). - - When dev server is running: viewing http://localhost:3100/ source contains `viewport-fit=cover` (e.g. `curl -s http://localhost:3100/ | grep -E 'viewport-fit=cover'` exits 0). Optional manual check; not strictly required for the automated gate. - - - `app/layout.tsx` exports both `metadata` (now with `manifest: "/manifest.json"`) and `viewport` (with `viewportFit: "cover"`, `width: "device-width"`, `initialScale: 1`, and themeColor light/dark pair). Type check passes. The `RootLayout` body is unchanged. PWA-02 (manifest reference) and PWA-03 (viewport-fit=cover) are satisfied. - - - - - - -## Trust Boundaries - -| Boundary | Description | -|----------|-------------| -| Browser ↔ static asset (/manifest.json) | Public client read of a manifest. No auth, no input. | -| Browser ↔ rendered HTML head | Public client read of `` and ``. | - -## STRIDE Threat Register - -| Threat ID | Category | Component | Disposition | Mitigation Plan | -|-----------|----------|-----------|-------------|-----------------| -| T-01-01 | Information Disclosure | public/manifest.json | accept | Manifest is intended to be world-readable per W3C Web App Manifest spec. Contains only public app branding (name, theme color, icon paths) — no secrets, no user data, no endpoints. | -| T-01-02 | Tampering | app/layout.tsx viewport export | accept | Server-rendered; no user input flows into the viewport meta. No injection vector. | -| T-01-03 | Denial of Service | manifest fetch | accept | Static file served by Next.js; same risk profile as `/favicon.png`. No new attack surface. | - -**Summary:** No new threat surface introduced. `manifest.json` is public per W3C spec; viewport meta is a public client hint; no auth, data, or endpoints are introduced. ASVS-L1 baseline preserved. - - - -With dev server running (`npm run dev` → port 3100), all of the following must pass: - -```bash -# Manifest is reachable and well-formed -curl -sf http://localhost:3100/manifest.json | jq -e '.name == "Pulse" and .display == "standalone" and .start_url == "/mobile"' - -# Manifest is referenced from root layout (Next emits the link tag automatically) -curl -s http://localhost:3100/ | grep -E 'rel="manifest"' - -# Viewport meta includes viewport-fit=cover -curl -s http://localhost:3100/ | grep -E 'viewport-fit=cover' - -# Type check passes -npx tsc --noEmit --pretty - -# No service worker file shipped (negative check — must be absent) -test ! -f public/sw.js && test ! -f public/service-worker.js - -# next-pwa is not in dependencies -! grep -E '"next-pwa"' package.json -``` - - - -- `public/manifest.json` exists with name "Pulse", short_name "Pulse", display "standalone", start_url "/mobile", theme_color "#0075AD", background_color "#FFFFFF", and at least one icon (PWA-01). -- `app/layout.tsx` references the manifest via `metadata.manifest = "/manifest.json"`, which makes Next.js emit `` in the rendered HTML head (PWA-02). -- `app/layout.tsx` exports `viewport: Viewport` with `viewportFit: "cover"` so the rendered `` tag contains `viewport-fit=cover` (PWA-03). -- `npx tsc --noEmit --pretty` passes. -- No service worker file or `next-pwa` dependency introduced. - - - -After completion, create `.planning/phases/01-pwa-scaffolding/01-01-SUMMARY.md` documenting: -- Files created/modified (paths and one-line descriptions) -- The exact `theme_color` and `background_color` values chosen (and why — Wulf brand blue + light shell background) -- The viewport export shape (so Phase 2 knows it can rely on `viewport-fit=cover` being present) -- Verification results (manifest curl, viewport grep, tsc result) -- Any deviations from the plan and rationale - diff --git a/.planning/phases/01-pwa-scaffolding/01-01-SUMMARY.md b/.planning/phases/01-pwa-scaffolding/01-01-SUMMARY.md deleted file mode 100644 index b652ff6..0000000 --- a/.planning/phases/01-pwa-scaffolding/01-01-SUMMARY.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -phase: 01-pwa-scaffolding -plan: 01 -subsystem: pwa-shell -tags: [pwa, manifest, viewport, mobile] -requires: - - app/layout.tsx (existing root layout with metadata export) - - public/wulff-logo.png, public/favicon.png, public/branding/wulf-mark.png (existing icon assets) -provides: - - public/manifest.json (Web App Manifest at /manifest.json) - - app/layout.tsx exports `viewport: Viewport` with viewportFit: "cover" - - app/layout.tsx exports `metadata.manifest = "/manifest.json"` (Next.js emits automatically) -affects: - - Phase 02 mobile shell (can rely on viewport-fit=cover for safe-area insets) - - All routes (root layout viewport applies app-wide) -tech-stack: - added: [] - patterns: - - Next.js 16 separate `viewport` export (replaces deprecated metadata.viewport) - - Next.js 16 metadata.manifest field (auto-emits ) -key-files: - created: - - public/manifest.json - modified: - - app/layout.tsx -decisions: - - theme_color #0075AD chosen as Wulf primary brand blue (sourced from app/styles/brand.css line 28, --wulf-blue) — gives consistent system UI tint in light and dark mode since manifest only allows one value - - background_color #FFFFFF chosen as the light shell background — manifest only allows one splash background, white matches Pulse's default light theme and is acceptable on dark devices (brief flash, not a regression) - - Used metadata.manifest field over hand-rolled — Next.js 16 emits the link tag automatically, satisfies spec wording, and keeps with the existing metadata API pattern - - Reused existing icon assets with `"sizes": "any"` (wulff-logo.png, branding/wulf-mark.png, favicon.png) instead of generating sized 192/512 variants — install tools accept this for PNGs; sized icons can be added in a future polish phase if install warns - - Added themeColor light/dark pair in viewport (one-line improvement) — paired with Next.js helper, emits per-scheme tags. Optional per the plan; kept since it costs nothing and improves dark-mode rendering - - orientation set to "portrait" — phone-first per spec §1/§2; tablet landscape is explicit out-of-scope per spec §7 - - scope set to "/" — allow standalone window to navigate anywhere in the app without falling out to browser -metrics: - duration: ~1m - tasks_completed: 2 - files_created: 1 - files_modified: 1 - completed: 2026-05-03T17:38:55Z ---- - -# Phase 01 Plan 01: PWA Scaffolding Summary - -PWA install surface added: a Web App Manifest at `/manifest.json` plus a Next.js 16 viewport export with `viewport-fit=cover` so the mobile shell can paint behind the device home indicator in future phases. - -## What Shipped - -### Task 1: `public/manifest.json` (NEW) - -Hand-written 31-line JSON manifest with all spec-mandated fields: - -| Field | Value | Why | -|-------|-------|-----| -| `name`, `short_name` | "Pulse" | Spec §4 verbatim | -| `description` | Wulf operations console blurb | Install dialog readability | -| `start_url` | `/mobile` | Spec §4 — phone install lands on mobile shell, not desktop dashboard | -| `scope` | `/` | Allow standalone window to navigate the whole app | -| `display` | `standalone` | Spec §4 — chromeless app surface | -| `orientation` | `portrait` | Phone-first (spec §1, §2); tablet landscape is OOS (§7) | -| `theme_color` | `#0075AD` | Wulf primary blue from `app/styles/brand.css` line 28 | -| `background_color` | `#FFFFFF` | Light shell background (manifest allows only one) | -| `icons` | 3 entries with `sizes: "any"` | Reuses `/wulff-logo.png`, `/branding/wulf-mark.png`, `/favicon.png` | - -No `serviceworker`, no `display_override`, no `prefer_related_applications`, no `next-pwa` — per spec §4 and CLAUDE.md. - -**Commit:** `3e3df24` - -### Task 2: `app/layout.tsx` (MODIFIED) - -Three minimal additions to the existing root layout, body unchanged: - -1. Import upgraded: `import type { Metadata, Viewport } from "next";` -2. `metadata.manifest = "/manifest.json"` added alongside the existing `icons` field — Next.js 16 emits `` in the rendered HTML head automatically (satisfies PWA-02 spec wording). -3. New `viewport` export: - - ```ts - export const viewport: Viewport = { - width: "device-width", - initialScale: 1, - viewportFit: "cover", - themeColor: [ - { media: "(prefers-color-scheme: light)", color: "#FFFFFF" }, - { media: "(prefers-color-scheme: dark)", color: "#0A0A0A" }, - ], - }; - ``` - - `viewportFit: "cover"` is the load-bearing field for PWA-03 — Next.js renders `viewport-fit=cover` in the `` tag so future phases can use safe-area-inset utilities to paint behind the home indicator. `width`, `initialScale`, and `themeColor` are baseline mobile defaults that prevent Next.js viewport warnings. - -**Commit:** `d196d22` - -## Verification Results - -| Gate | Result | -|------|--------| -| `test -f public/manifest.json` | PASS | -| `jq -e '.name == "Pulse" and .display == "standalone" and .start_url == "/mobile"' public/manifest.json` | PASS (true) | -| `jq -e '.theme_color == "#0075AD" and .background_color == "#FFFFFF"' public/manifest.json` | PASS | -| `jq -e '.icons \| length >= 1' public/manifest.json` | PASS (3 icons) | -| `jq -e '.serviceworker == null' public/manifest.json` | PASS | -| `jq empty public/manifest.json` | PASS (valid JSON) | -| `grep -E '^import type \{ Metadata, Viewport \} from "next"' app/layout.tsx` | PASS | -| `grep -E 'manifest:\s*"/manifest\.json"' app/layout.tsx` | PASS | -| `grep -E '^export const viewport: Viewport = \{' app/layout.tsx` | PASS | -| `grep -E 'viewportFit:\s*"cover"' app/layout.tsx` | PASS | -| `grep -E 'width:\s*"device-width"' app/layout.tsx` | PASS | -| `grep -E 'initialScale:\s*1' app/layout.tsx` | PASS | -| `grep -E 'apple:\s*"/wulff-logo\.png"' app/layout.tsx` (icons preserved) | PASS | -| `grep -E 'export default function RootLayout' app/layout.tsx` (body intact) | PASS | -| `! grep -E "^'use client'" app/layout.tsx` | PASS | -| `npx tsc --noEmit --pretty` | exit 0 | -| `test ! -f public/sw.js && test ! -f public/service-worker.js` | PASS | -| `! grep '"next-pwa"' package.json` | PASS | - -**Dev-server-only checks** (`curl http://localhost:3100/manifest.json`, `curl http://localhost:3100/ \| grep viewport-fit=cover`) were not run — this executor runs in a worktree without a dev server. The offline equivalents above are equivalent: the file is a static asset served verbatim by Next.js from `public/`, and `viewportFit: "cover"` is type-checked to render `viewport-fit=cover` per Next.js 16's documented metadata API. - -## Requirements Satisfied - -- **PWA-01:** `public/manifest.json` exists with name "Pulse", short_name "Pulse", display "standalone", start_url "/mobile", theme_color "#0075AD", background_color "#FFFFFF", and 3 icons. -- **PWA-02:** `app/layout.tsx` references the manifest via `metadata.manifest = "/manifest.json"` — Next.js 16 emits the `` tag automatically. -- **PWA-03:** `app/layout.tsx` exports `viewport: Viewport` with `viewportFit: "cover"` — Next.js renders `viewport-fit=cover` in the `` tag, unblocking safe-area painting in Phase 2. - -## Deviations from Plan - -None - plan executed exactly as written. - -No bugs encountered, no missing critical functionality, no blocking issues, no architectural decisions needed. - -## Threat Surface Scan - -No new threat surface introduced beyond the plan's ``. The manifest is world-readable per W3C Web App Manifest spec and contains only public branding (no secrets, no user data, no endpoints). The viewport export is server-rendered with no user input flow. ASVS-L1 baseline preserved. - -## Known Stubs - -None. All values are real (brand colors sourced from `app/styles/brand.css`, icons reference real public assets, start_url matches the existing `/mobile` route). - -## Self-Check: PASSED - -- `[ -f public/manifest.json ]` → FOUND -- `[ -f app/layout.tsx ]` → FOUND -- `git log --oneline | grep 3e3df24` → FOUND (Task 1 commit) -- `git log --oneline | grep d196d22` → FOUND (Task 2 commit) diff --git a/.planning/phases/01-pwa-scaffolding/01-02-PLAN.md b/.planning/phases/01-pwa-scaffolding/01-02-PLAN.md deleted file mode 100644 index 7c2dab8..0000000 --- a/.planning/phases/01-pwa-scaffolding/01-02-PLAN.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -phase: 01-pwa-scaffolding -plan: 02 -type: execute -wave: 1 -depends_on: [] -gap_closure: true -files_modified: - - app/styles/brand.css -autonomous: true -requirements: - - PWA-04 - -must_haves: - truths: - - "A shared @utility named pt-safe is defined in app/styles/brand.css that applies padding-top: env(safe-area-inset-top)" - - "A shared @utility named pb-safe is defined in app/styles/brand.css that applies padding-bottom: env(safe-area-inset-bottom)" - - "Phase 2's sticky header can opt into safe-area-inset-top padding by adding the pt-safe class" - - "Phase 2's fixed bottom nav can opt into safe-area-inset-bottom padding by adding the pb-safe class" - - "The Tailwind 4 build accepts the new @utility blocks (no CSS syntax errors, npm run build succeeds)" - artifacts: - - path: "app/styles/brand.css" - provides: "Two new @utility blocks (pt-safe, pb-safe) sitting alongside the existing num/metric/surface/rule/chrome/tagline utilities" - contains: "@utility pt-safe" - key_links: - - from: "app/styles/brand.css (@utility pt-safe)" - to: "rendered CSS class .pt-safe" - via: "Tailwind 4 @utility block — Tailwind compiles @utility name { ... } into a class .name { ... }" - pattern: "@utility pt-safe" - - from: "app/styles/brand.css (@utility pb-safe)" - to: "rendered CSS class .pb-safe" - via: "Tailwind 4 @utility block" - pattern: "@utility pb-safe" - - from: "app/globals.css" - to: "app/styles/brand.css" - via: "@import './styles/brand.css' on line 125 (already wired — no change required)" - pattern: '@import "./styles/brand.css"' ---- - - -Close the PWA-04 gap from Phase 01 verification by adding shared safe-area `@utility` blocks to `app/styles/brand.css`. Phase 2's sticky header and fixed bottom nav need to opt into `env(safe-area-inset-top)` / `env(safe-area-inset-bottom)` padding so content paints correctly under the iOS home indicator and Android gesture bar when `viewport-fit=cover` is in effect (already shipped by 01-01). - -Purpose: PWA-04 — make a safe-area utility available so any sticky top/bottom bar can opt in. ROADMAP Phase 1 SC #3 requires the utility to be **available in Phase 1**; Phase 2's contract (SHELL-05, SHELL-06) only mandates **consumption**. This plan restores the broken phase boundary identified by `01-VERIFICATION.md`. - -Output: `app/styles/brand.css` updated with two new `@utility` blocks (`pt-safe`, `pb-safe`) appended to the existing utility section. No other files touched. Verifiable by `grep -E '@utility (pt-safe|pb-safe)' app/styles/brand.css` and `npm run build`. - -Why `app/styles/brand.css` (not `app/globals.css`): -- All named project utilities (`num`, `num-lg`, `num-xl`, `metric-label`, `surface-brand`, `surface-brand-ink`, `rule-brand`, `text-chrome`, `border-chrome`, `tagline`, `has-mark-watermark`) already live there. -- `brand.css` is already imported into `globals.css` (line 125) — no extra wiring needed. -- Keeps utilities co-located so Phase 2 has a single file to scan when looking for project helpers. -- `globals.css` is reserved for Tailwind imports, `@theme inline` token mapping, and `:root` / `.dark` variable definitions — adding utility classes there would muddy that separation. - -Note on traceability: this plan claims `PWA-04` in its `requirements` frontmatter, restoring the orphaned-requirement state flagged by `01-VERIFICATION.md`. The executor's SUMMARY (`01-02-SUMMARY.md`) should explicitly call out that PWA-04 is now satisfied, closing the requirements traceability table. - - - -@$HOME/.claude/get-shit-done/workflows/execute-plan.md -@$HOME/.claude/get-shit-done/templates/summary.md - - - -@.planning/PROJECT.md -@.planning/ROADMAP.md -@.planning/REQUIREMENTS.md -@.planning/phases/01-pwa-scaffolding/01-VERIFICATION.md -@.planning/phases/01-pwa-scaffolding/01-01-SUMMARY.md -@docs/superpowers/specs/2026-05-03-mobile-shell-design.md -@CLAUDE.md -@app/styles/brand.css -@app/globals.css - - - - -**Tailwind 4 `@utility` syntax (already in use in this project):** -- This project is Tailwind 4 with NO `tailwind.config.*` file. Custom utilities are declared inline in CSS using the `@utility` at-rule. -- Pattern: `@utility name { /* CSS declarations */ }` — Tailwind compiles this to a class `.name { ... }` that participates in the variant system (`hover:name`, `md:name`, etc.). -- See existing examples in `app/styles/brand.css` lines 70-140 (e.g., `@utility num { ... }`, `@utility metric-label { ... }`, `@utility surface-brand { ... }`). -- Each `@utility` block holds plain CSS property declarations. No `@apply` is required for simple `padding-*` cases. - -**`env()` CSS environment variables for safe areas:** -- `env(safe-area-inset-top)` — top safe-area inset (e.g., iPhone notch / Dynamic Island area). -- `env(safe-area-inset-bottom)` — bottom safe-area inset (e.g., iPhone home indicator area). -- Browser-side CSS feature; no JavaScript involvement. Falls back to `0` on browsers/devices without safe-area insets. -- Requires `` to take non-zero values. **This is already shipped by 01-01** (`viewportFit: "cover"` in `app/layout.tsx`). - -**Existing `app/styles/brand.css` structure (line numbers from current file):** -- Lines 1-21: file header / brand documentation comment. -- Lines 23-52: `:root` overrides (`--wulf-blue`, etc.) for the light theme. -- Lines 54-62: `.dark` overrides. -- Lines 64-68: `/* === Utility classes === */` section header comment. -- Lines 70-140: existing `@utility` blocks — `num`, `num-lg`, `num-xl`, `metric-label`, `surface-brand`, `surface-brand-ink`, `rule-brand`, `text-chrome`, `border-chrome`, `tagline`. -- Lines 142-147: `/* === Wolf-mark watermark === */` section header comment. -- Lines 149-152: `@utility has-mark-watermark`. -- Lines 154-170: `.mark-watermark` plain rule + `.dark .mark-watermark` override. -- **Insertion point for new utilities:** after the `tagline` utility (line 140) and **before** the watermark section header (line 142). This keeps utilities grouped before the watermark block, which has its own thematic header. - -**`app/globals.css` import wiring (already in place — DO NOT change):** -- Line 125: `@import "./styles/brand.css";` — pulls `brand.css` into the global stylesheet at the end. Anything added to `brand.css` is automatically available app-wide. No additional wiring needed. - -**Spec wording (`docs/superpowers/specs/2026-05-03-mobile-shell-design.md` §5/§6):** -- The mobile shell's sticky top header must respect `env(safe-area-inset-top)`. -- The fixed bottom nav must respect `env(safe-area-inset-bottom)` (often combined with the bottom-nav height). -- The spec accepts either a named utility or Tailwind 4 arbitrary values (`pt-[env(safe-area-inset-top)]`). - -**Why ship a named utility (not arbitrary values):** -- Phase 2 will use these classes in 2+ places (header, bottom nav, drawer footer, possibly modals). A named utility is one source of truth — if the iOS rules ever change (e.g., add `max(env(safe-area-inset-top), 0.5rem)`), it's a one-line edit instead of a multi-file find-and-replace. -- `pt-safe` / `pb-safe` reads more clearly in JSX class lists than `pt-[env(safe-area-inset-top)]`. -- ROADMAP Phase 1 SC #3 explicitly mentions "shared utility class" as one acceptable form — picking that form removes ambiguity for Phase 2. - -**Verification commands the executor will use:** -- `grep -E '@utility pt-safe' app/styles/brand.css` -- `grep -E '@utility pb-safe' app/styles/brand.css` -- `grep -E 'env\(safe-area-inset-top\)' app/styles/brand.css` -- `grep -E 'env\(safe-area-inset-bottom\)' app/styles/brand.css` -- `npm run build` (CSS @utility blocks must parse — broken syntax fails the Tailwind compile step in Next.js) -- `npx tsc --noEmit --pretty` (sanity check; CSS doesn't affect TS but pre-existing baseline must hold) - - - - - - - Task 1: Append pt-safe and pb-safe @utility blocks to app/styles/brand.css - app/styles/brand.css - - - app/styles/brand.css (the entire file — confirm line numbers above match current state; the exact insertion point is between the existing `tagline` utility and the watermark section header) - - app/globals.css lines 1-5 and 125 (confirm `brand.css` is still imported; no change needed) - - .planning/phases/01-pwa-scaffolding/01-VERIFICATION.md (the gap source — frontmatter `gaps[0].missing`) - - .planning/REQUIREMENTS.md line 16 (PWA-04 wording) - - -Edit `app/styles/brand.css`. Append two new `@utility` blocks **after** the existing `@utility tagline { ... }` block (which ends around line 140) and **before** the `/* === Wolf-mark watermark === */` section header comment (around line 142). Do NOT touch any other part of the file. - -Insert exactly this block (including the leading section comment and the two `@utility` definitions): - -```css -/* === Safe-area insets ================================================= - * - * Opt-in padding helpers for sticky top / fixed bottom bars on devices - * with notches, dynamic islands, or gesture home indicators. Pair with - * the viewport-fit=cover viewport meta (set in app/layout.tsx) — without - * that, env(safe-area-inset-*) resolves to 0 and these utilities are - * no-ops, which is the desired fallback on non-PWA / non-mobile contexts. - * - * Usage: - *
// header clears notch - *