From 0e8eb4871e5b855e570851669a79ddd20509a313 Mon Sep 17 00:00:00 2001 From: lorentz Date: Sun, 5 Apr 2026 08:55:41 -0400 Subject: [PATCH] fix: prevent sync from nullifying assigned_resource_id on tickets - bulkUpsert now accepts preserveExistingOnNull column list, using COALESCE(EXCLUDED.col, table.col) so null incoming values never overwrite existing non-null DB values - bulkUpsertRecords passes resource ID columns as preserve-on-null for TICKETS and TASKS entities - getValidResourceIds now throws on DB error instead of returning empty set (which would nullify every resource reference) - Fix mimecast mailbox-remediate fetch handlers to check res.ok and content-type before calling res.json(), preventing JSON parse crash on 502 Bad Gateway responses --- app/admin/sync/mimecast/page.tsx | 14 ++++++++++---- lib/services/entity-sync.ts | 32 +++++++++++++------------------- lib/services/postgres-client.ts | 11 +++++++++-- lib/utils/db-helpers.ts | 9 ++++++++- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/app/admin/sync/mimecast/page.tsx b/app/admin/sync/mimecast/page.tsx index 51e6a89..f2c67eb 100644 --- a/app/admin/sync/mimecast/page.tsx +++ b/app/admin/sync/mimecast/page.tsx @@ -1144,11 +1144,13 @@ function DeliveredAnalysisDialog({ message, onClose, onFindSimilar, allMessages headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action: 'search', userEmail: message.to, fromAddress: message.from }), }); - const d = await res.json(); if (!res.ok) { - if (d.permissionRequired) { setPermError(d.detail); setRemedStep('idle'); return; } - throw new Error(d.error ?? `HTTP ${res.status}`); + const isJson = res.headers.get('content-type')?.includes('application/json'); + const d = isJson ? await res.json() : null; + if (d?.permissionRequired) { setPermError(d.detail); setRemedStep('idle'); return; } + throw new Error(d?.error ?? `HTTP ${res.status}`); } + const d = await res.json(); const matches = d.messages ?? []; setRemedMatches(matches); setRemedSelected(new Set(matches.map((m: any) => m.id))); @@ -1168,8 +1170,12 @@ function DeliveredAnalysisDialog({ message, onClose, onFindSimilar, allMessages headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action: 'move', userEmail: message.to, messageIds: [...remedSelected] }), }); + if (!res.ok) { + const isJson = res.headers.get('content-type')?.includes('application/json'); + const d = isJson ? await res.json() : null; + throw new Error(d?.error ?? `HTTP ${res.status}`); + } const d = await res.json(); - if (!res.ok) throw new Error(d.error ?? `HTTP ${res.status}`); setRemedResults({ succeeded: d.succeeded, failed: d.failed }); setRemedStep('done'); } catch (e: any) { diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 382e0b8..180a888 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -253,14 +253,16 @@ export class EntitySyncService { const validResourceIds = await this.getValidResourceIds(); // Filter tickets with invalid resource references + let skippedResourceCount = 0; mappedRecords = mappedRecords.map(ticket => { - // Set invalid resource IDs to null instead of filtering out the entire ticket + // Set to null — bulkUpsert uses COALESCE for these columns so existing DB value is preserved if (ticket.assigned_resource_id && !validResourceIds.has(ticket.assigned_resource_id)) { - entityLogger.debug(`Invalid assigned_resource_id, setting to null`, { + entityLogger.debug(`Unknown assigned_resource_id, will preserve existing DB value via COALESCE`, { ticketId: ticket.id, - invalidResourceId: ticket.assigned_resource_id, + unknownResourceId: ticket.assigned_resource_id, }); ticket.assigned_resource_id = null; + skippedResourceCount++; } if (ticket.first_response_assigned_resource_id && !validResourceIds.has(ticket.first_response_assigned_resource_id)) { ticket.first_response_assigned_resource_id = null; @@ -271,9 +273,8 @@ export class EntitySyncService { return ticket; }); - const nullifiedCount = initialCount - mappedRecords.filter(t => t.assigned_resource_id).length; - if (nullifiedCount > 0) { - entityLogger.warn('Nullified invalid resource references', { nullifiedCount }); + if (skippedResourceCount > 0) { + entityLogger.warn('Skipped unknown resource references (existing DB values preserved via COALESCE)', { skippedResourceCount }); } } @@ -595,7 +596,7 @@ export class EntitySyncService { chunkLogger.info('Cached valid resource IDs', { cacheSize: this.cachedValidResourceIds.size }); } - // Nullify invalid resource references + // Null out unknown resource references — bulkUpsert uses COALESCE so existing DB value is preserved mappedRecords = mappedRecords.map(ticket => { if (ticket.assigned_resource_id && !this.cachedValidResourceIds!.has(ticket.assigned_resource_id)) { ticket.assigned_resource_id = null; @@ -670,16 +671,9 @@ export class EntitySyncService { * @returns Set of valid resource IDs */ private async getValidResourceIds(): Promise> { - try { - const query = 'SELECT id FROM resources WHERE is_deleted = false'; - const result = await postgresClient.query<{ id: number }>(query); - return new Set(result.rows.map(row => row.id)); - } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); - this.logger.error('Failed to fetch valid resource IDs', {}, err); - // Return empty set on error - will cause all resource IDs to be nullified - return new Set(); - } + const query = 'SELECT id FROM resources WHERE is_deleted = false'; + const result = await postgresClient.query<{ id: number }>(query); + return new Set(result.rows.map(row => Number(row.id))); } /** @@ -690,7 +684,7 @@ export class EntitySyncService { try { const query = 'SELECT id FROM contacts WHERE is_deleted = false'; const result = await postgresClient.query<{ id: number }>(query); - return new Set(result.rows.map(row => row.id)); + return new Set(result.rows.map(row => Number(row.id))); } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); this.logger.error('Failed to fetch valid contact IDs', {}, err); @@ -708,7 +702,7 @@ export class EntitySyncService { try { const query = 'SELECT id FROM projects WHERE is_deleted = false'; const result = await postgresClient.query<{ id: number }>(query); - return new Set(result.rows.map(row => row.id)); + return new Set(result.rows.map(row => Number(row.id))); } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); this.logger.error('Failed to fetch valid project IDs', {}, err); diff --git a/lib/services/postgres-client.ts b/lib/services/postgres-client.ts index 7e69eb0..d7843d6 100644 --- a/lib/services/postgres-client.ts +++ b/lib/services/postgres-client.ts @@ -217,11 +217,14 @@ class PostgresClient { /** * Bulk upsert records + * For nullable FK columns (assigned_resource_id etc.), uses COALESCE so that + * a null incoming value does not overwrite an existing non-null DB value. */ async bulkUpsert( table: string, records: Record[], - conflictColumns: string[] = ['id'] + conflictColumns: string[] = ['id'], + preserveExistingOnNull: string[] = [] ): Promise { if (records.length === 0) return 0; @@ -240,7 +243,11 @@ class PostgresClient { // Build UPDATE clause for conflict resolution const updateKeys = keys.filter(k => !conflictColumns.includes(k)); const updateClause = updateKeys - .map(key => `${key} = EXCLUDED.${key}`) + .map(key => + preserveExistingOnNull.includes(key) + ? `${key} = COALESCE(EXCLUDED.${key}, ${table}.${key})` + : `${key} = EXCLUDED.${key}` + ) .join(', '); const query = ` diff --git a/lib/utils/db-helpers.ts b/lib/utils/db-helpers.ts index cd2b55f..bbee3e7 100644 --- a/lib/utils/db-helpers.ts +++ b/lib/utils/db-helpers.ts @@ -28,6 +28,12 @@ export async function upsertRecord( * @param batchSize Number of records per batch (default: 100) * @returns Total number of records upserted */ +// Columns where a null incoming value should preserve the existing DB value rather than overwrite it +const PRESERVE_EXISTING_ON_NULL: Partial> = { + [EntityType.TICKETS]: ['assigned_resource_id', 'first_response_assigned_resource_id', 'first_response_initiating_resource_id'], + [EntityType.TASKS]: ['assigned_resource_id'], +}; + export async function bulkUpsertRecords( entity: EntityType, records: Record[], @@ -36,12 +42,13 @@ export async function bulkUpsertRecords( if (records.length === 0) return 0; const tableName = getTableName(entity); + const preserveOnNull = PRESERVE_EXISTING_ON_NULL[entity] ?? []; let totalUpserted = 0; // Process in batches for (let i = 0; i < records.length; i += batchSize) { const batch = records.slice(i, i + batchSize); - const count = await postgresClient.bulkUpsert(tableName, batch); + const count = await postgresClient.bulkUpsert(tableName, batch, ['id'], preserveOnNull); totalUpserted += count; }