diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 12b3b53..e4b9bbb 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -18,6 +18,7 @@ import { buildContractsFilter, buildProjectsFilter, buildTimeEntriesFilter, + buildBillingItemsFilter, getTableName } from '../utils/sync-helpers'; import { createSyncLogger, SyncPhase, categorizeError } from '../utils/sync-logger'; @@ -112,6 +113,9 @@ export class EntitySyncService { } else if (entity === EntityType.TIME_ENTRIES) { filters.push(...buildTimeEntriesFilter(yearsBack)); entityLogger.info(`Full sync with dateWorked filter for last ${yearsBack} years`); + } else if (entity === EntityType.BILLING_ITEMS) { + filters.push(...buildBillingItemsFilter(yearsBack)); + entityLogger.info(`Full sync with itemDate filter for last ${yearsBack} years`); } else { // Add active filter if applicable const activeFilter = buildActiveFilter(entity); @@ -270,6 +274,32 @@ export class EntitySyncService { } } + // Validate project foreign keys for tasks + if (entity === EntityType.TASKS) { + const initialCount = mappedRecords.length; + + // Get all valid project IDs from database + const validProjectIds = await this.getValidProjectIds(); + + // Filter tasks with invalid project references + mappedRecords = mappedRecords.map(task => { + // Set invalid project IDs to null instead of filtering out the entire task + if (task.project_id && !validProjectIds.has(task.project_id)) { + entityLogger.debug(`Invalid project_id, setting to null`, { + taskId: task.id, + invalidProjectId: task.project_id, + }); + task.project_id = null; + } + return task; + }); + + const nullifiedCount = initialCount - mappedRecords.filter(t => t.project_id).length; + if (nullifiedCount > 0) { + entityLogger.warn('Nullified invalid project references', { nullifiedCount }); + } + } + entityLogger.info('Successfully mapped records', { mappedCount: mappedRecords.length }); // Bulk upsert to PostgreSQL @@ -547,6 +577,24 @@ export class EntitySyncService { } } + /** + * Get all valid project IDs from the database + * Used to validate foreign key references before insert + * @returns Set of valid project IDs + */ + private async getValidProjectIds(): Promise> { + try { + const query = 'SELECT id FROM projects WHERE is_deleted = false'; + const result = await postgresClient.query<{ id: number }>(query); + return new Set(result.rows.map(row => row.id)); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + this.logger.error('Failed to fetch valid project IDs', {}, err); + // Return empty set on error - will cause all project IDs to be nullified + return new Set(); + } + } + /** * Calculate monthly date chunks for a given time period * @param yearsBack Number of years to look back diff --git a/lib/utils/sync-helpers.ts b/lib/utils/sync-helpers.ts index 423abf0..fc80001 100644 --- a/lib/utils/sync-helpers.ts +++ b/lib/utils/sync-helpers.ts @@ -332,6 +332,30 @@ export function buildTimeEntriesFilter(yearsBack: number = 2): Array<{ field: st ]; } +/** + * Build special filter for billing items (requires filter) + * @param yearsBack Number of years to look back (default: 2) + * @returns Query filter array for billing items + */ +export function buildBillingItemsFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> { + // BillingItems API requires a filter. Use itemDate to limit the range + // Calculate date from X years ago + const cutoffDate = new Date(); + const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000; + const millisecondsBack = yearsBack * millisecondsPerYear; + cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack); + + console.log(`BillingItems filter: itemDate >= ${cutoffDate.toISOString()} (${yearsBack} years back)`); + + return [ + { + field: 'itemDate', + op: 'gte', + value: cutoffDate.toISOString(), + }, + ]; +} + /** * Calculate estimated sync duration based on record count * @param recordCount Number of records to sync