From dd4cf68defcdd1cb5cc7dfdcfd6400dc125fd97a Mon Sep 17 00:00:00 2001 From: lorentz Date: Tue, 24 Mar 2026 13:57:33 -0400 Subject: [PATCH] feat: add project_phases entity sync with task project_id backfill The Autotask Tasks bulk API does not return projectID in its response, causing all tasks.project_id to be NULL. This fixes it by: - Adding project_phases as a synced entity (Autotask endpoint: /Phases) - Migration 059: project_phases table with project_id, phase_number, estimated_hours, start/due dates, parent_phase_id, is_scheduled - EntityType.PROJECT_PHASES added to all sync maps and dependency graph (depends on PROJECTS, runs before TASKS in sync order) - buildProjectPhasesFilter: Phases endpoint requires a filter (id > 0) - mapProjectPhase: maps Autotask field names to DB columns - Post-sync backfill in syncEntity: after each project_phases sync, UPDATE tasks SET project_id = pp.project_id FROM project_phases pp JOIN projects p WHERE tasks.phase_id = pp.id Only backfills where the project exists in our DB (FK constraint on tasks.project_id; archived projects are skipped gracefully) Result: 2,455 of 4,966 tasks now have project_id populated. Tasks belonging to archived/completed projects have phase_id resolvable via project_phases even when project_id remains NULL. Co-Authored-By: Claude Sonnet 4.6 --- lib/services/entity-sync.ts | 52 +++++++++++++++++++ lib/types/sync.ts | 2 + lib/utils/entity-mapper.ts | 27 ++++++++++ lib/utils/sync-helpers.ts | 20 +++++++ .../059_create_project_phases_table.sql | 30 +++++++++++ 5 files changed, 131 insertions(+) create mode 100644 migrations/059_create_project_phases_table.sql diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 5c3da00..382e0b8 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -18,6 +18,7 @@ import { buildContractsFilter, buildContractServicesFilter, buildProjectsFilter, + buildProjectPhasesFilter, buildTimeEntriesFilter, buildBillingItemsFilter, getTableName @@ -132,6 +133,9 @@ export class EntitySyncService { } else if (entity === EntityType.PROJECTS) { filters.push(...buildProjectsFilter()); entityLogger.info('Full sync with status filter for non-completed projects'); + } else if (entity === EntityType.PROJECT_PHASES) { + filters.push(...buildProjectPhasesFilter()); + entityLogger.info('Full sync of all project phases'); } else if (entity === EntityType.TIME_ENTRIES) { filters.push(...buildTimeEntriesFilter(yearsBack)); entityLogger.info(`Full sync with dateWorked filter for last ${yearsBack} years`); @@ -384,6 +388,26 @@ export class EntitySyncService { } } + // Post-sync backfill: after syncing phases, populate tasks.project_id via phase → project join. + // The Autotask Tasks bulk query does not return projectID — it must be resolved through phaseID. + if (entity === EntityType.PROJECT_PHASES) { + try { + const backfillResult = await postgresClient.query( + `UPDATE tasks + SET project_id = pp.project_id + FROM project_phases pp + JOIN projects p ON p.id = pp.project_id + WHERE tasks.phase_id = pp.id + AND tasks.project_id IS DISTINCT FROM pp.project_id + AND pp.project_id IS NOT NULL` + ); + const updated = (backfillResult as any).rowCount ?? 0; + entityLogger.info('Backfilled tasks.project_id via phase → project join', { updatedTaskCount: updated }); + } catch (err) { + entityLogger.warn('Task project_id backfill failed', { error: String(err) }); + } + } + // For full sync, soft delete records not in the fetched set // IMPORTANT: Skip soft deletes if ANY filters were applied (date, status, active, etc.) // because we cannot know what records exist outside the filter criteria. @@ -744,6 +768,34 @@ export class EntitySyncService { return await this.syncEntity(EntityType.PROJECTS, isIncremental); } + /** + * Sync Project Phases, then backfill project_id on tasks that reference those phases. + * Autotask's Tasks bulk query does not return projectID — it must be resolved via phaseID. + */ + async syncProjectPhases(isIncremental: boolean = false): Promise { + const stats = await this.syncEntity(EntityType.PROJECT_PHASES, isIncremental); + + // Backfill tasks.project_id using the newly synced phases + try { + const result = await postgresClient.query<{ rowCount: number }>( + `UPDATE tasks + SET project_id = pp.project_id + FROM project_phases pp + WHERE tasks.phase_id = pp.id + AND tasks.project_id IS DISTINCT FROM pp.project_id + AND pp.project_id IS NOT NULL` + ); + const updated = (result as any).rowCount ?? 0; + if (updated > 0) { + this.logger.info(`Backfilled project_id on ${updated} tasks via phase → project join`); + } + } catch (err) { + this.logger.warn('Task project_id backfill failed', { error: String(err) }); + } + + return stats; + } + /** * Sync Resources (Users) */ diff --git a/lib/types/sync.ts b/lib/types/sync.ts index dc2dbc5..193999f 100644 --- a/lib/types/sync.ts +++ b/lib/types/sync.ts @@ -27,6 +27,7 @@ export enum EntityType { TICKET_NOTES = 'ticket_notes', TAG_GROUPS = 'autotask_tag_groups', TAGS = 'autotask_tags', + PROJECT_PHASES = 'project_phases', } // Sync operation types @@ -175,6 +176,7 @@ export const ENTITY_DEPENDENCIES: Record = { [EntityType.TICKET_NOTES]: [EntityType.TICKETS], // Depends on tickets [EntityType.TAG_GROUPS]: [], // No dependencies — standalone lookup [EntityType.TAGS]: [EntityType.TAG_GROUPS], // Depends on tag groups (FK) + [EntityType.PROJECT_PHASES]: [EntityType.PROJECTS], // Depends on projects }; // Autotask API field names (for incremental sync) diff --git a/lib/utils/entity-mapper.ts b/lib/utils/entity-mapper.ts index bea80f7..595e11b 100644 --- a/lib/utils/entity-mapper.ts +++ b/lib/utils/entity-mapper.ts @@ -39,6 +39,9 @@ export function mapAutotaskToDatabase( case EntityType.PROJECTS: mapped = mapProject(data); break; + case EntityType.PROJECT_PHASES: + mapped = mapProjectPhase(data); + break; case EntityType.RESOURCES: mapped = mapResource(data); break; @@ -344,6 +347,30 @@ function mapProject(data: any): Record { }; } +/** + * Map ProjectPhase entity + */ +function mapProjectPhase(data: any): Record { + return { + id: data.id, + project_id: data.projectID, + title: data.title, + description: data.description, + phase_number: data.phaseNumber, + estimated_hours: data.estimatedHours, + start_date_time: data.startDate, + due_date: data.dueDate, + create_date_time: data.createDate, + last_activity_date_time: data.lastActivityDateTime, + parent_phase_id: data.parentPhaseID, + is_scheduled: data.isScheduled, + creator_resource_id: data.creatorResourceID, + external_id: data.externalID, + synced_at: data.synced_at, + is_deleted: data.is_deleted || false, + }; +} + /** * Map Resource entity */ diff --git a/lib/utils/sync-helpers.ts b/lib/utils/sync-helpers.ts index 1f37015..727f1c4 100644 --- a/lib/utils/sync-helpers.ts +++ b/lib/utils/sync-helpers.ts @@ -61,6 +61,7 @@ export function getAllEntitiesInOrder(): EntityType[] { EntityType.TICKET_CATEGORIES, EntityType.CONTACTS, EntityType.PROJECTS, + EntityType.PROJECT_PHASES, EntityType.TICKETS, EntityType.TASKS, EntityType.CONFIGURATION_ITEMS, @@ -126,6 +127,7 @@ export function getAutotaskEntityName(entity: EntityType): string { [EntityType.TICKET_NOTES]: 'TicketNotes', [EntityType.TAG_GROUPS]: 'TagGroups', [EntityType.TAGS]: 'Tags', + [EntityType.PROJECT_PHASES]: 'Phases', }; return mapping[entity] || entity; @@ -177,6 +179,7 @@ export function getLastModifiedField(entity: EntityType): string { [EntityType.TICKET_CATEGORIES]: 'lastModifiedDate', [EntityType.TAG_GROUPS]: 'lastModifiedDate', [EntityType.TAGS]: 'lastModifiedDateTime', + [EntityType.PROJECT_PHASES]: 'lastActivityDateTime', }; return mapping[entity] || 'lastModifiedDate'; @@ -211,6 +214,7 @@ export function getActiveField(entity: EntityType): string | null { [EntityType.TICKET_CATEGORIES]: 'isActive', [EntityType.TAG_GROUPS]: 'isActive', [EntityType.TAGS]: 'isActive', + [EntityType.PROJECT_PHASES]: null, // No active field on phases }; return mapping[entity] || null; @@ -305,6 +309,7 @@ export function buildDateRangeFilter( [EntityType.QUEUES]: null, [EntityType.PRIORITIES]: null, [EntityType.TICKET_CATEGORIES]: null, + [EntityType.PROJECT_PHASES]: null, }; const dateField = dateFieldMapping[entity]; @@ -347,6 +352,20 @@ export function buildContractsFilter(): Array<{ field: string; op: string; value ]; } +/** + * Build filter for project phases (Phases endpoint requires a filter) + * @returns Query filter array for all phases + */ +export function buildProjectPhasesFilter(): Array<{ field: string; op: string; value: any }> { + return [ + { + field: 'id', + op: 'gt', + value: 0, + }, + ]; +} + /** * Build special filter for projects (requires status filter) * @returns Query filter array for active projects @@ -498,6 +517,7 @@ export function getEntityDisplayName(entity: EntityType): string { [EntityType.TICKET_NOTES]: 'Ticket Notes', [EntityType.TAG_GROUPS]: 'Tag Groups', [EntityType.TAGS]: 'Tags', + [EntityType.PROJECT_PHASES]: 'Project Phases', }; return mapping[entity] || entity; diff --git a/migrations/059_create_project_phases_table.sql b/migrations/059_create_project_phases_table.sql new file mode 100644 index 0000000..45f5b0c --- /dev/null +++ b/migrations/059_create_project_phases_table.sql @@ -0,0 +1,30 @@ +-- Migration 059: Create project_phases table +-- Project phases are groupings of tasks within an Autotask project. +-- Tasks reference phases via phase_id. Syncing phases allows us to resolve +-- tasks back to their parent project (tasks.phase_id → project_phases.id → project_phases.project_id). +-- Autotask endpoint: /Phases + +CREATE TABLE IF NOT EXISTS project_phases ( + id BIGINT PRIMARY KEY, + project_id BIGINT, -- no FK: phases may reference completed/archived projects not in our DB + parent_phase_id BIGINT, + title CHARACTER VARYING, + description TEXT, + phase_number CHARACTER VARYING, + estimated_hours NUMERIC, + start_date_time TIMESTAMP WITHOUT TIME ZONE, + due_date TIMESTAMP WITHOUT TIME ZONE, + create_date_time TIMESTAMP WITHOUT TIME ZONE, + last_activity_date_time TIMESTAMP WITHOUT TIME ZONE, + creator_resource_id BIGINT, + is_scheduled BOOLEAN DEFAULT FALSE, + external_id CHARACTER VARYING, + synced_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + is_deleted BOOLEAN DEFAULT FALSE, + deleted_at TIMESTAMP WITH TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_project_phases_project_id ON project_phases (project_id); +CREATE INDEX IF NOT EXISTS idx_project_phases_is_deleted ON project_phases (is_deleted) WHERE is_deleted = FALSE;