lorentz
7ce06803dc
Previously the incremental filter compared dateWorked >= lastSyncTime, meaning edits to existing entries (hours adjusted, notes changed, PTO blocks entered in advance) were never picked up after initial ingestion. Switch both the lastModifiedField mapping and buildTimeEntriesFilter to use lastModifiedDateTime so any created or modified entry is captured on the next scheduled sync.
492 lines
15 KiB
TypeScript
492 lines
15 KiB
TypeScript
/**
|
|
* Sync Helper Functions
|
|
* Utility functions for sync operations including dependency ordering
|
|
*/
|
|
|
|
import { EntityType, ENTITY_DEPENDENCIES } from '../types/sync';
|
|
|
|
/**
|
|
* Get entities in dependency order (parents before children)
|
|
* @param entities List of entities to sync
|
|
* @returns Ordered list of entities respecting dependencies
|
|
*/
|
|
export function getEntitySyncOrder(entities: EntityType[]): EntityType[] {
|
|
const ordered: EntityType[] = [];
|
|
const visited = new Set<EntityType>();
|
|
const visiting = new Set<EntityType>();
|
|
|
|
function visit(entity: EntityType) {
|
|
if (visited.has(entity)) return;
|
|
if (visiting.has(entity)) {
|
|
throw new Error(`Circular dependency detected for entity: ${entity}`);
|
|
}
|
|
|
|
visiting.add(entity);
|
|
|
|
// Visit dependencies first
|
|
const dependencies = ENTITY_DEPENDENCIES[entity] || [];
|
|
for (const dep of dependencies) {
|
|
if (entities.includes(dep)) {
|
|
visit(dep);
|
|
}
|
|
}
|
|
|
|
visiting.delete(entity);
|
|
visited.add(entity);
|
|
ordered.push(entity);
|
|
}
|
|
|
|
// Visit all entities
|
|
for (const entity of entities) {
|
|
visit(entity);
|
|
}
|
|
|
|
return ordered;
|
|
}
|
|
|
|
/**
|
|
* Get all entities in default sync order
|
|
* @returns All entities in dependency order
|
|
*/
|
|
export function getAllEntitiesInOrder(): EntityType[] {
|
|
return getEntitySyncOrder([
|
|
EntityType.COMPANIES,
|
|
EntityType.RESOURCES,
|
|
EntityType.STATUSES,
|
|
EntityType.ISSUE_TYPES,
|
|
EntityType.SUB_ISSUE_TYPES,
|
|
EntityType.WORK_TYPES,
|
|
EntityType.QUEUES,
|
|
EntityType.PRIORITIES,
|
|
EntityType.TICKET_CATEGORIES,
|
|
EntityType.CONTACTS,
|
|
EntityType.PROJECTS,
|
|
EntityType.TICKETS,
|
|
EntityType.TASKS,
|
|
EntityType.CONFIGURATION_ITEMS,
|
|
EntityType.CONTRACTS,
|
|
EntityType.CONTRACT_SERVICES,
|
|
EntityType.AUTOTASK_SERVICES,
|
|
EntityType.BILLING_ITEMS,
|
|
EntityType.TIME_ENTRIES,
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* Build filter for contract services (requires contractID filter — fetch all via contractIDs)
|
|
* @returns Query filter array for active contract services
|
|
*/
|
|
export function buildContractServicesFilter(): Array<{ field: string; op: string; value: any }> {
|
|
return [
|
|
{
|
|
field: 'contractID',
|
|
op: 'gt',
|
|
value: 0,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Get table name for entity type
|
|
* @param entity Entity type
|
|
* @returns PostgreSQL table name
|
|
*/
|
|
export function getTableName(entity: EntityType): string {
|
|
return entity;
|
|
}
|
|
|
|
/**
|
|
* Get Autotask API entity name
|
|
* @param entity Entity type
|
|
* @returns Autotask API entity name (PascalCase)
|
|
*/
|
|
export function getAutotaskEntityName(entity: EntityType): string {
|
|
const mapping: Record<EntityType, string> = {
|
|
[EntityType.COMPANIES]: 'Companies',
|
|
[EntityType.TICKETS]: 'Tickets',
|
|
[EntityType.TASKS]: 'Tasks',
|
|
[EntityType.PROJECTS]: 'Projects',
|
|
[EntityType.RESOURCES]: 'Resources',
|
|
[EntityType.STATUSES]: 'Statuses',
|
|
[EntityType.ISSUE_TYPES]: 'IssueTypes',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'SubIssueTypes',
|
|
[EntityType.WORK_TYPES]: 'WorkTypes',
|
|
[EntityType.QUEUES]: 'Queues',
|
|
[EntityType.PRIORITIES]: 'Priorities',
|
|
[EntityType.TICKET_CATEGORIES]: 'TicketCategories',
|
|
[EntityType.BILLING_ITEMS]: 'BillingItems',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'ConfigurationItems',
|
|
[EntityType.CONTACTS]: 'Contacts',
|
|
[EntityType.CONTRACTS]: 'Contracts',
|
|
[EntityType.CONTRACT_SERVICES]: 'ContractServices',
|
|
[EntityType.AUTOTASK_SERVICES]: 'Services',
|
|
[EntityType.TIME_ENTRIES]: 'TimeEntries',
|
|
[EntityType.TICKET_NOTES]: 'TicketNotes',
|
|
};
|
|
|
|
return mapping[entity] || entity;
|
|
}
|
|
|
|
/**
|
|
* Check if entity is a picklist type
|
|
* @param entity Entity type
|
|
* @returns True if entity is a picklist
|
|
*/
|
|
export function isPicklistEntity(entity: EntityType): boolean {
|
|
return [
|
|
EntityType.STATUSES,
|
|
EntityType.ISSUE_TYPES,
|
|
EntityType.SUB_ISSUE_TYPES,
|
|
EntityType.WORK_TYPES,
|
|
EntityType.QUEUES,
|
|
EntityType.PRIORITIES,
|
|
EntityType.TICKET_CATEGORIES,
|
|
].includes(entity);
|
|
}
|
|
|
|
/**
|
|
* Get field name for last modified date in Autotask
|
|
* @param entity Entity type
|
|
* @returns Field name for filtering by last modified date
|
|
*/
|
|
export function getLastModifiedField(entity: EntityType): string {
|
|
const mapping: Record<EntityType, string> = {
|
|
[EntityType.COMPANIES]: 'lastActivityDate',
|
|
[EntityType.TICKETS]: 'lastActivityDate',
|
|
[EntityType.TASKS]: 'lastActivityDateTime',
|
|
[EntityType.PROJECTS]: 'lastActivityDateTime',
|
|
[EntityType.RESOURCES]: 'lastModifiedDate',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'lastModifiedTime',
|
|
[EntityType.CONTACTS]: 'lastModifiedDate',
|
|
[EntityType.CONTRACTS]: 'lastModifiedDateTime',
|
|
[EntityType.CONTRACT_SERVICES]: 'lastModifiedDate',
|
|
[EntityType.AUTOTASK_SERVICES]: 'lastModifiedDate',
|
|
[EntityType.BILLING_ITEMS]: 'itemDate',
|
|
[EntityType.TIME_ENTRIES]: 'lastModifiedDateTime',
|
|
[EntityType.TICKET_NOTES]: 'lastActivityDate',
|
|
[EntityType.STATUSES]: 'lastModifiedDate',
|
|
[EntityType.ISSUE_TYPES]: 'lastModifiedDate',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'lastModifiedDate',
|
|
[EntityType.WORK_TYPES]: 'lastModifiedDate',
|
|
[EntityType.QUEUES]: 'lastModifiedDate',
|
|
[EntityType.PRIORITIES]: 'lastModifiedDate',
|
|
[EntityType.TICKET_CATEGORIES]: 'lastModifiedDate',
|
|
};
|
|
|
|
return mapping[entity] || 'lastModifiedDate';
|
|
}
|
|
|
|
/**
|
|
* Get active status field name for entity
|
|
* @param entity Entity type
|
|
* @returns Field name for active status
|
|
*/
|
|
export function getActiveField(entity: EntityType): string | null {
|
|
const mapping: Record<EntityType, string | null> = {
|
|
[EntityType.COMPANIES]: 'isActive',
|
|
[EntityType.TICKETS]: null, // Use status field instead
|
|
[EntityType.TASKS]: null, // Use status field instead
|
|
[EntityType.PROJECTS]: null, // Use status field instead
|
|
[EntityType.RESOURCES]: 'isActive',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'isActive',
|
|
[EntityType.CONTACTS]: 'isActive',
|
|
[EntityType.CONTRACTS]: null, // Use status field instead
|
|
[EntityType.CONTRACT_SERVICES]: null,
|
|
[EntityType.AUTOTASK_SERVICES]: 'isActive',
|
|
[EntityType.BILLING_ITEMS]: null,
|
|
[EntityType.TIME_ENTRIES]: null, // Time entries don't have active status
|
|
[EntityType.TICKET_NOTES]: null, // Ticket notes don't have active status
|
|
[EntityType.STATUSES]: 'isActive',
|
|
[EntityType.ISSUE_TYPES]: 'isActive',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'isActive',
|
|
[EntityType.WORK_TYPES]: 'isActive',
|
|
[EntityType.QUEUES]: 'isActive',
|
|
[EntityType.PRIORITIES]: 'isActive',
|
|
[EntityType.TICKET_CATEGORIES]: 'isActive',
|
|
};
|
|
|
|
return mapping[entity] || null;
|
|
}
|
|
|
|
/**
|
|
* Build Autotask query filter for incremental sync
|
|
* @param entity Entity type
|
|
* @param lastSyncTime Last successful sync timestamp
|
|
* @returns Query filter array
|
|
*/
|
|
export function buildIncrementalFilter(
|
|
entity: EntityType,
|
|
lastSyncTime: Date
|
|
): Array<{ field: string; op: string; value: any }> {
|
|
const lastModifiedField = getLastModifiedField(entity);
|
|
|
|
return [
|
|
{
|
|
field: lastModifiedField,
|
|
op: 'gte',
|
|
value: lastSyncTime.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build Autotask query filter for active records only
|
|
* @param entity Entity type
|
|
* @returns Query filter array or null if no active field
|
|
*/
|
|
export function buildActiveFilter(
|
|
entity: EntityType
|
|
): Array<{ field: string; op: string; value: any }> | null {
|
|
const activeField = getActiveField(entity);
|
|
|
|
if (!activeField) {
|
|
return null;
|
|
}
|
|
|
|
return [
|
|
{
|
|
field: activeField,
|
|
op: 'eq',
|
|
value: true,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build date range filter for entities to limit sync to recent records
|
|
* @param entity Entity type
|
|
* @param yearsBack Number of years to look back (default: 2)
|
|
* @returns Query filter array or null if entity doesn't support date filtering
|
|
*/
|
|
export function buildDateRangeFilter(
|
|
entity: EntityType,
|
|
yearsBack: number = 2
|
|
): Array<{ field: string; op: string; value: any }> | null {
|
|
// Only apply date range filters to time-based entities
|
|
// Note: TIME_ENTRIES removed because Autotask API doesn't support date filtering on TimeEntry
|
|
const timeBasedEntities = [
|
|
EntityType.TICKETS,
|
|
EntityType.TASKS,
|
|
];
|
|
|
|
if (!timeBasedEntities.includes(entity)) {
|
|
return null;
|
|
}
|
|
|
|
// Define which entities should have date range filters and which field to use
|
|
const dateFieldMapping: Record<EntityType, string | null> = {
|
|
[EntityType.TICKETS]: 'createDate',
|
|
[EntityType.TASKS]: 'createDateTime',
|
|
[EntityType.TIME_ENTRIES]: 'createDate', // TimeEntry uses createDate for filtering
|
|
[EntityType.PROJECTS]: 'startDateTime',
|
|
[EntityType.BILLING_ITEMS]: 'itemDate',
|
|
[EntityType.CONTRACTS]: 'startDate',
|
|
[EntityType.CONTRACT_SERVICES]: null,
|
|
[EntityType.AUTOTASK_SERVICES]: null,
|
|
[EntityType.COMPANIES]: null,
|
|
[EntityType.RESOURCES]: null,
|
|
[EntityType.CONTACTS]: null,
|
|
[EntityType.CONFIGURATION_ITEMS]: null,
|
|
[EntityType.TICKET_NOTES]: null,
|
|
[EntityType.STATUSES]: null,
|
|
[EntityType.ISSUE_TYPES]: null,
|
|
[EntityType.SUB_ISSUE_TYPES]: null,
|
|
[EntityType.WORK_TYPES]: null,
|
|
[EntityType.QUEUES]: null,
|
|
[EntityType.PRIORITIES]: null,
|
|
[EntityType.TICKET_CATEGORIES]: null,
|
|
};
|
|
|
|
const dateField = dateFieldMapping[entity];
|
|
|
|
if (!dateField) {
|
|
return null;
|
|
}
|
|
|
|
// Calculate date from X years ago
|
|
// Convert years to milliseconds for accurate calculation (including fractional years)
|
|
const cutoffDate = new Date();
|
|
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000; // Account for leap years
|
|
const millisecondsBack = yearsBack * millisecondsPerYear;
|
|
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
|
|
|
console.log(`Date range filter: ${dateField} >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
|
|
|
return [
|
|
{
|
|
field: dateField,
|
|
op: 'gte',
|
|
value: cutoffDate.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for contracts (requires status filter)
|
|
* @returns Query filter array for active contracts
|
|
*/
|
|
export function buildContractsFilter(): Array<{ field: string; op: string; value: any }> {
|
|
// Contracts API requires a filter. Use status = 1 for Active contracts
|
|
// Status values: 1 = Active, others are inactive/expired
|
|
return [
|
|
{
|
|
field: 'status',
|
|
op: 'eq',
|
|
value: 1,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for projects (requires status filter)
|
|
* @returns Query filter array for active projects
|
|
*/
|
|
export function buildProjectsFilter(): Array<{ field: string; op: string; value: any }> {
|
|
// Projects API requires a filter. Use status = 1 for New/Active projects
|
|
// Status values: 1 = New, others include Complete, Cancelled, etc.
|
|
// To get all active projects, we should filter for status NOT equal to Complete (5)
|
|
return [
|
|
{
|
|
field: 'status',
|
|
op: 'noteq',
|
|
value: 5, // 5 = Complete
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for time entries (requires filter)
|
|
* @param yearsBack Number of years to look back (default: 2)
|
|
* @returns Query filter array for time entries
|
|
*/
|
|
export function buildTimeEntriesFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> {
|
|
// TimeEntries API requires a filter. Use lastModifiedDateTime so that edits
|
|
// to existing entries (e.g. hours changed, notes updated) are picked up even
|
|
// if the dateWorked is in the past or future.
|
|
// We still bound it by yearsBack to avoid a full scan on first load.
|
|
const cutoffDate = new Date();
|
|
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000;
|
|
const millisecondsBack = yearsBack * millisecondsPerYear;
|
|
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
|
|
|
console.log(`TimeEntries filter: lastModifiedDateTime >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
|
|
|
return [
|
|
{
|
|
field: 'lastModifiedDateTime',
|
|
op: 'gte',
|
|
value: cutoffDate.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for billing items (requires filter)
|
|
* @param yearsBack Number of years to look back (default: 2)
|
|
* @returns Query filter array for billing items
|
|
*/
|
|
export function buildBillingItemsFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> {
|
|
// BillingItems API requires a filter. Use itemDate to limit the range
|
|
// Calculate date from X years ago
|
|
const cutoffDate = new Date();
|
|
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000;
|
|
const millisecondsBack = yearsBack * millisecondsPerYear;
|
|
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
|
|
|
console.log(`BillingItems filter: itemDate >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
|
|
|
return [
|
|
{
|
|
field: 'itemDate',
|
|
op: 'gte',
|
|
value: cutoffDate.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Calculate estimated sync duration based on record count
|
|
* @param recordCount Number of records to sync
|
|
* @param rateLimit Requests per second
|
|
* @param pageSize Records per page
|
|
* @returns Estimated duration in milliseconds
|
|
*/
|
|
export function estimateSyncDuration(
|
|
recordCount: number,
|
|
rateLimit: number = 10,
|
|
pageSize: number = 500
|
|
): number {
|
|
const totalPages = Math.ceil(recordCount / pageSize);
|
|
const secondsNeeded = totalPages / rateLimit;
|
|
const processingOverhead = recordCount * 0.001; // 1ms per record for processing
|
|
|
|
return (secondsNeeded * 1000) + processingOverhead;
|
|
}
|
|
|
|
/**
|
|
* Format sync duration for display
|
|
* @param milliseconds Duration in milliseconds
|
|
* @returns Formatted duration string
|
|
*/
|
|
export function formatDuration(milliseconds: number): string {
|
|
const seconds = Math.floor(milliseconds / 1000);
|
|
const minutes = Math.floor(seconds / 60);
|
|
const hours = Math.floor(minutes / 60);
|
|
|
|
if (hours > 0) {
|
|
return `${hours}h ${minutes % 60}m`;
|
|
} else if (minutes > 0) {
|
|
return `${minutes}m ${seconds % 60}s`;
|
|
} else {
|
|
return `${seconds}s`;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate unique sync ID
|
|
* @returns Unique sync identifier
|
|
*/
|
|
export function generateSyncId(): string {
|
|
return `sync_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
|
|
}
|
|
|
|
/**
|
|
* Validate entity type
|
|
* @param entity Entity type string
|
|
* @returns True if valid entity type
|
|
*/
|
|
export function isValidEntityType(entity: string): entity is EntityType {
|
|
return Object.values(EntityType).includes(entity as EntityType);
|
|
}
|
|
|
|
/**
|
|
* Get entity display name
|
|
* @param entity Entity type
|
|
* @returns Human-readable entity name
|
|
*/
|
|
export function getEntityDisplayName(entity: EntityType): string {
|
|
const mapping: Record<EntityType, string> = {
|
|
[EntityType.COMPANIES]: 'Companies',
|
|
[EntityType.TICKETS]: 'Tickets',
|
|
[EntityType.TASKS]: 'Tasks',
|
|
[EntityType.PROJECTS]: 'Projects',
|
|
[EntityType.RESOURCES]: 'Resources',
|
|
[EntityType.STATUSES]: 'Statuses',
|
|
[EntityType.ISSUE_TYPES]: 'Issue Types',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'Sub-Issue Types',
|
|
[EntityType.WORK_TYPES]: 'Work Types',
|
|
[EntityType.QUEUES]: 'Queues',
|
|
[EntityType.PRIORITIES]: 'Priorities',
|
|
[EntityType.TICKET_CATEGORIES]: 'Ticket Categories',
|
|
[EntityType.BILLING_ITEMS]: 'Billing Items',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'Configuration Items',
|
|
[EntityType.CONTACTS]: 'Contacts',
|
|
[EntityType.CONTRACTS]: 'Contracts',
|
|
[EntityType.CONTRACT_SERVICES]: 'Contract Services',
|
|
[EntityType.AUTOTASK_SERVICES]: 'Autotask Services',
|
|
[EntityType.TIME_ENTRIES]: 'Time Entries',
|
|
[EntityType.TICKET_NOTES]: 'Ticket Notes',
|
|
};
|
|
|
|
return mapping[entity] || entity;
|
|
}
|