lorentz
8f8b5ab7be
Multi-stage LLM pipeline that produces structured analyses of Autotask tickets from local Postgres. Migration 069 + Zod schemas, Stage 0 preprocessor, IT Glue redaction + search, Anthropic SDK wrapper, Stages 1/3/4 (Haiku/Sonnet/Opus), pipeline + cost circuit breaker, job worker (opt-in autostart), 6 API routes, 3 frontend pages, share-row persistence (email send deferred to phase 7). 128 vitest tests, tsc clean. Build journal in docs/wulf-pulse-ticket-analyzer-build-notes.md. Sync: adds syncTicketNotes() + ticket_notes to ordered/date-filtered entities so the analyzer's local mirror stays current via scheduler. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
543 lines
17 KiB
TypeScript
543 lines
17 KiB
TypeScript
/**
|
|
* Sync Helper Functions
|
|
* Utility functions for sync operations including dependency ordering
|
|
*/
|
|
|
|
import { EntityType, ENTITY_DEPENDENCIES } from '../types/sync';
|
|
|
|
/**
|
|
* Get entities in dependency order (parents before children)
|
|
* @param entities List of entities to sync
|
|
* @returns Ordered list of entities respecting dependencies
|
|
*/
|
|
export function getEntitySyncOrder(entities: EntityType[]): EntityType[] {
|
|
const ordered: EntityType[] = [];
|
|
const visited = new Set<EntityType>();
|
|
const visiting = new Set<EntityType>();
|
|
|
|
function visit(entity: EntityType) {
|
|
if (visited.has(entity)) return;
|
|
if (visiting.has(entity)) {
|
|
throw new Error(`Circular dependency detected for entity: ${entity}`);
|
|
}
|
|
|
|
visiting.add(entity);
|
|
|
|
// Visit dependencies first
|
|
const dependencies = ENTITY_DEPENDENCIES[entity] || [];
|
|
for (const dep of dependencies) {
|
|
if (entities.includes(dep)) {
|
|
visit(dep);
|
|
}
|
|
}
|
|
|
|
visiting.delete(entity);
|
|
visited.add(entity);
|
|
ordered.push(entity);
|
|
}
|
|
|
|
// Visit all entities
|
|
for (const entity of entities) {
|
|
visit(entity);
|
|
}
|
|
|
|
return ordered;
|
|
}
|
|
|
|
/**
|
|
* Get all entities in default sync order
|
|
* @returns All entities in dependency order
|
|
*/
|
|
export function getAllEntitiesInOrder(): EntityType[] {
|
|
return getEntitySyncOrder([
|
|
EntityType.COMPANIES,
|
|
EntityType.RESOURCES,
|
|
EntityType.STATUSES,
|
|
EntityType.ISSUE_TYPES,
|
|
EntityType.SUB_ISSUE_TYPES,
|
|
EntityType.WORK_TYPES,
|
|
EntityType.QUEUES,
|
|
EntityType.PRIORITIES,
|
|
EntityType.TICKET_CATEGORIES,
|
|
EntityType.CONTACTS,
|
|
EntityType.PROJECTS,
|
|
EntityType.PROJECT_PHASES,
|
|
EntityType.TICKETS,
|
|
EntityType.TICKET_NOTES,
|
|
EntityType.TASKS,
|
|
EntityType.CONFIGURATION_ITEMS,
|
|
EntityType.CONTRACTS,
|
|
EntityType.CONTRACT_SERVICES,
|
|
EntityType.AUTOTASK_SERVICES,
|
|
EntityType.BILLING_ITEMS,
|
|
EntityType.TIME_ENTRIES,
|
|
EntityType.TAG_GROUPS,
|
|
EntityType.TAGS,
|
|
EntityType.COMPANY_TEAMS,
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* Build filter for contract services (requires contractID filter — fetch all via contractIDs)
|
|
* @returns Query filter array for active contract services
|
|
*/
|
|
export function buildContractServicesFilter(): Array<{ field: string; op: string; value: any }> {
|
|
return [
|
|
{
|
|
field: 'contractID',
|
|
op: 'gt',
|
|
value: 0,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Get table name for entity type
|
|
* @param entity Entity type
|
|
* @returns PostgreSQL table name
|
|
*/
|
|
export function getTableName(entity: EntityType): string {
|
|
return entity;
|
|
}
|
|
|
|
/**
|
|
* Get Autotask API entity name
|
|
* @param entity Entity type
|
|
* @returns Autotask API entity name (PascalCase)
|
|
*/
|
|
export function getAutotaskEntityName(entity: EntityType): string {
|
|
const mapping: Record<EntityType, string> = {
|
|
[EntityType.COMPANIES]: 'Companies',
|
|
[EntityType.TICKETS]: 'Tickets',
|
|
[EntityType.TASKS]: 'Tasks',
|
|
[EntityType.PROJECTS]: 'Projects',
|
|
[EntityType.RESOURCES]: 'Resources',
|
|
[EntityType.STATUSES]: 'Statuses',
|
|
[EntityType.ISSUE_TYPES]: 'IssueTypes',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'SubIssueTypes',
|
|
[EntityType.WORK_TYPES]: 'WorkTypes',
|
|
[EntityType.QUEUES]: 'Queues',
|
|
[EntityType.PRIORITIES]: 'Priorities',
|
|
[EntityType.TICKET_CATEGORIES]: 'TicketCategories',
|
|
[EntityType.BILLING_ITEMS]: 'BillingItems',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'ConfigurationItems',
|
|
[EntityType.CONTACTS]: 'Contacts',
|
|
[EntityType.CONTRACTS]: 'Contracts',
|
|
[EntityType.CONTRACT_SERVICES]: 'ContractServices',
|
|
[EntityType.AUTOTASK_SERVICES]: 'Services',
|
|
[EntityType.TIME_ENTRIES]: 'TimeEntries',
|
|
[EntityType.TICKET_NOTES]: 'TicketNotes',
|
|
[EntityType.TAG_GROUPS]: 'TagGroups',
|
|
[EntityType.TAGS]: 'Tags',
|
|
[EntityType.PROJECT_PHASES]: 'Phases',
|
|
[EntityType.COMPANY_CATEGORIES]: 'CompanyCategories',
|
|
[EntityType.COMPANY_TYPES]: 'CompanyTypes',
|
|
[EntityType.COMPANY_TEAMS]: 'CompanyTeams',
|
|
};
|
|
|
|
return mapping[entity] || entity;
|
|
}
|
|
|
|
/**
|
|
* Check if entity is a picklist type
|
|
* @param entity Entity type
|
|
* @returns True if entity is a picklist
|
|
*/
|
|
export function isPicklistEntity(entity: EntityType): boolean {
|
|
return [
|
|
EntityType.STATUSES,
|
|
EntityType.ISSUE_TYPES,
|
|
EntityType.SUB_ISSUE_TYPES,
|
|
EntityType.WORK_TYPES,
|
|
EntityType.QUEUES,
|
|
EntityType.PRIORITIES,
|
|
EntityType.TICKET_CATEGORIES,
|
|
EntityType.COMPANY_TYPES,
|
|
].includes(entity);
|
|
}
|
|
|
|
/**
|
|
* Get field name for last modified date in Autotask
|
|
* @param entity Entity type
|
|
* @returns Field name for filtering by last modified date
|
|
*/
|
|
export function getLastModifiedField(entity: EntityType): string {
|
|
const mapping: Record<EntityType, string> = {
|
|
[EntityType.COMPANIES]: 'lastActivityDate',
|
|
[EntityType.TICKETS]: 'lastActivityDate',
|
|
[EntityType.TASKS]: 'lastActivityDateTime',
|
|
[EntityType.PROJECTS]: 'lastActivityDateTime',
|
|
[EntityType.RESOURCES]: 'lastModifiedDate',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'lastModifiedTime',
|
|
[EntityType.CONTACTS]: 'lastModifiedDate',
|
|
[EntityType.CONTRACTS]: 'lastModifiedDateTime',
|
|
[EntityType.CONTRACT_SERVICES]: 'lastModifiedDate',
|
|
[EntityType.AUTOTASK_SERVICES]: 'lastModifiedDate',
|
|
[EntityType.BILLING_ITEMS]: 'itemDate',
|
|
[EntityType.TIME_ENTRIES]: 'lastModifiedDateTime',
|
|
[EntityType.TICKET_NOTES]: 'lastActivityDate',
|
|
[EntityType.STATUSES]: 'lastModifiedDate',
|
|
[EntityType.ISSUE_TYPES]: 'lastModifiedDate',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'lastModifiedDate',
|
|
[EntityType.WORK_TYPES]: 'lastModifiedDate',
|
|
[EntityType.QUEUES]: 'lastModifiedDate',
|
|
[EntityType.PRIORITIES]: 'lastModifiedDate',
|
|
[EntityType.TICKET_CATEGORIES]: 'lastModifiedDate',
|
|
[EntityType.TAG_GROUPS]: 'lastModifiedDate',
|
|
[EntityType.TAGS]: 'lastModifiedDateTime',
|
|
[EntityType.PROJECT_PHASES]: 'lastActivityDateTime',
|
|
[EntityType.COMPANY_CATEGORIES]: 'lastModifiedDate',
|
|
[EntityType.COMPANY_TYPES]: 'lastModifiedDate',
|
|
[EntityType.COMPANY_TEAMS]: 'lastModifiedDate', // No date field; incremental not supported
|
|
};
|
|
|
|
return mapping[entity] || 'lastModifiedDate';
|
|
}
|
|
|
|
/**
|
|
* Get active status field name for entity
|
|
* @param entity Entity type
|
|
* @returns Field name for active status
|
|
*/
|
|
export function getActiveField(entity: EntityType): string | null {
|
|
const mapping: Record<EntityType, string | null> = {
|
|
[EntityType.COMPANIES]: 'isActive',
|
|
[EntityType.TICKETS]: null, // Use status field instead
|
|
[EntityType.TASKS]: null, // Use status field instead
|
|
[EntityType.PROJECTS]: null, // Use status field instead
|
|
[EntityType.RESOURCES]: 'isActive',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'isActive',
|
|
[EntityType.CONTACTS]: 'isActive',
|
|
[EntityType.CONTRACTS]: null, // Use status field instead
|
|
[EntityType.CONTRACT_SERVICES]: null,
|
|
[EntityType.AUTOTASK_SERVICES]: 'isActive',
|
|
[EntityType.BILLING_ITEMS]: null,
|
|
[EntityType.TIME_ENTRIES]: null, // Time entries don't have active status
|
|
[EntityType.TICKET_NOTES]: null, // Ticket notes don't have active status
|
|
[EntityType.STATUSES]: 'isActive',
|
|
[EntityType.ISSUE_TYPES]: 'isActive',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'isActive',
|
|
[EntityType.WORK_TYPES]: 'isActive',
|
|
[EntityType.QUEUES]: 'isActive',
|
|
[EntityType.PRIORITIES]: 'isActive',
|
|
[EntityType.TICKET_CATEGORIES]: 'isActive',
|
|
[EntityType.TAG_GROUPS]: 'isActive',
|
|
[EntityType.TAGS]: 'isActive',
|
|
[EntityType.PROJECT_PHASES]: null, // No active field on phases
|
|
[EntityType.COMPANY_CATEGORIES]: 'isActive',
|
|
[EntityType.COMPANY_TYPES]: 'isActive',
|
|
[EntityType.COMPANY_TEAMS]: null,
|
|
};
|
|
|
|
return mapping[entity] || null;
|
|
}
|
|
|
|
/**
|
|
* Build Autotask query filter for incremental sync
|
|
* @param entity Entity type
|
|
* @param lastSyncTime Last successful sync timestamp
|
|
* @returns Query filter array
|
|
*/
|
|
export function buildIncrementalFilter(
|
|
entity: EntityType,
|
|
lastSyncTime: Date
|
|
): Array<{ field: string; op: string; value: any }> {
|
|
const lastModifiedField = getLastModifiedField(entity);
|
|
|
|
return [
|
|
{
|
|
field: lastModifiedField,
|
|
op: 'gte',
|
|
value: lastSyncTime.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build Autotask query filter for active records only
|
|
* @param entity Entity type
|
|
* @returns Query filter array or null if no active field
|
|
*/
|
|
export function buildActiveFilter(
|
|
entity: EntityType
|
|
): Array<{ field: string; op: string; value: any }> | null {
|
|
const activeField = getActiveField(entity);
|
|
|
|
if (!activeField) {
|
|
return null;
|
|
}
|
|
|
|
return [
|
|
{
|
|
field: activeField,
|
|
op: 'eq',
|
|
value: true,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build date range filter for entities to limit sync to recent records
|
|
* @param entity Entity type
|
|
* @param yearsBack Number of years to look back (default: 2)
|
|
* @returns Query filter array or null if entity doesn't support date filtering
|
|
*/
|
|
export function buildDateRangeFilter(
|
|
entity: EntityType,
|
|
yearsBack: number = 2
|
|
): Array<{ field: string; op: string; value: any }> | null {
|
|
// Only apply date range filters to time-based entities
|
|
// Note: TIME_ENTRIES removed because Autotask API doesn't support date filtering on TimeEntry
|
|
const timeBasedEntities = [
|
|
EntityType.TICKETS,
|
|
EntityType.TICKET_NOTES,
|
|
EntityType.TASKS,
|
|
];
|
|
|
|
if (!timeBasedEntities.includes(entity)) {
|
|
return null;
|
|
}
|
|
|
|
// Define which entities should have date range filters and which field to use
|
|
const dateFieldMapping: Record<EntityType, string | null> = {
|
|
[EntityType.TICKETS]: 'createDate',
|
|
[EntityType.TASKS]: 'createDateTime',
|
|
[EntityType.TIME_ENTRIES]: 'createDate', // TimeEntry uses createDate for filtering
|
|
[EntityType.PROJECTS]: 'startDateTime',
|
|
[EntityType.BILLING_ITEMS]: 'itemDate',
|
|
[EntityType.CONTRACTS]: 'startDate',
|
|
[EntityType.CONTRACT_SERVICES]: null,
|
|
[EntityType.AUTOTASK_SERVICES]: null,
|
|
[EntityType.COMPANIES]: null,
|
|
[EntityType.RESOURCES]: null,
|
|
[EntityType.CONTACTS]: null,
|
|
[EntityType.CONFIGURATION_ITEMS]: null,
|
|
[EntityType.TICKET_NOTES]: 'lastActivityDate',
|
|
[EntityType.TAG_GROUPS]: null,
|
|
[EntityType.TAGS]: null,
|
|
[EntityType.STATUSES]: null,
|
|
[EntityType.ISSUE_TYPES]: null,
|
|
[EntityType.SUB_ISSUE_TYPES]: null,
|
|
[EntityType.WORK_TYPES]: null,
|
|
[EntityType.QUEUES]: null,
|
|
[EntityType.PRIORITIES]: null,
|
|
[EntityType.TICKET_CATEGORIES]: null,
|
|
[EntityType.PROJECT_PHASES]: null,
|
|
[EntityType.COMPANY_CATEGORIES]: null,
|
|
[EntityType.COMPANY_TYPES]: null,
|
|
[EntityType.COMPANY_TEAMS]: null,
|
|
};
|
|
|
|
const dateField = dateFieldMapping[entity];
|
|
|
|
if (!dateField) {
|
|
return null;
|
|
}
|
|
|
|
// Calculate date from X years ago
|
|
// Convert years to milliseconds for accurate calculation (including fractional years)
|
|
const cutoffDate = new Date();
|
|
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000; // Account for leap years
|
|
const millisecondsBack = yearsBack * millisecondsPerYear;
|
|
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
|
|
|
console.log(`Date range filter: ${dateField} >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
|
|
|
return [
|
|
{
|
|
field: dateField,
|
|
op: 'gte',
|
|
value: cutoffDate.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for contracts (requires status filter)
|
|
* @returns Query filter array for active contracts
|
|
*/
|
|
export function buildContractsFilter(): Array<{ field: string; op: string; value: any }> {
|
|
// Contracts API requires a filter. Use status = 1 for Active contracts
|
|
// Status values: 1 = Active, others are inactive/expired
|
|
return [
|
|
{
|
|
field: 'status',
|
|
op: 'eq',
|
|
value: 1,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build filter for project phases (Phases endpoint requires a filter)
|
|
* @returns Query filter array for all phases
|
|
*/
|
|
export function buildProjectPhasesFilter(): Array<{ field: string; op: string; value: any }> {
|
|
return [
|
|
{
|
|
field: 'id',
|
|
op: 'gt',
|
|
value: 0,
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for projects (requires status filter)
|
|
* @returns Query filter array for active projects
|
|
*/
|
|
export function buildProjectsFilter(): Array<{ field: string; op: string; value: any }> {
|
|
// Projects API requires a filter. Use status = 1 for New/Active projects
|
|
// Status values: 1 = New, others include Complete, Cancelled, etc.
|
|
// To get all active projects, we should filter for status NOT equal to Complete (5)
|
|
return [
|
|
{
|
|
field: 'status',
|
|
op: 'noteq',
|
|
value: 5, // 5 = Complete
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for time entries (requires filter)
|
|
* @param yearsBack Number of years to look back (default: 2)
|
|
* @returns Query filter array for time entries
|
|
*/
|
|
export function buildTimeEntriesFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> {
|
|
// TimeEntries API requires a filter. Use lastModifiedDateTime so that edits
|
|
// to existing entries (e.g. hours changed, notes updated) are picked up even
|
|
// if the dateWorked is in the past or future.
|
|
// We still bound it by yearsBack to avoid a full scan on first load.
|
|
const cutoffDate = new Date();
|
|
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000;
|
|
const millisecondsBack = yearsBack * millisecondsPerYear;
|
|
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
|
|
|
console.log(`TimeEntries filter: lastModifiedDateTime >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
|
|
|
return [
|
|
{
|
|
field: 'lastModifiedDateTime',
|
|
op: 'gte',
|
|
value: cutoffDate.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Build special filter for billing items (requires filter)
|
|
* @param yearsBack Number of years to look back (default: 2)
|
|
* @returns Query filter array for billing items
|
|
*/
|
|
export function buildBillingItemsFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> {
|
|
// BillingItems API requires a filter. Use itemDate to limit the range
|
|
// Calculate date from X years ago
|
|
const cutoffDate = new Date();
|
|
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000;
|
|
const millisecondsBack = yearsBack * millisecondsPerYear;
|
|
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
|
|
|
console.log(`BillingItems filter: itemDate >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
|
|
|
return [
|
|
{
|
|
field: 'itemDate',
|
|
op: 'gte',
|
|
value: cutoffDate.toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Calculate estimated sync duration based on record count
|
|
* @param recordCount Number of records to sync
|
|
* @param rateLimit Requests per second
|
|
* @param pageSize Records per page
|
|
* @returns Estimated duration in milliseconds
|
|
*/
|
|
export function estimateSyncDuration(
|
|
recordCount: number,
|
|
rateLimit: number = 10,
|
|
pageSize: number = 500
|
|
): number {
|
|
const totalPages = Math.ceil(recordCount / pageSize);
|
|
const secondsNeeded = totalPages / rateLimit;
|
|
const processingOverhead = recordCount * 0.001; // 1ms per record for processing
|
|
|
|
return (secondsNeeded * 1000) + processingOverhead;
|
|
}
|
|
|
|
/**
|
|
* Format sync duration for display
|
|
* @param milliseconds Duration in milliseconds
|
|
* @returns Formatted duration string
|
|
*/
|
|
export function formatDuration(milliseconds: number): string {
|
|
const seconds = Math.floor(milliseconds / 1000);
|
|
const minutes = Math.floor(seconds / 60);
|
|
const hours = Math.floor(minutes / 60);
|
|
|
|
if (hours > 0) {
|
|
return `${hours}h ${minutes % 60}m`;
|
|
} else if (minutes > 0) {
|
|
return `${minutes}m ${seconds % 60}s`;
|
|
} else {
|
|
return `${seconds}s`;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate unique sync ID
|
|
* @returns Unique sync identifier
|
|
*/
|
|
export function generateSyncId(): string {
|
|
return `sync_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
|
|
}
|
|
|
|
/**
|
|
* Validate entity type
|
|
* @param entity Entity type string
|
|
* @returns True if valid entity type
|
|
*/
|
|
export function isValidEntityType(entity: string): entity is EntityType {
|
|
return Object.values(EntityType).includes(entity as EntityType);
|
|
}
|
|
|
|
/**
|
|
* Get entity display name
|
|
* @param entity Entity type
|
|
* @returns Human-readable entity name
|
|
*/
|
|
export function getEntityDisplayName(entity: EntityType): string {
|
|
const mapping: Record<EntityType, string> = {
|
|
[EntityType.COMPANIES]: 'Companies',
|
|
[EntityType.TICKETS]: 'Tickets',
|
|
[EntityType.TASKS]: 'Tasks',
|
|
[EntityType.PROJECTS]: 'Projects',
|
|
[EntityType.RESOURCES]: 'Resources',
|
|
[EntityType.STATUSES]: 'Statuses',
|
|
[EntityType.ISSUE_TYPES]: 'Issue Types',
|
|
[EntityType.SUB_ISSUE_TYPES]: 'Sub-Issue Types',
|
|
[EntityType.WORK_TYPES]: 'Work Types',
|
|
[EntityType.QUEUES]: 'Queues',
|
|
[EntityType.PRIORITIES]: 'Priorities',
|
|
[EntityType.TICKET_CATEGORIES]: 'Ticket Categories',
|
|
[EntityType.BILLING_ITEMS]: 'Billing Items',
|
|
[EntityType.CONFIGURATION_ITEMS]: 'Configuration Items',
|
|
[EntityType.CONTACTS]: 'Contacts',
|
|
[EntityType.CONTRACTS]: 'Contracts',
|
|
[EntityType.CONTRACT_SERVICES]: 'Contract Services',
|
|
[EntityType.AUTOTASK_SERVICES]: 'Autotask Services',
|
|
[EntityType.TIME_ENTRIES]: 'Time Entries',
|
|
[EntityType.TICKET_NOTES]: 'Ticket Notes',
|
|
[EntityType.TAG_GROUPS]: 'Tag Groups',
|
|
[EntityType.TAGS]: 'Tags',
|
|
[EntityType.PROJECT_PHASES]: 'Project Phases',
|
|
[EntityType.COMPANY_CATEGORIES]: 'Company Categories',
|
|
[EntityType.COMPANY_TYPES]: 'Company Types',
|
|
[EntityType.COMPANY_TEAMS]: 'Company Teams',
|
|
};
|
|
|
|
return mapping[entity] || entity;
|
|
}
|