Add comprehensive admin features and multi-system integration
- Add admin dashboard with sync controls and data browser - Implement RMM, Auvik, and Addigy organization mappings - Add chunked ticket sync with progress tracking - Implement entity sync service with rate limiting - Add analytics engine and performance optimizer - Create data browser for all PSA entities - Add navigation components and UI improvements - Implement background processing and sync services - Add comprehensive documentation and migration scripts - Update configuration items with multi-system support - Enhance contact management and purchase history - Add issue type assignment and LLM analyzer - Improve error handling and logging utilities
This commit is contained in:
root
parent
e8462ef301
commit
6eee14f8af
171 changed files with 32671 additions and 621 deletions
414
lib/utils/sync-helpers.ts
Normal file
414
lib/utils/sync-helpers.ts
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
/**
|
||||
* Sync Helper Functions
|
||||
* Utility functions for sync operations including dependency ordering
|
||||
*/
|
||||
|
||||
import { EntityType, ENTITY_DEPENDENCIES } from '../types/sync';
|
||||
|
||||
/**
|
||||
* Get entities in dependency order (parents before children)
|
||||
* @param entities List of entities to sync
|
||||
* @returns Ordered list of entities respecting dependencies
|
||||
*/
|
||||
export function getEntitySyncOrder(entities: EntityType[]): EntityType[] {
|
||||
const ordered: EntityType[] = [];
|
||||
const visited = new Set<EntityType>();
|
||||
const visiting = new Set<EntityType>();
|
||||
|
||||
function visit(entity: EntityType) {
|
||||
if (visited.has(entity)) return;
|
||||
if (visiting.has(entity)) {
|
||||
throw new Error(`Circular dependency detected for entity: ${entity}`);
|
||||
}
|
||||
|
||||
visiting.add(entity);
|
||||
|
||||
// Visit dependencies first
|
||||
const dependencies = ENTITY_DEPENDENCIES[entity] || [];
|
||||
for (const dep of dependencies) {
|
||||
if (entities.includes(dep)) {
|
||||
visit(dep);
|
||||
}
|
||||
}
|
||||
|
||||
visiting.delete(entity);
|
||||
visited.add(entity);
|
||||
ordered.push(entity);
|
||||
}
|
||||
|
||||
// Visit all entities
|
||||
for (const entity of entities) {
|
||||
visit(entity);
|
||||
}
|
||||
|
||||
return ordered;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all entities in default sync order
|
||||
* @returns All entities in dependency order
|
||||
*/
|
||||
export function getAllEntitiesInOrder(): EntityType[] {
|
||||
return getEntitySyncOrder([
|
||||
EntityType.COMPANIES,
|
||||
EntityType.RESOURCES,
|
||||
EntityType.STATUSES,
|
||||
EntityType.ISSUE_TYPES,
|
||||
EntityType.SUB_ISSUE_TYPES,
|
||||
EntityType.WORK_TYPES,
|
||||
EntityType.CONTACTS,
|
||||
EntityType.PROJECTS,
|
||||
EntityType.TICKETS,
|
||||
EntityType.TASKS,
|
||||
EntityType.CONFIGURATION_ITEMS,
|
||||
EntityType.CONTRACTS,
|
||||
EntityType.BILLING_ITEMS,
|
||||
EntityType.TIME_ENTRIES,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table name for entity type
|
||||
* @param entity Entity type
|
||||
* @returns PostgreSQL table name
|
||||
*/
|
||||
export function getTableName(entity: EntityType): string {
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Autotask API entity name
|
||||
* @param entity Entity type
|
||||
* @returns Autotask API entity name (PascalCase)
|
||||
*/
|
||||
export function getAutotaskEntityName(entity: EntityType): string {
|
||||
const mapping: Record<EntityType, string> = {
|
||||
[EntityType.COMPANIES]: 'Companies',
|
||||
[EntityType.TICKETS]: 'Tickets',
|
||||
[EntityType.TASKS]: 'Tasks',
|
||||
[EntityType.PROJECTS]: 'Projects',
|
||||
[EntityType.RESOURCES]: 'Resources',
|
||||
[EntityType.STATUSES]: 'Statuses',
|
||||
[EntityType.ISSUE_TYPES]: 'IssueTypes',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'SubIssueTypes',
|
||||
[EntityType.WORK_TYPES]: 'WorkTypes',
|
||||
[EntityType.BILLING_ITEMS]: 'BillingItems',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'ConfigurationItems',
|
||||
[EntityType.CONTACTS]: 'Contacts',
|
||||
[EntityType.CONTRACTS]: 'Contracts',
|
||||
[EntityType.TIME_ENTRIES]: 'TimeEntries',
|
||||
};
|
||||
|
||||
return mapping[entity] || entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if entity is a picklist type
|
||||
* @param entity Entity type
|
||||
* @returns True if entity is a picklist
|
||||
*/
|
||||
export function isPicklistEntity(entity: EntityType): boolean {
|
||||
return [
|
||||
EntityType.STATUSES,
|
||||
EntityType.ISSUE_TYPES,
|
||||
EntityType.SUB_ISSUE_TYPES,
|
||||
EntityType.WORK_TYPES,
|
||||
].includes(entity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get field name for last modified date in Autotask
|
||||
* @param entity Entity type
|
||||
* @returns Field name for filtering by last modified date
|
||||
*/
|
||||
export function getLastModifiedField(entity: EntityType): string {
|
||||
const mapping: Record<EntityType, string> = {
|
||||
[EntityType.COMPANIES]: 'lastTrackedModificationDateTime',
|
||||
[EntityType.TICKETS]: 'lastActivityDate',
|
||||
[EntityType.TASKS]: 'lastActivityDateTime',
|
||||
[EntityType.PROJECTS]: 'lastActivityDateTime',
|
||||
[EntityType.RESOURCES]: 'lastModifiedDate',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'lastModifiedTime',
|
||||
[EntityType.CONTACTS]: 'lastModifiedDate',
|
||||
[EntityType.CONTRACTS]: 'lastModifiedDateTime',
|
||||
[EntityType.BILLING_ITEMS]: 'createDate',
|
||||
[EntityType.TIME_ENTRIES]: 'lastModifiedDate',
|
||||
[EntityType.STATUSES]: 'lastModifiedDate',
|
||||
[EntityType.ISSUE_TYPES]: 'lastModifiedDate',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'lastModifiedDate',
|
||||
[EntityType.WORK_TYPES]: 'lastModifiedDate',
|
||||
};
|
||||
|
||||
return mapping[entity] || 'lastModifiedDate';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get active status field name for entity
|
||||
* @param entity Entity type
|
||||
* @returns Field name for active status
|
||||
*/
|
||||
export function getActiveField(entity: EntityType): string | null {
|
||||
const mapping: Record<EntityType, string | null> = {
|
||||
[EntityType.COMPANIES]: 'isActive',
|
||||
[EntityType.TICKETS]: null, // Use status field instead
|
||||
[EntityType.TASKS]: null, // Use status field instead
|
||||
[EntityType.PROJECTS]: null, // Use status field instead
|
||||
[EntityType.RESOURCES]: 'isActive',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'isActive',
|
||||
[EntityType.CONTACTS]: 'isActive',
|
||||
[EntityType.CONTRACTS]: null, // Use status field instead
|
||||
[EntityType.BILLING_ITEMS]: null,
|
||||
[EntityType.TIME_ENTRIES]: null, // Time entries don't have active status
|
||||
[EntityType.STATUSES]: 'isActive',
|
||||
[EntityType.ISSUE_TYPES]: 'isActive',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'isActive',
|
||||
[EntityType.WORK_TYPES]: 'isActive',
|
||||
};
|
||||
|
||||
return mapping[entity] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Autotask query filter for incremental sync
|
||||
* @param entity Entity type
|
||||
* @param lastSyncTime Last successful sync timestamp
|
||||
* @returns Query filter array
|
||||
*/
|
||||
export function buildIncrementalFilter(
|
||||
entity: EntityType,
|
||||
lastSyncTime: Date
|
||||
): Array<{ field: string; op: string; value: any }> {
|
||||
const lastModifiedField = getLastModifiedField(entity);
|
||||
|
||||
return [
|
||||
{
|
||||
field: lastModifiedField,
|
||||
op: 'gte',
|
||||
value: lastSyncTime.toISOString(),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Autotask query filter for active records only
|
||||
* @param entity Entity type
|
||||
* @returns Query filter array or null if no active field
|
||||
*/
|
||||
export function buildActiveFilter(
|
||||
entity: EntityType
|
||||
): Array<{ field: string; op: string; value: any }> | null {
|
||||
const activeField = getActiveField(entity);
|
||||
|
||||
if (!activeField) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
field: activeField,
|
||||
op: 'eq',
|
||||
value: true,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build date range filter for entities to limit sync to recent records
|
||||
* @param entity Entity type
|
||||
* @param yearsBack Number of years to look back (default: 2)
|
||||
* @returns Query filter array or null if entity doesn't support date filtering
|
||||
*/
|
||||
export function buildDateRangeFilter(
|
||||
entity: EntityType,
|
||||
yearsBack: number = 2
|
||||
): Array<{ field: string; op: string; value: any }> | null {
|
||||
// Only apply date range filters to time-based entities
|
||||
// Note: TIME_ENTRIES removed because Autotask API doesn't support date filtering on TimeEntry
|
||||
const timeBasedEntities = [
|
||||
EntityType.TICKETS,
|
||||
EntityType.TASKS,
|
||||
];
|
||||
|
||||
if (!timeBasedEntities.includes(entity)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Define which entities should have date range filters and which field to use
|
||||
const dateFieldMapping: Record<EntityType, string | null> = {
|
||||
[EntityType.TICKETS]: 'createDate',
|
||||
[EntityType.TASKS]: 'createDateTime',
|
||||
[EntityType.TIME_ENTRIES]: 'createDate', // TimeEntry uses createDate for filtering
|
||||
[EntityType.PROJECTS]: 'startDateTime',
|
||||
[EntityType.BILLING_ITEMS]: 'itemDate',
|
||||
[EntityType.CONTRACTS]: 'startDate', // Contracts use startDate
|
||||
[EntityType.COMPANIES]: null,
|
||||
[EntityType.RESOURCES]: null,
|
||||
[EntityType.CONTACTS]: null,
|
||||
[EntityType.CONFIGURATION_ITEMS]: null,
|
||||
[EntityType.STATUSES]: null,
|
||||
[EntityType.ISSUE_TYPES]: null,
|
||||
[EntityType.SUB_ISSUE_TYPES]: null,
|
||||
[EntityType.WORK_TYPES]: null,
|
||||
};
|
||||
|
||||
const dateField = dateFieldMapping[entity];
|
||||
|
||||
if (!dateField) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculate date from X years ago
|
||||
// Convert years to milliseconds for accurate calculation (including fractional years)
|
||||
const cutoffDate = new Date();
|
||||
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000; // Account for leap years
|
||||
const millisecondsBack = yearsBack * millisecondsPerYear;
|
||||
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
||||
|
||||
console.log(`Date range filter: ${dateField} >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
||||
|
||||
return [
|
||||
{
|
||||
field: dateField,
|
||||
op: 'gte',
|
||||
value: cutoffDate.toISOString(),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build special filter for contracts (requires status filter)
|
||||
* @returns Query filter array for active contracts
|
||||
*/
|
||||
export function buildContractsFilter(): Array<{ field: string; op: string; value: any }> {
|
||||
// Contracts API requires a filter. Use status = 1 for Active contracts
|
||||
// Status values: 1 = Active, others are inactive/expired
|
||||
return [
|
||||
{
|
||||
field: 'status',
|
||||
op: 'eq',
|
||||
value: 1,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build special filter for projects (requires status filter)
|
||||
* @returns Query filter array for active projects
|
||||
*/
|
||||
export function buildProjectsFilter(): Array<{ field: string; op: string; value: any }> {
|
||||
// Projects API requires a filter. Use status = 1 for New/Active projects
|
||||
// Status values: 1 = New, others include Complete, Cancelled, etc.
|
||||
// To get all active projects, we should filter for status NOT equal to Complete (5)
|
||||
return [
|
||||
{
|
||||
field: 'status',
|
||||
op: 'noteq',
|
||||
value: 5, // 5 = Complete
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build special filter for time entries (requires filter)
|
||||
* @param yearsBack Number of years to look back (default: 2)
|
||||
* @returns Query filter array for time entries
|
||||
*/
|
||||
export function buildTimeEntriesFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> {
|
||||
// TimeEntries API requires a filter. Use dateWorked to limit the range
|
||||
// Calculate date from X years ago
|
||||
const cutoffDate = new Date();
|
||||
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000;
|
||||
const millisecondsBack = yearsBack * millisecondsPerYear;
|
||||
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
||||
|
||||
console.log(`TimeEntries filter: dateWorked >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
||||
|
||||
return [
|
||||
{
|
||||
field: 'dateWorked',
|
||||
op: 'gte',
|
||||
value: cutoffDate.toISOString(),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate estimated sync duration based on record count
|
||||
* @param recordCount Number of records to sync
|
||||
* @param rateLimit Requests per second
|
||||
* @param pageSize Records per page
|
||||
* @returns Estimated duration in milliseconds
|
||||
*/
|
||||
export function estimateSyncDuration(
|
||||
recordCount: number,
|
||||
rateLimit: number = 10,
|
||||
pageSize: number = 500
|
||||
): number {
|
||||
const totalPages = Math.ceil(recordCount / pageSize);
|
||||
const secondsNeeded = totalPages / rateLimit;
|
||||
const processingOverhead = recordCount * 0.001; // 1ms per record for processing
|
||||
|
||||
return (secondsNeeded * 1000) + processingOverhead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format sync duration for display
|
||||
* @param milliseconds Duration in milliseconds
|
||||
* @returns Formatted duration string
|
||||
*/
|
||||
export function formatDuration(milliseconds: number): string {
|
||||
const seconds = Math.floor(milliseconds / 1000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const hours = Math.floor(minutes / 60);
|
||||
|
||||
if (hours > 0) {
|
||||
return `${hours}h ${minutes % 60}m`;
|
||||
} else if (minutes > 0) {
|
||||
return `${minutes}m ${seconds % 60}s`;
|
||||
} else {
|
||||
return `${seconds}s`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate unique sync ID
|
||||
* @returns Unique sync identifier
|
||||
*/
|
||||
export function generateSyncId(): string {
|
||||
return `sync_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate entity type
|
||||
* @param entity Entity type string
|
||||
* @returns True if valid entity type
|
||||
*/
|
||||
export function isValidEntityType(entity: string): entity is EntityType {
|
||||
return Object.values(EntityType).includes(entity as EntityType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get entity display name
|
||||
* @param entity Entity type
|
||||
* @returns Human-readable entity name
|
||||
*/
|
||||
export function getEntityDisplayName(entity: EntityType): string {
|
||||
const mapping: Record<EntityType, string> = {
|
||||
[EntityType.COMPANIES]: 'Companies',
|
||||
[EntityType.TICKETS]: 'Tickets',
|
||||
[EntityType.TASKS]: 'Tasks',
|
||||
[EntityType.PROJECTS]: 'Projects',
|
||||
[EntityType.RESOURCES]: 'Resources',
|
||||
[EntityType.STATUSES]: 'Statuses',
|
||||
[EntityType.ISSUE_TYPES]: 'Issue Types',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'Sub-Issue Types',
|
||||
[EntityType.WORK_TYPES]: 'Work Types',
|
||||
[EntityType.BILLING_ITEMS]: 'Billing Items',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'Configuration Items',
|
||||
[EntityType.CONTACTS]: 'Contacts',
|
||||
[EntityType.CONTRACTS]: 'Contracts',
|
||||
[EntityType.TIME_ENTRIES]: 'Time Entries',
|
||||
};
|
||||
|
||||
return mapping[entity] || entity;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue