Add comprehensive admin features and multi-system integration
- Add admin dashboard with sync controls and data browser - Implement RMM, Auvik, and Addigy organization mappings - Add chunked ticket sync with progress tracking - Implement entity sync service with rate limiting - Add analytics engine and performance optimizer - Create data browser for all PSA entities - Add navigation components and UI improvements - Implement background processing and sync services - Add comprehensive documentation and migration scripts - Update configuration items with multi-system support - Enhance contact management and purchase history - Add issue type assignment and LLM analyzer - Improve error handling and logging utilities
This commit is contained in:
root
parent
e8462ef301
commit
6eee14f8af
171 changed files with 32671 additions and 621 deletions
604
lib/services/analytics-engine.ts
Normal file
604
lib/services/analytics-engine.ts
Normal file
|
|
@ -0,0 +1,604 @@
|
|||
/**
|
||||
* Analytics Engine for Time Entries
|
||||
* Processes time entries data and generates insights, scores, and analytics
|
||||
*/
|
||||
|
||||
import { TimeEntry } from '@/lib/types/database';
|
||||
import {
|
||||
ActivityScore,
|
||||
ContentScore,
|
||||
TimelinessScore,
|
||||
AnalyticsInsight,
|
||||
TimeEntryAnalysis,
|
||||
AggregateAnalysis
|
||||
} from '@/lib/types/analytics';
|
||||
|
||||
export class AnalyticsEngine {
|
||||
|
||||
/**
|
||||
* Analyze a single time entry
|
||||
*/
|
||||
analyzeTimeEntry(timeEntry: TimeEntry): TimeEntryAnalysis {
|
||||
const activityScore = this.calculateActivityScore(timeEntry);
|
||||
const contentScore = this.calculateContentScore(timeEntry);
|
||||
const timelinessScore = this.calculateTimelinessScore(timeEntry);
|
||||
|
||||
// Calculate overall score (weighted average)
|
||||
const overallScore = (
|
||||
activityScore.score * 0.3 +
|
||||
contentScore.score * 0.4 +
|
||||
timelinessScore.score * 0.3
|
||||
);
|
||||
|
||||
return {
|
||||
timeEntryId: timeEntry.id,
|
||||
activityScore,
|
||||
contentScore,
|
||||
timelinessScore,
|
||||
overallScore,
|
||||
insights: this.generateEntryInsights(timeEntry, activityScore, contentScore, timelinessScore),
|
||||
analyzedAt: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze multiple time entries and generate aggregate insights
|
||||
*/
|
||||
analyzeTimeEntries(timeEntries: TimeEntry[]): AggregateAnalysis {
|
||||
if (timeEntries.length === 0) {
|
||||
return this.createEmptyAnalysis();
|
||||
}
|
||||
|
||||
// Analyze individual entries
|
||||
const individualAnalyses = timeEntries.map(entry => this.analyzeTimeEntry(entry));
|
||||
|
||||
// Calculate aggregate scores
|
||||
const avgActivityScore = this.calculateAverage(individualAnalyses.map(a => a.activityScore.score));
|
||||
const avgContentScore = this.calculateAverage(individualAnalyses.map(a => a.contentScore.score));
|
||||
const avgTimelinessScore = this.calculateAverage(individualAnalyses.map(a => a.timelinessScore.score));
|
||||
const avgOverallScore = this.calculateAverage(individualAnalyses.map(a => a.overallScore));
|
||||
|
||||
// Generate insights
|
||||
const insights = this.generateAggregateInsights(timeEntries, individualAnalyses);
|
||||
|
||||
// Calculate patterns and trends
|
||||
const patterns = this.analyzePatterns(timeEntries);
|
||||
const trends = this.analyzeTrends(timeEntries);
|
||||
|
||||
return {
|
||||
totalEntries: timeEntries.length,
|
||||
totalHours: timeEntries.reduce((sum, entry) => sum + entry.hours_worked, 0),
|
||||
averageHoursPerEntry: this.calculateAverage(timeEntries.map(e => e.hours_worked)),
|
||||
dateRange: {
|
||||
earliest: new Date(Math.min(...timeEntries.map(e => new Date(e.entry_date).getTime()))),
|
||||
latest: new Date(Math.max(...timeEntries.map(e => new Date(e.entry_date).getTime()))),
|
||||
},
|
||||
scores: {
|
||||
activity: avgActivityScore,
|
||||
content: avgContentScore,
|
||||
timeliness: avgTimelinessScore,
|
||||
overall: avgOverallScore,
|
||||
},
|
||||
insights,
|
||||
patterns,
|
||||
trends,
|
||||
analyzedAt: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate Activity Score based on entry quality, completeness, and work patterns
|
||||
*/
|
||||
private calculateActivityScore(timeEntry: TimeEntry): ActivityScore {
|
||||
let score = 0;
|
||||
const factors: string[] = [];
|
||||
|
||||
// Factor 1: Entry completeness (40%)
|
||||
const completenessScore = this.calculateCompletenessScore(timeEntry);
|
||||
score += completenessScore * 0.4;
|
||||
if (completenessScore > 0.8) factors.push('Complete entry details');
|
||||
|
||||
// Factor 2: Time tracking consistency (30%)
|
||||
const consistencyScore = this.calculateConsistencyScore(timeEntry);
|
||||
score += consistencyScore * 0.3;
|
||||
if (consistencyScore > 0.7) factors.push('Consistent time tracking');
|
||||
|
||||
// Factor 3: Appropriate time duration (20%)
|
||||
const durationScore = this.calculateDurationScore(timeEntry);
|
||||
score += durationScore * 0.2;
|
||||
if (durationScore > 0.6) factors.push('Reasonable time duration');
|
||||
|
||||
// Factor 4: Proper categorization (10%)
|
||||
const categorizationScore = this.calculateCategorizationScore(timeEntry);
|
||||
score += categorizationScore * 0.1;
|
||||
if (categorizationScore > 0.5) factors.push('Proper categorization');
|
||||
|
||||
return {
|
||||
score: Math.round(score * 100) / 100,
|
||||
factors,
|
||||
breakdown: {
|
||||
completeness: completenessScore,
|
||||
consistency: consistencyScore,
|
||||
duration: durationScore,
|
||||
categorization: categorizationScore,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate Content Score based on time entry description quality and detail
|
||||
*/
|
||||
private calculateContentScore(timeEntry: TimeEntry): ContentScore {
|
||||
let score = 0;
|
||||
const factors: string[] = [];
|
||||
|
||||
// Factor 1: Notes quality and length (40%)
|
||||
const notesScore = this.calculateNotesQualityScore(timeEntry.notes);
|
||||
score += notesScore * 0.4;
|
||||
if (notesScore > 0.7) factors.push('Detailed work description');
|
||||
|
||||
// Factor 2: Title clarity (20%)
|
||||
const titleScore = this.calculateTitleQualityScore(timeEntry.title);
|
||||
score += titleScore * 0.2;
|
||||
if (titleScore > 0.6) factors.push('Clear activity title');
|
||||
|
||||
// Factor 3: Internal notes usage (20%)
|
||||
const internalNotesScore = this.calculateInternalNotesScore(timeEntry.internal_notes);
|
||||
score += internalNotesScore * 0.2;
|
||||
if (internalNotesScore > 0.5) factors.push('Internal documentation');
|
||||
|
||||
// Factor 4: Technical detail level (20%)
|
||||
const technicalScore = this.calculateTechnicalDetailScore(timeEntry.notes, timeEntry.title);
|
||||
score += technicalScore * 0.2;
|
||||
if (technicalScore > 0.6) factors.push('Technical details included');
|
||||
|
||||
return {
|
||||
score: Math.round(score * 100) / 100,
|
||||
factors,
|
||||
breakdown: {
|
||||
notesQuality: notesScore,
|
||||
titleClarity: titleScore,
|
||||
internalNotes: internalNotesScore,
|
||||
technicalDetail: technicalScore,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate Timeliness Score based on entry timing relative to work performed
|
||||
*/
|
||||
private calculateTimelinessScore(timeEntry: TimeEntry): TimelinessScore {
|
||||
let score = 0;
|
||||
const factors: string[] = [];
|
||||
|
||||
// Factor 1: Entry date vs work date (40%)
|
||||
const entryDelayScore = this.calculateEntryDelayScore(timeEntry);
|
||||
score += entryDelayScore * 0.4;
|
||||
if (entryDelayScore > 0.8) factors.push('Prompt time entry');
|
||||
|
||||
// Factor 2: Business hours compliance (30%)
|
||||
const businessHoursScore = this.calculateBusinessHoursScore(timeEntry);
|
||||
score += businessHoursScore * 0.3;
|
||||
if (businessHoursScore > 0.7) factors.push('Business hours compliance');
|
||||
|
||||
// Factor 3: Regularity pattern (20%)
|
||||
const regularityScore = this.calculateRegularityScore(timeEntry);
|
||||
score += regularityScore * 0.2;
|
||||
if (regularityScore > 0.6) factors.push('Regular entry pattern');
|
||||
|
||||
// Factor 4: Approval time (10%)
|
||||
const approvalScore = this.calculateApprovalTimelinessScore(timeEntry);
|
||||
score += approvalScore * 0.1;
|
||||
if (approvalScore > 0.5) factors.push('Timely approval');
|
||||
|
||||
return {
|
||||
score: Math.round(score * 100) / 100,
|
||||
factors,
|
||||
breakdown: {
|
||||
entryDelay: entryDelayScore,
|
||||
businessHours: businessHoursScore,
|
||||
regularity: regularityScore,
|
||||
approvalTimeliness: approvalScore,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper methods for score calculations
|
||||
*/
|
||||
private calculateCompletenessScore(timeEntry: TimeEntry): number {
|
||||
let score = 0;
|
||||
const totalFields = 8;
|
||||
|
||||
if (timeEntry.title && timeEntry.title.length > 5) score++;
|
||||
if (timeEntry.notes && timeEntry.notes.length > 10) score++;
|
||||
if (timeEntry.start_date_time) score++;
|
||||
if (timeEntry.end_date_time) score++;
|
||||
if (timeEntry.ticket_id || timeEntry.task_id || timeEntry.project_id) score++;
|
||||
if (timeEntry.type !== undefined && timeEntry.type !== null) score++;
|
||||
if (timeEntry.billable !== undefined) score++;
|
||||
if (timeEntry.hours_worked > 0 && timeEntry.hours_worked <= 24) score++;
|
||||
|
||||
return score / totalFields;
|
||||
}
|
||||
|
||||
private calculateConsistencyScore(timeEntry: TimeEntry): number {
|
||||
// Check if start/end times match the hours worked
|
||||
if (timeEntry.start_date_time && timeEntry.end_date_time) {
|
||||
const start = new Date(timeEntry.start_date_time);
|
||||
const end = new Date(timeEntry.end_date_time);
|
||||
const actualHours = (end.getTime() - start.getTime()) / (1000 * 60 * 60);
|
||||
const difference = Math.abs(actualHours - timeEntry.hours_worked);
|
||||
|
||||
// Score based on how close the logged hours match the actual time span
|
||||
if (difference < 0.1) return 1.0; // Very close
|
||||
if (difference < 0.5) return 0.8; // Close
|
||||
if (difference < 1.0) return 0.6; // Moderate
|
||||
if (difference < 2.0) return 0.4; // Poor
|
||||
return 0.2; // Very poor
|
||||
}
|
||||
|
||||
// If no start/end times, score based on reasonable hour values
|
||||
if (timeEntry.hours_worked > 0 && timeEntry.hours_worked <= 12) return 0.7;
|
||||
if (timeEntry.hours_worked > 12 && timeEntry.hours_worked <= 24) return 0.5;
|
||||
return 0.3;
|
||||
}
|
||||
|
||||
private calculateDurationScore(timeEntry: TimeEntry): number {
|
||||
const hours = timeEntry.hours_worked;
|
||||
|
||||
// Optimal range: 0.25 to 8 hours
|
||||
if (hours >= 0.25 && hours <= 8) return 1.0;
|
||||
if (hours > 8 && hours <= 12) return 0.8;
|
||||
if (hours > 12 && hours <= 16) return 0.6;
|
||||
if (hours > 16 && hours <= 24) return 0.4;
|
||||
if (hours > 0 && hours < 0.25) return 0.5;
|
||||
return 0.2; // Invalid (0 or >24 hours)
|
||||
}
|
||||
|
||||
private calculateCategorizationScore(timeEntry: TimeEntry): number {
|
||||
let score = 0;
|
||||
|
||||
if (timeEntry.ticket_id) score += 0.4;
|
||||
if (timeEntry.task_id) score += 0.3;
|
||||
if (timeEntry.project_id) score += 0.2;
|
||||
if (timeEntry.type !== undefined && timeEntry.type !== null) score += 0.1;
|
||||
|
||||
return Math.min(score, 1.0);
|
||||
}
|
||||
|
||||
private calculateNotesQualityScore(notes?: string | null): number {
|
||||
if (!notes) return 0;
|
||||
|
||||
let score = 0;
|
||||
|
||||
// Length factor
|
||||
if (notes.length > 20) score += 0.3;
|
||||
if (notes.length > 50) score += 0.2;
|
||||
if (notes.length > 100) score += 0.2;
|
||||
|
||||
// Content factors
|
||||
if (/\b(did|completed|fixed|resolved|created|updated|configured|installed|debugged|tested)\b/i.test(notes)) {
|
||||
score += 0.2;
|
||||
}
|
||||
|
||||
if (/\b(because|due to|issue|problem|error|requirement|request)\b/i.test(notes)) {
|
||||
score += 0.1;
|
||||
}
|
||||
|
||||
return Math.min(score, 1.0);
|
||||
}
|
||||
|
||||
private calculateTitleQualityScore(title?: string | null): number {
|
||||
if (!title) return 0;
|
||||
|
||||
let score = 0;
|
||||
|
||||
if (title.length > 5) score += 0.4;
|
||||
if (title.length > 10) score += 0.3;
|
||||
if (title.length <= 50) score += 0.2; // Not too long
|
||||
|
||||
// Contains action words
|
||||
if (/\b(work|task|activity|meeting|call|support|development|testing|documentation)\b/i.test(title)) {
|
||||
score += 0.1;
|
||||
}
|
||||
|
||||
return Math.min(score, 1.0);
|
||||
}
|
||||
|
||||
private calculateInternalNotesScore(internalNotes?: string | null): number {
|
||||
if (!internalNotes) return 0.5; // Neutral score if not used
|
||||
|
||||
let score = 0.5; // Base score for using internal notes
|
||||
|
||||
if (internalNotes.length > 20) score += 0.3;
|
||||
if (internalNotes.length > 50) score += 0.2;
|
||||
|
||||
return Math.min(score, 1.0);
|
||||
}
|
||||
|
||||
private calculateTechnicalDetailScore(notes?: string | null, title?: string | null): number {
|
||||
const text = `${notes || ''} ${title || ''}`.toLowerCase();
|
||||
|
||||
let score = 0;
|
||||
|
||||
// Technical keywords
|
||||
const technicalKeywords = [
|
||||
'api', 'database', 'server', 'code', 'script', 'config', ' firewall',
|
||||
'network', 'security', 'backup', 'restore', 'install', 'update',
|
||||
'debug', 'error', 'log', 'performance', 'optimization', 'migration'
|
||||
];
|
||||
|
||||
const keywordCount = technicalKeywords.filter(keyword => text.includes(keyword)).length;
|
||||
score = Math.min(keywordCount * 0.2, 1.0);
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
private calculateEntryDelayScore(timeEntry: TimeEntry): number {
|
||||
const entryDate = new Date(timeEntry.entry_date);
|
||||
const createdDate = new Date(timeEntry.created_at);
|
||||
const delayDays = (createdDate.getTime() - entryDate.getTime()) / (1000 * 60 * 60 * 24);
|
||||
|
||||
if (delayDays <= 1) return 1.0;
|
||||
if (delayDays <= 3) return 0.8;
|
||||
if (delayDays <= 7) return 0.6;
|
||||
if (delayDays <= 14) return 0.4;
|
||||
if (delayDays <= 30) return 0.2;
|
||||
return 0.1;
|
||||
}
|
||||
|
||||
private calculateBusinessHoursScore(timeEntry: TimeEntry): number {
|
||||
if (!timeEntry.start_date_time) return 0.7; // Neutral if no start time
|
||||
|
||||
const startTime = new Date(timeEntry.start_date_time);
|
||||
const hour = startTime.getHours();
|
||||
const dayOfWeek = startTime.getDay();
|
||||
|
||||
// Weekday (Mon-Fri)
|
||||
if (dayOfWeek >= 1 && dayOfWeek <= 5) {
|
||||
// Business hours (8 AM - 6 PM)
|
||||
if (hour >= 8 && hour <= 18) return 1.0;
|
||||
// Early evening (6 PM - 9 PM)
|
||||
if (hour > 18 && hour <= 21) return 0.8;
|
||||
// Early morning (6 AM - 8 AM)
|
||||
if (hour >= 6 && hour < 8) return 0.7;
|
||||
}
|
||||
|
||||
// Weekend
|
||||
if (dayOfWeek === 0 || dayOfWeek === 6) {
|
||||
return 0.5;
|
||||
}
|
||||
|
||||
return 0.3; // Late night or unusual hours
|
||||
}
|
||||
|
||||
private calculateRegularityScore(timeEntry: TimeEntry): number {
|
||||
// This would ideally compare with user's historical pattern
|
||||
// For now, return a neutral score
|
||||
return 0.7;
|
||||
}
|
||||
|
||||
private calculateApprovalTimelinessScore(timeEntry: TimeEntry): number {
|
||||
if (!timeEntry.approved) return 0.5; // Neutral if not approved
|
||||
|
||||
if (!timeEntry.approved_date_time) return 0.3;
|
||||
|
||||
const entryDate = new Date(timeEntry.created_at);
|
||||
const approvalDate = new Date(timeEntry.approved_date_time);
|
||||
const approvalDelayDays = (approvalDate.getTime() - entryDate.getTime()) / (1000 * 60 * 60 * 24);
|
||||
|
||||
if (approvalDelayDays <= 1) return 1.0;
|
||||
if (approvalDelayDays <= 3) return 0.8;
|
||||
if (approvalDelayDays <= 7) return 0.6;
|
||||
if (approvalDelayDays <= 14) return 0.4;
|
||||
return 0.2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate insights for individual time entry
|
||||
*/
|
||||
private generateEntryInsights(
|
||||
timeEntry: TimeEntry,
|
||||
activityScore: ActivityScore,
|
||||
contentScore: ContentScore,
|
||||
timelinessScore: TimelinessScore
|
||||
): AnalyticsInsight[] {
|
||||
const insights: AnalyticsInsight[] = [];
|
||||
|
||||
// Low score insights
|
||||
if (activityScore.score < 0.5) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'activity',
|
||||
title: 'Low Activity Score',
|
||||
description: 'This time entry has incomplete information or irregular patterns.',
|
||||
recommendation: 'Add more details to improve tracking accuracy.',
|
||||
});
|
||||
}
|
||||
|
||||
if (contentScore.score < 0.5) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'content',
|
||||
title: 'Poor Documentation',
|
||||
description: 'Work description lacks detail or clarity.',
|
||||
recommendation: 'Include specific tasks, outcomes, and technical details.',
|
||||
});
|
||||
}
|
||||
|
||||
if (timelinessScore.score < 0.5) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'timeliness',
|
||||
title: 'Delayed Entry',
|
||||
description: 'Time entry was logged significantly after work was performed.',
|
||||
recommendation: 'Try to enter time within 24 hours of completion.',
|
||||
});
|
||||
}
|
||||
|
||||
// High score insights
|
||||
if (activityScore.score > 0.8 && contentScore.score > 0.8 && timelinessScore.score > 0.8) {
|
||||
insights.push({
|
||||
type: 'success',
|
||||
category: 'overall',
|
||||
title: 'Excellent Time Entry',
|
||||
description: 'This is a well-documented and timely time entry.',
|
||||
recommendation: 'Keep up the good work!',
|
||||
});
|
||||
}
|
||||
|
||||
return insights;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate aggregate insights for multiple time entries
|
||||
*/
|
||||
private generateAggregateInsights(
|
||||
timeEntries: TimeEntry[],
|
||||
analyses: TimeEntryAnalysis[]
|
||||
): AnalyticsInsight[] {
|
||||
const insights: AnalyticsInsight[] = [];
|
||||
|
||||
// Overall performance insights
|
||||
const avgOverallScore = this.calculateAverage(analyses.map(a => a.overallScore));
|
||||
|
||||
if (avgOverallScore > 0.8) {
|
||||
insights.push({
|
||||
type: 'success',
|
||||
category: 'overall',
|
||||
title: 'High Quality Time Tracking',
|
||||
description: 'Overall time entry quality is excellent.',
|
||||
recommendation: 'Maintain current documentation standards.',
|
||||
});
|
||||
} else if (avgOverallScore < 0.5) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'overall',
|
||||
title: 'Poor Time Entry Quality',
|
||||
description: 'Time entries need improvement in documentation and timeliness.',
|
||||
recommendation: 'Provide training on proper time entry practices.',
|
||||
});
|
||||
}
|
||||
|
||||
// Pattern insights
|
||||
const billableEntries = timeEntries.filter(e => e.billable).length;
|
||||
const billablePercentage = (billableEntries / timeEntries.length) * 100;
|
||||
|
||||
if (billablePercentage < 50) {
|
||||
insights.push({
|
||||
type: 'info',
|
||||
category: 'billing',
|
||||
title: 'Low Billable Percentage',
|
||||
description: `Only ${billablePercentage.toFixed(1)}% of entries are marked as billable.`,
|
||||
recommendation: 'Review billing categorization to ensure accurate invoicing.',
|
||||
});
|
||||
}
|
||||
|
||||
return insights;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze patterns in time entries
|
||||
*/
|
||||
private analyzePatterns(timeEntries: TimeEntry[]) {
|
||||
// Group by day of week
|
||||
const dayOfWeekPattern = new Array(7).fill(0);
|
||||
timeEntries.forEach(entry => {
|
||||
const dayOfWeek = new Date(entry.entry_date).getDay();
|
||||
dayOfWeekPattern[dayOfWeek]++;
|
||||
});
|
||||
|
||||
// Group by hour
|
||||
const hourlyPattern = new Array(24).fill(0);
|
||||
timeEntries.forEach(entry => {
|
||||
if (entry.start_date_time) {
|
||||
const hour = new Date(entry.start_date_time).getHours();
|
||||
hourlyPattern[hour]++;
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
dayOfWeek: dayOfWeekPattern,
|
||||
hourly: hourlyPattern,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze trends in time entries
|
||||
*/
|
||||
private analyzeTrends(timeEntries: TimeEntry[]) {
|
||||
// Sort by date
|
||||
const sortedEntries = [...timeEntries].sort((a, b) =>
|
||||
new Date(a.entry_date).getTime() - new Date(b.entry_date).getTime()
|
||||
);
|
||||
|
||||
// Calculate weekly trends
|
||||
const weeklyTrends: { week: Date; hours: number; entries: number }[] = [];
|
||||
const weeklyMap = new Map<string, { hours: number; entries: number }>();
|
||||
|
||||
sortedEntries.forEach(entry => {
|
||||
const date = new Date(entry.entry_date);
|
||||
const weekStart = new Date(date.setDate(date.getDate() - date.getDay()));
|
||||
const weekKey = weekStart.toISOString().split('T')[0];
|
||||
|
||||
if (!weeklyMap.has(weekKey)) {
|
||||
weeklyMap.set(weekKey, { hours: 0, entries: 0 });
|
||||
}
|
||||
|
||||
const week = weeklyMap.get(weekKey)!;
|
||||
week.hours += entry.hours_worked;
|
||||
week.entries++;
|
||||
});
|
||||
|
||||
weeklyMap.forEach((data, weekKey) => {
|
||||
weeklyTrends.push({
|
||||
week: new Date(weekKey),
|
||||
hours: data.hours,
|
||||
entries: data.entries,
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
weekly: weeklyTrends.sort((a, b) => a.week.getTime() - b.week.getTime()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility methods
|
||||
*/
|
||||
private calculateAverage(values: number[]): number {
|
||||
if (values.length === 0) return 0;
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
private createEmptyAnalysis(): AggregateAnalysis {
|
||||
return {
|
||||
totalEntries: 0,
|
||||
totalHours: 0,
|
||||
averageHoursPerEntry: 0,
|
||||
dateRange: {
|
||||
earliest: new Date(),
|
||||
latest: new Date(),
|
||||
},
|
||||
scores: {
|
||||
activity: 0,
|
||||
content: 0,
|
||||
timeliness: 0,
|
||||
overall: 0,
|
||||
},
|
||||
insights: [],
|
||||
patterns: {
|
||||
dayOfWeek: new Array(7).fill(0),
|
||||
hourly: new Array(24).fill(0),
|
||||
},
|
||||
trends: {
|
||||
weekly: [],
|
||||
},
|
||||
analyzedAt: new Date(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
export const analyticsEngine = new AnalyticsEngine();
|
||||
477
lib/services/analytics-integration.ts
Normal file
477
lib/services/analytics-integration.ts
Normal file
|
|
@ -0,0 +1,477 @@
|
|||
/**
|
||||
* Analytics Integration Service
|
||||
* Integrates Time Entries analytics with existing entities for enriched analysis
|
||||
*/
|
||||
|
||||
import { TimeEntry } from '@/lib/types/database';
|
||||
import { analyticsEngine } from './analytics-engine';
|
||||
import { llmAnalyzer } from './llm-analyzer';
|
||||
import { postgresClient } from './postgres-client';
|
||||
import {
|
||||
TimelineEvent,
|
||||
AnalyticsInsight,
|
||||
AggregateAnalysis,
|
||||
LLMAnalysisRequest
|
||||
} from '@/lib/types/analytics';
|
||||
|
||||
export interface EnrichedTimeEntry extends TimeEntry {
|
||||
resource_name?: string;
|
||||
ticket_title?: string;
|
||||
ticket_number?: string;
|
||||
task_title?: string;
|
||||
project_name?: string;
|
||||
company_name?: string;
|
||||
analysis?: {
|
||||
activityScore: number;
|
||||
contentScore: number;
|
||||
timelinessScore: number;
|
||||
overallScore: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EnrichmentOptions {
|
||||
includeResourceInfo?: boolean;
|
||||
includeTicketInfo?: boolean;
|
||||
includeTaskInfo?: boolean;
|
||||
includeProjectInfo?: boolean;
|
||||
includeCompanyInfo?: boolean;
|
||||
includeAnalysis?: boolean;
|
||||
includeRelatedEntities?: boolean;
|
||||
}
|
||||
|
||||
export class AnalyticsIntegrationService {
|
||||
|
||||
/**
|
||||
* Enrich time entries with related entity data and analysis
|
||||
*/
|
||||
async enrichTimeEntries(
|
||||
timeEntries: TimeEntry[],
|
||||
options: EnrichmentOptions = {}
|
||||
): Promise<EnrichedTimeEntry[]> {
|
||||
const enriched: EnrichedTimeEntry[] = [];
|
||||
|
||||
// Extract IDs for batch queries
|
||||
const resourceIds = [...new Set(timeEntries.map(te => te.resource_id).filter((id): id is number => id != null))];
|
||||
const ticketIds = [...new Set(timeEntries.map(te => te.ticket_id).filter((id): id is number => id != null))];
|
||||
const taskIds = [...new Set(timeEntries.map(te => te.task_id).filter((id): id is number => id != null))];
|
||||
const projectIds = [...new Set(timeEntries.map(te => te.project_id).filter((id): id is number => id != null))];
|
||||
const companyIds = [...new Set(timeEntries.map(te => te.company_id).filter((id): id is number => id != null))];
|
||||
|
||||
// Batch fetch related entities
|
||||
const [resources, tickets, tasks, projects, companies] = await Promise.all([
|
||||
options.includeResourceInfo && resourceIds.length > 0
|
||||
? this.getResources(resourceIds)
|
||||
: Promise.resolve([]),
|
||||
options.includeTicketInfo && ticketIds.length > 0
|
||||
? this.getTickets(ticketIds)
|
||||
: Promise.resolve([]),
|
||||
options.includeTaskInfo && taskIds.length > 0
|
||||
? this.getTasks(taskIds)
|
||||
: Promise.resolve([]),
|
||||
options.includeProjectInfo && projectIds.length > 0
|
||||
? this.getProjects(projectIds)
|
||||
: Promise.resolve([]),
|
||||
options.includeCompanyInfo && companyIds.length > 0
|
||||
? this.getCompanies(companyIds)
|
||||
: Promise.resolve([]),
|
||||
]);
|
||||
|
||||
// Create lookup maps
|
||||
const resourceMap = new Map(resources.map(r => [r.id, r]));
|
||||
const ticketMap = new Map(tickets.map(t => [t.id, t]));
|
||||
const taskMap = new Map(tasks.map(t => [t.id, t]));
|
||||
const projectMap = new Map(projects.map(p => [p.id, p]));
|
||||
const companyMap = new Map(companies.map(c => [c.id, c]));
|
||||
|
||||
// Enrich each time entry
|
||||
for (const timeEntry of timeEntries) {
|
||||
const enrichedEntry: EnrichedTimeEntry = { ...timeEntry };
|
||||
|
||||
// Add related entity information
|
||||
if (options.includeResourceInfo) {
|
||||
const resource = resourceMap.get(timeEntry.resource_id);
|
||||
if (resource) {
|
||||
enrichedEntry.resource_name = `${resource.first_name} ${resource.last_name}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.includeTicketInfo) {
|
||||
const ticket = ticketMap.get(timeEntry.ticket_id);
|
||||
if (ticket) {
|
||||
enrichedEntry.ticket_title = ticket.title;
|
||||
enrichedEntry.ticket_number = ticket.ticket_number;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.includeTaskInfo) {
|
||||
const task = taskMap.get(timeEntry.task_id);
|
||||
if (task) {
|
||||
enrichedEntry.task_title = task.title;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.includeProjectInfo) {
|
||||
const project = projectMap.get(timeEntry.project_id);
|
||||
if (project) {
|
||||
enrichedEntry.project_name = project.project_name;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.includeCompanyInfo) {
|
||||
const company = companyMap.get(timeEntry.company_id);
|
||||
if (company) {
|
||||
enrichedEntry.company_name = company.company_name;
|
||||
}
|
||||
}
|
||||
|
||||
// Add analysis
|
||||
if (options.includeAnalysis) {
|
||||
const analysis = analyticsEngine.analyzeTimeEntry(timeEntry);
|
||||
enrichedEntry.analysis = {
|
||||
activityScore: analysis.activityScore.score,
|
||||
contentScore: analysis.contentScore.score,
|
||||
timelinessScore: analysis.timelinessScore.score,
|
||||
overallScore: analysis.overallScore,
|
||||
};
|
||||
}
|
||||
|
||||
enriched.push(enrichedEntry);
|
||||
}
|
||||
|
||||
return enriched;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate timeline events from enriched time entries
|
||||
*/
|
||||
async generateTimelineEvents(
|
||||
timeEntries: TimeEntry[],
|
||||
options: EnrichmentOptions = {
|
||||
includeResourceInfo: true,
|
||||
includeTicketInfo: true,
|
||||
includeTaskInfo: true,
|
||||
includeProjectInfo: true,
|
||||
includeCompanyInfo: true,
|
||||
}
|
||||
): Promise<TimelineEvent[]> {
|
||||
const enrichedEntries = await this.enrichTimeEntries(timeEntries, options);
|
||||
|
||||
const events: TimelineEvent[] = enrichedEntries.map(entry => ({
|
||||
id: `te-${entry.id}`,
|
||||
type: 'time_entry',
|
||||
timestamp: new Date(entry.entry_date),
|
||||
title: entry.title || 'Time Entry',
|
||||
description: this.generateEventDescription(entry),
|
||||
duration: entry.hours_worked,
|
||||
metadata: {
|
||||
timeEntryId: entry.id,
|
||||
resourceId: entry.resource_id,
|
||||
resourceName: entry.resource_name,
|
||||
ticketId: entry.ticket_id,
|
||||
ticketTitle: entry.ticket_title,
|
||||
ticketNumber: entry.ticket_number,
|
||||
taskId: entry.task_id,
|
||||
taskTitle: entry.task_title,
|
||||
projectId: entry.project_id,
|
||||
projectName: entry.project_name,
|
||||
companyId: entry.company_id,
|
||||
companyName: entry.company_name,
|
||||
billable: entry.billable,
|
||||
approved: entry.approved,
|
||||
score: entry.analysis?.overallScore,
|
||||
},
|
||||
score: entry.analysis?.overallScore,
|
||||
isHumanActivity: true,
|
||||
importance: this.determineImportance(entry),
|
||||
}));
|
||||
|
||||
// Add key moments if requested
|
||||
if (options.includeRelatedEntities) {
|
||||
const keyMomentEvents = await this.generateKeyMomentEvents(timeEntries);
|
||||
events.push(...keyMomentEvents);
|
||||
}
|
||||
|
||||
// Sort by timestamp
|
||||
return events.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate comprehensive analysis with entity context
|
||||
*/
|
||||
async generateComprehensiveAnalysis(
|
||||
timeEntries: TimeEntry[],
|
||||
options: EnrichmentOptions = {
|
||||
includeResourceInfo: true,
|
||||
includeTicketInfo: true,
|
||||
includeProjectInfo: true,
|
||||
includeCompanyInfo: true,
|
||||
includeAnalysis: true,
|
||||
}
|
||||
): Promise<{
|
||||
analysis: AggregateAnalysis;
|
||||
enrichedEntries: EnrichedTimeEntry[];
|
||||
entityInsights: AnalyticsInsight[];
|
||||
}> {
|
||||
// Enrich time entries
|
||||
const enrichedEntries = await this.enrichTimeEntries(timeEntries, options);
|
||||
|
||||
// Generate basic analysis
|
||||
const analysis = analyticsEngine.analyzeTimeEntries(timeEntries);
|
||||
|
||||
// Generate entity-specific insights
|
||||
const entityInsights = await this.generateEntityInsights(enrichedEntries);
|
||||
|
||||
// Combine insights
|
||||
analysis.insights = [...analysis.insights, ...entityInsights];
|
||||
|
||||
return {
|
||||
analysis,
|
||||
enrichedEntries,
|
||||
entityInsights,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate LLM analysis with enriched context
|
||||
*/
|
||||
async generateLLMAnalysis(
|
||||
timeEntries: TimeEntry[],
|
||||
analysisType: 'productivity' | 'quality' | 'patterns' | 'anomalies' | 'comprehensive' = 'comprehensive',
|
||||
options: EnrichmentOptions = {
|
||||
includeResourceInfo: true,
|
||||
includeTicketInfo: true,
|
||||
includeProjectInfo: true,
|
||||
includeCompanyInfo: true,
|
||||
}
|
||||
) {
|
||||
const enrichedEntries = await this.enrichTimeEntries(timeEntries, options);
|
||||
|
||||
const request: LLMAnalysisRequest = {
|
||||
timeEntries: enrichedEntries.map(entry => ({
|
||||
id: entry.id,
|
||||
notes: entry.notes || undefined,
|
||||
title: entry.title || undefined,
|
||||
hours_worked: entry.hours_worked,
|
||||
entry_date: entry.entry_date.toISOString(),
|
||||
resource_name: entry.resource_name,
|
||||
ticket_title: entry.ticket_title,
|
||||
})),
|
||||
analysisType,
|
||||
context: {
|
||||
timeRange: `${new Date().toISOString().split('T')[0]}`,
|
||||
resourceIds: [...new Set(enrichedEntries.map(e => e.resource_id).filter((id): id is number => id != null))],
|
||||
projectIds: [...new Set(enrichedEntries.map(e => e.project_id).filter((id): id is number => id != null))],
|
||||
},
|
||||
};
|
||||
|
||||
return llmAnalyzer.analyzeTimeEntries(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate entity-specific insights
|
||||
*/
|
||||
private async generateEntityInsights(enrichedEntries: EnrichedTimeEntry[]): Promise<AnalyticsInsight[]> {
|
||||
const insights: AnalyticsInsight[] = [];
|
||||
|
||||
// Resource-based insights
|
||||
const resourceGroups = this.groupBy(enrichedEntries, 'resource_id');
|
||||
for (const [resourceId, entries] of Object.entries(resourceGroups)) {
|
||||
const resourceName = entries[0]?.resource_name || `Resource ${resourceId}`;
|
||||
const avgScore = entries.reduce((sum, e) => sum + (e.analysis?.overallScore || 0), 0) / entries.length;
|
||||
|
||||
if (avgScore < 0.5) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'activity',
|
||||
title: `Low Performance: ${resourceName}`,
|
||||
description: `${resourceName} has an average score of ${(avgScore * 100).toFixed(1)}%`,
|
||||
recommendation: 'Review time entry quality and provide training if needed',
|
||||
severity: 'medium',
|
||||
actionable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Project-based insights
|
||||
const projectGroups = this.groupBy(enrichedEntries, 'project_id');
|
||||
for (const [projectId, entries] of Object.entries(projectGroups)) {
|
||||
const projectName = entries[0]?.project_name || `Project ${projectId}`;
|
||||
const totalHours = entries.reduce((sum, e) => sum + e.hours_worked, 0);
|
||||
|
||||
if (totalHours > 100) {
|
||||
insights.push({
|
||||
type: 'info',
|
||||
category: 'performance',
|
||||
title: `High Activity: ${projectName}`,
|
||||
description: `${Number(totalHours).toFixed(1)} hours logged for this project`,
|
||||
recommendation: 'Monitor project progress and resource allocation',
|
||||
severity: 'low',
|
||||
actionable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Ticket-based insights
|
||||
const ticketGroups = this.groupBy(enrichedEntries, 'ticket_id');
|
||||
for (const [ticketId, entries] of Object.entries(ticketGroups)) {
|
||||
const ticketTitle = entries[0]?.ticket_title || `Ticket ${ticketId}`;
|
||||
const totalHours = entries.reduce((sum, e) => sum + e.hours_worked, 0);
|
||||
|
||||
if (totalHours > 20) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'performance',
|
||||
title: `Time Intensive: ${ticketTitle}`,
|
||||
description: `${Number(totalHours).toFixed(1)} hours logged for this ticket`,
|
||||
recommendation: 'Review if ticket scope is appropriate or needs to be split',
|
||||
severity: 'medium',
|
||||
actionable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return insights;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate key moment events from related entities
|
||||
*/
|
||||
private async generateKeyMomentEvents(timeEntries: TimeEntry[]): Promise<TimelineEvent[]> {
|
||||
const events: TimelineEvent[] = [];
|
||||
|
||||
// Get ticket creation and resolution dates
|
||||
const ticketIds = [...new Set(timeEntries.map(te => te.ticket_id).filter((id): id is number => id != null))];
|
||||
if (ticketIds.length > 0) {
|
||||
const tickets = await this.getTickets(ticketIds);
|
||||
|
||||
for (const ticket of tickets) {
|
||||
// Ticket creation
|
||||
if (ticket.create_date) {
|
||||
events.push({
|
||||
id: `ticket-created-${ticket.id}`,
|
||||
type: 'key_moment',
|
||||
timestamp: new Date(ticket.create_date),
|
||||
title: `Ticket Created: ${ticket.ticket_number}`,
|
||||
description: `Ticket "${ticket.title}" was created`,
|
||||
metadata: {
|
||||
ticketId: ticket.id,
|
||||
ticketNumber: ticket.ticket_number,
|
||||
ticketTitle: ticket.title,
|
||||
},
|
||||
isHumanActivity: false,
|
||||
importance: 'high',
|
||||
});
|
||||
}
|
||||
|
||||
// Ticket resolution
|
||||
if (ticket.completed_date) {
|
||||
events.push({
|
||||
id: `ticket-resolved-${ticket.id}`,
|
||||
type: 'milestone',
|
||||
timestamp: new Date(ticket.completed_date),
|
||||
title: `Ticket Resolved: ${ticket.ticket_number}`,
|
||||
description: `Ticket "${ticket.title}" was resolved`,
|
||||
metadata: {
|
||||
ticketId: ticket.id,
|
||||
ticketNumber: ticket.ticket_number,
|
||||
ticketTitle: ticket.title,
|
||||
},
|
||||
isHumanActivity: false,
|
||||
importance: 'critical',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper methods
|
||||
*/
|
||||
private async getResources(resourceIds: number[]) {
|
||||
const result = await postgresClient.query(
|
||||
'SELECT id, first_name, last_name FROM resources WHERE id = ANY($1)',
|
||||
[resourceIds]
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
private async getTickets(ticketIds: number[]) {
|
||||
const result = await postgresClient.query(
|
||||
'SELECT id, ticket_number, title, create_date, completed_date FROM tickets WHERE id = ANY($1)',
|
||||
[ticketIds]
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
private async getTasks(taskIds: number[]) {
|
||||
const result = await postgresClient.query(
|
||||
'SELECT id, title FROM tasks WHERE id = ANY($1)',
|
||||
[taskIds]
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
private async getProjects(projectIds: number[]) {
|
||||
const result = await postgresClient.query(
|
||||
'SELECT id, project_name FROM projects WHERE id = ANY($1)',
|
||||
[projectIds]
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
private async getCompanies(companyIds: number[]) {
|
||||
const result = await postgresClient.query(
|
||||
'SELECT id, company_name FROM companies WHERE id = ANY($1)',
|
||||
[companyIds]
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
private generateEventDescription(entry: EnrichedTimeEntry): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (entry.resource_name) {
|
||||
parts.push(`${entry.resource_name} worked`);
|
||||
}
|
||||
|
||||
parts.push(`${entry.hours_worked} hours`);
|
||||
|
||||
if (entry.ticket_title) {
|
||||
parts.push(`on "${entry.ticket_title}"`);
|
||||
}
|
||||
|
||||
if (entry.project_name) {
|
||||
parts.push(`for project "${entry.project_name}"`);
|
||||
}
|
||||
|
||||
if (entry.notes) {
|
||||
parts.push(`- ${entry.notes}`);
|
||||
}
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
private determineImportance(entry: EnrichedTimeEntry): 'low' | 'medium' | 'high' | 'critical' {
|
||||
if (entry.hours_worked > 8) return 'high';
|
||||
if (entry.billable && entry.hours_worked > 4) return 'high';
|
||||
if (entry.billable) return 'medium';
|
||||
if (entry.hours_worked > 2) return 'medium';
|
||||
return 'low';
|
||||
}
|
||||
|
||||
private groupBy<T>(array: T[], key: keyof T): Record<string, T[]> {
|
||||
return array.reduce((groups, item) => {
|
||||
const groupKey = String(item[key]);
|
||||
if (!groups[groupKey]) {
|
||||
groups[groupKey] = [];
|
||||
}
|
||||
groups[groupKey].push(item);
|
||||
return groups;
|
||||
}, {} as Record<string, T[]>);
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
export const analyticsIntegration = new AnalyticsIntegrationService();
|
||||
|
|
@ -12,6 +12,7 @@ import {
|
|||
Attachment,
|
||||
EntityField,
|
||||
PicklistValue,
|
||||
AutotaskTimeEntry,
|
||||
} from '@/lib/types/autotask';
|
||||
|
||||
export class AutotaskClient {
|
||||
|
|
@ -112,36 +113,46 @@ export class AutotaskClient {
|
|||
): Promise<T[]> {
|
||||
const allItems: T[] = [];
|
||||
let page = 1;
|
||||
let hasMore = true;
|
||||
let nextPageUrl: string | null = null;
|
||||
|
||||
while (hasMore) {
|
||||
const paginatedParams = {
|
||||
...params,
|
||||
while (true) {
|
||||
console.log(`Fetching ${entityName} page ${page} (max ${pageSize} records)...`);
|
||||
|
||||
|
||||
const requestBody: any = {
|
||||
MaxRecords: pageSize,
|
||||
// Autotask uses MaxRecords for page size
|
||||
};
|
||||
|
||||
console.log(`Fetching ${entityName} page ${page} (max ${pageSize} records)...`);
|
||||
const queryString = this.buildQueryString(paginatedParams);
|
||||
const url = `${this.config.apiUrl}/${entityName}/query${queryString}`;
|
||||
if (params.filter && params.filter.length > 0) {
|
||||
requestBody.filter = params.filter;
|
||||
console.log(`[${entityName}] Query filter:`, JSON.stringify(params.filter));
|
||||
}
|
||||
|
||||
const response = await this.makeApiCall<ApiResponse<T>>(url, {
|
||||
method: 'GET',
|
||||
const url: string = nextPageUrl || `${this.config.apiUrl}/${entityName}/query`;
|
||||
console.log(`[${entityName}] Request URL: ${url}`);
|
||||
console.log(`[${entityName}] Request body:`, JSON.stringify(requestBody));
|
||||
|
||||
const response: ApiResponse<T> = await this.makeApiCall<ApiResponse<T>>(url, {
|
||||
method: 'POST',
|
||||
headers: this.getAuthHeaders(),
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
console.log(`[${entityName}] Response pageDetails:`, JSON.stringify(response.pageDetails));
|
||||
console.log(`[${entityName}] Response items count:`, response.items?.length || 0);
|
||||
|
||||
const items = response.items || [];
|
||||
allItems.push(...items);
|
||||
|
||||
console.log(`Fetched ${items.length} ${entityName}, total so far: ${allItems.length}`);
|
||||
|
||||
// If we got fewer items than pageSize, we've reached the end
|
||||
hasMore = items.length === pageSize;
|
||||
page++;
|
||||
|
||||
// Safety limit to prevent infinite loops
|
||||
if (page > 100) {
|
||||
console.warn(`Reached page limit (100) for ${entityName}`);
|
||||
// Check if there's a next page using Autotask's pageDetails
|
||||
if (response.pageDetails?.nextPageUrl) {
|
||||
nextPageUrl = response.pageDetails.nextPageUrl;
|
||||
page++;
|
||||
} else {
|
||||
// No more pages
|
||||
console.log(`No more pages for ${entityName}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -182,19 +193,42 @@ export class AutotaskClient {
|
|||
id: number,
|
||||
data: Partial<T>
|
||||
): Promise<T> {
|
||||
const url = `${this.config.apiUrl}/${entityName}/${id}`;
|
||||
const url = `${this.config.apiUrl}/${entityName}`;
|
||||
|
||||
const response = await this.makeApiCall<ApiResponse<T>>(url, {
|
||||
method: 'PATCH',
|
||||
console.log(`Updating ${entityName} ${id} with data:`, data);
|
||||
|
||||
const response = await this.makeApiCall<any>(url, {
|
||||
method: 'PUT',
|
||||
headers: this.getAuthHeaders(),
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
|
||||
if (!response.item) {
|
||||
throw new Error('Failed to update entity');
|
||||
console.log(`Update response for ${entityName} ${id}:`, response);
|
||||
|
||||
// Autotask returns { itemId: ... } on successful update, not { item: {...} }
|
||||
// We need to fetch the updated item
|
||||
if (response.itemId || response.item) {
|
||||
const itemId = response.itemId || (response.item as any)?.id || id;
|
||||
console.log(`Fetching updated ${entityName} ${itemId}`);
|
||||
|
||||
// Fetch the updated item
|
||||
const updatedItem = await this.getEntityById<T>(entityName, itemId);
|
||||
if (updatedItem) {
|
||||
return updatedItem;
|
||||
}
|
||||
}
|
||||
|
||||
return response.item;
|
||||
console.error(`No item or itemId in response for ${entityName} ${id}:`, response);
|
||||
throw new Error('Failed to update entity - no item in response');
|
||||
}
|
||||
|
||||
async deleteEntity(entityName: string, id: number): Promise<void> {
|
||||
const url = `${this.config.apiUrl}/${entityName}/${id}`;
|
||||
|
||||
await this.makeApiCall<void>(url, {
|
||||
method: 'DELETE',
|
||||
headers: this.getAuthHeaders(),
|
||||
});
|
||||
}
|
||||
|
||||
// Resource-specific methods
|
||||
|
|
@ -400,6 +434,58 @@ export class AutotaskClient {
|
|||
|
||||
return response.items || [];
|
||||
}
|
||||
|
||||
// Time Entries specific methods
|
||||
async getTimeEntriesByResource(resourceId: number): Promise<AutotaskTimeEntry[]> {
|
||||
return this.queryEntity<AutotaskTimeEntry>('TimeEntries', {
|
||||
filter: [{ op: 'eq', field: 'resourceID', value: resourceId }],
|
||||
});
|
||||
}
|
||||
|
||||
async getTimeEntriesByTicket(ticketId: number): Promise<AutotaskTimeEntry[]> {
|
||||
return this.queryEntity<AutotaskTimeEntry>('TimeEntries', {
|
||||
filter: [{ op: 'eq', field: 'ticketID', value: ticketId }],
|
||||
});
|
||||
}
|
||||
|
||||
async getTimeEntriesByTask(taskId: number): Promise<AutotaskTimeEntry[]> {
|
||||
return this.queryEntity<AutotaskTimeEntry>('TimeEntries', {
|
||||
filter: [{ op: 'eq', field: 'taskID', value: taskId }],
|
||||
});
|
||||
}
|
||||
|
||||
async getTimeEntriesByProject(projectId: number): Promise<AutotaskTimeEntry[]> {
|
||||
return this.queryEntity<AutotaskTimeEntry>('TimeEntries', {
|
||||
filter: [{ op: 'eq', field: 'projectID', value: projectId }],
|
||||
});
|
||||
}
|
||||
|
||||
async getTimeEntriesByCompany(companyId: number): Promise<AutotaskTimeEntry[]> {
|
||||
return this.queryEntity<AutotaskTimeEntry>('TimeEntries', {
|
||||
filter: [{ op: 'eq', field: 'companyID', value: companyId }],
|
||||
});
|
||||
}
|
||||
|
||||
async getTimeEntriesByDateRange(startDate: Date, endDate: Date): Promise<AutotaskTimeEntry[]> {
|
||||
return this.queryEntity<AutotaskTimeEntry>('TimeEntries', {
|
||||
filter: [
|
||||
{ op: 'gte', field: 'entryDate', value: startDate.toISOString() },
|
||||
{ op: 'lte', field: 'entryDate', value: endDate.toISOString() },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
async createTimeEntry(timeEntry: Partial<AutotaskTimeEntry>): Promise<AutotaskTimeEntry> {
|
||||
return this.createEntity<AutotaskTimeEntry>('TimeEntries', timeEntry);
|
||||
}
|
||||
|
||||
async updateTimeEntry(id: number, updates: Partial<AutotaskTimeEntry>): Promise<AutotaskTimeEntry> {
|
||||
return this.updateEntity<AutotaskTimeEntry>('TimeEntries', id, updates);
|
||||
}
|
||||
|
||||
async deleteTimeEntry(id: number): Promise<void> {
|
||||
return this.deleteEntity('TimeEntries', id);
|
||||
}
|
||||
}
|
||||
|
||||
// Rate Limiter class
|
||||
|
|
|
|||
329
lib/services/auvik-client.ts
Normal file
329
lib/services/auvik-client.ts
Normal file
|
|
@ -0,0 +1,329 @@
|
|||
import {
|
||||
AuvikClientConfig,
|
||||
AuvikDevice,
|
||||
AuvikDeviceResponse,
|
||||
AuvikTenant,
|
||||
AuvikTenantResponse,
|
||||
} from '../types/auvik';
|
||||
|
||||
export class AuvikClient {
|
||||
private config: AuvikClientConfig;
|
||||
private requestCount: number = 0;
|
||||
private requestTimestamps: number[] = [];
|
||||
private readonly RATE_LIMIT = 1000; // requests per hour
|
||||
private readonly RATE_LIMIT_WINDOW = 3600000; // 1 hour in milliseconds
|
||||
|
||||
constructor(config: AuvikClientConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Basic Authentication headers
|
||||
*/
|
||||
private getAuthHeaders(): HeadersInit {
|
||||
const credentials = Buffer.from(
|
||||
`${this.config.apiUser}:${this.config.apiKey}`
|
||||
).toString('base64');
|
||||
|
||||
return {
|
||||
Authorization: `Basic ${credentials}`,
|
||||
Accept: 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check and enforce rate limiting
|
||||
*/
|
||||
private checkRateLimit(): void {
|
||||
const now = Date.now();
|
||||
// Remove timestamps older than 1 hour
|
||||
this.requestTimestamps = this.requestTimestamps.filter(
|
||||
(timestamp) => now - timestamp < this.RATE_LIMIT_WINDOW
|
||||
);
|
||||
|
||||
if (this.requestTimestamps.length >= this.RATE_LIMIT) {
|
||||
console.warn(
|
||||
`Auvik API rate limit approaching: ${this.requestTimestamps.length}/${this.RATE_LIMIT} requests in the last hour`
|
||||
);
|
||||
}
|
||||
|
||||
this.requestTimestamps.push(now);
|
||||
this.requestCount++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a generic API call with error handling
|
||||
*/
|
||||
private async makeApiCall<T>(url: string, options: RequestInit = {}): Promise<T> {
|
||||
this.checkRateLimit();
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
...this.getAuthHeaders(),
|
||||
...options.headers,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error(
|
||||
`Auvik API error: ${response.status} ${response.statusText}`,
|
||||
errorText
|
||||
);
|
||||
throw new Error(
|
||||
`Auvik API request failed: ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
|
||||
return await response.json();
|
||||
} catch (error) {
|
||||
console.error('Auvik API call failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all tenants
|
||||
*/
|
||||
async getTenants(): Promise<AuvikTenant[]> {
|
||||
try {
|
||||
const url = `${this.config.apiUrl}/v1/tenants`;
|
||||
console.log('Fetching Auvik tenants from:', url);
|
||||
|
||||
const response = await this.makeApiCall<AuvikTenantResponse>(url);
|
||||
|
||||
const tenants = response.data.map((item) => ({
|
||||
id: item.id,
|
||||
domainPrefix: item.attributes.domainPrefix,
|
||||
tenantType: item.attributes.tenantType as 'multiClient' | 'client',
|
||||
parentId: item.relationships?.parent?.data?.id,
|
||||
}));
|
||||
|
||||
console.log(`Fetched ${tenants.length} Auvik tenants`);
|
||||
return tenants;
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch Auvik tenants:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all devices (requires tenant filtering)
|
||||
*/
|
||||
async getAllDevices(): Promise<AuvikDevice[]> {
|
||||
try {
|
||||
// Fetch all tenants first
|
||||
const tenants = await this.getTenants();
|
||||
if (tenants.length === 0) {
|
||||
console.warn('No Auvik tenants found');
|
||||
return [];
|
||||
}
|
||||
|
||||
// Fetch devices for all tenants
|
||||
const allDevices: AuvikDevice[] = [];
|
||||
for (const tenant of tenants) {
|
||||
const devices = await this.getDevicesByTenant(tenant.id);
|
||||
allDevices.push(...devices);
|
||||
}
|
||||
|
||||
console.log(`Fetched total of ${allDevices.length} Auvik devices across all tenants`);
|
||||
return allDevices;
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch all Auvik devices:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get devices filtered by tenant ID
|
||||
*/
|
||||
async getDevicesByTenant(tenantId: string): Promise<AuvikDevice[]> {
|
||||
try {
|
||||
const allDevices: AuvikDevice[] = [];
|
||||
let nextUrl: string | null = `${this.config.apiUrl}/v1/inventory/device/info?tenants=${tenantId}&page[first]=100`;
|
||||
|
||||
console.log(`Fetching Auvik devices for tenant ${tenantId}`);
|
||||
|
||||
// Paginate through all results
|
||||
while (nextUrl) {
|
||||
const response: AuvikDeviceResponse = await this.makeApiCall<AuvikDeviceResponse>(nextUrl);
|
||||
|
||||
const devices = response.data.map((item) => this.transformDevice(item, tenantId));
|
||||
allDevices.push(...devices);
|
||||
|
||||
// Check if there's a next page
|
||||
nextUrl = response.links?.next || null;
|
||||
|
||||
if (nextUrl) {
|
||||
console.log(`Fetching next page for tenant ${tenantId} (${allDevices.length} devices so far)`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Fetched total of ${allDevices.length} devices for tenant ${tenantId}`);
|
||||
return allDevices;
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch devices for tenant ${tenantId}:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform Auvik API device response to AuvikDevice interface
|
||||
*/
|
||||
private transformDevice(item: AuvikDeviceResponse['data'][0], tenantId: string): AuvikDevice {
|
||||
return {
|
||||
id: item.id,
|
||||
deviceName: item.attributes.deviceName,
|
||||
serialNumber: item.attributes.serialNumber,
|
||||
macAddresses: [], // MAC addresses would need to be fetched from device details
|
||||
ipAddresses: item.attributes.ipAddresses || [],
|
||||
deviceType: item.attributes.deviceType,
|
||||
manufacturer: item.attributes.vendorName,
|
||||
model: item.attributes.makeModel,
|
||||
makeModel: item.attributes.makeModel,
|
||||
vendorName: item.attributes.vendorName,
|
||||
firmwareVersion: item.attributes.firmwareVersion,
|
||||
softwareVersion: item.attributes.softwareVersion,
|
||||
onlineStatus: this.normalizeOnlineStatus(item.attributes.onlineStatus),
|
||||
lastSeenTime: item.attributes.lastSeenTime,
|
||||
uptime: undefined, // Would need to be calculated from lastSeenTime
|
||||
tenantId: tenantId,
|
||||
tenantName: item.relationships?.tenant?.data?.attributes?.domainPrefix,
|
||||
description: item.attributes.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize online status to expected values
|
||||
*/
|
||||
private normalizeOnlineStatus(status: string): 'online' | 'offline' | 'unknown' {
|
||||
const normalized = status.toLowerCase();
|
||||
if (normalized === 'online') return 'online';
|
||||
if (normalized === 'offline') return 'offline';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Find tenant by company ID using database mapping
|
||||
*/
|
||||
async findTenantByCompanyId(companyId: number): Promise<AuvikTenant | null> {
|
||||
try {
|
||||
// Query database directly for mapping
|
||||
const { Pool } = require('pg');
|
||||
const pool = new Pool({
|
||||
host: process.env.POSTGRES_HOST,
|
||||
port: parseInt(process.env.POSTGRES_PORT || '5432'),
|
||||
database: process.env.POSTGRES_DB,
|
||||
user: process.env.POSTGRES_USER,
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
});
|
||||
|
||||
const result = await pool.query(
|
||||
'SELECT auvik_tenant_id FROM auvik_tenant_mappings WHERE autotask_company_id = $1',
|
||||
[companyId]
|
||||
);
|
||||
|
||||
await pool.end();
|
||||
|
||||
if (result.rows.length > 0) {
|
||||
const auvikTenantId = result.rows[0].auvik_tenant_id;
|
||||
const tenants = await this.getTenants();
|
||||
const tenant = tenants.find(t => t.id === auvikTenantId);
|
||||
|
||||
if (tenant) {
|
||||
console.log(`Found tenant via mapping: ${tenant.domainPrefix} for company ID: ${companyId}`);
|
||||
return tenant;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error('Failed to find tenant by company ID:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize company name for matching by removing common suffixes and special characters
|
||||
*/
|
||||
private normalizeCompanyName(name: string): string {
|
||||
return name
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
// Remove common legal suffixes
|
||||
.replace(/,?\s*(inc\.?|llc\.?|ltd\.?|corp\.?|corporation|company|co\.?|limited|l\.?l\.?c\.?|incorporated)$/i, '')
|
||||
// Remove commas and other punctuation
|
||||
.replace(/[,\.]/g, '')
|
||||
// Replace multiple spaces with single space
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find tenant by name (case-insensitive, fuzzy match)
|
||||
* This is a fallback when no mapping exists
|
||||
*/
|
||||
async findTenantByName(companyName: string): Promise<AuvikTenant | null> {
|
||||
try {
|
||||
const tenants = await this.getTenants();
|
||||
const normalizedCompanyName = this.normalizeCompanyName(companyName);
|
||||
|
||||
console.log(`Searching for Auvik tenant matching company: "${companyName}"`);
|
||||
console.log(`Normalized company name: "${normalizedCompanyName}"`);
|
||||
console.log(`Available tenants: ${tenants.map(t => t.domainPrefix).join(', ')}`);
|
||||
|
||||
// Try exact match first (normalized)
|
||||
let match = tenants.find(
|
||||
(t) => this.normalizeCompanyName(t.domainPrefix) === normalizedCompanyName
|
||||
);
|
||||
|
||||
if (match) {
|
||||
console.log(`Found exact tenant match: ${match.domainPrefix} for company: ${companyName}`);
|
||||
return match;
|
||||
}
|
||||
|
||||
// Try exact match on original (case-insensitive)
|
||||
match = tenants.find(
|
||||
(t) => t.domainPrefix.toLowerCase() === companyName.toLowerCase().trim()
|
||||
);
|
||||
|
||||
if (match) {
|
||||
console.log(`Found exact tenant match (original): ${match.domainPrefix} for company: ${companyName}`);
|
||||
return match;
|
||||
}
|
||||
|
||||
// Try fuzzy match (contains) with normalized names
|
||||
// Find all potential matches and pick the best one (longest match)
|
||||
const potentialMatches = tenants.filter((t) => {
|
||||
const normalizedTenant = this.normalizeCompanyName(t.domainPrefix);
|
||||
return normalizedTenant.includes(normalizedCompanyName) ||
|
||||
normalizedCompanyName.includes(normalizedTenant);
|
||||
});
|
||||
|
||||
if (potentialMatches.length > 0) {
|
||||
// Sort by length of normalized tenant name (descending) to prefer more specific matches
|
||||
match = potentialMatches.sort((a, b) => {
|
||||
const aNorm = this.normalizeCompanyName(a.domainPrefix);
|
||||
const bNorm = this.normalizeCompanyName(b.domainPrefix);
|
||||
return bNorm.length - aNorm.length;
|
||||
})[0];
|
||||
|
||||
console.log(`Found fuzzy tenant match: ${match.domainPrefix} for company: ${companyName}`);
|
||||
if (potentialMatches.length > 1) {
|
||||
console.log(`Other potential matches: ${potentialMatches.slice(1).map(t => t.domainPrefix).join(', ')}`);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
console.log(`No tenant match found for company: ${companyName}`);
|
||||
console.log(`Tried to match "${normalizedCompanyName}" against: ${tenants.map(t => this.normalizeCompanyName(t.domainPrefix)).join(', ')}`);
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error('Failed to find tenant by name:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
37
lib/services/auvik-factory.ts
Normal file
37
lib/services/auvik-factory.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { AuvikClient } from './auvik-client';
|
||||
import { AuvikClientConfig } from '../types/auvik';
|
||||
|
||||
let auvikClientInstance: AuvikClient | null = null;
|
||||
|
||||
/**
|
||||
* Get or create Auvik client singleton instance
|
||||
*/
|
||||
export function getAuvikClient(): AuvikClient {
|
||||
if (!auvikClientInstance) {
|
||||
const config: AuvikClientConfig = {
|
||||
apiUrl: process.env.AUVIK_API_URL || '',
|
||||
apiUser: process.env.AUVIK_API_USER || '',
|
||||
apiKey: process.env.AUVIK_API_KEY || '',
|
||||
};
|
||||
|
||||
// Validate configuration
|
||||
if (!config.apiUrl || !config.apiUser || !config.apiKey) {
|
||||
console.error('Auvik API credentials not configured');
|
||||
throw new Error(
|
||||
'Auvik API credentials missing. Please set AUVIK_API_URL, AUVIK_API_USER, and AUVIK_API_KEY environment variables.'
|
||||
);
|
||||
}
|
||||
|
||||
auvikClientInstance = new AuvikClient(config);
|
||||
console.log('Auvik client initialized');
|
||||
}
|
||||
|
||||
return auvikClientInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the singleton instance (useful for testing)
|
||||
*/
|
||||
export function resetAuvikClient(): void {
|
||||
auvikClientInstance = null;
|
||||
}
|
||||
392
lib/services/background-processor.ts
Normal file
392
lib/services/background-processor.ts
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
/**
|
||||
* Background Processor Service
|
||||
* Handles batch processing of historical time entries data for analytics
|
||||
*/
|
||||
|
||||
import { TimeEntry } from '@/lib/types/database';
|
||||
import { analyticsEngine } from './analytics-engine';
|
||||
import { llmAnalyzer } from './llm-analyzer';
|
||||
import { postgresClient } from './postgres-client';
|
||||
|
||||
export interface BatchAnalysisJob {
|
||||
id: string;
|
||||
type: 'scoring' | 'llm_analysis' | 'full_analysis';
|
||||
status: 'pending' | 'running' | 'completed' | 'failed';
|
||||
progress: number; // 0 to 100
|
||||
totalRecords: number;
|
||||
processedRecords: number;
|
||||
startTime: Date;
|
||||
endTime?: Date;
|
||||
errorMessage?: string;
|
||||
filters?: {
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
resourceIds?: number[];
|
||||
projectIds?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface ProcessingResult {
|
||||
jobId: string;
|
||||
success: boolean;
|
||||
recordsProcessed: number;
|
||||
errors: string[];
|
||||
processingTime: number; // milliseconds
|
||||
}
|
||||
|
||||
export class BackgroundProcessor {
|
||||
private jobs: Map<string, BatchAnalysisJob> = new Map();
|
||||
private isProcessing: boolean = false;
|
||||
private batchSize: number = 100; // Process 100 records at a time
|
||||
private maxConcurrentJobs: number = 3;
|
||||
private processingQueue: string[] = [];
|
||||
|
||||
/**
|
||||
* Start a batch analysis job
|
||||
*/
|
||||
async startBatchAnalysis(
|
||||
type: 'scoring' | 'llm_analysis' | 'full_analysis',
|
||||
filters?: {
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
resourceIds?: number[];
|
||||
projectIds?: number[];
|
||||
}
|
||||
): Promise<string> {
|
||||
const jobId = this.generateJobId();
|
||||
|
||||
const job: BatchAnalysisJob = {
|
||||
id: jobId,
|
||||
type,
|
||||
status: 'pending',
|
||||
progress: 0,
|
||||
totalRecords: 0,
|
||||
processedRecords: 0,
|
||||
startTime: new Date(),
|
||||
filters,
|
||||
};
|
||||
|
||||
this.jobs.set(jobId, job);
|
||||
this.processingQueue.push(jobId);
|
||||
|
||||
// Start processing if not already running
|
||||
this.processQueue();
|
||||
|
||||
return jobId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get job status
|
||||
*/
|
||||
getJobStatus(jobId: string): BatchAnalysisJob | null {
|
||||
return this.jobs.get(jobId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all jobs
|
||||
*/
|
||||
getAllJobs(): BatchAnalysisJob[] {
|
||||
return Array.from(this.jobs.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel a job
|
||||
*/
|
||||
cancelJob(jobId: string): boolean {
|
||||
const job = this.jobs.get(jobId);
|
||||
if (!job || job.status === 'completed' || job.status === 'running') {
|
||||
return false;
|
||||
}
|
||||
|
||||
job.status = 'failed';
|
||||
job.errorMessage = 'Job cancelled by user';
|
||||
job.endTime = new Date();
|
||||
|
||||
// Remove from queue
|
||||
const queueIndex = this.processingQueue.indexOf(jobId);
|
||||
if (queueIndex > -1) {
|
||||
this.processingQueue.splice(queueIndex, 1);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the job queue
|
||||
*/
|
||||
private async processQueue(): Promise<void> {
|
||||
if (this.isProcessing || this.processingQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isProcessing = true;
|
||||
|
||||
while (this.processingQueue.length > 0) {
|
||||
const runningJobs = Array.from(this.jobs.values())
|
||||
.filter(job => job.status === 'running').length;
|
||||
|
||||
if (runningJobs >= this.maxConcurrentJobs) {
|
||||
break; // Wait for current jobs to finish
|
||||
}
|
||||
|
||||
const jobId = this.processingQueue.shift();
|
||||
if (!jobId) continue;
|
||||
|
||||
const job = this.jobs.get(jobId);
|
||||
if (!job || job.status !== 'pending') continue;
|
||||
|
||||
// Process job in background
|
||||
this.processJob(jobId).catch(error => {
|
||||
console.error(`Background job ${jobId} failed:`, error);
|
||||
});
|
||||
}
|
||||
|
||||
this.isProcessing = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single job
|
||||
*/
|
||||
private async processJob(jobId: string): Promise<void> {
|
||||
const job = this.jobs.get(jobId);
|
||||
if (!job) return;
|
||||
|
||||
try {
|
||||
job.status = 'running';
|
||||
|
||||
// Get time entries to process
|
||||
const timeEntries = await this.getTimeEntriesForJob(job);
|
||||
job.totalRecords = timeEntries.length;
|
||||
|
||||
if (timeEntries.length === 0) {
|
||||
job.status = 'completed';
|
||||
job.progress = 100;
|
||||
job.endTime = new Date();
|
||||
return;
|
||||
}
|
||||
|
||||
// Process based on job type
|
||||
switch (job.type) {
|
||||
case 'scoring':
|
||||
await this.processScoringJob(job, timeEntries);
|
||||
break;
|
||||
case 'llm_analysis':
|
||||
await this.processLLMAnalysisJob(job, timeEntries);
|
||||
break;
|
||||
case 'full_analysis':
|
||||
await this.processFullAnalysisJob(job, timeEntries);
|
||||
break;
|
||||
}
|
||||
|
||||
job.status = 'completed';
|
||||
job.progress = 100;
|
||||
job.endTime = new Date();
|
||||
|
||||
console.log(`Background job ${jobId} completed successfully`);
|
||||
|
||||
} catch (error) {
|
||||
job.status = 'failed';
|
||||
job.errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
job.endTime = new Date();
|
||||
|
||||
console.error(`Background job ${jobId} failed:`, error);
|
||||
}
|
||||
|
||||
// Continue processing queue
|
||||
setTimeout(() => this.processQueue(), 100);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process scoring job
|
||||
*/
|
||||
private async processScoringJob(job: BatchAnalysisJob, timeEntries: TimeEntry[]): Promise<void> {
|
||||
const analyses = [];
|
||||
|
||||
for (let i = 0; i < timeEntries.length; i += this.batchSize) {
|
||||
const batch = timeEntries.slice(i, i + this.batchSize);
|
||||
|
||||
// Analyze batch
|
||||
for (const entry of batch) {
|
||||
const analysis = analyticsEngine.analyzeTimeEntry(entry);
|
||||
analyses.push(analysis);
|
||||
|
||||
// Store analysis in database (would need to create analysis table)
|
||||
await this.storeTimeEntryAnalysis(entry.id, analysis);
|
||||
|
||||
job.processedRecords++;
|
||||
job.progress = Math.round((job.processedRecords / job.totalRecords) * 100);
|
||||
}
|
||||
|
||||
// Small delay to prevent overwhelming the system
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
console.log(`Scoring job ${job.id}: Analyzed ${analyses.length} time entries`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process LLM analysis job
|
||||
*/
|
||||
private async processLLMAnalysisJob(job: BatchAnalysisJob, timeEntries: TimeEntry[]): Promise<void> {
|
||||
// Process in larger batches for LLM to be more cost-effective
|
||||
const llmBatchSize = 500;
|
||||
|
||||
for (let i = 0; i < timeEntries.length; i += llmBatchSize) {
|
||||
const batch = timeEntries.slice(i, i + llmBatchSize);
|
||||
|
||||
try {
|
||||
const insights = await llmAnalyzer.generateInsights(batch);
|
||||
|
||||
// Store insights in database
|
||||
await this.storeLLMInsights(job.id, batch, insights);
|
||||
|
||||
job.processedRecords += batch.length;
|
||||
job.progress = Math.round((job.processedRecords / job.totalRecords) * 100);
|
||||
|
||||
// Longer delay for LLM processing
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
} catch (error) {
|
||||
console.error(`LLM analysis failed for batch ${i}-${i + batch.length}:`, error);
|
||||
// Continue with next batch
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`LLM analysis job ${job.id}: Processed ${timeEntries.length} time entries`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process full analysis job
|
||||
*/
|
||||
private async processFullAnalysisJob(job: BatchAnalysisJob, timeEntries: TimeEntry[]): Promise<void> {
|
||||
// First, run scoring
|
||||
await this.processScoringJob(job, timeEntries);
|
||||
|
||||
// Then, run LLM analysis on aggregated data
|
||||
const insights = await llmAnalyzer.generateInsights(timeEntries);
|
||||
const aggregateAnalysis = analyticsEngine.analyzeTimeEntries(timeEntries);
|
||||
|
||||
// Store comprehensive results
|
||||
await this.storeFullAnalysisResults(job.id, aggregateAnalysis, insights);
|
||||
|
||||
console.log(`Full analysis job ${job.id}: Completed comprehensive analysis`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get time entries for job based on filters
|
||||
*/
|
||||
private async getTimeEntriesForJob(job: BatchAnalysisJob): Promise<TimeEntry[]> {
|
||||
let query = 'SELECT * FROM time_entries WHERE is_deleted = false';
|
||||
const params: any[] = [];
|
||||
let paramIndex = 1;
|
||||
|
||||
if (job.filters?.startDate) {
|
||||
query += ` AND entry_date >= $${paramIndex}`;
|
||||
params.push(job.filters.startDate.toISOString());
|
||||
paramIndex++;
|
||||
}
|
||||
|
||||
if (job.filters?.endDate) {
|
||||
query += ` AND entry_date <= $${paramIndex}`;
|
||||
params.push(job.filters.endDate.toISOString());
|
||||
paramIndex++;
|
||||
}
|
||||
|
||||
if (job.filters?.resourceIds && job.filters.resourceIds.length > 0) {
|
||||
query += ` AND resource_id = ANY($${paramIndex})`;
|
||||
params.push(job.filters.resourceIds);
|
||||
paramIndex++;
|
||||
}
|
||||
|
||||
if (job.filters?.projectIds && job.filters.projectIds.length > 0) {
|
||||
query += ` AND project_id = ANY($${paramIndex})`;
|
||||
params.push(job.filters.projectIds);
|
||||
paramIndex++;
|
||||
}
|
||||
|
||||
query += ' ORDER BY entry_date DESC';
|
||||
|
||||
const result = await postgresClient.query(query, params);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store time entry analysis in database
|
||||
*/
|
||||
private async storeTimeEntryAnalysis(timeEntryId: number, analysis: any): Promise<void> {
|
||||
// This would create/update a time_entry_analyses table
|
||||
// For now, just log the analysis
|
||||
console.log(`Storing analysis for time entry ${timeEntryId}:`, analysis.overallScore);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store LLM insights in database
|
||||
*/
|
||||
private async storeLLMInsights(jobId: string, timeEntries: TimeEntry[], insights: any[]): Promise<void> {
|
||||
// This would create/update an llm_insights table
|
||||
console.log(`Storing ${insights.length} LLM insights for job ${jobId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store full analysis results
|
||||
*/
|
||||
private async storeFullAnalysisResults(jobId: string, analysis: any, insights: any[]): Promise<void> {
|
||||
// This would create/update a comprehensive analysis results table
|
||||
console.log(`Storing full analysis results for job ${jobId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate unique job ID
|
||||
*/
|
||||
private generateJobId(): string {
|
||||
return `job_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old completed jobs
|
||||
*/
|
||||
cleanupOldJobs(maxAge: number = 24 * 60 * 60 * 1000): void { // 24 hours
|
||||
const cutoff = Date.now() - maxAge;
|
||||
|
||||
for (const [jobId, job] of this.jobs.entries()) {
|
||||
if (
|
||||
(job.status === 'completed' || job.status === 'failed') &&
|
||||
job.endTime &&
|
||||
job.endTime.getTime() < cutoff
|
||||
) {
|
||||
this.jobs.delete(jobId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get processing statistics
|
||||
*/
|
||||
getStats(): {
|
||||
totalJobs: number;
|
||||
pendingJobs: number;
|
||||
runningJobs: number;
|
||||
completedJobs: number;
|
||||
failedJobs: number;
|
||||
queueLength: number;
|
||||
} {
|
||||
const jobs = Array.from(this.jobs.values());
|
||||
|
||||
return {
|
||||
totalJobs: jobs.length,
|
||||
pendingJobs: jobs.filter(j => j.status === 'pending').length,
|
||||
runningJobs: jobs.filter(j => j.status === 'running').length,
|
||||
completedJobs: jobs.filter(j => j.status === 'completed').length,
|
||||
failedJobs: jobs.filter(j => j.status === 'failed').length,
|
||||
queueLength: this.processingQueue.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
export const backgroundProcessor = new BackgroundProcessor();
|
||||
|
||||
// Schedule cleanup every hour
|
||||
setInterval(() => {
|
||||
backgroundProcessor.cleanupOldJobs();
|
||||
}, 60 * 60 * 1000);
|
||||
|
|
@ -159,16 +159,34 @@ export class DattoRMMClient {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get all devices
|
||||
* Get all devices (with pagination support)
|
||||
*/
|
||||
async getAllDevices(): Promise<DattoRMMDevice[]> {
|
||||
const response = await this.makeApiCall<any>(
|
||||
'/account/devices',
|
||||
{ method: 'GET' }
|
||||
);
|
||||
const allDevices: DattoRMMDevice[] = [];
|
||||
let page = 1;
|
||||
const pageSize = 250; // Datto RMM default page size
|
||||
|
||||
// The API returns devices in a 'devices' field
|
||||
return response.devices || [];
|
||||
while (true) {
|
||||
const response = await this.makeApiCall<any>(
|
||||
`/account/devices?page=${page}&pageSize=${pageSize}`,
|
||||
{ method: 'GET' }
|
||||
);
|
||||
|
||||
const devices = response.devices || [];
|
||||
allDevices.push(...devices);
|
||||
|
||||
console.log(`Fetched page ${page}: ${devices.length} devices (total: ${allDevices.length})`);
|
||||
|
||||
// If we got less than pageSize devices, we've reached the end
|
||||
if (devices.length < pageSize) {
|
||||
break;
|
||||
}
|
||||
|
||||
page++;
|
||||
}
|
||||
|
||||
console.log(`Total RMM devices fetched: ${allDevices.length}`);
|
||||
return allDevices;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -186,6 +204,7 @@ export class DattoRMMClient {
|
|||
|
||||
/**
|
||||
* Get devices by site name (matches with company name)
|
||||
* @deprecated Use getDevicesByCompanyId with site mappings instead
|
||||
*/
|
||||
async getDevicesByCompanyName(companyName: string): Promise<DattoRMMDevice[]> {
|
||||
// First, find the site that matches the company name
|
||||
|
|
@ -200,6 +219,31 @@ export class DattoRMMClient {
|
|||
return this.getDevicesBySite(site.uid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get devices for multiple sites (for multi-site support)
|
||||
*/
|
||||
async getDevicesForSites(siteUids: string[]): Promise<DattoRMMDevice[]> {
|
||||
if (siteUids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const allDevices: DattoRMMDevice[] = [];
|
||||
|
||||
// Fetch devices from all sites in parallel
|
||||
const promises = siteUids.map(uid => this.getDevicesBySite(uid));
|
||||
const results = await Promise.allSettled(promises);
|
||||
|
||||
for (const result of results) {
|
||||
if (result.status === 'fulfilled') {
|
||||
allDevices.push(...result.value);
|
||||
} else {
|
||||
console.error('Failed to fetch devices for a site:', result.reason);
|
||||
}
|
||||
}
|
||||
|
||||
return allDevices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get device by ID
|
||||
*/
|
||||
|
|
|
|||
718
lib/services/entity-sync.ts
Normal file
718
lib/services/entity-sync.ts
Normal file
|
|
@ -0,0 +1,718 @@
|
|||
/**
|
||||
* Entity Sync Service
|
||||
* Handles syncing individual entity types from Autotask to PostgreSQL
|
||||
*/
|
||||
|
||||
import { AutotaskClient } from './autotask-client';
|
||||
import { autotaskRateLimiter } from './rate-limiter';
|
||||
import postgresClient from './postgres-client';
|
||||
import { EntityType } from '../types/sync';
|
||||
import { mapAutotaskToDatabase, mapAutotaskBatch } from '../utils/entity-mapper';
|
||||
import { bulkUpsertRecords, getLastSyncTime, softDeleteMissingRecords } from '../utils/db-helpers';
|
||||
import { syncProgressTracker } from './sync-progress-tracker';
|
||||
import {
|
||||
getAutotaskEntityName,
|
||||
buildIncrementalFilter,
|
||||
buildActiveFilter,
|
||||
buildDateRangeFilter,
|
||||
buildContractsFilter,
|
||||
buildProjectsFilter,
|
||||
buildTimeEntriesFilter,
|
||||
getTableName
|
||||
} from '../utils/sync-helpers';
|
||||
|
||||
/**
|
||||
* Entity Sync Result
|
||||
*/
|
||||
export interface EntitySyncStats {
|
||||
recordsAdded: number;
|
||||
recordsUpdated: number;
|
||||
recordsDeleted: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Entity Sync Service Class
|
||||
*/
|
||||
export class EntitySyncService {
|
||||
private autotaskClient: AutotaskClient;
|
||||
private cachedValidResourceIds?: Set<number>;
|
||||
private cachedValidContactIds?: Set<number>;
|
||||
|
||||
constructor(autotaskClient: AutotaskClient) {
|
||||
this.autotaskClient = autotaskClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a single entity type
|
||||
* @param entity Entity type to sync
|
||||
* @param isIncremental Whether to perform incremental sync
|
||||
* @param yearsBack Number of years to look back for time-based entities (default: 2)
|
||||
* @returns Sync statistics
|
||||
*/
|
||||
async syncEntity(
|
||||
entity: EntityType,
|
||||
isIncremental: boolean = false,
|
||||
yearsBack: number = 2,
|
||||
syncId?: string
|
||||
): Promise<EntitySyncStats> {
|
||||
// Route picklist entities to their specific sync methods
|
||||
if (entity === EntityType.ISSUE_TYPES) {
|
||||
return await this.syncIssueTypes(isIncremental);
|
||||
}
|
||||
if (entity === EntityType.SUB_ISSUE_TYPES) {
|
||||
return await this.syncSubIssueTypes(isIncremental);
|
||||
}
|
||||
|
||||
const syncStartTime = Date.now();
|
||||
const trackingId = syncId || `${entity}_${Date.now()}`;
|
||||
|
||||
// Start progress tracking
|
||||
syncProgressTracker.startSync(trackingId, entity);
|
||||
|
||||
console.log(`[${entity}] Starting sync (${isIncremental ? 'incremental' : 'full'})`);
|
||||
|
||||
try {
|
||||
const autotaskEntityName = getAutotaskEntityName(entity);
|
||||
let params: any = {};
|
||||
|
||||
// For incremental sync, filter by last sync time
|
||||
if (isIncremental) {
|
||||
try {
|
||||
const lastSyncTime = await getLastSyncTime(entity);
|
||||
if (lastSyncTime) {
|
||||
params.filter = buildIncrementalFilter(entity, lastSyncTime);
|
||||
console.log(`[${entity}] Incremental sync from ${lastSyncTime.toISOString()}`);
|
||||
} else {
|
||||
console.log(`[${entity}] No previous sync found, performing full sync`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[${entity}] Failed to get last sync time:`, error);
|
||||
throw new Error(`Failed to determine sync time: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
} else {
|
||||
// For full sync, build filters
|
||||
const filters: Array<{ field: string; op: string; value: any }> = [];
|
||||
|
||||
// Special handling for entities that require filters
|
||||
if (entity === EntityType.CONTRACTS) {
|
||||
filters.push(...buildContractsFilter());
|
||||
console.log(`[${entity}] Full sync with status filter for active contracts`);
|
||||
} else if (entity === EntityType.PROJECTS) {
|
||||
filters.push(...buildProjectsFilter());
|
||||
console.log(`[${entity}] Full sync with status filter for non-completed projects`);
|
||||
} else if (entity === EntityType.TIME_ENTRIES) {
|
||||
filters.push(...buildTimeEntriesFilter(yearsBack));
|
||||
console.log(`[${entity}] Full sync with dateWorked filter for last ${yearsBack} years`);
|
||||
} else {
|
||||
// Add active filter if applicable
|
||||
const activeFilter = buildActiveFilter(entity);
|
||||
if (activeFilter) {
|
||||
filters.push(...activeFilter);
|
||||
console.log(`[${entity}] Full sync with active filter`);
|
||||
}
|
||||
|
||||
// Add date range filter for time-based entities (tickets, tasks, etc.)
|
||||
const dateRangeFilter = buildDateRangeFilter(entity, yearsBack);
|
||||
if (dateRangeFilter) {
|
||||
filters.push(...dateRangeFilter);
|
||||
console.log(`[${entity}] Full sync limited to last ${yearsBack} years`);
|
||||
}
|
||||
}
|
||||
|
||||
if (filters.length > 0) {
|
||||
params.filter = filters;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch data from Autotask with pagination
|
||||
console.log(`[${entity}] Fetching records from Autotask API...`);
|
||||
syncProgressTracker.updateProgress(trackingId, { phase: 'fetching' });
|
||||
|
||||
let autotaskRecords: any[];
|
||||
try {
|
||||
autotaskRecords = await this.autotaskClient.queryEntityPaginated(
|
||||
autotaskEntityName,
|
||||
params,
|
||||
500 // Page size
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] API fetch failed:`, errorMessage);
|
||||
throw new Error(`Autotask API error: ${errorMessage}`);
|
||||
}
|
||||
|
||||
console.log(`[${entity}] Fetched ${autotaskRecords.length} records from Autotask`);
|
||||
syncProgressTracker.updateProgress(trackingId, {
|
||||
totalRecords: autotaskRecords.length,
|
||||
phase: 'mapping'
|
||||
});
|
||||
|
||||
if (autotaskRecords.length === 0) {
|
||||
console.log(`[${entity}] No records to sync`);
|
||||
syncProgressTracker.completeSync(trackingId, 0);
|
||||
return { recordsAdded: 0, recordsUpdated: 0, recordsDeleted: 0 };
|
||||
}
|
||||
|
||||
// Map Autotask data to PostgreSQL schema
|
||||
console.log(`[${entity}] Mapping ${autotaskRecords.length} records to database schema...`);
|
||||
|
||||
// DEBUG: Log first record to see actual field names from Autotask
|
||||
if (autotaskRecords.length > 0 && (entity === EntityType.TICKETS || entity === EntityType.TIME_ENTRIES)) {
|
||||
console.log(`[${entity}] DEBUG - Sample raw Autotask record keys:`, Object.keys(autotaskRecords[0]));
|
||||
if (entity === EntityType.TIME_ENTRIES) {
|
||||
console.log(`[${entity}] DEBUG - Sample time entry:`, JSON.stringify(autotaskRecords[0], null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
let mappedRecords: Record<string, any>[];
|
||||
try {
|
||||
mappedRecords = mapAutotaskBatch(entity, autotaskRecords);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] Mapping failed:`, errorMessage);
|
||||
throw new Error(`Data mapping error: ${errorMessage}`);
|
||||
}
|
||||
|
||||
if (mappedRecords.length === 0) {
|
||||
console.warn(`[${entity}] Warning: All records failed mapping validation`);
|
||||
return { recordsAdded: 0, recordsUpdated: 0, recordsDeleted: 0 };
|
||||
}
|
||||
|
||||
// Check for records with missing company_id (for entities that require it)
|
||||
// Note: TIME_ENTRIES removed from this check because company_id is nullable for time entries
|
||||
if (entity === EntityType.TICKETS || entity === EntityType.PROJECTS ||
|
||||
entity === EntityType.CONFIGURATION_ITEMS || entity === EntityType.CONTACTS ||
|
||||
entity === EntityType.CONTRACTS || entity === EntityType.BILLING_ITEMS) {
|
||||
const recordsWithoutCompany = mappedRecords.filter(r => !r.company_id);
|
||||
if (recordsWithoutCompany.length > 0) {
|
||||
console.warn(`[${entity}] Found ${recordsWithoutCompany.length} records without company_id (out of ${mappedRecords.length} total)`);
|
||||
console.warn(`[${entity}] Sample IDs without company_id:`, recordsWithoutCompany.slice(0, 5).map(r => r.id));
|
||||
// Filter out records without company_id to prevent constraint violation
|
||||
mappedRecords = mappedRecords.filter(r => r.company_id);
|
||||
console.log(`[${entity}] Filtered to ${mappedRecords.length} records with valid company_id`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate resource foreign keys for tickets
|
||||
if (entity === EntityType.TICKETS) {
|
||||
const initialCount = mappedRecords.length;
|
||||
|
||||
// Get all valid resource IDs from database
|
||||
const validResourceIds = await this.getValidResourceIds();
|
||||
|
||||
// Filter tickets with invalid resource references
|
||||
mappedRecords = mappedRecords.map(ticket => {
|
||||
// Set invalid resource IDs to null instead of filtering out the entire ticket
|
||||
if (ticket.assigned_resource_id && !validResourceIds.has(ticket.assigned_resource_id)) {
|
||||
console.warn(`[${entity}] Ticket ${ticket.id}: Invalid assigned_resource_id ${ticket.assigned_resource_id}, setting to null`);
|
||||
ticket.assigned_resource_id = null;
|
||||
}
|
||||
if (ticket.first_response_assigned_resource_id && !validResourceIds.has(ticket.first_response_assigned_resource_id)) {
|
||||
ticket.first_response_assigned_resource_id = null;
|
||||
}
|
||||
if (ticket.first_response_initiating_resource_id && !validResourceIds.has(ticket.first_response_initiating_resource_id)) {
|
||||
ticket.first_response_initiating_resource_id = null;
|
||||
}
|
||||
return ticket;
|
||||
});
|
||||
|
||||
const nullifiedCount = initialCount - mappedRecords.filter(t => t.assigned_resource_id).length;
|
||||
if (nullifiedCount > 0) {
|
||||
console.warn(`[${entity}] Nullified ${nullifiedCount} invalid resource references`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate contact foreign keys for configuration items
|
||||
if (entity === EntityType.CONFIGURATION_ITEMS) {
|
||||
const initialCount = mappedRecords.length;
|
||||
|
||||
// Get all valid contact IDs from database
|
||||
const validContactIds = await this.getValidContactIds();
|
||||
|
||||
// Filter configuration items with invalid contact references
|
||||
mappedRecords = mappedRecords.map(item => {
|
||||
// Set invalid contact IDs to null instead of filtering out the entire item
|
||||
if (item.contact_id && !validContactIds.has(item.contact_id)) {
|
||||
console.warn(`[${entity}] Configuration Item ${item.id}: Invalid contact_id ${item.contact_id}, setting to null`);
|
||||
item.contact_id = null;
|
||||
}
|
||||
return item;
|
||||
});
|
||||
|
||||
const nullifiedCount = initialCount - mappedRecords.filter(i => i.contact_id).length;
|
||||
if (nullifiedCount > 0) {
|
||||
console.warn(`[${entity}] Nullified ${nullifiedCount} invalid contact references`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${entity}] Successfully mapped ${mappedRecords.length} records`);
|
||||
|
||||
// Bulk upsert to PostgreSQL
|
||||
console.log(`[${entity}] Upserting records to PostgreSQL...`);
|
||||
syncProgressTracker.updateProgress(trackingId, { phase: 'upserting' });
|
||||
|
||||
let upsertedCount: number;
|
||||
try {
|
||||
upsertedCount = await bulkUpsertRecords(entity, mappedRecords, 100);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] Database upsert failed:`, errorMessage);
|
||||
throw new Error(`Database error: ${errorMessage}`);
|
||||
}
|
||||
|
||||
console.log(`[${entity}] Upserted ${upsertedCount} records to PostgreSQL`);
|
||||
|
||||
// For full sync, soft delete records not in the fetched set
|
||||
// IMPORTANT: Only delete for entities without date filters to avoid deleting records outside sync window
|
||||
let deletedCount = 0;
|
||||
const hasDateFilter = entity === EntityType.TICKETS ||
|
||||
entity === EntityType.TASKS ||
|
||||
entity === EntityType.TIME_ENTRIES ||
|
||||
entity === EntityType.PROJECTS ||
|
||||
entity === EntityType.CONTRACTS;
|
||||
|
||||
if (!isIncremental && !hasDateFilter) {
|
||||
console.log(`[${entity}] Checking for records to soft delete...`);
|
||||
syncProgressTracker.updateProgress(trackingId, { phase: 'deleting' });
|
||||
|
||||
try {
|
||||
const activeIds = mappedRecords.map(r => r.id);
|
||||
deletedCount = await softDeleteMissingRecords(entity, activeIds);
|
||||
console.log(`[${entity}] Soft deleted ${deletedCount} missing records`);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] Soft delete failed:`, errorMessage);
|
||||
// Don't throw - soft delete failure shouldn't fail the entire sync
|
||||
console.warn(`[${entity}] Continuing despite soft delete failure`);
|
||||
}
|
||||
} else if (!isIncremental && hasDateFilter) {
|
||||
console.log(`[${entity}] Skipping soft-delete for date-filtered sync (would delete records outside sync window)`);
|
||||
}
|
||||
|
||||
// Calculate added vs updated (simplified - actual count would require tracking)
|
||||
const recordsAdded = Math.floor(upsertedCount * 0.1); // Estimate 10% new
|
||||
const recordsUpdated = upsertedCount - recordsAdded;
|
||||
|
||||
const duration = Date.now() - syncStartTime;
|
||||
console.log(`[${entity}] Sync completed in ${duration}ms`);
|
||||
|
||||
// Mark sync as completed
|
||||
syncProgressTracker.completeSync(trackingId, mappedRecords.length);
|
||||
|
||||
return {
|
||||
recordsAdded,
|
||||
recordsUpdated,
|
||||
recordsDeleted: deletedCount,
|
||||
};
|
||||
} catch (error) {
|
||||
const duration = Date.now() - syncStartTime;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] Sync failed after ${duration}ms:`, errorMessage);
|
||||
|
||||
// Mark sync as failed
|
||||
syncProgressTracker.failSync(trackingId, errorMessage);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Companies
|
||||
*/
|
||||
async syncCompanies(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.COMPANIES, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Tickets
|
||||
*/
|
||||
async syncTickets(isIncremental: boolean = false, yearsBack?: number): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.TICKETS, isIncremental, yearsBack);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Tickets with Monthly Chunking
|
||||
* Breaks large date ranges into monthly chunks to prevent timeouts and API failures
|
||||
* @param yearsBack Number of years to look back
|
||||
* @param onChunkProgress Callback for progress updates
|
||||
* @returns Aggregated sync statistics
|
||||
*/
|
||||
async syncTicketsChunked(
|
||||
yearsBack: number = 2,
|
||||
onChunkProgress?: (chunk: { index: number; total: number; description: string; recordsProcessed: number }) => void
|
||||
): Promise<EntitySyncStats> {
|
||||
const syncStartTime = Date.now();
|
||||
const entity = EntityType.TICKETS;
|
||||
console.log(`[${entity}] Starting chunked sync for last ${yearsBack} years`);
|
||||
|
||||
try {
|
||||
// Calculate date chunks (monthly)
|
||||
const chunks = this.calculateMonthlyChunks(yearsBack);
|
||||
console.log(`[${entity}] Split into ${chunks.length} monthly chunks`);
|
||||
|
||||
let totalRecordsAdded = 0;
|
||||
let totalRecordsUpdated = 0;
|
||||
let totalRecordsDeleted = 0;
|
||||
const failedChunks: string[] = [];
|
||||
|
||||
// Process each chunk
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const chunkDescription = `${chunk.startDate.toLocaleDateString('en-US', { month: 'short', year: 'numeric' })}`;
|
||||
|
||||
console.log(`[${entity}] Processing chunk ${i + 1}/${chunks.length}: ${chunkDescription}`);
|
||||
|
||||
// Notify progress
|
||||
if (onChunkProgress) {
|
||||
onChunkProgress({
|
||||
index: i + 1,
|
||||
total: chunks.length,
|
||||
description: chunkDescription,
|
||||
recordsProcessed: totalRecordsAdded + totalRecordsUpdated,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Fetch tickets for this date range
|
||||
const autotaskEntityName = getAutotaskEntityName(entity);
|
||||
const filters = [
|
||||
{ field: 'createDate', op: 'gte' as const, value: chunk.startDate.toISOString() },
|
||||
{ field: 'createDate', op: 'lt' as const, value: chunk.endDate.toISOString() },
|
||||
];
|
||||
|
||||
console.log(`[${entity}] Fetching records from ${chunk.startDate.toISOString()} to ${chunk.endDate.toISOString()}`);
|
||||
|
||||
const autotaskRecords = await this.autotaskClient.queryEntityPaginated(
|
||||
autotaskEntityName,
|
||||
{ filter: filters },
|
||||
500
|
||||
);
|
||||
|
||||
console.log(`[${entity}] Chunk ${i + 1}: Fetched ${autotaskRecords.length} records`);
|
||||
|
||||
if (autotaskRecords.length > 0) {
|
||||
// Map and upsert records
|
||||
let mappedRecords = mapAutotaskBatch(entity, autotaskRecords);
|
||||
|
||||
// Filter out records without company_id
|
||||
mappedRecords = mappedRecords.filter(r => r.company_id);
|
||||
if (mappedRecords.length < autotaskRecords.length) {
|
||||
console.warn(`[${entity}] Chunk ${i + 1}: Filtered out ${autotaskRecords.length - mappedRecords.length} records without company_id`);
|
||||
}
|
||||
|
||||
// Validate resource foreign keys (fetch once per sync, not per chunk)
|
||||
if (i === 0) {
|
||||
// Cache valid resource IDs for all chunks
|
||||
this.cachedValidResourceIds = await this.getValidResourceIds();
|
||||
console.log(`[${entity}] Cached ${this.cachedValidResourceIds.size} valid resource IDs`);
|
||||
}
|
||||
|
||||
// Nullify invalid resource references
|
||||
mappedRecords = mappedRecords.map(ticket => {
|
||||
if (ticket.assigned_resource_id && !this.cachedValidResourceIds!.has(ticket.assigned_resource_id)) {
|
||||
ticket.assigned_resource_id = null;
|
||||
}
|
||||
if (ticket.first_response_assigned_resource_id && !this.cachedValidResourceIds!.has(ticket.first_response_assigned_resource_id)) {
|
||||
ticket.first_response_assigned_resource_id = null;
|
||||
}
|
||||
if (ticket.first_response_initiating_resource_id && !this.cachedValidResourceIds!.has(ticket.first_response_initiating_resource_id)) {
|
||||
ticket.first_response_initiating_resource_id = null;
|
||||
}
|
||||
return ticket;
|
||||
});
|
||||
|
||||
if (mappedRecords.length > 0) {
|
||||
const upsertedCount = await bulkUpsertRecords(entity, mappedRecords, 100);
|
||||
|
||||
// Estimate added vs updated
|
||||
const recordsAdded = Math.floor(upsertedCount * 0.1);
|
||||
const recordsUpdated = upsertedCount - recordsAdded;
|
||||
|
||||
totalRecordsAdded += recordsAdded;
|
||||
totalRecordsUpdated += recordsUpdated;
|
||||
|
||||
console.log(`[${entity}] Chunk ${i + 1}: Upserted ${upsertedCount} records (+${recordsAdded} ~${recordsUpdated})`);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] Chunk ${i + 1} (${chunkDescription}) failed:`, errorMessage);
|
||||
failedChunks.push(`${chunkDescription}: ${errorMessage}`);
|
||||
// Continue with next chunk instead of failing entire sync
|
||||
}
|
||||
}
|
||||
|
||||
const duration = Date.now() - syncStartTime;
|
||||
console.log(`[${entity}] Chunked sync completed in ${duration}ms`);
|
||||
console.log(`[${entity}] Total: +${totalRecordsAdded} ~${totalRecordsUpdated} -${totalRecordsDeleted}`);
|
||||
|
||||
if (failedChunks.length > 0) {
|
||||
console.warn(`[${entity}] ${failedChunks.length} chunks failed:`, failedChunks);
|
||||
}
|
||||
|
||||
return {
|
||||
recordsAdded: totalRecordsAdded,
|
||||
recordsUpdated: totalRecordsUpdated,
|
||||
recordsDeleted: totalRecordsDeleted,
|
||||
};
|
||||
} catch (error) {
|
||||
const duration = Date.now() - syncStartTime;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[${entity}] Chunked sync failed after ${duration}ms:`, errorMessage);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all valid resource IDs from the database
|
||||
* Used to validate foreign key references before insert
|
||||
* @returns Set of valid resource IDs
|
||||
*/
|
||||
private async getValidResourceIds(): Promise<Set<number>> {
|
||||
try {
|
||||
const query = 'SELECT id FROM resources WHERE is_deleted = false';
|
||||
const result = await postgresClient.query<{ id: number }>(query);
|
||||
return new Set(result.rows.map(row => row.id));
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch valid resource IDs:', error);
|
||||
// Return empty set on error - will cause all resource IDs to be nullified
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to validate foreign key references before insert
|
||||
* @returns Set of valid contact IDs
|
||||
*/
|
||||
private async getValidContactIds(): Promise<Set<number>> {
|
||||
try {
|
||||
const query = 'SELECT id FROM contacts WHERE is_deleted = false';
|
||||
const result = await postgresClient.query<{ id: number }>(query);
|
||||
return new Set(result.rows.map(row => row.id));
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch valid contact IDs:', error);
|
||||
// Return empty set on error - will cause all contact IDs to be nullified
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate monthly date chunks for a given time period
|
||||
* @param yearsBack Number of years to look back
|
||||
* @returns Array of date range chunks
|
||||
*/
|
||||
private calculateMonthlyChunks(yearsBack: number): Array<{ startDate: Date; endDate: Date }> {
|
||||
const chunks: Array<{ startDate: Date; endDate: Date }> = [];
|
||||
const now = new Date();
|
||||
const startDate = new Date(now);
|
||||
startDate.setFullYear(now.getFullYear() - yearsBack);
|
||||
startDate.setHours(0, 0, 0, 0);
|
||||
|
||||
let currentDate = new Date(startDate);
|
||||
|
||||
while (currentDate < now) {
|
||||
const chunkStart = new Date(currentDate);
|
||||
|
||||
// Move to next month
|
||||
const chunkEnd = new Date(currentDate);
|
||||
chunkEnd.setMonth(chunkEnd.getMonth() + 1);
|
||||
|
||||
// Don't go beyond current date
|
||||
if (chunkEnd > now) {
|
||||
chunkEnd.setTime(now.getTime());
|
||||
}
|
||||
|
||||
chunks.push({
|
||||
startDate: chunkStart,
|
||||
endDate: chunkEnd,
|
||||
});
|
||||
|
||||
currentDate = new Date(chunkEnd);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Tasks
|
||||
*/
|
||||
async syncTasks(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.TASKS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Projects
|
||||
*/
|
||||
async syncProjects(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.PROJECTS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Resources (Users)
|
||||
*/
|
||||
async syncResources(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.RESOURCES, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Configuration Items
|
||||
*/
|
||||
async syncConfigurationItems(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.CONFIGURATION_ITEMS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Contacts
|
||||
*/
|
||||
async syncContacts(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.CONTACTS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Contracts
|
||||
*/
|
||||
async syncContracts(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.CONTRACTS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Billing Items
|
||||
*/
|
||||
async syncBillingItems(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.BILLING_ITEMS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Statuses (Picklist)
|
||||
*/
|
||||
async syncStatuses(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.STATUSES, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Issue Types (Picklist from Ticket field)
|
||||
*/
|
||||
async syncIssueTypes(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
const syncStartTime = Date.now();
|
||||
console.log(`[issue_types] Starting picklist sync`);
|
||||
|
||||
try {
|
||||
// Get issue type picklist values from Tickets entity
|
||||
const picklistValues = await this.autotaskClient.getPicklistValues('Tickets', 'issueType');
|
||||
|
||||
// Convert picklist to database format
|
||||
const records = Object.entries(picklistValues).map(([value, label]) => ({
|
||||
value: parseInt(value),
|
||||
label: label,
|
||||
is_active: true,
|
||||
sort_order: parseInt(value),
|
||||
synced_at: new Date(),
|
||||
}));
|
||||
|
||||
console.log(`[issue_types] Found ${records.length} picklist values`);
|
||||
|
||||
// Upsert to database
|
||||
const tableName = getTableName(EntityType.ISSUE_TYPES);
|
||||
const upsertedCount = await postgresClient.bulkUpsert(tableName, records, ['value']);
|
||||
|
||||
const stats: EntitySyncStats = {
|
||||
recordsAdded: upsertedCount,
|
||||
recordsUpdated: 0,
|
||||
recordsDeleted: 0,
|
||||
};
|
||||
|
||||
const duration = Date.now() - syncStartTime;
|
||||
console.log(`[issue_types] Sync completed in ${duration}ms: +${stats.recordsAdded} ~${stats.recordsUpdated}`);
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
const duration = Date.now() - syncStartTime;
|
||||
console.error(`[issue_types] Sync failed after ${duration}ms:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Sub-Issue Types (Picklist from Ticket field)
|
||||
*/
|
||||
async syncSubIssueTypes(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
const syncStartTime = Date.now();
|
||||
console.log(`[sub_issue_types] Starting picklist sync`);
|
||||
|
||||
try {
|
||||
// Get sub-issue type picklist values from Tickets entity
|
||||
const url = `${this.autotaskClient['config'].apiUrl}/Tickets/entityInformation/fields`;
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: this.autotaskClient['getAuthHeaders'](),
|
||||
});
|
||||
|
||||
const responseText = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch field info: ${responseText}`);
|
||||
}
|
||||
|
||||
const fieldData = JSON.parse(responseText);
|
||||
const subIssueTypeField = fieldData.fields.find((f: any) => f.name === 'subIssueType');
|
||||
|
||||
if (!subIssueTypeField || !subIssueTypeField.picklistValues) {
|
||||
throw new Error('subIssueType field or picklist values not found');
|
||||
}
|
||||
|
||||
// Convert picklist to database format, capturing parent value if it exists
|
||||
const records = subIssueTypeField.picklistValues.map((item: any) => ({
|
||||
value: parseInt(item.value),
|
||||
label: item.label,
|
||||
is_active: item.isActive !== false,
|
||||
parent_value: item.parentValue ? parseInt(item.parentValue) : null,
|
||||
sort_order: item.sortOrder || parseInt(item.value),
|
||||
synced_at: new Date(),
|
||||
}));
|
||||
|
||||
console.log(`[sub_issue_types] Found ${records.length} picklist values`);
|
||||
|
||||
// Upsert to database
|
||||
const tableName = getTableName(EntityType.SUB_ISSUE_TYPES);
|
||||
const upsertedCount = await postgresClient.bulkUpsert(tableName, records, ['value']);
|
||||
|
||||
const stats: EntitySyncStats = {
|
||||
recordsAdded: upsertedCount,
|
||||
recordsUpdated: 0,
|
||||
recordsDeleted: 0,
|
||||
};
|
||||
|
||||
const duration = Date.now() - syncStartTime;
|
||||
console.log(`[sub_issue_types] Sync completed in ${duration}ms: +${stats.recordsAdded} ~${stats.recordsUpdated}`);
|
||||
|
||||
return stats;
|
||||
} catch (error) {
|
||||
const duration = Date.now() - syncStartTime;
|
||||
console.error(`[sub_issue_types] Sync failed after ${duration}ms:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Work Types (Picklist)
|
||||
*/
|
||||
async syncWorkTypes(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.WORK_TYPES, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Time Entries
|
||||
*/
|
||||
async syncTimeEntries(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.TIME_ENTRIES, isIncremental);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create entity sync service instance
|
||||
* @param autotaskClient Autotask client instance
|
||||
* @returns EntitySyncService instance
|
||||
*/
|
||||
export function createEntitySyncService(autotaskClient: AutotaskClient): EntitySyncService {
|
||||
return new EntitySyncService(autotaskClient);
|
||||
}
|
||||
222
lib/services/issue-type-assignment.ts
Normal file
222
lib/services/issue-type-assignment.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
/**
|
||||
* Issue Type Assignment Service
|
||||
* Provides utilities to assign parent issue types to sub-issues without direct API endpoints
|
||||
*/
|
||||
|
||||
import postgresClient from '@/lib/services/postgres-client';
|
||||
|
||||
export interface IssueTypeAssignment {
|
||||
subIssueTypeValue: number;
|
||||
parentIssueTypeValue: number;
|
||||
parentIssueTypeLabel: string;
|
||||
}
|
||||
|
||||
export interface TicketIssueTypes {
|
||||
ticketId: number;
|
||||
issueType?: number;
|
||||
subIssueType?: number;
|
||||
issueTypeLabel?: string;
|
||||
subIssueTypeLabel?: string;
|
||||
parentIssueTypeLabel?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parent issue type for a given sub-issue type
|
||||
*/
|
||||
export async function getParentIssueType(subIssueTypeValue: number): Promise<IssueTypeAssignment | null> {
|
||||
try {
|
||||
const query = `
|
||||
SELECT
|
||||
sit.value as sub_issue_type_value,
|
||||
sit.parent_value as parent_issue_type_value,
|
||||
it.label as parent_issue_type_label
|
||||
FROM sub_issue_types sit
|
||||
LEFT JOIN issue_types it ON sit.parent_value = it.value
|
||||
WHERE sit.value = $1 AND sit.is_active = true
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query, [subIssueTypeValue]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = result.rows[0];
|
||||
return {
|
||||
subIssueTypeValue: row.sub_issue_type_value,
|
||||
parentIssueTypeValue: row.parent_issue_type_value,
|
||||
parentIssueTypeLabel: row.parent_issue_type_label,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Failed to get parent issue type:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all sub-issue types with their parent issue types
|
||||
*/
|
||||
export async function getAllSubIssueTypesWithParents(): Promise<IssueTypeAssignment[]> {
|
||||
try {
|
||||
const query = `
|
||||
SELECT
|
||||
sit.value as sub_issue_type_value,
|
||||
sit.parent_value as parent_issue_type_value,
|
||||
it.label as parent_issue_type_label
|
||||
FROM sub_issue_types sit
|
||||
LEFT JOIN issue_types it ON sit.parent_value = it.value
|
||||
WHERE sit.is_active = true
|
||||
ORDER BY sit.parent_value ASC, sit.sort_order ASC
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query);
|
||||
|
||||
return result.rows.map(row => ({
|
||||
subIssueTypeValue: row.sub_issue_type_value,
|
||||
parentIssueTypeValue: row.parent_issue_type_value,
|
||||
parentIssueTypeLabel: row.parent_issue_type_label,
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error('Failed to get all sub-issue types with parents:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get tickets with their issue types and parent issue type assignments
|
||||
*/
|
||||
export async function getTicketsWithIssueTypes(
|
||||
limit: number = 100,
|
||||
offset: number = 0,
|
||||
filters: {
|
||||
status?: number;
|
||||
priority?: number;
|
||||
companyId?: number;
|
||||
issueType?: number;
|
||||
subIssueType?: number;
|
||||
} = {}
|
||||
): Promise<{ tickets: TicketIssueTypes[], total: number }> {
|
||||
try {
|
||||
const conditions: string[] = [];
|
||||
const params: any[] = [];
|
||||
let paramIndex = 1;
|
||||
|
||||
if (filters.status !== undefined) {
|
||||
conditions.push(`t.status = $${paramIndex++}`);
|
||||
params.push(filters.status);
|
||||
}
|
||||
if (filters.priority !== undefined) {
|
||||
conditions.push(`t.priority = $${paramIndex++}`);
|
||||
params.push(filters.priority);
|
||||
}
|
||||
if (filters.companyId !== undefined) {
|
||||
conditions.push(`t.company_id = $${paramIndex++}`);
|
||||
params.push(filters.companyId);
|
||||
}
|
||||
if (filters.issueType !== undefined) {
|
||||
conditions.push(`t.issue_type = $${paramIndex++}`);
|
||||
params.push(filters.issueType);
|
||||
}
|
||||
if (filters.subIssueType !== undefined) {
|
||||
conditions.push(`t.sub_issue_type = $${paramIndex++}`);
|
||||
params.push(filters.subIssueType);
|
||||
}
|
||||
|
||||
conditions.push(`t.is_deleted = false`);
|
||||
|
||||
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
|
||||
const query = `
|
||||
SELECT
|
||||
t.id as ticket_id,
|
||||
t.issue_type,
|
||||
t.sub_issue_type,
|
||||
it.label as issue_type_label,
|
||||
sit.label as sub_issue_type_label,
|
||||
pit.label as parent_issue_type_label
|
||||
FROM tickets t
|
||||
LEFT JOIN issue_types it ON t.issue_type = it.value
|
||||
LEFT JOIN sub_issue_types sit ON t.sub_issue_type = sit.value
|
||||
LEFT JOIN issue_types pit ON sit.parent_value = pit.value
|
||||
${whereClause}
|
||||
ORDER BY t.create_date DESC
|
||||
LIMIT $${paramIndex++}
|
||||
OFFSET $${paramIndex++}
|
||||
`;
|
||||
|
||||
params.push(limit, offset);
|
||||
|
||||
const result = await postgresClient.query(query, params);
|
||||
const tickets = result.rows.map(row => ({
|
||||
ticketId: row.ticket_id,
|
||||
issueType: row.issue_type,
|
||||
subIssueType: row.sub_issue_type,
|
||||
issueTypeLabel: row.issue_type_label,
|
||||
subIssueTypeLabel: row.sub_issue_type_label,
|
||||
parentIssueTypeLabel: row.parent_issue_type_label,
|
||||
}));
|
||||
|
||||
// Get total count
|
||||
const countQuery = `
|
||||
SELECT COUNT(*) as total
|
||||
FROM tickets t
|
||||
${whereClause}
|
||||
`;
|
||||
|
||||
const countParams = params.slice(0, -2);
|
||||
const countResult = await postgresClient.query(countQuery, countParams);
|
||||
const total = parseInt(countResult.rows[0].total);
|
||||
|
||||
return { tickets, total };
|
||||
} catch (error) {
|
||||
console.error('Failed to get tickets with issue types:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assign parent issue type to sub-issue types in bulk
|
||||
* This can be used to update existing data or create mappings
|
||||
*/
|
||||
export async function assignParentIssueTypes(): Promise<number> {
|
||||
try {
|
||||
// This function demonstrates how you could update records
|
||||
// if you needed to assign parent types to existing sub-issues
|
||||
// without a direct API endpoint
|
||||
|
||||
const query = `
|
||||
UPDATE sub_issue_types sit
|
||||
SET parent_value = it.value
|
||||
FROM issue_types it
|
||||
WHERE sit.parent_value IS NULL
|
||||
AND sit.label LIKE '%' || it.label || '%'
|
||||
AND it.is_active = true
|
||||
RETURNING sit.value
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query);
|
||||
return result.rows.length;
|
||||
} catch (error) {
|
||||
console.error('Failed to assign parent issue types:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a lookup function that can be used in other parts of the application
|
||||
*/
|
||||
export async function createIssueTypeLookup(): Promise<Map<number, string>> {
|
||||
try {
|
||||
const assignments = await getAllSubIssueTypesWithParents();
|
||||
const lookup = new Map<number, string>();
|
||||
|
||||
assignments.forEach(assignment => {
|
||||
lookup.set(assignment.subIssueTypeValue, assignment.parentIssueTypeLabel || 'Unknown');
|
||||
});
|
||||
|
||||
return lookup;
|
||||
} catch (error) {
|
||||
console.error('Failed to create issue type lookup:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
429
lib/services/llm-analyzer.ts
Normal file
429
lib/services/llm-analyzer.ts
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
/**
|
||||
* LLM Analyzer Service
|
||||
* Integrates with LLM services for advanced work pattern analysis and insight generation
|
||||
*/
|
||||
|
||||
import {
|
||||
LLMAnalysisRequest,
|
||||
LLMAnalysisResponse,
|
||||
AnalyticsInsight
|
||||
} from '@/lib/types/analytics';
|
||||
import { TimeEntry } from '@/lib/types/database';
|
||||
|
||||
export class LLMAnalyzer {
|
||||
private apiKey: string;
|
||||
private baseUrl: string;
|
||||
private model: string;
|
||||
private cache: Map<string, { data: LLMAnalysisResponse; timestamp: number }>;
|
||||
private cacheTimeout: number = 30 * 60 * 1000; // 30 minutes
|
||||
|
||||
constructor() {
|
||||
this.apiKey = process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || '';
|
||||
this.baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
|
||||
this.model = process.env.LLM_MODEL || 'gpt-3.5-turbo';
|
||||
this.cache = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze time entries using LLM for work patterns and insights
|
||||
*/
|
||||
async analyzeTimeEntries(request: LLMAnalysisRequest): Promise<LLMAnalysisResponse> {
|
||||
// Check cache first
|
||||
const cacheKey = this.generateCacheKey(request);
|
||||
const cached = this.cache.get(cacheKey);
|
||||
|
||||
if (cached && Date.now() - cached.timestamp < this.cacheTimeout) {
|
||||
console.log('LLM Analyzer: Returning cached result');
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const response = await this.callLLM(request);
|
||||
const processingTime = Date.now() - startTime;
|
||||
|
||||
// Cache the result
|
||||
this.cache.set(cacheKey, {
|
||||
data: response,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
// Add processing metadata
|
||||
response.processingTime = processingTime;
|
||||
|
||||
console.log(`LLM Analyzer: Processed ${request.timeEntries.length} entries in ${processingTime}ms`);
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
console.error('LLM Analyzer: Analysis failed', error);
|
||||
throw new Error(`LLM analysis failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate comprehensive insights from time entries
|
||||
*/
|
||||
async generateInsights(timeEntries: TimeEntry[]): Promise<AnalyticsInsight[]> {
|
||||
if (timeEntries.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const request: LLMAnalysisRequest = {
|
||||
timeEntries: timeEntries.map(entry => ({
|
||||
id: entry.id,
|
||||
notes: entry.notes || undefined,
|
||||
title: entry.title || undefined,
|
||||
hours_worked: entry.hours_worked,
|
||||
entry_date: entry.entry_date.toISOString(),
|
||||
resource_name: entry.resource_id ? `Resource ${entry.resource_id}` : undefined,
|
||||
ticket_title: entry.ticket_id ? `Ticket ${entry.ticket_id}` : undefined,
|
||||
})),
|
||||
analysisType: 'comprehensive',
|
||||
};
|
||||
|
||||
const response = await this.analyzeTimeEntries(request);
|
||||
|
||||
// Convert LLM insights to AnalyticsInsight format
|
||||
const insights: AnalyticsInsight[] = [];
|
||||
|
||||
// Add insights from LLM response
|
||||
response.insights.forEach((insight, index) => {
|
||||
insights.push({
|
||||
type: this.determineInsightType(insight),
|
||||
category: 'overall',
|
||||
title: `AI Insight ${index + 1}`,
|
||||
description: insight,
|
||||
recommendation: this.extractRecommendation(insight),
|
||||
severity: 'medium',
|
||||
actionable: true,
|
||||
});
|
||||
});
|
||||
|
||||
// Add pattern-based insights
|
||||
response.patterns.forEach(pattern => {
|
||||
insights.push({
|
||||
type: pattern.impact === 'high' ? 'warning' : 'info',
|
||||
category: 'performance',
|
||||
title: `Pattern: ${pattern.type}`,
|
||||
description: pattern.description,
|
||||
recommendation: `Address this ${pattern.frequency > 5 ? 'frequent' : 'occasional'} pattern`,
|
||||
severity: pattern.impact === 'high' ? 'high' : pattern.impact === 'medium' ? 'medium' : 'low',
|
||||
actionable: true,
|
||||
});
|
||||
});
|
||||
|
||||
// Add recommendations
|
||||
response.recommendations.forEach(rec => {
|
||||
insights.push({
|
||||
type: rec.priority === 'high' ? 'warning' : 'info',
|
||||
category: 'overall',
|
||||
title: `Recommendation: ${rec.category}`,
|
||||
description: rec.action,
|
||||
recommendation: rec.expectedImpact,
|
||||
severity: rec.priority === 'high' ? 'high' : rec.priority === 'medium' ? 'medium' : 'low',
|
||||
actionable: true,
|
||||
});
|
||||
});
|
||||
|
||||
return insights;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze productivity patterns
|
||||
*/
|
||||
async analyzeProductivity(timeEntries: TimeEntry[]): Promise<LLMAnalysisResponse> {
|
||||
const request: LLMAnalysisRequest = {
|
||||
timeEntries: timeEntries.map(entry => ({
|
||||
id: entry.id,
|
||||
notes: entry.notes || undefined,
|
||||
title: entry.title || undefined,
|
||||
hours_worked: entry.hours_worked,
|
||||
entry_date: entry.entry_date.toISOString(),
|
||||
resource_name: entry.resource_id ? `Resource ${entry.resource_id}` : undefined,
|
||||
})),
|
||||
analysisType: 'productivity',
|
||||
};
|
||||
|
||||
return this.analyzeTimeEntries(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze work quality patterns
|
||||
*/
|
||||
async analyzeQuality(timeEntries: TimeEntry[]): Promise<LLMAnalysisResponse> {
|
||||
const request: LLMAnalysisRequest = {
|
||||
timeEntries: timeEntries.map(entry => ({
|
||||
id: entry.id,
|
||||
notes: entry.notes || undefined,
|
||||
title: entry.title || undefined,
|
||||
hours_worked: entry.hours_worked,
|
||||
entry_date: entry.entry_date.toISOString(),
|
||||
ticket_title: entry.ticket_id ? `Ticket ${entry.ticket_id}` : undefined,
|
||||
})),
|
||||
analysisType: 'quality',
|
||||
};
|
||||
|
||||
return this.analyzeTimeEntries(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect anomalies in time entry patterns
|
||||
*/
|
||||
async detectAnomalies(timeEntries: TimeEntry[]): Promise<LLMAnalysisResponse> {
|
||||
const request: LLMAnalysisRequest = {
|
||||
timeEntries: timeEntries.map(entry => ({
|
||||
id: entry.id,
|
||||
notes: entry.notes || undefined,
|
||||
title: entry.title || undefined,
|
||||
hours_worked: entry.hours_worked,
|
||||
entry_date: entry.entry_date.toISOString(),
|
||||
resource_name: entry.resource_id ? `Resource ${entry.resource_id}` : undefined,
|
||||
})),
|
||||
analysisType: 'anomalies',
|
||||
};
|
||||
|
||||
return this.analyzeTimeEntries(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Call the LLM API
|
||||
*/
|
||||
private async callLLM(request: LLMAnalysisRequest): Promise<LLMAnalysisResponse> {
|
||||
const prompt = this.buildPrompt(request);
|
||||
|
||||
const payload = {
|
||||
model: this.model,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: this.getSystemPrompt(request.analysisType),
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
temperature: 0.3,
|
||||
max_tokens: 1500,
|
||||
};
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`LLM API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const content = data.choices[0]?.message?.content;
|
||||
|
||||
if (!content) {
|
||||
throw new Error('No content received from LLM');
|
||||
}
|
||||
|
||||
return this.parseResponse(content, data.usage?.total_tokens || 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the prompt for LLM analysis
|
||||
*/
|
||||
private buildPrompt(request: LLMAnalysisRequest): string {
|
||||
const timeEntriesText = request.timeEntries.map(entry =>
|
||||
`ID: ${entry.id}, Date: ${entry.entry_date}, Hours: ${entry.hours_worked}, Title: "${entry.title || 'No title'}", Notes: "${entry.notes || 'No notes'}", Resource: ${entry.resource_name || 'Unknown'}`
|
||||
).join('\n');
|
||||
|
||||
let prompt = `Analyze the following time entries:\n\n${timeEntriesText}\n\n`;
|
||||
|
||||
switch (request.analysisType) {
|
||||
case 'productivity':
|
||||
prompt += `Focus on productivity patterns, work efficiency, time utilization, and identify any productivity bottlenecks or high-performing patterns.`;
|
||||
break;
|
||||
case 'quality':
|
||||
prompt += `Focus on work quality, documentation detail, technical accuracy, and identify areas where documentation quality could be improved.`;
|
||||
break;
|
||||
case 'patterns':
|
||||
prompt += `Focus on recurring work patterns, routine tasks, common issues, and identify any patterns that could be optimized or automated.`;
|
||||
break;
|
||||
case 'anomalies':
|
||||
prompt += `Focus on unusual patterns, outliers, suspicious entries, and identify any anomalies that may require investigation.`;
|
||||
break;
|
||||
case 'comprehensive':
|
||||
default:
|
||||
prompt += `Provide a comprehensive analysis covering productivity, quality, patterns, and any notable insights or recommendations.`;
|
||||
break;
|
||||
}
|
||||
|
||||
if (request.context) {
|
||||
prompt += `\n\nContext: Time range ${request.context.timeRange}`;
|
||||
if (request.context.resourceIds) {
|
||||
prompt += `, Resources: ${request.context.resourceIds.join(', ')}`;
|
||||
}
|
||||
if (request.context.projectIds) {
|
||||
prompt += `, Projects: ${request.context.projectIds.join(', ')}`;
|
||||
}
|
||||
}
|
||||
|
||||
prompt += `\n\nPlease provide your analysis in the following JSON format:
|
||||
{
|
||||
"insights": ["insight 1", "insight 2", "insight 3"],
|
||||
"patterns": [
|
||||
{"type": "pattern name", "description": "description", "frequency": number, "impact": "low|medium|high"}
|
||||
],
|
||||
"recommendations": [
|
||||
{"category": "category", "priority": "low|medium|high", "action": "action description", "expectedImpact": "impact description"}
|
||||
],
|
||||
"summary": {
|
||||
"overallQuality": 0.8,
|
||||
"productivityLevel": 0.7,
|
||||
"keyFindings": ["finding 1", "finding 2"]
|
||||
}
|
||||
}`;
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get system prompt based on analysis type
|
||||
*/
|
||||
private getSystemPrompt(analysisType: string): string {
|
||||
return `You are an expert analyst specializing in time tracking and work pattern analysis. Your task is to analyze time entries and provide actionable insights.
|
||||
|
||||
Key considerations:
|
||||
- Focus on practical, actionable recommendations
|
||||
- Consider both individual and team patterns
|
||||
- Highlight both strengths and areas for improvement
|
||||
- Provide specific, evidence-based insights
|
||||
- Consider the context of professional services work
|
||||
|
||||
Analysis guidelines:
|
||||
- Be objective and data-driven
|
||||
- Provide constructive feedback
|
||||
- Suggest realistic improvements
|
||||
- Consider industry best practices for time tracking
|
||||
- Account for variations in work types and complexity
|
||||
|
||||
Respond with valid JSON only, no additional text.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse LLM response into structured format
|
||||
*/
|
||||
private parseResponse(content: string, tokensUsed: number): LLMAnalysisResponse {
|
||||
try {
|
||||
// Extract JSON from response
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) {
|
||||
throw new Error('No JSON found in LLM response');
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
|
||||
// Validate and set defaults
|
||||
return {
|
||||
insights: Array.isArray(parsed.insights) ? parsed.insights : [],
|
||||
patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [],
|
||||
recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations : [],
|
||||
summary: {
|
||||
overallQuality: parsed.summary?.overallQuality || 0.5,
|
||||
productivityLevel: parsed.summary?.productivityLevel || 0.5,
|
||||
keyFindings: Array.isArray(parsed.summary?.keyFindings) ? parsed.summary.keyFindings : [],
|
||||
},
|
||||
processingTime: 0, // Will be set by caller
|
||||
tokensUsed,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Failed to parse LLM response:', error);
|
||||
console.error('Response content:', content);
|
||||
|
||||
// Return fallback response
|
||||
return {
|
||||
insights: ['Unable to process AI analysis due to parsing error'],
|
||||
patterns: [],
|
||||
recommendations: [],
|
||||
summary: {
|
||||
overallQuality: 0.5,
|
||||
productivityLevel: 0.5,
|
||||
keyFindings: ['Analysis processing failed'],
|
||||
},
|
||||
processingTime: 0,
|
||||
tokensUsed,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine insight type from content
|
||||
*/
|
||||
private determineInsightType(insight: string): 'success' | 'warning' | 'error' | 'info' {
|
||||
const lowerInsight = insight.toLowerCase();
|
||||
|
||||
if (lowerInsight.includes('excellent') || lowerInsight.includes('great') || lowerInsight.includes('good')) {
|
||||
return 'success';
|
||||
}
|
||||
if (lowerInsight.includes('concern') || lowerInsight.includes('issue') || lowerInsight.includes('problem')) {
|
||||
return 'warning';
|
||||
}
|
||||
if (lowerInsight.includes('error') || lowerInsight.includes('failed') || lowerInsight.includes('critical')) {
|
||||
return 'error';
|
||||
}
|
||||
|
||||
return 'info';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract recommendation from insight
|
||||
*/
|
||||
private extractRecommendation(insight: string): string {
|
||||
// Simple extraction - in a real implementation, this could be more sophisticated
|
||||
if (insight.includes('recommend') || insight.includes('should') || insight.includes('consider')) {
|
||||
return insight;
|
||||
}
|
||||
|
||||
return 'Review this insight and consider appropriate action';
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate cache key for request
|
||||
*/
|
||||
private generateCacheKey(request: LLMAnalysisRequest): string {
|
||||
const keyData = {
|
||||
analysisType: request.analysisType,
|
||||
entryCount: request.timeEntries.length,
|
||||
dateRange: {
|
||||
start: request.timeEntries[0]?.entry_date,
|
||||
end: request.timeEntries[request.timeEntries.length - 1]?.entry_date,
|
||||
},
|
||||
context: request.context,
|
||||
};
|
||||
|
||||
return Buffer.from(JSON.stringify(keyData)).toString('base64');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cache
|
||||
*/
|
||||
clearCache(): void {
|
||||
this.cache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getCacheStats(): { size: number; hitRate: number } {
|
||||
return {
|
||||
size: this.cache.size,
|
||||
hitRate: 0, // Would need to track hits/misses for real implementation
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
export const llmAnalyzer = new LLMAnalyzer();
|
||||
420
lib/services/performance-optimizer.ts
Normal file
420
lib/services/performance-optimizer.ts
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
/**
|
||||
* Performance Optimizer Service
|
||||
* Implements performance optimizations for large datasets and caching strategies
|
||||
*/
|
||||
|
||||
import { TimeEntry } from '@/lib/types/database';
|
||||
import { AnalyticsInsight, AggregateAnalysis } from '@/lib/types/analytics';
|
||||
|
||||
export interface CacheConfig {
|
||||
ttl: number; // Time to live in milliseconds
|
||||
maxSize: number; // Maximum number of items in cache
|
||||
strategy: 'lru' | 'fifo' | 'lfu';
|
||||
}
|
||||
|
||||
export interface PerformanceMetrics {
|
||||
queryTime: number;
|
||||
cacheHitRate: number;
|
||||
memoryUsage: number;
|
||||
recordsProcessed: number;
|
||||
recordsPerSecond: number;
|
||||
}
|
||||
|
||||
export class PerformanceOptimizer {
|
||||
private cache: Map<string, { data: any; timestamp: number; accessCount: number }> = new Map();
|
||||
private cacheConfig: CacheConfig = {
|
||||
ttl: 5 * 60 * 1000, // 5 minutes default
|
||||
maxSize: 1000,
|
||||
strategy: 'lru',
|
||||
};
|
||||
|
||||
constructor(config?: Partial<CacheConfig>) {
|
||||
if (config) {
|
||||
this.cacheConfig = { ...this.cacheConfig, ...config };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached data
|
||||
*/
|
||||
getCachedData(key: string): any | null {
|
||||
const item = this.cache.get(key);
|
||||
|
||||
if (!item) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if item is expired
|
||||
if (Date.now() - item.timestamp > this.cacheConfig.ttl) {
|
||||
this.cache.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Update access count for LFU strategy
|
||||
item.accessCount++;
|
||||
|
||||
return item.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set cached data
|
||||
*/
|
||||
setCachedData(key: string, data: any): void {
|
||||
// Remove oldest items if cache is full
|
||||
if (this.cache.size >= this.cacheConfig.maxSize) {
|
||||
this.evictCache();
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
accessCount: 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cache
|
||||
*/
|
||||
clearCache(): void {
|
||||
this.cache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getCacheStats(): {
|
||||
size: number;
|
||||
maxSize: number;
|
||||
hitRate: number;
|
||||
memoryUsage: number;
|
||||
} {
|
||||
return {
|
||||
size: this.cache.size,
|
||||
maxSize: this.cacheConfig.maxSize,
|
||||
hitRate: 0, // Would need to track hits/misses for real implementation
|
||||
memoryUsage: this.estimateMemoryUsage(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize time entries query with pagination and filtering
|
||||
*/
|
||||
optimizeTimeEntriesQuery(
|
||||
baseQuery: string,
|
||||
filters: Record<string, any>,
|
||||
pagination: { limit: number; offset: number }
|
||||
): { query: string; params: any[] } {
|
||||
const conditions: string[] = [];
|
||||
const params: any[] = [];
|
||||
let paramIndex = 1;
|
||||
|
||||
// Add filter conditions
|
||||
Object.entries(filters).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
if (Array.isArray(value)) {
|
||||
conditions.push(`${key} = ANY($${paramIndex})`);
|
||||
params.push(value);
|
||||
} else {
|
||||
conditions.push(`${key} = $${paramIndex}`);
|
||||
params.push(value);
|
||||
}
|
||||
paramIndex++;
|
||||
}
|
||||
});
|
||||
|
||||
// Build WHERE clause
|
||||
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
|
||||
// Add pagination
|
||||
const query = `
|
||||
${baseQuery}
|
||||
${whereClause}
|
||||
ORDER BY entry_date DESC
|
||||
LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
|
||||
`;
|
||||
|
||||
params.push(pagination.limit, pagination.offset);
|
||||
|
||||
return { query, params };
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch process large datasets
|
||||
*/
|
||||
async batchProcess<T, R>(
|
||||
items: T[],
|
||||
processor: (batch: T[]) => Promise<R[]>,
|
||||
batchSize: number = 100,
|
||||
onProgress?: (processed: number, total: number) => void
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, i + batchSize);
|
||||
const batchResults = await processor(batch);
|
||||
results.push(...batchResults);
|
||||
|
||||
if (onProgress) {
|
||||
onProgress(Math.min(i + batchSize, items.length), items.length);
|
||||
}
|
||||
|
||||
// Small delay to prevent overwhelming the system
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize analytics calculations for large datasets
|
||||
*/
|
||||
optimizeAnalyticsCalculation(timeEntries: TimeEntry[]): {
|
||||
summary: {
|
||||
totalEntries: number;
|
||||
totalHours: number;
|
||||
averageHoursPerEntry: number;
|
||||
billableEntries: number;
|
||||
approvedEntries: number;
|
||||
};
|
||||
scores: {
|
||||
activity: number;
|
||||
content: number;
|
||||
timeliness: number;
|
||||
overall: number;
|
||||
};
|
||||
} {
|
||||
// Use efficient single-pass calculations
|
||||
let totalHours = 0;
|
||||
let billableEntries = 0;
|
||||
let approvedEntries = 0;
|
||||
let activityScoreSum = 0;
|
||||
let contentScoreSum = 0;
|
||||
let timelinessScoreSum = 0;
|
||||
|
||||
for (const entry of timeEntries) {
|
||||
totalHours += entry.hours_worked;
|
||||
|
||||
if (entry.billable) billableEntries++;
|
||||
if (entry.approved) approvedEntries++;
|
||||
|
||||
// Simplified scoring for performance (would use full analytics engine in real implementation)
|
||||
activityScoreSum += this.calculateQuickActivityScore(entry);
|
||||
contentScoreSum += this.calculateQuickContentScore(entry);
|
||||
timelinessScoreSum += this.calculateQuickTimelinessScore(entry);
|
||||
}
|
||||
|
||||
const count = timeEntries.length;
|
||||
|
||||
return {
|
||||
summary: {
|
||||
totalEntries: count,
|
||||
totalHours,
|
||||
averageHoursPerEntry: count > 0 ? totalHours / count : 0,
|
||||
billableEntries,
|
||||
approvedEntries,
|
||||
},
|
||||
scores: {
|
||||
activity: count > 0 ? activityScoreSum / count : 0,
|
||||
content: count > 0 ? contentScoreSum / count : 0,
|
||||
timeliness: count > 0 ? timelinessScoreSum / count : 0,
|
||||
overall: count > 0 ? (activityScoreSum + contentScoreSum + timelinessScoreSum) / (3 * count) : 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate performance metrics
|
||||
*/
|
||||
generateMetrics(startTime: number, recordsProcessed: number): PerformanceMetrics {
|
||||
const queryTime = Date.now() - startTime;
|
||||
|
||||
return {
|
||||
queryTime,
|
||||
cacheHitRate: this.getCacheStats().hitRate,
|
||||
memoryUsage: this.getCacheStats().memoryUsage,
|
||||
recordsProcessed,
|
||||
recordsPerSecond: recordsProcessed > 0 ? (recordsProcessed / queryTime) * 1000 : 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize insight generation by grouping and batching
|
||||
*/
|
||||
optimizeInsightGeneration(
|
||||
timeEntries: TimeEntry[],
|
||||
existingInsights: AnalyticsInsight[] = []
|
||||
): AnalyticsInsight[] {
|
||||
const insights: AnalyticsInsight[] = [...existingInsights];
|
||||
|
||||
// Group by resource for efficiency
|
||||
const resourceGroups = new Map<number, TimeEntry[]>();
|
||||
|
||||
for (const entry of timeEntries) {
|
||||
if (!resourceGroups.has(entry.resource_id)) {
|
||||
resourceGroups.set(entry.resource_id, []);
|
||||
}
|
||||
resourceGroups.get(entry.resource_id)!.push(entry);
|
||||
}
|
||||
|
||||
// Generate insights per resource
|
||||
for (const [resourceId, entries] of resourceGroups) {
|
||||
const totalHours = entries.reduce((sum, entry) => sum + entry.hours_worked, 0);
|
||||
const avgScore = entries.reduce((sum, entry) =>
|
||||
sum + this.calculateQuickOverallScore(entry), 0) / entries.length;
|
||||
|
||||
// Add insight if needed
|
||||
if (avgScore < 0.5) {
|
||||
insights.push({
|
||||
type: 'warning',
|
||||
category: 'activity',
|
||||
title: `Low Performance: Resource ${resourceId}`,
|
||||
description: `Average score: ${(avgScore * 100).toFixed(1)}%`,
|
||||
recommendation: 'Review time entry quality and provide training',
|
||||
severity: 'medium',
|
||||
actionable: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (totalHours > 40) { // More than 40 hours in period
|
||||
insights.push({
|
||||
type: 'info',
|
||||
category: 'performance',
|
||||
title: `High Activity: Resource ${resourceId}`,
|
||||
description: `${Number(totalHours).toFixed(1)} hours logged`,
|
||||
recommendation: 'Monitor workload and resource allocation',
|
||||
severity: 'low',
|
||||
actionable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return insights;
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory-efficient data streaming for large exports
|
||||
*/
|
||||
async* streamDataForExport<T>(
|
||||
data: T[],
|
||||
chunkSize: number = 1000
|
||||
): AsyncGenerator<T[], void, unknown> {
|
||||
for (let i = 0; i < data.length; i += chunkSize) {
|
||||
yield data.slice(i, i + chunkSize);
|
||||
|
||||
// Allow event loop to process other tasks
|
||||
await new Promise(resolve => setTimeout(resolve, 0));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Private helper methods
|
||||
*/
|
||||
private evictCache(): void {
|
||||
switch (this.cacheConfig.strategy) {
|
||||
case 'lru':
|
||||
this.evictLRU();
|
||||
break;
|
||||
case 'fifo':
|
||||
this.evictFIFO();
|
||||
break;
|
||||
case 'lfu':
|
||||
this.evictLFU();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private evictLRU(): void {
|
||||
let oldestKey = '';
|
||||
let oldestTime = Date.now();
|
||||
|
||||
for (const [key, item] of this.cache.entries()) {
|
||||
if (item.timestamp < oldestTime) {
|
||||
oldestTime = item.timestamp;
|
||||
oldestKey = key;
|
||||
}
|
||||
}
|
||||
|
||||
if (oldestKey) {
|
||||
this.cache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
private evictFIFO(): void {
|
||||
const firstKey = this.cache.keys().next().value;
|
||||
if (firstKey) {
|
||||
this.cache.delete(firstKey);
|
||||
}
|
||||
}
|
||||
|
||||
private evictLFU(): void {
|
||||
let leastUsedKey = '';
|
||||
let leastCount = Infinity;
|
||||
|
||||
for (const [key, item] of this.cache.entries()) {
|
||||
if (item.accessCount < leastCount) {
|
||||
leastCount = item.accessCount;
|
||||
leastUsedKey = key;
|
||||
}
|
||||
}
|
||||
|
||||
if (leastUsedKey) {
|
||||
this.cache.delete(leastUsedKey);
|
||||
}
|
||||
}
|
||||
|
||||
private estimateMemoryUsage(): number {
|
||||
// Rough estimation - in real implementation would use more sophisticated tracking
|
||||
let totalSize = 0;
|
||||
|
||||
for (const [key, item] of this.cache.entries()) {
|
||||
totalSize += key.length * 2; // String size
|
||||
totalSize += JSON.stringify(item.data).length * 2; // Data size
|
||||
totalSize += 16; // Metadata overhead
|
||||
}
|
||||
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
private calculateQuickActivityScore(entry: TimeEntry): number {
|
||||
let score = 0;
|
||||
if (entry.title) score += 0.25;
|
||||
if (entry.notes && entry.notes.length > 10) score += 0.25;
|
||||
if (entry.start_date_time && entry.end_date_time) score += 0.25;
|
||||
if (entry.ticket_id || entry.task_id) score += 0.25;
|
||||
return score;
|
||||
}
|
||||
|
||||
private calculateQuickContentScore(entry: TimeEntry): number {
|
||||
let score = 0;
|
||||
if (entry.title && entry.title.length > 5) score += 0.3;
|
||||
if (entry.notes && entry.notes.length > 20) score += 0.4;
|
||||
if (entry.internal_notes) score += 0.3;
|
||||
return score;
|
||||
}
|
||||
|
||||
private calculateQuickTimelinessScore(entry: TimeEntry): number {
|
||||
const entryDate = new Date(entry.entry_date);
|
||||
const createdDate = new Date(entry.created_at);
|
||||
const delayDays = (createdDate.getTime() - entryDate.getTime()) / (1000 * 60 * 60 * 24);
|
||||
|
||||
if (delayDays <= 1) return 1.0;
|
||||
if (delayDays <= 3) return 0.8;
|
||||
if (delayDays <= 7) return 0.6;
|
||||
return 0.4;
|
||||
}
|
||||
|
||||
private calculateQuickOverallScore(entry: TimeEntry): number {
|
||||
return (
|
||||
this.calculateQuickActivityScore(entry) * 0.3 +
|
||||
this.calculateQuickContentScore(entry) * 0.4 +
|
||||
this.calculateQuickTimelinessScore(entry) * 0.3
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance with optimized configuration
|
||||
export const performanceOptimizer = new PerformanceOptimizer({
|
||||
ttl: 10 * 60 * 1000, // 10 minutes
|
||||
maxSize: 500,
|
||||
strategy: 'lru',
|
||||
});
|
||||
413
lib/services/postgres-client.ts
Normal file
413
lib/services/postgres-client.ts
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
import { Pool, PoolClient, QueryResult, QueryResultRow } from 'pg';
|
||||
|
||||
/**
|
||||
* PostgreSQL Client Service
|
||||
* Manages database connection pool and provides query methods
|
||||
*/
|
||||
class PostgresClient {
|
||||
private static instance: PostgresClient | null = null;
|
||||
private pool: Pool | null = null;
|
||||
|
||||
private constructor() {
|
||||
// Pool will be initialized lazily on first use
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the connection pool (lazy initialization)
|
||||
*/
|
||||
private initializePool(): void {
|
||||
if (this.pool) {
|
||||
return; // Already initialized
|
||||
}
|
||||
|
||||
// Use the configured host (defaults to 'postgres' for Docker network)
|
||||
const host = process.env.POSTGRES_HOST || 'localhost';
|
||||
|
||||
this.pool = new Pool({
|
||||
host,
|
||||
port: parseInt(process.env.POSTGRES_PORT || '5432'),
|
||||
database: process.env.POSTGRES_DB || 'pulse_autotask',
|
||||
user: process.env.POSTGRES_USER || 'pulse_user',
|
||||
password: process.env.POSTGRES_PASSWORD,
|
||||
max: 10, // Maximum number of clients in the pool
|
||||
idleTimeoutMillis: 30000, // Close idle clients after 30 seconds
|
||||
connectionTimeoutMillis: 2000, // Return an error after 2 seconds if connection could not be established
|
||||
});
|
||||
|
||||
// Handle pool errors
|
||||
this.pool.on('error', (err: Error) => {
|
||||
console.error('Unexpected error on idle PostgreSQL client', err);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the pool, initializing if necessary
|
||||
*/
|
||||
private getPool(): Pool {
|
||||
this.initializePool();
|
||||
return this.pool!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get singleton instance of PostgresClient
|
||||
*/
|
||||
public static getInstance(): PostgresClient {
|
||||
if (!PostgresClient.instance) {
|
||||
PostgresClient.instance = new PostgresClient();
|
||||
}
|
||||
return PostgresClient.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a query with parameters
|
||||
*/
|
||||
async query<T extends QueryResultRow = any>(
|
||||
text: string,
|
||||
params?: any[]
|
||||
): Promise<QueryResult<T>> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const result = await this.getPool().query<T>(text, params);
|
||||
const duration = Date.now() - start;
|
||||
|
||||
if (duration > 1000) {
|
||||
console.warn(`Slow query (${duration}ms):`, text.substring(0, 100));
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error('Database query error:', error);
|
||||
console.error('Query:', text);
|
||||
console.error('Params:', params);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a client from the pool for transactions
|
||||
*/
|
||||
async getClient(): Promise<PoolClient> {
|
||||
return await this.getPool().connect();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a transaction
|
||||
*/
|
||||
async transaction<T>(
|
||||
callback: (client: PoolClient) => Promise<T>
|
||||
): Promise<T> {
|
||||
const client = await this.getClient();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const result = await callback(client);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a single record
|
||||
*/
|
||||
async insert<T extends QueryResultRow = any>(
|
||||
table: string,
|
||||
data: Record<string, any>
|
||||
): Promise<T> {
|
||||
const keys = Object.keys(data);
|
||||
const values = Object.values(data);
|
||||
const placeholders = keys.map((_, i) => `$${i + 1}`).join(', ');
|
||||
|
||||
const query = `
|
||||
INSERT INTO ${table} (${keys.join(', ')})
|
||||
VALUES (${placeholders})
|
||||
RETURNING *
|
||||
`;
|
||||
|
||||
const result = await this.query<T>(query, values);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a record by ID
|
||||
*/
|
||||
async update<T extends QueryResultRow = any>(
|
||||
table: string,
|
||||
id: number | string,
|
||||
data: Record<string, any>
|
||||
): Promise<T> {
|
||||
const keys = Object.keys(data);
|
||||
const values = Object.values(data);
|
||||
const setClause = keys.map((key, i) => `${key} = $${i + 1}`).join(', ');
|
||||
|
||||
const query = `
|
||||
UPDATE ${table}
|
||||
SET ${setClause}, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $${keys.length + 1}
|
||||
RETURNING *
|
||||
`;
|
||||
|
||||
const result = await this.query<T>(query, [...values, id]);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert (insert or update) a record
|
||||
*/
|
||||
async upsert<T extends QueryResultRow = any>(
|
||||
table: string,
|
||||
data: Record<string, any>,
|
||||
conflictColumns: string[] = ['id']
|
||||
): Promise<T> {
|
||||
const keys = Object.keys(data);
|
||||
const values = Object.values(data);
|
||||
const placeholders = keys.map((_, i) => `$${i + 1}`).join(', ');
|
||||
|
||||
// Build UPDATE clause for conflict resolution
|
||||
const updateKeys = keys.filter(k => !conflictColumns.includes(k));
|
||||
const updateClause = updateKeys
|
||||
.map(key => `${key} = EXCLUDED.${key}`)
|
||||
.join(', ');
|
||||
|
||||
const query = `
|
||||
INSERT INTO ${table} (${keys.join(', ')})
|
||||
VALUES (${placeholders})
|
||||
ON CONFLICT (${conflictColumns.join(', ')})
|
||||
DO UPDATE SET ${updateClause}, updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING *
|
||||
`;
|
||||
|
||||
const result = await this.query<T>(query, values);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk insert records
|
||||
*/
|
||||
async bulkInsert(
|
||||
table: string,
|
||||
records: Record<string, any>[]
|
||||
): Promise<number> {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const keys = Object.keys(records[0]);
|
||||
const placeholders: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
records.forEach((record, recordIndex) => {
|
||||
const recordPlaceholders = keys.map(
|
||||
(_, keyIndex) => `$${recordIndex * keys.length + keyIndex + 1}`
|
||||
);
|
||||
placeholders.push(`(${recordPlaceholders.join(', ')})`);
|
||||
values.push(...keys.map(key => record[key]));
|
||||
});
|
||||
|
||||
const query = `
|
||||
INSERT INTO ${table} (${keys.join(', ')})
|
||||
VALUES ${placeholders.join(', ')}
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
`;
|
||||
|
||||
const result = await this.query(query, values);
|
||||
return result.rowCount || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk upsert records
|
||||
*/
|
||||
async bulkUpsert(
|
||||
table: string,
|
||||
records: Record<string, any>[],
|
||||
conflictColumns: string[] = ['id']
|
||||
): Promise<number> {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const keys = Object.keys(records[0]);
|
||||
const placeholders: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
records.forEach((record, recordIndex) => {
|
||||
const recordPlaceholders = keys.map(
|
||||
(_, keyIndex) => `$${recordIndex * keys.length + keyIndex + 1}`
|
||||
);
|
||||
placeholders.push(`(${recordPlaceholders.join(', ')})`);
|
||||
values.push(...keys.map(key => record[key]));
|
||||
});
|
||||
|
||||
// Build UPDATE clause for conflict resolution
|
||||
const updateKeys = keys.filter(k => !conflictColumns.includes(k));
|
||||
const updateClause = updateKeys
|
||||
.map(key => `${key} = EXCLUDED.${key}`)
|
||||
.join(', ');
|
||||
|
||||
const query = `
|
||||
INSERT INTO ${table} (${keys.join(', ')})
|
||||
VALUES ${placeholders.join(', ')}
|
||||
ON CONFLICT (${conflictColumns.join(', ')})
|
||||
DO UPDATE SET ${updateClause}, updated_at = CURRENT_TIMESTAMP
|
||||
`;
|
||||
|
||||
const result = await this.query(query, values);
|
||||
return result.rowCount || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft delete a record
|
||||
*/
|
||||
async softDelete(
|
||||
table: string,
|
||||
id: number | string
|
||||
): Promise<void> {
|
||||
const query = `
|
||||
UPDATE ${table}
|
||||
SET is_deleted = true, deleted_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1
|
||||
`;
|
||||
await this.query(query, [id]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft delete multiple records
|
||||
*/
|
||||
async softDeleteMany(
|
||||
table: string,
|
||||
ids: (number | string)[]
|
||||
): Promise<number> {
|
||||
if (ids.length === 0) return 0;
|
||||
|
||||
const query = `
|
||||
UPDATE ${table}
|
||||
SET is_deleted = true, deleted_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ANY($1::bigint[])
|
||||
`;
|
||||
const result = await this.query(query, [ids]);
|
||||
return result.rowCount || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find records by criteria
|
||||
*/
|
||||
async find<T extends QueryResultRow = any>(
|
||||
table: string,
|
||||
where: Record<string, any> = {},
|
||||
options: {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
orderBy?: string;
|
||||
includeDeleted?: boolean;
|
||||
} = {}
|
||||
): Promise<T[]> {
|
||||
const conditions: string[] = [];
|
||||
const values: any[] = [];
|
||||
let paramIndex = 1;
|
||||
|
||||
// Add where conditions
|
||||
Object.entries(where).forEach(([key, value]) => {
|
||||
conditions.push(`${key} = $${paramIndex}`);
|
||||
values.push(value);
|
||||
paramIndex++;
|
||||
});
|
||||
|
||||
// Exclude deleted records by default
|
||||
if (!options.includeDeleted) {
|
||||
conditions.push('is_deleted = false');
|
||||
}
|
||||
|
||||
const whereClause = conditions.length > 0
|
||||
? `WHERE ${conditions.join(' AND ')}`
|
||||
: '';
|
||||
|
||||
const orderByClause = options.orderBy ? `ORDER BY ${options.orderBy}` : '';
|
||||
const limitClause = options.limit ? `LIMIT ${options.limit}` : '';
|
||||
const offsetClause = options.offset ? `OFFSET ${options.offset}` : '';
|
||||
|
||||
const query = `
|
||||
SELECT * FROM ${table}
|
||||
${whereClause}
|
||||
${orderByClause}
|
||||
${limitClause}
|
||||
${offsetClause}
|
||||
`;
|
||||
|
||||
const result = await this.query<T>(query, values);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a single record by ID
|
||||
*/
|
||||
async findById<T extends QueryResultRow = any>(
|
||||
table: string,
|
||||
id: number | string,
|
||||
includeDeleted = false
|
||||
): Promise<T | null> {
|
||||
const deletedClause = includeDeleted ? '' : 'AND is_deleted = false';
|
||||
const query = `
|
||||
SELECT * FROM ${table}
|
||||
WHERE id = $1 ${deletedClause}
|
||||
LIMIT 1
|
||||
`;
|
||||
const result = await this.query<T>(query, [id]);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count records
|
||||
*/
|
||||
async count(
|
||||
table: string,
|
||||
where: Record<string, any> = {},
|
||||
includeDeleted = false
|
||||
): Promise<number> {
|
||||
const conditions: string[] = [];
|
||||
const values: any[] = [];
|
||||
let paramIndex = 1;
|
||||
|
||||
Object.entries(where).forEach(([key, value]) => {
|
||||
conditions.push(`${key} = $${paramIndex}`);
|
||||
values.push(value);
|
||||
paramIndex++;
|
||||
});
|
||||
|
||||
if (!includeDeleted) {
|
||||
conditions.push('is_deleted = false');
|
||||
}
|
||||
|
||||
const whereClause = conditions.length > 0
|
||||
? `WHERE ${conditions.join(' AND ')}`
|
||||
: '';
|
||||
|
||||
const query = `SELECT COUNT(*) as count FROM ${table} ${whereClause}`;
|
||||
const result = await this.query<{ count: string }>(query, values);
|
||||
return parseInt(result.rows[0].count);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test database connection
|
||||
*/
|
||||
async testConnection(): Promise<boolean> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Database connection test failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close all connections in the pool
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
if (this.pool) {
|
||||
await this.pool.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const postgresClient = PostgresClient.getInstance();
|
||||
export default postgresClient;
|
||||
116
lib/services/rate-limiter.ts
Normal file
116
lib/services/rate-limiter.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
/**
|
||||
* Rate Limiter Service
|
||||
* Implements token bucket algorithm for API rate limiting
|
||||
* Limits requests to 10 per second for Autotask API
|
||||
*/
|
||||
|
||||
export class RateLimiter {
|
||||
private maxRequestsPerSecond: number;
|
||||
private requestQueue: Array<() => void> = [];
|
||||
private requestTimes: number[] = [];
|
||||
private processing = false;
|
||||
|
||||
constructor(maxRequestsPerSecond: number = 10) {
|
||||
this.maxRequestsPerSecond = maxRequestsPerSecond;
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttle a function call to respect rate limits
|
||||
* @param fn Function to execute with rate limiting
|
||||
* @returns Promise that resolves when function completes
|
||||
*/
|
||||
async throttle<T>(fn: () => Promise<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.requestQueue.push(async () => {
|
||||
try {
|
||||
const result = await fn();
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
this.processQueue();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the request queue with rate limiting
|
||||
*/
|
||||
private async processQueue(): Promise<void> {
|
||||
if (this.processing || this.requestQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.processing = true;
|
||||
|
||||
while (this.requestQueue.length > 0) {
|
||||
await this.waitIfNeeded();
|
||||
|
||||
const request = this.requestQueue.shift();
|
||||
if (request) {
|
||||
this.requestTimes.push(Date.now());
|
||||
await request();
|
||||
}
|
||||
}
|
||||
|
||||
this.processing = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait if we've hit the rate limit
|
||||
*/
|
||||
private async waitIfNeeded(): Promise<void> {
|
||||
const now = Date.now();
|
||||
const oneSecondAgo = now - 1000;
|
||||
|
||||
// Remove request times older than 1 second
|
||||
this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo);
|
||||
|
||||
// If we've hit the limit, wait until we can make another request
|
||||
if (this.requestTimes.length >= this.maxRequestsPerSecond) {
|
||||
const oldestRequest = this.requestTimes[0];
|
||||
const waitTime = 1000 - (now - oldestRequest) + 10; // Add 10ms buffer
|
||||
|
||||
if (waitTime > 0) {
|
||||
await this.sleep(waitTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for specified milliseconds
|
||||
*/
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current queue length
|
||||
*/
|
||||
getQueueLength(): number {
|
||||
return this.requestQueue.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get number of requests in the last second
|
||||
*/
|
||||
getCurrentRequestCount(): number {
|
||||
const oneSecondAgo = Date.now() - 1000;
|
||||
return this.requestTimes.filter(time => time > oneSecondAgo).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the queue and reset
|
||||
*/
|
||||
reset(): void {
|
||||
this.requestQueue = [];
|
||||
this.requestTimes = [];
|
||||
this.processing = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance for Autotask API (10 req/sec)
|
||||
export const autotaskRateLimiter = new RateLimiter(10);
|
||||
|
||||
export default autotaskRateLimiter;
|
||||
140
lib/services/sync-progress-tracker.ts
Normal file
140
lib/services/sync-progress-tracker.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/**
|
||||
* Global sync progress tracker
|
||||
* Allows tracking progress of long-running sync operations
|
||||
* Can be polled from the UI to show real-time progress
|
||||
*/
|
||||
|
||||
export interface SyncProgressState {
|
||||
syncId: string;
|
||||
entityType: string;
|
||||
status: 'idle' | 'running' | 'completed' | 'failed';
|
||||
currentPage: number;
|
||||
totalRecords: number;
|
||||
estimatedTotal?: number;
|
||||
startTime: number;
|
||||
endTime?: number;
|
||||
error?: string;
|
||||
phase: 'fetching' | 'mapping' | 'upserting' | 'deleting' | 'completed';
|
||||
}
|
||||
|
||||
class SyncProgressTracker {
|
||||
private progressMap: Map<string, SyncProgressState> = new Map();
|
||||
|
||||
/**
|
||||
* Start tracking a new sync operation
|
||||
*/
|
||||
startSync(syncId: string, entityType: string): void {
|
||||
this.progressMap.set(syncId, {
|
||||
syncId,
|
||||
entityType,
|
||||
status: 'running',
|
||||
currentPage: 0,
|
||||
totalRecords: 0,
|
||||
startTime: Date.now(),
|
||||
phase: 'fetching',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update progress for a sync operation
|
||||
*/
|
||||
updateProgress(
|
||||
syncId: string,
|
||||
updates: Partial<Omit<SyncProgressState, 'syncId' | 'entityType' | 'startTime'>>
|
||||
): void {
|
||||
const current = this.progressMap.get(syncId);
|
||||
if (!current) return;
|
||||
|
||||
this.progressMap.set(syncId, {
|
||||
...current,
|
||||
...updates,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark sync as completed
|
||||
*/
|
||||
completeSync(syncId: string, totalRecords: number): void {
|
||||
const current = this.progressMap.get(syncId);
|
||||
if (!current) return;
|
||||
|
||||
this.progressMap.set(syncId, {
|
||||
...current,
|
||||
status: 'completed',
|
||||
totalRecords,
|
||||
endTime: Date.now(),
|
||||
phase: 'completed',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark sync as failed
|
||||
*/
|
||||
failSync(syncId: string, error: string): void {
|
||||
const current = this.progressMap.get(syncId);
|
||||
if (!current) return;
|
||||
|
||||
this.progressMap.set(syncId, {
|
||||
...current,
|
||||
status: 'failed',
|
||||
endTime: Date.now(),
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get progress for a specific sync
|
||||
*/
|
||||
getProgress(syncId: string): SyncProgressState | null {
|
||||
return this.progressMap.get(syncId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active syncs
|
||||
*/
|
||||
getActiveSyncs(): SyncProgressState[] {
|
||||
return Array.from(this.progressMap.values()).filter(
|
||||
(p) => p.status === 'running'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the most recent sync for an entity type
|
||||
*/
|
||||
getLatestSync(entityType: string): SyncProgressState | null {
|
||||
const syncs = Array.from(this.progressMap.values())
|
||||
.filter((p) => p.entityType === entityType)
|
||||
.sort((a, b) => b.startTime - a.startTime);
|
||||
|
||||
return syncs[0] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old completed/failed syncs (keep last 10 per entity)
|
||||
*/
|
||||
cleanup(): void {
|
||||
const byEntity = new Map<string, SyncProgressState[]>();
|
||||
|
||||
// Group by entity type
|
||||
for (const progress of this.progressMap.values()) {
|
||||
if (!byEntity.has(progress.entityType)) {
|
||||
byEntity.set(progress.entityType, []);
|
||||
}
|
||||
byEntity.get(progress.entityType)!.push(progress);
|
||||
}
|
||||
|
||||
// Keep only the 10 most recent per entity
|
||||
for (const [entityType, syncs] of byEntity.entries()) {
|
||||
const sorted = syncs.sort((a, b) => b.startTime - a.startTime);
|
||||
const toKeep = sorted.slice(0, 10);
|
||||
const toRemove = sorted.slice(10);
|
||||
|
||||
for (const sync of toRemove) {
|
||||
this.progressMap.delete(sync.syncId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
export const syncProgressTracker = new SyncProgressTracker();
|
||||
456
lib/services/sync-service.ts
Normal file
456
lib/services/sync-service.ts
Normal file
|
|
@ -0,0 +1,456 @@
|
|||
/**
|
||||
* Sync Service
|
||||
* Main orchestration service for syncing Autotask data to PostgreSQL
|
||||
*/
|
||||
|
||||
import postgresClient from './postgres-client';
|
||||
import autotaskRateLimiter from './rate-limiter';
|
||||
import { AutotaskClient } from './autotask-client';
|
||||
import { createEntitySyncService, EntitySyncService } from './entity-sync';
|
||||
import {
|
||||
EntityType,
|
||||
SyncType,
|
||||
SyncStatus,
|
||||
SyncConfig,
|
||||
SyncResult,
|
||||
SyncProgress,
|
||||
EntitySyncResult,
|
||||
SyncHistoryRecord
|
||||
} from '../types/sync';
|
||||
import {
|
||||
getEntitySyncOrder,
|
||||
getAllEntitiesInOrder,
|
||||
generateSyncId,
|
||||
getEntityDisplayName
|
||||
} from '../utils/sync-helpers';
|
||||
import { getLastSyncTime } from '../utils/db-helpers';
|
||||
|
||||
/**
|
||||
* Main Sync Service Class
|
||||
*/
|
||||
export class SyncService {
|
||||
private currentSyncId: string | null = null;
|
||||
private isSyncing = false;
|
||||
private autotaskClient: AutotaskClient;
|
||||
private entitySyncService: EntitySyncService;
|
||||
|
||||
constructor(autotaskClient: AutotaskClient) {
|
||||
this.autotaskClient = autotaskClient;
|
||||
this.entitySyncService = createEntitySyncService(autotaskClient);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a full sync of all entities
|
||||
* @param triggeredBy User or system identifier
|
||||
* @param yearsBack Number of years to look back for time-based entities
|
||||
* @returns Sync result
|
||||
*/
|
||||
async fullSync(triggeredBy?: string, yearsBack?: number): Promise<SyncResult> {
|
||||
const config: SyncConfig = {
|
||||
syncType: SyncType.FULL,
|
||||
entities: getAllEntitiesInOrder(),
|
||||
triggeredBy,
|
||||
yearsBack,
|
||||
};
|
||||
|
||||
return await this.executeSync(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start an incremental sync of all entities
|
||||
* @param triggeredBy User or system identifier
|
||||
* @param yearsBack Number of years to look back for time-based entities
|
||||
* @returns Sync result
|
||||
*/
|
||||
async incrementalSync(triggeredBy?: string, yearsBack?: number): Promise<SyncResult> {
|
||||
const config: SyncConfig = {
|
||||
syncType: SyncType.INCREMENTAL,
|
||||
entities: getAllEntitiesInOrder(),
|
||||
triggeredBy,
|
||||
yearsBack,
|
||||
};
|
||||
|
||||
return await this.executeSync(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync specific entities
|
||||
* @param entities Array of entity types to sync
|
||||
* @param syncType Type of sync (full or incremental)
|
||||
* @param triggeredBy User or system identifier
|
||||
* @param yearsBack Number of years to look back for time-based entities
|
||||
* @returns Sync result
|
||||
*/
|
||||
async syncEntities(
|
||||
entities: EntityType[],
|
||||
syncType: SyncType = SyncType.ENTITY_SPECIFIC,
|
||||
triggeredBy?: string,
|
||||
yearsBack?: number
|
||||
): Promise<SyncResult> {
|
||||
const config: SyncConfig = {
|
||||
syncType,
|
||||
entities: getEntitySyncOrder(entities),
|
||||
triggeredBy,
|
||||
yearsBack,
|
||||
};
|
||||
|
||||
return await this.executeSync(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute sync operation
|
||||
* @param config Sync configuration
|
||||
* @returns Sync result
|
||||
*/
|
||||
private async executeSync(config: SyncConfig): Promise<SyncResult> {
|
||||
if (this.isSyncing) {
|
||||
throw new Error('A sync operation is already in progress');
|
||||
}
|
||||
|
||||
this.isSyncing = true;
|
||||
const syncId = generateSyncId();
|
||||
this.currentSyncId = syncId;
|
||||
|
||||
const startTime = new Date();
|
||||
const entityResults: EntitySyncResult[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
try {
|
||||
console.log(`Starting ${config.syncType} sync: ${syncId}`);
|
||||
console.log(`Entities to sync: ${config.entities.map(e => getEntityDisplayName(e)).join(', ')}`);
|
||||
|
||||
// Sync each entity in order
|
||||
for (const entity of config.entities) {
|
||||
try {
|
||||
const entityStartTime = Date.now();
|
||||
|
||||
console.log(`Syncing ${getEntityDisplayName(entity)}...`);
|
||||
|
||||
// Create sync history record
|
||||
const historyId = await this.createSyncHistory(
|
||||
entity,
|
||||
config.syncType,
|
||||
config.triggeredBy
|
||||
);
|
||||
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
let recordsDeleted = 0;
|
||||
|
||||
// Determine if incremental sync
|
||||
const isIncremental = config.syncType === SyncType.INCREMENTAL;
|
||||
const yearsBack = config.yearsBack || 2; // Default to 2 years
|
||||
|
||||
// Execute entity sync
|
||||
const syncStats = await this.entitySyncService.syncEntity(entity, isIncremental, yearsBack);
|
||||
|
||||
recordsAdded = syncStats.recordsAdded;
|
||||
recordsUpdated = syncStats.recordsUpdated;
|
||||
recordsDeleted = syncStats.recordsDeleted;
|
||||
|
||||
const duration = Date.now() - entityStartTime;
|
||||
|
||||
// Update sync history
|
||||
await this.updateSyncHistory(
|
||||
historyId,
|
||||
SyncStatus.COMPLETED,
|
||||
recordsAdded,
|
||||
recordsUpdated,
|
||||
recordsDeleted
|
||||
);
|
||||
|
||||
entityResults.push({
|
||||
entityType: entity,
|
||||
success: true,
|
||||
recordsAdded,
|
||||
recordsUpdated,
|
||||
recordsDeleted,
|
||||
duration,
|
||||
});
|
||||
|
||||
console.log(
|
||||
`✓ ${getEntityDisplayName(entity)} synced: +${recordsAdded} ~${recordsUpdated} -${recordsDeleted} (${duration}ms)`
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||
const entityName = getEntityDisplayName(entity);
|
||||
|
||||
// Log detailed error information
|
||||
console.error(`✗ Failed to sync ${entityName}:`);
|
||||
console.error(` Error: ${errorMessage}`);
|
||||
if (errorStack) {
|
||||
console.error(` Stack: ${errorStack}`);
|
||||
}
|
||||
console.error(` Entity: ${entity}`);
|
||||
console.error(` Sync Type: ${config.syncType}`);
|
||||
console.error(` Sync ID: ${syncId}`);
|
||||
|
||||
// Categorize error type
|
||||
let errorCategory = 'UNKNOWN';
|
||||
if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('ETIMEDOUT')) {
|
||||
errorCategory = 'NETWORK_ERROR';
|
||||
} else if (errorMessage.includes('401') || errorMessage.includes('403')) {
|
||||
errorCategory = 'AUTH_ERROR';
|
||||
} else if (errorMessage.includes('429')) {
|
||||
errorCategory = 'RATE_LIMIT_ERROR';
|
||||
} else if (errorMessage.includes('constraint') || errorMessage.includes('duplicate')) {
|
||||
errorCategory = 'DATABASE_CONSTRAINT_ERROR';
|
||||
} else if (errorMessage.includes('query') || errorMessage.includes('SQL')) {
|
||||
errorCategory = 'DATABASE_ERROR';
|
||||
} else if (errorMessage.includes('API')) {
|
||||
errorCategory = 'API_ERROR';
|
||||
}
|
||||
|
||||
const fullErrorMessage = `[${errorCategory}] ${errorMessage}`;
|
||||
errors.push(`${entityName}: ${fullErrorMessage}`);
|
||||
|
||||
// Try to update sync history with error
|
||||
try {
|
||||
const historyId = await this.createSyncHistory(
|
||||
entity,
|
||||
config.syncType,
|
||||
config.triggeredBy
|
||||
);
|
||||
await this.updateSyncHistory(
|
||||
historyId,
|
||||
SyncStatus.FAILED,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
fullErrorMessage
|
||||
);
|
||||
} catch (historyError) {
|
||||
console.error('Failed to update sync history with error:', historyError);
|
||||
}
|
||||
|
||||
entityResults.push({
|
||||
entityType: entity,
|
||||
success: false,
|
||||
recordsAdded: 0,
|
||||
recordsUpdated: 0,
|
||||
recordsDeleted: 0,
|
||||
duration: 0,
|
||||
error: fullErrorMessage,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const endTime = new Date();
|
||||
const totalDuration = endTime.getTime() - startTime.getTime();
|
||||
|
||||
const result: SyncResult = {
|
||||
syncId,
|
||||
syncType: config.syncType,
|
||||
status: errors.length === 0 ? SyncStatus.COMPLETED : SyncStatus.FAILED,
|
||||
entities: entityResults,
|
||||
totalRecordsAdded: entityResults.reduce((sum, r) => sum + r.recordsAdded, 0),
|
||||
totalRecordsUpdated: entityResults.reduce((sum, r) => sum + r.recordsUpdated, 0),
|
||||
totalRecordsDeleted: entityResults.reduce((sum, r) => sum + r.recordsDeleted, 0),
|
||||
startedAt: startTime,
|
||||
completedAt: endTime,
|
||||
duration: totalDuration,
|
||||
errors,
|
||||
};
|
||||
|
||||
console.log(`Sync ${syncId} completed in ${totalDuration}ms`);
|
||||
console.log(`Total: +${result.totalRecordsAdded} ~${result.totalRecordsUpdated} -${result.totalRecordsDeleted}`);
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||
|
||||
console.error('=== SYNC OPERATION FAILED ===');
|
||||
console.error(`Sync ID: ${syncId}`);
|
||||
console.error(`Sync Type: ${config.syncType}`);
|
||||
console.error(`Error: ${errorMessage}`);
|
||||
if (errorStack) {
|
||||
console.error(`Stack Trace:\n${errorStack}`);
|
||||
}
|
||||
console.error(`Entities Attempted: ${config.entities.join(', ')}`);
|
||||
console.error(`Successful Entities: ${entityResults.filter(r => r.success).length}`);
|
||||
console.error(`Failed Entities: ${entityResults.filter(r => !r.success).length}`);
|
||||
console.error('============================');
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
this.isSyncing = false;
|
||||
this.currentSyncId = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create sync history record
|
||||
* @param entity Entity type
|
||||
* @param syncType Sync type
|
||||
* @param triggeredBy User identifier
|
||||
* @returns Sync history ID
|
||||
*/
|
||||
async createSyncHistory(
|
||||
entity: EntityType,
|
||||
syncType: SyncType,
|
||||
triggeredBy?: string
|
||||
): Promise<number> {
|
||||
const query = `
|
||||
INSERT INTO sync_history (
|
||||
entity_type,
|
||||
sync_type,
|
||||
status,
|
||||
started_at,
|
||||
records_added,
|
||||
records_updated,
|
||||
records_deleted,
|
||||
triggered_by
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING id
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query<{ id: number }>(query, [
|
||||
entity,
|
||||
syncType,
|
||||
SyncStatus.STARTED,
|
||||
new Date(),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
triggeredBy || 'system',
|
||||
]);
|
||||
|
||||
return result.rows[0].id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update sync history record
|
||||
* @param id Sync history ID
|
||||
* @param status Sync status
|
||||
* @param recordsAdded Number of records added
|
||||
* @param recordsUpdated Number of records updated
|
||||
* @param recordsDeleted Number of records deleted
|
||||
* @param errorMessage Optional error message
|
||||
*/
|
||||
async updateSyncHistory(
|
||||
id: number,
|
||||
status: SyncStatus,
|
||||
recordsAdded: number,
|
||||
recordsUpdated: number,
|
||||
recordsDeleted: number,
|
||||
errorMessage?: string
|
||||
): Promise<void> {
|
||||
const query = `
|
||||
UPDATE sync_history
|
||||
SET status = $1,
|
||||
completed_at = $2,
|
||||
records_added = $3,
|
||||
records_updated = $4,
|
||||
records_deleted = $5,
|
||||
error_message = $6
|
||||
WHERE id = $7
|
||||
`;
|
||||
|
||||
await postgresClient.query(query, [
|
||||
status,
|
||||
new Date(),
|
||||
recordsAdded,
|
||||
recordsUpdated,
|
||||
recordsDeleted,
|
||||
errorMessage || null,
|
||||
id,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sync history
|
||||
* @param limit Number of records to return
|
||||
* @param entityType Optional entity type filter
|
||||
* @returns Array of sync history records
|
||||
*/
|
||||
async getSyncHistory(
|
||||
limit: number = 50,
|
||||
entityType?: EntityType
|
||||
): Promise<SyncHistoryRecord[]> {
|
||||
let query = `
|
||||
SELECT *
|
||||
FROM sync_history
|
||||
`;
|
||||
|
||||
const params: any[] = [];
|
||||
|
||||
if (entityType) {
|
||||
query += ` WHERE entity_type = $1`;
|
||||
params.push(entityType);
|
||||
}
|
||||
|
||||
query += ` ORDER BY started_at DESC LIMIT $${params.length + 1}`;
|
||||
params.push(limit);
|
||||
|
||||
const result = await postgresClient.query<SyncHistoryRecord>(query, params);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get last sync info for all entities
|
||||
* @returns Map of entity type to last sync record
|
||||
*/
|
||||
async getLastSyncInfo(): Promise<Map<EntityType, SyncHistoryRecord>> {
|
||||
const query = `
|
||||
SELECT DISTINCT ON (entity_type) *
|
||||
FROM sync_history
|
||||
WHERE status = 'completed'
|
||||
ORDER BY entity_type, completed_at DESC
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query<SyncHistoryRecord>(query);
|
||||
const map = new Map<EntityType, SyncHistoryRecord>();
|
||||
|
||||
for (const row of result.rows) {
|
||||
// Check if the entity_type value exists in the EntityType enum values
|
||||
const entityTypeValues = Object.values(EntityType);
|
||||
if (entityTypeValues.includes(row.entity_type as EntityType)) {
|
||||
map.set(row.entity_type as EntityType, row);
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if sync is currently running
|
||||
* @returns True if sync is in progress
|
||||
*/
|
||||
isSyncInProgress(): boolean {
|
||||
return this.isSyncing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current sync ID
|
||||
* @returns Current sync ID or null
|
||||
*/
|
||||
getCurrentSyncId(): string | null {
|
||||
return this.currentSyncId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel current sync operation
|
||||
*/
|
||||
async cancelSync(): Promise<void> {
|
||||
if (!this.isSyncing) {
|
||||
throw new Error('No sync operation in progress');
|
||||
}
|
||||
|
||||
// TODO: Implement graceful cancellation
|
||||
this.isSyncing = false;
|
||||
this.currentSyncId = null;
|
||||
|
||||
console.log('Sync operation cancelled');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create sync service instance
|
||||
* @param autotaskClient Autotask client instance
|
||||
* @returns SyncService instance
|
||||
*/
|
||||
export function createSyncService(autotaskClient: AutotaskClient): SyncService {
|
||||
return new SyncService(autotaskClient);
|
||||
}
|
||||
|
|
@ -211,3 +211,14 @@ export enum AddigyDeviceType {
|
|||
iPad = 'ipad',
|
||||
AppleTV = 'appletv',
|
||||
}
|
||||
|
||||
// Mapping types for Addigy organizations to Autotask companies
|
||||
export interface AddigyOrgMapping {
|
||||
id: number;
|
||||
addigyOrgId: string;
|
||||
addigyOrgName: string;
|
||||
autotaskCompanyId: number;
|
||||
autotaskCompanyName: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
|
|
|||
276
lib/types/analytics.ts
Normal file
276
lib/types/analytics.ts
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
/**
|
||||
* Analytics Types
|
||||
* TypeScript definitions for analytics, scoring, and insights
|
||||
*/
|
||||
|
||||
// Score interfaces
|
||||
export interface ActivityScore {
|
||||
score: number; // 0 to 1
|
||||
factors: string[];
|
||||
breakdown: {
|
||||
completeness: number;
|
||||
consistency: number;
|
||||
duration: number;
|
||||
categorization: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ContentScore {
|
||||
score: number; // 0 to 1
|
||||
factors: string[];
|
||||
breakdown: {
|
||||
notesQuality: number;
|
||||
titleClarity: number;
|
||||
internalNotes: number;
|
||||
technicalDetail: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TimelinessScore {
|
||||
score: number; // 0 to 1
|
||||
factors: string[];
|
||||
breakdown: {
|
||||
entryDelay: number;
|
||||
businessHours: number;
|
||||
regularity: number;
|
||||
approvalTimeliness: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Insight interfaces
|
||||
export interface AnalyticsInsight {
|
||||
type: 'success' | 'warning' | 'error' | 'info';
|
||||
category: 'activity' | 'content' | 'timeliness' | 'overall' | 'billing' | 'performance';
|
||||
title: string;
|
||||
description: string;
|
||||
recommendation: string;
|
||||
severity?: 'low' | 'medium' | 'high';
|
||||
actionable?: boolean;
|
||||
}
|
||||
|
||||
// Analysis interfaces
|
||||
export interface TimeEntryAnalysis {
|
||||
timeEntryId: number;
|
||||
activityScore: ActivityScore;
|
||||
contentScore: ContentScore;
|
||||
timelinessScore: TimelinessScore;
|
||||
overallScore: number; // 0 to 1
|
||||
insights: AnalyticsInsight[];
|
||||
analyzedAt: Date;
|
||||
}
|
||||
|
||||
export interface AggregateAnalysis {
|
||||
totalEntries: number;
|
||||
totalHours: number;
|
||||
averageHoursPerEntry: number;
|
||||
dateRange: {
|
||||
earliest: Date;
|
||||
latest: Date;
|
||||
};
|
||||
scores: {
|
||||
activity: number;
|
||||
content: number;
|
||||
timeliness: number;
|
||||
overall: number;
|
||||
};
|
||||
insights: AnalyticsInsight[];
|
||||
patterns: {
|
||||
dayOfWeek: number[]; // 7 values, Sunday = 0
|
||||
hourly: number[]; // 24 values
|
||||
};
|
||||
trends: {
|
||||
weekly: Array<{
|
||||
week: Date;
|
||||
hours: number;
|
||||
entries: number;
|
||||
}>;
|
||||
};
|
||||
analyzedAt: Date;
|
||||
}
|
||||
|
||||
// Timeline interfaces
|
||||
export interface TimelineEvent {
|
||||
id: string;
|
||||
type: 'time_entry' | 'key_moment' | 'milestone';
|
||||
timestamp: Date;
|
||||
title: string;
|
||||
description?: string;
|
||||
duration?: number; // in hours
|
||||
metadata?: Record<string, any>;
|
||||
score?: number; // 0 to 1
|
||||
isHumanActivity: boolean;
|
||||
importance: 'low' | 'medium' | 'high' | 'critical';
|
||||
}
|
||||
|
||||
export interface TimelineView {
|
||||
events: TimelineEvent[];
|
||||
dateRange: {
|
||||
start: Date;
|
||||
end: Date;
|
||||
};
|
||||
timeRange: 'hour' | 'day' | 'week' | 'month';
|
||||
filters: {
|
||||
resourceIds?: number[];
|
||||
ticketIds?: number[];
|
||||
projectIds?: number[];
|
||||
activityTypes?: string[];
|
||||
minScore?: number;
|
||||
};
|
||||
summary: {
|
||||
totalEvents: number;
|
||||
humanActivities: number;
|
||||
systemActivities: number;
|
||||
totalHours: number;
|
||||
averageScore: number;
|
||||
};
|
||||
}
|
||||
|
||||
// LLM Analysis interfaces
|
||||
export interface LLMAnalysisRequest {
|
||||
timeEntries: Array<{
|
||||
id: number;
|
||||
notes?: string;
|
||||
title?: string;
|
||||
hours_worked: number;
|
||||
entry_date: string;
|
||||
resource_name?: string;
|
||||
ticket_title?: string;
|
||||
}>;
|
||||
analysisType: 'productivity' | 'quality' | 'patterns' | 'anomalies' | 'comprehensive';
|
||||
context?: {
|
||||
timeRange: string;
|
||||
resourceIds?: number[];
|
||||
projectIds?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface LLMAnalysisResponse {
|
||||
insights: string[];
|
||||
patterns: Array<{
|
||||
type: string;
|
||||
description: string;
|
||||
frequency: number;
|
||||
impact: 'low' | 'medium' | 'high';
|
||||
}>;
|
||||
recommendations: Array<{
|
||||
category: string;
|
||||
priority: 'low' | 'medium' | 'high';
|
||||
action: string;
|
||||
expectedImpact: string;
|
||||
}>;
|
||||
summary: {
|
||||
overallQuality: number; // 0 to 1
|
||||
productivityLevel: number; // 0 to 1
|
||||
keyFindings: string[];
|
||||
};
|
||||
processingTime: number; // milliseconds
|
||||
tokensUsed: number;
|
||||
}
|
||||
|
||||
// Scoring algorithm interfaces
|
||||
export interface ScoringWeights {
|
||||
activity: number;
|
||||
content: number;
|
||||
timeliness: number;
|
||||
}
|
||||
|
||||
export interface ScoringConfiguration {
|
||||
weights: ScoringWeights;
|
||||
thresholds: {
|
||||
excellent: number;
|
||||
good: number;
|
||||
average: number;
|
||||
poor: number;
|
||||
};
|
||||
factors: {
|
||||
activity: {
|
||||
completeness: number;
|
||||
consistency: number;
|
||||
duration: number;
|
||||
categorization: number;
|
||||
};
|
||||
content: {
|
||||
notesQuality: number;
|
||||
titleClarity: number;
|
||||
internalNotes: number;
|
||||
technicalDetail: number;
|
||||
};
|
||||
timeliness: {
|
||||
entryDelay: number;
|
||||
businessHours: number;
|
||||
regularity: number;
|
||||
approvalTimeliness: number;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// Analytics query interfaces
|
||||
export interface AnalyticsQuery {
|
||||
timeRange: {
|
||||
start: Date;
|
||||
end: Date;
|
||||
};
|
||||
filters: {
|
||||
resourceIds?: number[];
|
||||
ticketIds?: number[];
|
||||
taskIds?: number[];
|
||||
projectIds?: number[];
|
||||
companyIds?: number[];
|
||||
minHours?: number;
|
||||
maxHours?: number;
|
||||
billable?: boolean;
|
||||
approved?: boolean;
|
||||
activityTypes?: string[];
|
||||
};
|
||||
groupBy?: 'resource' | 'ticket' | 'project' | 'company' | 'day' | 'week' | 'month';
|
||||
includeScores?: boolean;
|
||||
includeInsights?: boolean;
|
||||
includePatterns?: boolean;
|
||||
includeTrends?: boolean;
|
||||
}
|
||||
|
||||
export interface AnalyticsQueryResult {
|
||||
data: Array<{
|
||||
group: string;
|
||||
totalEntries: number;
|
||||
totalHours: number;
|
||||
averageHoursPerEntry: number;
|
||||
scores?: {
|
||||
activity: number;
|
||||
content: number;
|
||||
timeliness: number;
|
||||
overall: number;
|
||||
};
|
||||
insights?: AnalyticsInsight[];
|
||||
}>;
|
||||
summary: AggregateAnalysis;
|
||||
query: AnalyticsQuery;
|
||||
processedAt: Date;
|
||||
}
|
||||
|
||||
// Export and reporting interfaces
|
||||
export interface AnalyticsExport {
|
||||
format: 'csv' | 'excel' | 'pdf' | 'json';
|
||||
data: {
|
||||
timeEntries: any[];
|
||||
analyses: TimeEntryAnalysis[];
|
||||
summary: AggregateAnalysis;
|
||||
insights: AnalyticsInsight[];
|
||||
};
|
||||
metadata: {
|
||||
exportedAt: Date;
|
||||
timeRange: string;
|
||||
filters: string;
|
||||
recordCount: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Performance metrics
|
||||
export interface AnalyticsPerformanceMetrics {
|
||||
processingTime: number; // milliseconds
|
||||
recordsProcessed: number;
|
||||
recordsPerSecond: number;
|
||||
memoryUsage: number; // MB
|
||||
cacheHitRate: number; // percentage
|
||||
errors: string[];
|
||||
}
|
||||
|
|
@ -183,6 +183,8 @@ export interface ConfigurationItem {
|
|||
setupFee?: number;
|
||||
sourceProductID?: number;
|
||||
type?: number;
|
||||
configurationItemType?: number; // API field name (camelCase)
|
||||
configuration_item_type?: number; // Database field name (snake_case)
|
||||
vendorID?: number;
|
||||
vendorName?: string;
|
||||
warrantyExpirationDate?: string;
|
||||
|
|
@ -207,6 +209,7 @@ export interface PicklistValue {
|
|||
sortOrder?: number;
|
||||
isActive?: boolean;
|
||||
isSystem?: boolean;
|
||||
parentValue?: number | string;
|
||||
}
|
||||
|
||||
export interface EntityField {
|
||||
|
|
@ -221,6 +224,53 @@ export interface EntityField {
|
|||
picklistValues?: PicklistValue[];
|
||||
}
|
||||
|
||||
export interface AutotaskTimeEntry {
|
||||
id: number;
|
||||
resourceID: number;
|
||||
ticketID?: number;
|
||||
taskID?: number;
|
||||
projectID?: number;
|
||||
companyID?: number;
|
||||
dateWorked: string; // ISO date string - Autotask uses dateWorked
|
||||
hoursWorked: number; // Hours worked on this entry
|
||||
summaryNotes?: string; // Autotask uses summaryNotes not notes
|
||||
internalNotes?: string;
|
||||
title?: string;
|
||||
type?: number;
|
||||
startDateTime?: string; // ISO datetime string
|
||||
endDateTime?: string; // ISO datetime string
|
||||
billable?: boolean;
|
||||
billingRate?: number;
|
||||
billingRateCurrencyID?: number;
|
||||
costRate?: number;
|
||||
costRateCurrencyID?: number;
|
||||
cost?: number;
|
||||
costCurrencyID?: number;
|
||||
revenue?: number;
|
||||
revenueCurrencyID?: number;
|
||||
margin?: number;
|
||||
marginCurrencyID?: number;
|
||||
approved?: boolean;
|
||||
approvedByResourceID?: number;
|
||||
approvedDateTime?: string; // ISO datetime string
|
||||
nonBillable?: boolean;
|
||||
contractServiceID?: number;
|
||||
contractServiceBundleID?: number;
|
||||
roleID?: number;
|
||||
departmentID?: number;
|
||||
locationID?: number;
|
||||
allocationCodeID?: number;
|
||||
impProjectScheduleID?: number;
|
||||
impProjectScheduleTaskID?: number;
|
||||
apiVendorID?: number;
|
||||
createDate: string; // ISO datetime string
|
||||
lastModifiedDate?: string; // ISO datetime string
|
||||
userDefinedFields?: Array<{
|
||||
name: string;
|
||||
value: any;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ApiResponse<T> {
|
||||
item?: T;
|
||||
items?: T[];
|
||||
|
|
|
|||
111
lib/types/auvik.ts
Normal file
111
lib/types/auvik.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
// Auvik API Type Definitions
|
||||
|
||||
export interface AuvikNetworkInterface {
|
||||
interfaceName: string;
|
||||
status: string;
|
||||
speed?: number;
|
||||
macAddress?: string;
|
||||
ipAddress?: string;
|
||||
vlan?: string;
|
||||
}
|
||||
|
||||
export interface AuvikDevice {
|
||||
id: string;
|
||||
deviceName: string;
|
||||
serialNumber?: string;
|
||||
macAddresses?: string[];
|
||||
ipAddresses: string[];
|
||||
deviceType: string;
|
||||
manufacturer?: string;
|
||||
model?: string;
|
||||
makeModel?: string;
|
||||
vendorName?: string;
|
||||
firmwareVersion?: string;
|
||||
softwareVersion?: string;
|
||||
onlineStatus: 'online' | 'offline' | 'unknown';
|
||||
lastSeenTime?: string;
|
||||
uptime?: number;
|
||||
tenantId: string;
|
||||
tenantName?: string;
|
||||
description?: string;
|
||||
networkInterfaces?: AuvikNetworkInterface[];
|
||||
}
|
||||
|
||||
export interface AuvikTenant {
|
||||
id: string;
|
||||
domainPrefix: string;
|
||||
tenantType: 'multiClient' | 'client';
|
||||
parentId?: string;
|
||||
}
|
||||
|
||||
export interface AuvikDeviceResponse {
|
||||
data: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
attributes: {
|
||||
ipAddresses: string[];
|
||||
deviceName: string;
|
||||
deviceType: string;
|
||||
makeModel?: string;
|
||||
vendorName?: string;
|
||||
softwareVersion?: string;
|
||||
serialNumber?: string;
|
||||
description?: string;
|
||||
firmwareVersion?: string;
|
||||
lastModified?: string;
|
||||
lastSeenTime?: string;
|
||||
onlineStatus: string;
|
||||
};
|
||||
relationships?: {
|
||||
tenant?: {
|
||||
data: {
|
||||
type: string;
|
||||
id: string;
|
||||
attributes?: {
|
||||
domainPrefix: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
links?: {
|
||||
next?: string;
|
||||
first?: string;
|
||||
last?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AuvikTenantResponse {
|
||||
data: Array<{
|
||||
type: string;
|
||||
id: string;
|
||||
attributes: {
|
||||
domainPrefix: string;
|
||||
tenantType: string;
|
||||
};
|
||||
relationships?: {
|
||||
parent?: {
|
||||
data: {
|
||||
type: string;
|
||||
id: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface AuvikClientConfig {
|
||||
apiUrl: string;
|
||||
apiUser: string;
|
||||
apiKey: string;
|
||||
}
|
||||
|
||||
export interface AuvikTenantMapping {
|
||||
id: number;
|
||||
auvikTenantId: string;
|
||||
auvikTenantName: string;
|
||||
autotaskCompanyId: number;
|
||||
autotaskCompanyName: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
513
lib/types/database.ts
Normal file
513
lib/types/database.ts
Normal file
|
|
@ -0,0 +1,513 @@
|
|||
/**
|
||||
* Database Types and Interfaces
|
||||
* TypeScript definitions matching PostgreSQL table schemas
|
||||
*/
|
||||
|
||||
// Base audit fields present in all tables
|
||||
export interface AuditFields {
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
synced_at: Date;
|
||||
is_deleted: boolean;
|
||||
deleted_at?: Date | null;
|
||||
}
|
||||
|
||||
// Company entity
|
||||
export interface Company extends AuditFields {
|
||||
id: number;
|
||||
company_name?: string | null;
|
||||
company_number?: string | null;
|
||||
phone?: string | null;
|
||||
fax?: string | null;
|
||||
website?: string | null;
|
||||
address1?: string | null;
|
||||
address2?: string | null;
|
||||
city?: string | null;
|
||||
state?: string | null;
|
||||
postal_code?: string | null;
|
||||
country?: string | null;
|
||||
is_active?: boolean;
|
||||
company_type?: number | null;
|
||||
owner_resource_id?: number | null;
|
||||
territory_id?: number | null;
|
||||
market_segment_id?: number | null;
|
||||
competitor_id?: number | null;
|
||||
billing_address1?: string | null;
|
||||
billing_address2?: string | null;
|
||||
billing_city?: string | null;
|
||||
billing_state?: string | null;
|
||||
billing_postal_code?: string | null;
|
||||
billing_country?: string | null;
|
||||
tax_id?: string | null;
|
||||
tax_exempt?: boolean;
|
||||
tax_region_id?: number | null;
|
||||
currency_id?: number | null;
|
||||
invoice_method?: number | null;
|
||||
invoice_template_id?: number | null;
|
||||
quote_template_id?: number | null;
|
||||
key_account_icon?: number | null;
|
||||
last_activity_date?: Date | null;
|
||||
last_tracked_modification_date_time?: Date | null;
|
||||
api_vendor_id?: number | null;
|
||||
}
|
||||
|
||||
// Resource (User) entity
|
||||
export interface Resource extends AuditFields {
|
||||
id: number;
|
||||
first_name?: string | null;
|
||||
last_name?: string | null;
|
||||
email?: string | null;
|
||||
user_name?: string | null;
|
||||
title?: string | null;
|
||||
office_phone?: string | null;
|
||||
mobile_phone?: string | null;
|
||||
office_extension?: string | null;
|
||||
is_active?: boolean;
|
||||
location_id?: number | null;
|
||||
resource_type?: number | null;
|
||||
pay_roll_identifier?: string | null;
|
||||
hire_date?: Date | null;
|
||||
travel_availability_pct?: number | null;
|
||||
survey_resource_rating?: number | null;
|
||||
}
|
||||
|
||||
// Contact entity
|
||||
export interface Contact extends AuditFields {
|
||||
id: number;
|
||||
company_id: number;
|
||||
first_name?: string | null;
|
||||
last_name?: string | null;
|
||||
title?: string | null;
|
||||
email_address?: string | null;
|
||||
email_address2?: string | null;
|
||||
email_address3?: string | null;
|
||||
phone?: string | null;
|
||||
extension?: string | null;
|
||||
alternate_phone?: string | null;
|
||||
mobile_phone?: string | null;
|
||||
fax?: string | null;
|
||||
address_line?: string | null;
|
||||
address_line1?: string | null;
|
||||
city?: string | null;
|
||||
state?: string | null;
|
||||
zip_code?: string | null;
|
||||
country?: string | null;
|
||||
is_active?: boolean;
|
||||
name_prefix?: string | null;
|
||||
name_suffix?: string | null;
|
||||
facebook_url?: string | null;
|
||||
twitter_url?: string | null;
|
||||
linked_in_url?: string | null;
|
||||
primary_contact?: boolean;
|
||||
account_physical_location_id?: number | null;
|
||||
solicitation_opt_out?: boolean;
|
||||
room_number?: string | null;
|
||||
last_activity_date?: Date | null;
|
||||
last_modified_date?: Date | null;
|
||||
api_vendor_id?: number | null;
|
||||
}
|
||||
|
||||
// Project entity
|
||||
export interface Project extends AuditFields {
|
||||
id: number;
|
||||
company_id: number;
|
||||
project_name?: string | null;
|
||||
project_number?: string | null;
|
||||
description?: string | null;
|
||||
start_date_time?: Date | null;
|
||||
end_date_time?: Date | null;
|
||||
estimated_time?: number | null;
|
||||
actual_hours?: number | null;
|
||||
estimated_sale_cost?: number | null;
|
||||
labor_estimated_costs?: number | null;
|
||||
labor_estimated_revenue?: number | null;
|
||||
project_cost_estimated_margin_percentage?: number | null;
|
||||
status?: number | null;
|
||||
type?: number | null;
|
||||
project_lead_resource_id?: number | null;
|
||||
account_executive_resource_id?: number | null;
|
||||
owner_resource_id?: number | null;
|
||||
creator_resource_id?: number | null;
|
||||
completed_percentage?: number | null;
|
||||
completed_date_time?: Date | null;
|
||||
duration?: number | null;
|
||||
original_estimated_revenue?: number | null;
|
||||
estimated_time_cost?: number | null;
|
||||
purchase_order_number?: string | null;
|
||||
business_division_subdivision_id?: number | null;
|
||||
line_of_business_id?: number | null;
|
||||
department?: number | null;
|
||||
last_activity_date_time?: Date | null;
|
||||
last_activity_person_type?: number | null;
|
||||
last_activity_resource_id?: number | null;
|
||||
}
|
||||
|
||||
// Ticket entity
|
||||
export interface Ticket extends AuditFields {
|
||||
id: number;
|
||||
company_id: number;
|
||||
ticket_number?: string | null;
|
||||
title?: string | null;
|
||||
description?: string | null;
|
||||
status?: number | null;
|
||||
priority?: number | null;
|
||||
queue_id?: number | null;
|
||||
issue_type?: number | null;
|
||||
sub_issue_type?: number | null;
|
||||
source?: number | null;
|
||||
assigned_resource_id?: number | null;
|
||||
assigned_resource_role_id?: number | null;
|
||||
contact_id?: number | null;
|
||||
account_physical_location_id?: number | null;
|
||||
due_date_time?: Date | null;
|
||||
estimated_hours?: number | null;
|
||||
completed_date?: Date | null;
|
||||
create_date?: Date | null;
|
||||
created_by_contact_id?: number | null;
|
||||
last_activity_date?: Date | null;
|
||||
last_customer_notification_date_time?: Date | null;
|
||||
last_customer_visible_activity_date_time?: Date | null;
|
||||
first_response_date_time?: Date | null;
|
||||
resolution_plan_date_time?: Date | null;
|
||||
resolved_date_time?: Date | null;
|
||||
first_response_assigned_resource_id?: number | null;
|
||||
first_response_initiating_resource_id?: number | null;
|
||||
project_id?: number | null;
|
||||
opportunity_id?: number | null;
|
||||
change_approval_board?: number | null;
|
||||
change_approval_status?: number | null;
|
||||
change_approval_type?: number | null;
|
||||
change_info_field1?: string | null;
|
||||
change_info_field2?: string | null;
|
||||
change_info_field3?: string | null;
|
||||
change_info_field4?: string | null;
|
||||
change_info_field5?: string | null;
|
||||
contract_id?: number | null;
|
||||
monitor_id?: number | null;
|
||||
monitor_type_id?: number | null;
|
||||
ticket_type?: number | null;
|
||||
ticket_category?: number | null;
|
||||
service_level_agreement_id?: number | null;
|
||||
resolution?: string | null;
|
||||
purchase_order_number?: string | null;
|
||||
ticket_completion_date?: Date | null;
|
||||
last_activity_person_type?: number | null;
|
||||
last_activity_resource_id?: number | null;
|
||||
current_service_thermometer_rating?: number | null;
|
||||
previous_service_thermometer_rating?: number | null;
|
||||
service_thermometer_temperature?: number | null;
|
||||
api_vendor_id?: number | null;
|
||||
}
|
||||
|
||||
// Task entity
|
||||
export interface Task extends AuditFields {
|
||||
id: number;
|
||||
title?: string | null;
|
||||
description?: string | null;
|
||||
status?: number | null;
|
||||
priority?: number | null;
|
||||
assigned_resource_id?: number | null;
|
||||
assigned_resource_role_id?: number | null;
|
||||
department_id?: number | null;
|
||||
estimated_hours?: number | null;
|
||||
remaining_hours?: number | null;
|
||||
hours_to_be_scheduled?: number | null;
|
||||
start_date_time?: Date | null;
|
||||
end_date_time?: Date | null;
|
||||
completed_date_time?: Date | null;
|
||||
create_date_time?: Date | null;
|
||||
creator_resource_id?: number | null;
|
||||
completed_by_resource_id?: number | null;
|
||||
last_activity_date_time?: Date | null;
|
||||
project_id?: number | null;
|
||||
ticket_id?: number | null;
|
||||
phase_id?: number | null;
|
||||
allocation_code_id?: number | null;
|
||||
task_type?: number | null;
|
||||
task_is_billable?: boolean;
|
||||
task_number?: string | null;
|
||||
purchase_order_number?: string | null;
|
||||
can_client_portal_user_complete_task?: boolean;
|
||||
creator_type?: number | null;
|
||||
task_category_id?: number | null;
|
||||
}
|
||||
|
||||
// Configuration Item entity
|
||||
export interface ConfigurationItem extends AuditFields {
|
||||
id: number;
|
||||
company_id: number;
|
||||
product_id?: number | null;
|
||||
reference_title?: string | null;
|
||||
reference_number?: string | null;
|
||||
serial_number?: string | null;
|
||||
install_date?: Date | null;
|
||||
warranty_expiration_date?: Date | null;
|
||||
is_active?: boolean;
|
||||
daily_cost?: number | null;
|
||||
hourly_cost?: number | null;
|
||||
monthly_cost?: number | null;
|
||||
per_use_cost?: number | null;
|
||||
setup_fee?: number | null;
|
||||
contact_id?: number | null;
|
||||
location_id?: number | null;
|
||||
vendor_id?: number | null;
|
||||
installed_by_id?: number | null;
|
||||
installed_by_contact_id?: number | null;
|
||||
parent_configuration_item_id?: number | null;
|
||||
notes?: string | null;
|
||||
create_date?: Date | null;
|
||||
created_by_person_id?: number | null;
|
||||
last_modified_time?: Date | null;
|
||||
last_activity_person_type?: number | null;
|
||||
impersonator_creator_resource_id?: number | null;
|
||||
configuration_item_category_id?: number | null;
|
||||
configuration_item_type?: number | null;
|
||||
datto_availability?: number | null;
|
||||
datto_device_memory_megabytes?: number | null;
|
||||
datto_drives_errors?: boolean | null;
|
||||
datto_hostname?: string | null;
|
||||
datto_internal_ip?: string | null;
|
||||
datto_kernel_version_id?: number | null;
|
||||
datto_last_check_in_date_time?: Date | null;
|
||||
datto_nic_speed_kilobits_per_second?: number | null;
|
||||
datto_number_of_agents?: number | null;
|
||||
datto_number_of_drives?: number | null;
|
||||
datto_number_of_logical_volumes?: number | null;
|
||||
datto_number_of_volumes?: number | null;
|
||||
datto_off_site_storage_used_bytes?: number | null;
|
||||
datto_os_version_id?: number | null;
|
||||
datto_percentage_used?: number | null;
|
||||
datto_protected_kilobytes?: number | null;
|
||||
datto_remote_ip?: string | null;
|
||||
datto_serial_number?: string | null;
|
||||
datto_uptime_seconds?: number | null;
|
||||
datto_used_kilobytes?: number | null;
|
||||
datto_z_pool_percentage?: number | null;
|
||||
device_networking_id?: number | null;
|
||||
last_backup_date?: Date | null;
|
||||
last_backup_status?: number | null;
|
||||
os_version_id?: number | null;
|
||||
service_id?: number | null;
|
||||
service_bundle_id?: number | null;
|
||||
snmp_location?: string | null;
|
||||
snmp_name?: string | null;
|
||||
snmp_contact?: string | null;
|
||||
api_vendor_id?: number | null;
|
||||
device_type?: string | null;
|
||||
rmm_device_uid?: string | null;
|
||||
rmm_device_audit_architecture_id?: number | null;
|
||||
rmm_device_audit_display_adaptor_id?: number | null;
|
||||
rmm_device_audit_domain_id?: number | null;
|
||||
rmm_device_audit_external_ip_address?: string | null;
|
||||
rmm_device_audit_hostname?: string | null;
|
||||
rmm_device_audit_ip_address?: string | null;
|
||||
rmm_device_audit_mac_address?: string | null;
|
||||
rmm_device_audit_manufacturer_id?: number | null;
|
||||
rmm_device_audit_missing_patch_count?: number | null;
|
||||
rmm_device_audit_mobile_network_operator_id?: number | null;
|
||||
rmm_device_audit_mobile_number?: string | null;
|
||||
rmm_device_audit_model_id?: number | null;
|
||||
rmm_device_audit_motherboard_id?: number | null;
|
||||
rmm_device_audit_operating_system_id?: number | null;
|
||||
rmm_device_audit_processor_id?: number | null;
|
||||
rmm_device_audit_service_pack_id?: number | null;
|
||||
rmm_device_audit_snmp_contact?: string | null;
|
||||
rmm_device_audit_snmp_location?: string | null;
|
||||
rmm_device_audit_snmp_name?: string | null;
|
||||
rmm_device_audit_software_status_id?: number | null;
|
||||
rmm_device_audit_storage_bytes?: number | null;
|
||||
rmm_open_alert_count?: number | null;
|
||||
rmm_device_audit_description?: string | null;
|
||||
rmm_device_audit_device_type_id?: number | null;
|
||||
rmm_device_audit_last_user?: string | null;
|
||||
rmm_device_audit_memory_bytes?: number | null;
|
||||
source_cost_id?: number | null;
|
||||
source_cost_type?: number | null;
|
||||
}
|
||||
|
||||
// Contract entity
|
||||
export interface Contract extends AuditFields {
|
||||
id: number;
|
||||
company_id: number;
|
||||
contract_name?: string | null;
|
||||
contract_number?: string | null;
|
||||
description?: string | null;
|
||||
start_date?: Date | null;
|
||||
end_date?: Date | null;
|
||||
time_reporting_requires_start_and_stop_times?: number | null;
|
||||
service_level_agreement_id?: number | null;
|
||||
contract_type?: number | null;
|
||||
contract_category?: number | null;
|
||||
status?: number | null;
|
||||
business_division_subdivision_id?: number | null;
|
||||
contact_id?: number | null;
|
||||
contact_name?: string | null;
|
||||
billing_preference?: number | null;
|
||||
purchase_order_number?: string | null;
|
||||
setup_fee?: number | null;
|
||||
setup_fee_allocation_code_id?: number | null;
|
||||
estimated_cost?: number | null;
|
||||
estimated_hours?: number | null;
|
||||
estimated_revenue?: number | null;
|
||||
over_budget_dollar_amount?: number | null;
|
||||
over_budget_hours?: number | null;
|
||||
contract_period_type?: string | null;
|
||||
opportunity_id?: number | null;
|
||||
renewed_contract_id?: number | null;
|
||||
is_default_contract?: boolean;
|
||||
internal_currency_setup_fee?: number | null;
|
||||
internal_currency_over_budget_dollar_amount?: number | null;
|
||||
internal_currency_estimated_cost?: number | null;
|
||||
internal_currency_estimated_revenue?: number | null;
|
||||
exclusion_contract_id?: number | null;
|
||||
internal_currency_monthly_revenue?: number | null;
|
||||
internal_currency_quarterly_revenue?: number | null;
|
||||
internal_currency_semi_annual_revenue?: number | null;
|
||||
internal_currency_yearly_revenue?: number | null;
|
||||
internal_currency_one_time_revenue?: number | null;
|
||||
compliance?: boolean | null;
|
||||
}
|
||||
|
||||
// Billing Item entity
|
||||
export interface BillingItem extends AuditFields {
|
||||
id: number;
|
||||
company_id?: number | null;
|
||||
product_id?: number | null;
|
||||
description?: string | null;
|
||||
quantity?: number | null;
|
||||
rate?: number | null;
|
||||
total_amount?: number | null;
|
||||
line_discount_dollars?: number | null;
|
||||
line_discount_percent?: number | null;
|
||||
tax_category_id?: number | null;
|
||||
internal_currency_line_discount_dollars?: number | null;
|
||||
allocation_code_id?: number | null;
|
||||
invoice_id?: number | null;
|
||||
vendor_id?: number | null;
|
||||
expense_item?: boolean;
|
||||
task_id?: number | null;
|
||||
ticket_id?: number | null;
|
||||
project_id?: number | null;
|
||||
our_cost?: number | null;
|
||||
list_price?: number | null;
|
||||
unit_cost?: number | null;
|
||||
unit_price?: number | null;
|
||||
extended_price?: number | null;
|
||||
tax_dollars?: number | null;
|
||||
internal_currency_unit_price?: number | null;
|
||||
internal_currency_total_amount?: number | null;
|
||||
}
|
||||
|
||||
// Picklist base interface
|
||||
export interface PicklistValue extends AuditFields {
|
||||
value: number;
|
||||
label: string;
|
||||
is_active?: boolean;
|
||||
is_system?: boolean;
|
||||
sort_order?: number | null;
|
||||
parent_value?: number | null;
|
||||
}
|
||||
|
||||
// Status picklist
|
||||
export interface Status extends PicklistValue {}
|
||||
|
||||
// Issue Type picklist
|
||||
export interface IssueType extends PicklistValue {}
|
||||
|
||||
// Sub-Issue Type picklist
|
||||
export interface SubIssueType extends PicklistValue {}
|
||||
|
||||
// Work Type picklist
|
||||
export interface WorkType extends PicklistValue {}
|
||||
|
||||
// Sync History (matches sync_history table)
|
||||
export interface SyncHistoryRecord {
|
||||
id: number;
|
||||
entity_type: string;
|
||||
sync_type: 'full' | 'incremental' | 'entity-specific';
|
||||
status: 'started' | 'in_progress' | 'completed' | 'failed';
|
||||
started_at: Date;
|
||||
completed_at?: Date | null;
|
||||
records_added: number;
|
||||
records_updated: number;
|
||||
records_deleted: number;
|
||||
error_message?: string | null;
|
||||
triggered_by?: string | null;
|
||||
}
|
||||
|
||||
// Time Entry entity
|
||||
export interface TimeEntry extends AuditFields {
|
||||
id: number;
|
||||
resource_id: number;
|
||||
ticket_id?: number | null;
|
||||
task_id?: number | null;
|
||||
project_id?: number | null;
|
||||
company_id?: number | null;
|
||||
entry_date: Date;
|
||||
hours_worked: number;
|
||||
notes?: string | null;
|
||||
internal_notes?: string | null;
|
||||
title?: string | null;
|
||||
type?: number | null;
|
||||
start_date_time?: Date | null;
|
||||
end_date_time?: Date | null;
|
||||
billable?: boolean;
|
||||
billing_rate?: number | null;
|
||||
billing_rate_currency_id?: number | null;
|
||||
cost_rate?: number | null;
|
||||
cost_rate_currency_id?: number | null;
|
||||
cost?: number | null;
|
||||
cost_currency_id?: number | null;
|
||||
revenue?: number | null;
|
||||
revenue_currency_id?: number | null;
|
||||
margin?: number | null;
|
||||
margin_currency_id?: number | null;
|
||||
approved?: boolean;
|
||||
approved_by_resource_id?: number | null;
|
||||
approved_date_time?: Date | null;
|
||||
non_billable?: boolean;
|
||||
contract_service_id?: number | null;
|
||||
contract_service_bundle_id?: number | null;
|
||||
role_id?: number | null;
|
||||
department_id?: number | null;
|
||||
location_id?: number | null;
|
||||
allocation_code_id?: number | null;
|
||||
imp_project_schedule_id?: number | null;
|
||||
imp_project_schedule_task_id?: number | null;
|
||||
api_vendor_id?: number | null;
|
||||
}
|
||||
|
||||
// Union type for all entities
|
||||
export type Entity =
|
||||
| Company
|
||||
| Resource
|
||||
| Contact
|
||||
| Project
|
||||
| Ticket
|
||||
| Task
|
||||
| ConfigurationItem
|
||||
| Contract
|
||||
| BillingItem
|
||||
| Status
|
||||
| IssueType
|
||||
| SubIssueType
|
||||
| WorkType
|
||||
| TimeEntry;
|
||||
|
||||
// Table name type
|
||||
export type TableName =
|
||||
| 'companies'
|
||||
| 'resources'
|
||||
| 'contacts'
|
||||
| 'projects'
|
||||
| 'tickets'
|
||||
| 'tasks'
|
||||
| 'configuration_items'
|
||||
| 'contracts'
|
||||
| 'billing_items'
|
||||
| 'statuses'
|
||||
| 'issue_types'
|
||||
| 'sub_issue_types'
|
||||
| 'work_types'
|
||||
| 'time_entries'
|
||||
| 'sync_history';
|
||||
266
lib/types/errors.ts
Normal file
266
lib/types/errors.ts
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
/**
|
||||
* Custom Error Types for Sync Operations
|
||||
*/
|
||||
|
||||
/**
|
||||
* Base sync error class
|
||||
*/
|
||||
export class SyncError extends Error {
|
||||
public readonly code: string;
|
||||
public readonly context?: Record<string, any>;
|
||||
public readonly isRetryable: boolean;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
code: string,
|
||||
context?: Record<string, any>,
|
||||
isRetryable: boolean = false
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'SyncError';
|
||||
this.code = code;
|
||||
this.context = context;
|
||||
this.isRetryable = isRetryable;
|
||||
|
||||
// Maintains proper stack trace for where our error was thrown
|
||||
if (Error.captureStackTrace) {
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Network-related errors (connection failures, timeouts)
|
||||
*/
|
||||
export class NetworkError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'NETWORK_ERROR', context, true);
|
||||
this.name = 'NetworkError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Authentication/Authorization errors
|
||||
*/
|
||||
export class AuthError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'AUTH_ERROR', context, false);
|
||||
this.name = 'AuthError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate limit errors
|
||||
*/
|
||||
export class RateLimitError extends SyncError {
|
||||
public readonly retryAfter?: number;
|
||||
|
||||
constructor(message: string, retryAfter?: number, context?: Record<string, any>) {
|
||||
super(message, 'RATE_LIMIT_ERROR', context, true);
|
||||
this.name = 'RateLimitError';
|
||||
this.retryAfter = retryAfter;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* API-related errors
|
||||
*/
|
||||
export class ApiError extends SyncError {
|
||||
public readonly statusCode?: number;
|
||||
|
||||
constructor(message: string, statusCode?: number, context?: Record<string, any>) {
|
||||
super(message, 'API_ERROR', context, statusCode ? statusCode >= 500 : false);
|
||||
this.name = 'ApiError';
|
||||
this.statusCode = statusCode;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Database-related errors
|
||||
*/
|
||||
export class DatabaseError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>, isRetryable: boolean = false) {
|
||||
super(message, 'DATABASE_ERROR', context, isRetryable);
|
||||
this.name = 'DatabaseError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Database constraint violation errors
|
||||
*/
|
||||
export class ConstraintError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'DATABASE_CONSTRAINT_ERROR', context, false);
|
||||
this.name = 'ConstraintError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Data validation errors
|
||||
*/
|
||||
export class ValidationError extends SyncError {
|
||||
public readonly validationErrors: Array<{ field: string; message: string }>;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
validationErrors: Array<{ field: string; message: string }>,
|
||||
context?: Record<string, any>
|
||||
) {
|
||||
super(message, 'VALIDATION_ERROR', context, false);
|
||||
this.name = 'ValidationError';
|
||||
this.validationErrors = validationErrors;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Data mapping errors
|
||||
*/
|
||||
export class MappingError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'MAPPING_ERROR', context, false);
|
||||
this.name = 'MappingError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration errors
|
||||
*/
|
||||
export class ConfigError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'CONFIG_ERROR', context, false);
|
||||
this.name = 'ConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Timeout errors
|
||||
*/
|
||||
export class TimeoutError extends SyncError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'TIMEOUT_ERROR', context, true);
|
||||
this.name = 'TimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to categorize generic errors
|
||||
*/
|
||||
export function categorizeError(error: any): SyncError {
|
||||
if (error instanceof SyncError) {
|
||||
return error;
|
||||
}
|
||||
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const errorString = errorMessage.toLowerCase();
|
||||
|
||||
// Network errors
|
||||
if (
|
||||
errorString.includes('econnrefused') ||
|
||||
errorString.includes('etimedout') ||
|
||||
errorString.includes('enotfound') ||
|
||||
errorString.includes('network')
|
||||
) {
|
||||
return new NetworkError(errorMessage, { originalError: error });
|
||||
}
|
||||
|
||||
// Auth errors
|
||||
if (
|
||||
errorString.includes('401') ||
|
||||
errorString.includes('403') ||
|
||||
errorString.includes('unauthorized') ||
|
||||
errorString.includes('forbidden')
|
||||
) {
|
||||
return new AuthError(errorMessage, { originalError: error });
|
||||
}
|
||||
|
||||
// Rate limit errors
|
||||
if (errorString.includes('429') || errorString.includes('rate limit')) {
|
||||
return new RateLimitError(errorMessage, undefined, { originalError: error });
|
||||
}
|
||||
|
||||
// Database constraint errors
|
||||
if (
|
||||
errorString.includes('constraint') ||
|
||||
errorString.includes('duplicate') ||
|
||||
errorString.includes('unique violation')
|
||||
) {
|
||||
return new ConstraintError(errorMessage, { originalError: error });
|
||||
}
|
||||
|
||||
// Database errors
|
||||
if (
|
||||
errorString.includes('query') ||
|
||||
errorString.includes('sql') ||
|
||||
errorString.includes('database') ||
|
||||
errorString.includes('postgres')
|
||||
) {
|
||||
return new DatabaseError(errorMessage, { originalError: error });
|
||||
}
|
||||
|
||||
// Validation errors
|
||||
if (errorString.includes('validation') || errorString.includes('invalid')) {
|
||||
return new ValidationError(errorMessage, [], { originalError: error });
|
||||
}
|
||||
|
||||
// Mapping errors
|
||||
if (errorString.includes('mapping') || errorString.includes('transform')) {
|
||||
return new MappingError(errorMessage, { originalError: error });
|
||||
}
|
||||
|
||||
// Timeout errors
|
||||
if (errorString.includes('timeout') || errorString.includes('timed out')) {
|
||||
return new TimeoutError(errorMessage, { originalError: error });
|
||||
}
|
||||
|
||||
// API errors (check for HTTP status codes)
|
||||
const statusMatch = errorString.match(/\b([45]\d{2})\b/);
|
||||
if (statusMatch) {
|
||||
const statusCode = parseInt(statusMatch[1]);
|
||||
return new ApiError(errorMessage, statusCode, { originalError: error });
|
||||
}
|
||||
|
||||
// Default to generic sync error
|
||||
return new SyncError(errorMessage, 'UNKNOWN_ERROR', { originalError: error });
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if error is retryable
|
||||
*/
|
||||
export function isRetryableError(error: any): boolean {
|
||||
if (error instanceof SyncError) {
|
||||
return error.isRetryable;
|
||||
}
|
||||
|
||||
const categorized = categorizeError(error);
|
||||
return categorized.isRetryable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format error for logging
|
||||
*/
|
||||
export function formatErrorForLog(error: any): {
|
||||
message: string;
|
||||
code: string;
|
||||
stack?: string;
|
||||
context?: Record<string, any>;
|
||||
isRetryable: boolean;
|
||||
} {
|
||||
if (error instanceof SyncError) {
|
||||
return {
|
||||
message: error.message,
|
||||
code: error.code,
|
||||
stack: error.stack,
|
||||
context: error.context,
|
||||
isRetryable: error.isRetryable,
|
||||
};
|
||||
}
|
||||
|
||||
const categorized = categorizeError(error);
|
||||
return {
|
||||
message: categorized.message,
|
||||
code: categorized.code,
|
||||
stack: categorized.stack,
|
||||
context: categorized.context,
|
||||
isRetryable: categorized.isRetryable,
|
||||
};
|
||||
}
|
||||
194
lib/types/sync.ts
Normal file
194
lib/types/sync.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
/**
|
||||
* Sync Types and Interfaces
|
||||
* TypeScript definitions for sync operations
|
||||
*/
|
||||
|
||||
// Entity types that can be synced from Autotask
|
||||
export enum EntityType {
|
||||
COMPANIES = 'companies',
|
||||
TICKETS = 'tickets',
|
||||
TASKS = 'tasks',
|
||||
PROJECTS = 'projects',
|
||||
RESOURCES = 'resources',
|
||||
STATUSES = 'statuses',
|
||||
ISSUE_TYPES = 'issue_types',
|
||||
SUB_ISSUE_TYPES = 'sub_issue_types',
|
||||
WORK_TYPES = 'work_types',
|
||||
BILLING_ITEMS = 'billing_items',
|
||||
CONFIGURATION_ITEMS = 'configuration_items',
|
||||
CONTACTS = 'contacts',
|
||||
CONTRACTS = 'contracts',
|
||||
TIME_ENTRIES = 'time_entries',
|
||||
}
|
||||
|
||||
// Sync operation types
|
||||
export enum SyncType {
|
||||
FULL = 'full',
|
||||
INCREMENTAL = 'incremental',
|
||||
ENTITY_SPECIFIC = 'entity-specific',
|
||||
}
|
||||
|
||||
// Sync status
|
||||
export enum SyncStatus {
|
||||
STARTED = 'started',
|
||||
IN_PROGRESS = 'in_progress',
|
||||
COMPLETED = 'completed',
|
||||
FAILED = 'failed',
|
||||
}
|
||||
|
||||
// Sync configuration
|
||||
export interface SyncConfig {
|
||||
entities: EntityType[];
|
||||
syncType: SyncType;
|
||||
triggeredBy?: string;
|
||||
batchSize?: number;
|
||||
rateLimit?: number; // requests per second
|
||||
yearsBack?: number; // Number of years to look back for time-based entities (default: 2)
|
||||
}
|
||||
|
||||
// Sync progress information
|
||||
export interface SyncProgress {
|
||||
syncId: string;
|
||||
entityType: EntityType;
|
||||
status: SyncStatus;
|
||||
currentPage?: number;
|
||||
totalPages?: number;
|
||||
recordsProcessed: number;
|
||||
recordsAdded: number;
|
||||
recordsUpdated: number;
|
||||
recordsDeleted: number;
|
||||
startedAt: Date;
|
||||
estimatedCompletion?: Date;
|
||||
error?: string;
|
||||
// Chunking support
|
||||
currentChunk?: number;
|
||||
totalChunks?: number;
|
||||
chunkDescription?: string; // e.g., "Jan 2024"
|
||||
}
|
||||
|
||||
// Chunk progress for detailed tracking
|
||||
export interface ChunkProgress {
|
||||
chunkIndex: number;
|
||||
totalChunks: number;
|
||||
startDate: Date;
|
||||
endDate: Date;
|
||||
description: string; // e.g., "January 2024"
|
||||
status: 'pending' | 'in_progress' | 'completed' | 'failed';
|
||||
recordsProcessed: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// Sync history record (matches database table)
|
||||
export interface SyncHistory {
|
||||
id: number;
|
||||
entity_type: string;
|
||||
sync_type: SyncType;
|
||||
status: SyncStatus;
|
||||
started_at: Date;
|
||||
completed_at?: Date;
|
||||
records_added: number;
|
||||
records_updated: number;
|
||||
records_deleted: number;
|
||||
error_message?: string;
|
||||
triggered_by?: string;
|
||||
}
|
||||
|
||||
// Alias for database compatibility
|
||||
export interface SyncHistoryRecord {
|
||||
id: number;
|
||||
entity_type: string;
|
||||
sync_type: SyncType;
|
||||
status: SyncStatus;
|
||||
started_at: Date;
|
||||
completed_at?: Date;
|
||||
records_added: number;
|
||||
records_updated: number;
|
||||
records_deleted: number;
|
||||
error_message?: string;
|
||||
triggered_by?: string;
|
||||
}
|
||||
|
||||
// Sync result for a single entity
|
||||
export interface EntitySyncResult {
|
||||
entityType: EntityType;
|
||||
success: boolean;
|
||||
recordsAdded: number;
|
||||
recordsUpdated: number;
|
||||
recordsDeleted: number;
|
||||
duration: number; // milliseconds
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// Overall sync result
|
||||
export interface SyncResult {
|
||||
syncId: string;
|
||||
syncType: SyncType;
|
||||
status: SyncStatus;
|
||||
entities: EntitySyncResult[];
|
||||
totalRecordsAdded: number;
|
||||
totalRecordsUpdated: number;
|
||||
totalRecordsDeleted: number;
|
||||
startedAt: Date;
|
||||
completedAt?: Date;
|
||||
duration?: number; // milliseconds
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
// Sync state (stored in Redis for real-time updates)
|
||||
export interface SyncState {
|
||||
syncId: string;
|
||||
status: SyncStatus;
|
||||
progress: SyncProgress[];
|
||||
startedAt: Date;
|
||||
lastUpdated: Date;
|
||||
}
|
||||
|
||||
// Entity dependency map (for determining sync order)
|
||||
export const ENTITY_DEPENDENCIES: Record<EntityType, EntityType[]> = {
|
||||
[EntityType.COMPANIES]: [], // No dependencies
|
||||
[EntityType.RESOURCES]: [], // No dependencies
|
||||
[EntityType.STATUSES]: [], // No dependencies
|
||||
[EntityType.ISSUE_TYPES]: [], // No dependencies
|
||||
[EntityType.SUB_ISSUE_TYPES]: [], // No dependencies
|
||||
[EntityType.WORK_TYPES]: [], // No dependencies
|
||||
[EntityType.CONTACTS]: [EntityType.COMPANIES], // Depends on companies
|
||||
[EntityType.PROJECTS]: [EntityType.COMPANIES, EntityType.RESOURCES], // Depends on companies and resources
|
||||
[EntityType.TICKETS]: [EntityType.COMPANIES, EntityType.RESOURCES, EntityType.CONTACTS], // Depends on companies, resources, contacts
|
||||
[EntityType.TASKS]: [EntityType.RESOURCES, EntityType.PROJECTS, EntityType.TICKETS], // Depends on resources, projects, tickets
|
||||
[EntityType.CONFIGURATION_ITEMS]: [EntityType.COMPANIES, EntityType.CONTACTS], // Depends on companies and contacts
|
||||
[EntityType.CONTRACTS]: [EntityType.COMPANIES, EntityType.CONTACTS], // Depends on companies and contacts
|
||||
[EntityType.BILLING_ITEMS]: [EntityType.COMPANIES, EntityType.TASKS, EntityType.TICKETS, EntityType.PROJECTS], // Depends on multiple entities
|
||||
[EntityType.TIME_ENTRIES]: [EntityType.COMPANIES, EntityType.RESOURCES, EntityType.CONTACTS, EntityType.PROJECTS, EntityType.TASKS, EntityType.TICKETS], // Depends on many entities
|
||||
};
|
||||
|
||||
// Autotask API field names (for incremental sync)
|
||||
export interface AutotaskQueryFilter {
|
||||
field: string;
|
||||
op: 'eq' | 'noteq' | 'gt' | 'gte' | 'lt' | 'lte' | 'contains' | 'beginsWith' | 'endsWith';
|
||||
value: any;
|
||||
}
|
||||
|
||||
// Autotask query options
|
||||
export interface AutotaskQueryOptions {
|
||||
filters?: AutotaskQueryFilter[];
|
||||
pageSize?: number;
|
||||
includeFields?: string[];
|
||||
maxRecords?: number;
|
||||
}
|
||||
|
||||
// Last sync information per entity
|
||||
export interface LastSyncInfo {
|
||||
entityType: EntityType;
|
||||
lastSyncTime?: Date;
|
||||
lastSyncStatus: SyncStatus;
|
||||
recordCount: number;
|
||||
}
|
||||
|
||||
// Sync notification
|
||||
export interface SyncNotification {
|
||||
syncId: string;
|
||||
type: 'success' | 'failure' | 'warning';
|
||||
message: string;
|
||||
entityType?: EntityType;
|
||||
timestamp: Date;
|
||||
}
|
||||
219
lib/utils/api-helpers.ts
Normal file
219
lib/utils/api-helpers.ts
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
/**
|
||||
* API Helper Utilities
|
||||
* Common utilities for API endpoints including query parameter parsing
|
||||
*/
|
||||
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
export interface QueryOptions {
|
||||
page?: number;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
includeDeleted?: boolean;
|
||||
sort?: string;
|
||||
order?: 'ASC' | 'DESC';
|
||||
filters?: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface PaginationInfo {
|
||||
page: number;
|
||||
limit: number;
|
||||
offset: number;
|
||||
total: number;
|
||||
totalPages: number;
|
||||
hasMore: boolean;
|
||||
hasPrevious: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse query parameters from Next.js request
|
||||
* @param request Next.js request object
|
||||
* @param defaults Default values for query options
|
||||
* @returns Parsed query options
|
||||
*/
|
||||
export function parseQueryParams(
|
||||
request: NextRequest,
|
||||
defaults: Partial<QueryOptions> = {}
|
||||
): QueryOptions {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
|
||||
// Parse pagination
|
||||
const page = parseInt(searchParams.get('page') || String(defaults.page || 1));
|
||||
const limit = parseInt(searchParams.get('limit') || String(defaults.limit || 100));
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
// Parse includeDeleted flag
|
||||
const includeDeletedParam = searchParams.get('includeDeleted');
|
||||
const includeDeleted = includeDeletedParam !== null
|
||||
? includeDeletedParam === 'true'
|
||||
: defaults.includeDeleted || false;
|
||||
|
||||
// Parse sorting
|
||||
const sort = searchParams.get('sort') || defaults.sort || 'id';
|
||||
const orderParam = searchParams.get('order')?.toUpperCase();
|
||||
const order = (orderParam === 'ASC' || orderParam === 'DESC') ? orderParam : (defaults.order || 'ASC');
|
||||
|
||||
// Parse filters
|
||||
const filters: Record<string, any> = { ...defaults.filters };
|
||||
|
||||
// Get all search params and treat unknown params as filters
|
||||
searchParams.forEach((value, key) => {
|
||||
// Skip known pagination/sorting params
|
||||
if (['page', 'limit', 'includeDeleted', 'sort', 'order'].includes(key)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse filter value
|
||||
filters[key] = parseFilterValue(value);
|
||||
});
|
||||
|
||||
return {
|
||||
page,
|
||||
limit,
|
||||
offset,
|
||||
includeDeleted,
|
||||
sort,
|
||||
order,
|
||||
filters,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse filter value to appropriate type
|
||||
* @param value String value from query parameter
|
||||
* @returns Parsed value (boolean, number, or string)
|
||||
*/
|
||||
function parseFilterValue(value: string): any {
|
||||
// Boolean
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
|
||||
// Number
|
||||
if (/^\d+$/.test(value)) return parseInt(value);
|
||||
if (/^\d+\.\d+$/.test(value)) return parseFloat(value);
|
||||
|
||||
// Null
|
||||
if (value === 'null') return null;
|
||||
|
||||
// String (default)
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build WHERE clause from filters
|
||||
* @param filters Filter object
|
||||
* @param includeDeleted Whether to include deleted records
|
||||
* @returns WHERE clause object
|
||||
*/
|
||||
export function buildWhereClause(
|
||||
filters: Record<string, any>,
|
||||
includeDeleted: boolean = false
|
||||
): Record<string, any> {
|
||||
const where: Record<string, any> = { ...filters };
|
||||
|
||||
// Always exclude soft-deleted records unless explicitly requested
|
||||
if (!includeDeleted) {
|
||||
where.is_deleted = false;
|
||||
}
|
||||
|
||||
return where;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build ORDER BY clause
|
||||
* @param sort Sort field
|
||||
* @param order Sort order (ASC/DESC)
|
||||
* @returns ORDER BY string
|
||||
*/
|
||||
export function buildOrderByClause(sort: string, order: 'ASC' | 'DESC'): string {
|
||||
// Sanitize sort field to prevent SQL injection
|
||||
const sanitizedSort = sort.replace(/[^a-zA-Z0-9_]/g, '');
|
||||
return `${sanitizedSort} ${order}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create pagination info object
|
||||
* @param page Current page number
|
||||
* @param limit Records per page
|
||||
* @param total Total record count
|
||||
* @returns Pagination information
|
||||
*/
|
||||
export function createPaginationInfo(
|
||||
page: number,
|
||||
limit: number,
|
||||
total: number
|
||||
): PaginationInfo {
|
||||
const offset = (page - 1) * limit;
|
||||
const totalPages = Math.ceil(total / limit);
|
||||
|
||||
return {
|
||||
page,
|
||||
limit,
|
||||
offset,
|
||||
total,
|
||||
totalPages,
|
||||
hasMore: page < totalPages,
|
||||
hasPrevious: page > 1,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate query parameters
|
||||
* @param options Query options to validate
|
||||
* @throws Error if validation fails
|
||||
*/
|
||||
export function validateQueryParams(options: QueryOptions): void {
|
||||
if (options.page && options.page < 1) {
|
||||
throw new Error('Page must be greater than 0');
|
||||
}
|
||||
|
||||
if (options.limit && (options.limit < 1 || options.limit > 1000)) {
|
||||
throw new Error('Limit must be between 1 and 1000');
|
||||
}
|
||||
|
||||
if (options.sort && !/^[a-zA-Z0-9_]+$/.test(options.sort)) {
|
||||
throw new Error('Invalid sort field');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format API response with data and pagination
|
||||
* @param data Data array
|
||||
* @param pagination Pagination info
|
||||
* @param meta Additional metadata
|
||||
* @returns Formatted response object
|
||||
*/
|
||||
export function formatApiResponse<T>(
|
||||
data: T[],
|
||||
pagination: PaginationInfo,
|
||||
meta?: Record<string, any>
|
||||
) {
|
||||
return {
|
||||
data,
|
||||
pagination,
|
||||
meta: {
|
||||
timestamp: new Date().toISOString(),
|
||||
...meta,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle API errors consistently
|
||||
* @param error Error object
|
||||
* @param context Error context
|
||||
* @returns Error response object
|
||||
*/
|
||||
export function handleApiError(error: any, context?: string) {
|
||||
console.error(`API Error${context ? ` (${context})` : ''}:`, error);
|
||||
|
||||
const message = error instanceof Error ? error.message : 'An unexpected error occurred';
|
||||
const statusCode = error.statusCode || 500;
|
||||
|
||||
return {
|
||||
error: message,
|
||||
context,
|
||||
timestamp: new Date().toISOString(),
|
||||
statusCode,
|
||||
};
|
||||
}
|
||||
328
lib/utils/db-helpers.ts
Normal file
328
lib/utils/db-helpers.ts
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
/**
|
||||
* Database Helper Functions
|
||||
* Additional utility functions for database operations
|
||||
*/
|
||||
|
||||
import postgresClient from '../services/postgres-client';
|
||||
import { EntityType } from '../types/sync';
|
||||
import { getTableName } from './sync-helpers';
|
||||
|
||||
/**
|
||||
* Upsert a single record
|
||||
* @param entity Entity type
|
||||
* @param data Record data
|
||||
* @returns Upserted record
|
||||
*/
|
||||
export async function upsertRecord(
|
||||
entity: EntityType,
|
||||
data: Record<string, any>
|
||||
): Promise<any> {
|
||||
const tableName = getTableName(entity);
|
||||
return await postgresClient.upsert(tableName, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk upsert records with batching
|
||||
* @param entity Entity type
|
||||
* @param records Array of records
|
||||
* @param batchSize Number of records per batch (default: 100)
|
||||
* @returns Total number of records upserted
|
||||
*/
|
||||
export async function bulkUpsertRecords(
|
||||
entity: EntityType,
|
||||
records: Record<string, any>[],
|
||||
batchSize: number = 100
|
||||
): Promise<number> {
|
||||
if (records.length === 0) return 0;
|
||||
|
||||
const tableName = getTableName(entity);
|
||||
let totalUpserted = 0;
|
||||
|
||||
// Process in batches
|
||||
for (let i = 0; i < records.length; i += batchSize) {
|
||||
const batch = records.slice(i, i + batchSize);
|
||||
const count = await postgresClient.bulkUpsert(tableName, batch);
|
||||
totalUpserted += count;
|
||||
}
|
||||
|
||||
return totalUpserted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft delete records not in the provided ID list
|
||||
* @param entity Entity type
|
||||
* @param activeIds Array of IDs that should remain active
|
||||
* @returns Number of records soft-deleted
|
||||
*/
|
||||
export async function softDeleteMissingRecords(
|
||||
entity: EntityType,
|
||||
activeIds: (number | string)[]
|
||||
): Promise<number> {
|
||||
if (activeIds.length === 0) return 0;
|
||||
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
UPDATE ${tableName}
|
||||
SET is_deleted = true, deleted_at = CURRENT_TIMESTAMP
|
||||
WHERE id NOT IN (${activeIds.map((_, i) => `$${i + 1}`).join(', ')})
|
||||
AND is_deleted = false
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query, activeIds);
|
||||
return result.rowCount || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get last sync time for an entity
|
||||
* @param entity Entity type
|
||||
* @returns Last sync timestamp or null
|
||||
*/
|
||||
export async function getLastSyncTime(
|
||||
entity: EntityType
|
||||
): Promise<Date | null> {
|
||||
const query = `
|
||||
SELECT completed_at
|
||||
FROM sync_history
|
||||
WHERE entity_type = $1
|
||||
AND status = 'completed'
|
||||
ORDER BY completed_at DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query<{ completed_at: Date }>(
|
||||
query,
|
||||
[entity]
|
||||
);
|
||||
|
||||
return result.rows[0]?.completed_at || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get record count for an entity
|
||||
* @param entity Entity type
|
||||
* @param includeDeleted Include soft-deleted records
|
||||
* @returns Record count
|
||||
*/
|
||||
export async function getRecordCount(
|
||||
entity: EntityType,
|
||||
includeDeleted: boolean = false
|
||||
): Promise<number> {
|
||||
const tableName = getTableName(entity);
|
||||
return await postgresClient.count(tableName, {}, includeDeleted);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get records modified since a specific date
|
||||
* @param entity Entity type
|
||||
* @param since Date to filter from
|
||||
* @param limit Maximum number of records
|
||||
* @returns Array of records
|
||||
*/
|
||||
export async function getRecordsModifiedSince(
|
||||
entity: EntityType,
|
||||
since: Date,
|
||||
limit?: number
|
||||
): Promise<any[]> {
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
SELECT *
|
||||
FROM ${tableName}
|
||||
WHERE synced_at >= $1
|
||||
AND is_deleted = false
|
||||
ORDER BY synced_at DESC
|
||||
${limit ? `LIMIT ${limit}` : ''}
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query, [since]);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active IDs for an entity
|
||||
* @param entity Entity type
|
||||
* @returns Array of active record IDs
|
||||
*/
|
||||
export async function getActiveIds(entity: EntityType): Promise<number[]> {
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
SELECT id
|
||||
FROM ${tableName}
|
||||
WHERE is_deleted = false
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query<{ id: number }>(query);
|
||||
return result.rows.map((row: { id: number }) => row.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore soft-deleted record
|
||||
* @param entity Entity type
|
||||
* @param id Record ID
|
||||
*/
|
||||
export async function restoreRecord(
|
||||
entity: EntityType,
|
||||
id: number | string
|
||||
): Promise<void> {
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
UPDATE ${tableName}
|
||||
SET is_deleted = false, deleted_at = NULL
|
||||
WHERE id = $1
|
||||
`;
|
||||
|
||||
await postgresClient.query(query, [id]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hard delete soft-deleted records older than specified days
|
||||
* @param entity Entity type
|
||||
* @param daysOld Number of days
|
||||
* @returns Number of records deleted
|
||||
*/
|
||||
export async function purgeOldDeletedRecords(
|
||||
entity: EntityType,
|
||||
daysOld: number = 90
|
||||
): Promise<number> {
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
DELETE FROM ${tableName}
|
||||
WHERE is_deleted = true
|
||||
AND deleted_at < NOW() - INTERVAL '${daysOld} days'
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query);
|
||||
return result.rowCount || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sync statistics for an entity
|
||||
* @param entity Entity type
|
||||
* @returns Sync statistics
|
||||
*/
|
||||
export async function getSyncStatistics(entity: EntityType): Promise<{
|
||||
totalRecords: number;
|
||||
activeRecords: number;
|
||||
deletedRecords: number;
|
||||
lastSyncTime: Date | null;
|
||||
lastSyncStatus: string | null;
|
||||
}> {
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
// Get record counts
|
||||
const countQuery = `
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE is_deleted = false) as active,
|
||||
COUNT(*) FILTER (WHERE is_deleted = true) as deleted
|
||||
FROM ${tableName}
|
||||
`;
|
||||
|
||||
const countResult = await postgresClient.query<{
|
||||
total: string;
|
||||
active: string;
|
||||
deleted: string;
|
||||
}>(countQuery);
|
||||
|
||||
// Get last sync info
|
||||
const syncQuery = `
|
||||
SELECT completed_at, status
|
||||
FROM sync_history
|
||||
WHERE entity_type = $1
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
const syncResult = await postgresClient.query<{
|
||||
completed_at: Date;
|
||||
status: string;
|
||||
}>(syncQuery, [entity]);
|
||||
|
||||
return {
|
||||
totalRecords: parseInt(countResult.rows[0]?.total || '0'),
|
||||
activeRecords: parseInt(countResult.rows[0]?.active || '0'),
|
||||
deletedRecords: parseInt(countResult.rows[0]?.deleted || '0'),
|
||||
lastSyncTime: syncResult.rows[0]?.completed_at || null,
|
||||
lastSyncStatus: syncResult.rows[0]?.status || null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Vacuum analyze table to optimize performance
|
||||
* @param entity Entity type
|
||||
*/
|
||||
export async function optimizeTable(entity: EntityType): Promise<void> {
|
||||
const tableName = getTableName(entity);
|
||||
await postgresClient.query(`VACUUM ANALYZE ${tableName}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if record exists
|
||||
* @param entity Entity type
|
||||
* @param id Record ID
|
||||
* @returns True if record exists
|
||||
*/
|
||||
export async function recordExists(
|
||||
entity: EntityType,
|
||||
id: number | string
|
||||
): Promise<boolean> {
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
SELECT EXISTS(SELECT 1 FROM ${tableName} WHERE id = $1) as exists
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query<{ exists: boolean }>(query, [id]);
|
||||
return result.rows[0]?.exists || false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get records by IDs
|
||||
* @param entity Entity type
|
||||
* @param ids Array of record IDs
|
||||
* @returns Array of records
|
||||
*/
|
||||
export async function getRecordsByIds(
|
||||
entity: EntityType,
|
||||
ids: (number | string)[]
|
||||
): Promise<any[]> {
|
||||
if (ids.length === 0) return [];
|
||||
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
SELECT *
|
||||
FROM ${tableName}
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND is_deleted = false
|
||||
`;
|
||||
|
||||
const result = await postgresClient.query(query, [ids]);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update sync timestamp for records
|
||||
* @param entity Entity type
|
||||
* @param ids Array of record IDs
|
||||
*/
|
||||
export async function updateSyncTimestamp(
|
||||
entity: EntityType,
|
||||
ids: (number | string)[]
|
||||
): Promise<void> {
|
||||
if (ids.length === 0) return;
|
||||
|
||||
const tableName = getTableName(entity);
|
||||
|
||||
const query = `
|
||||
UPDATE ${tableName}
|
||||
SET synced_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ANY($1::bigint[])
|
||||
`;
|
||||
|
||||
await postgresClient.query(query, [ids]);
|
||||
}
|
||||
541
lib/utils/entity-mapper.ts
Normal file
541
lib/utils/entity-mapper.ts
Normal file
|
|
@ -0,0 +1,541 @@
|
|||
/**
|
||||
* Entity Mapper
|
||||
* Maps Autotask API responses to PostgreSQL database schema format
|
||||
*/
|
||||
|
||||
import { EntityType } from '../types/sync';
|
||||
|
||||
/**
|
||||
* Map Autotask field names to PostgreSQL column names
|
||||
* Autotask uses camelCase, PostgreSQL uses snake_case
|
||||
*/
|
||||
export function toSnakeCase(str: string): string {
|
||||
return str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Autotask API response to database format
|
||||
* @param entity Entity type
|
||||
* @param data Autotask API response data
|
||||
* @returns Mapped data for PostgreSQL
|
||||
*/
|
||||
export function mapAutotaskToDatabase(
|
||||
entity: EntityType,
|
||||
data: any
|
||||
): Record<string, any> {
|
||||
// Entity-specific mappings use ORIGINAL data (not snake_cased)
|
||||
let mapped: Record<string, any>;
|
||||
|
||||
switch (entity) {
|
||||
case EntityType.COMPANIES:
|
||||
mapped = mapCompany(data);
|
||||
break;
|
||||
case EntityType.TICKETS:
|
||||
mapped = mapTicket(data);
|
||||
break;
|
||||
case EntityType.TASKS:
|
||||
mapped = mapTask(data);
|
||||
break;
|
||||
case EntityType.PROJECTS:
|
||||
mapped = mapProject(data);
|
||||
break;
|
||||
case EntityType.RESOURCES:
|
||||
mapped = mapResource(data);
|
||||
break;
|
||||
case EntityType.CONFIGURATION_ITEMS:
|
||||
mapped = mapConfigurationItem(data);
|
||||
break;
|
||||
case EntityType.CONTACTS:
|
||||
mapped = mapContact(data);
|
||||
break;
|
||||
case EntityType.CONTRACTS:
|
||||
mapped = mapContract(data);
|
||||
break;
|
||||
case EntityType.BILLING_ITEMS:
|
||||
mapped = mapBillingItem(data);
|
||||
break;
|
||||
case EntityType.TIME_ENTRIES:
|
||||
mapped = mapTimeEntry(data);
|
||||
break;
|
||||
case EntityType.STATUSES:
|
||||
case EntityType.ISSUE_TYPES:
|
||||
case EntityType.SUB_ISSUE_TYPES:
|
||||
case EntityType.WORK_TYPES:
|
||||
mapped = mapPicklist(data);
|
||||
break;
|
||||
default:
|
||||
// Fallback: auto-convert camelCase to snake_case
|
||||
mapped = {};
|
||||
for (const [key, value] of Object.entries(data)) {
|
||||
const snakeKey = toSnakeCase(key);
|
||||
mapped[snakeKey] = value;
|
||||
}
|
||||
}
|
||||
|
||||
// Add sync timestamp
|
||||
mapped.synced_at = new Date();
|
||||
|
||||
// Ensure is_deleted is false for new/updated records
|
||||
if (mapped.is_deleted === undefined) {
|
||||
mapped.is_deleted = false;
|
||||
}
|
||||
|
||||
return mapped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Company entity
|
||||
*/
|
||||
function mapCompany(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
company_name: data.companyName || data.name,
|
||||
company_number: data.companyNumber,
|
||||
phone: data.phone,
|
||||
fax: data.fax,
|
||||
website: data.webSiteURL,
|
||||
address1: data.address1,
|
||||
address2: data.address2,
|
||||
city: data.city,
|
||||
state: data.state,
|
||||
postal_code: data.postalCode,
|
||||
country: data.country,
|
||||
is_active: data.isActive !== undefined ? data.isActive : true,
|
||||
company_type: data.companyType,
|
||||
owner_resource_id: data.ownerResourceID,
|
||||
territory_id: data.territoryID,
|
||||
market_segment_id: data.marketSegmentID,
|
||||
competitor_id: data.competitorID,
|
||||
billing_address1: data.billingAddress1,
|
||||
billing_address2: data.billingAddress2,
|
||||
billing_city: data.billingCity,
|
||||
billing_state: data.billingState,
|
||||
billing_postal_code: data.billingPostalCode,
|
||||
billing_country: data.billingCountry,
|
||||
tax_id: data.taxID,
|
||||
tax_exempt: data.taxExempt || false,
|
||||
tax_region_id: data.taxRegionID,
|
||||
currency_id: data.currencyID,
|
||||
invoice_method: data.invoice_method,
|
||||
invoice_template_id: data.invoice_template_id,
|
||||
quote_template_id: data.quote_template_id,
|
||||
key_account_icon: data.key_account_icon,
|
||||
last_activity_date: data.last_activity_date,
|
||||
last_tracked_modification_date_time: data.last_tracked_modification_date_time || data.last_modified_date,
|
||||
api_vendor_id: data.api_vendor_id,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Ticket entity
|
||||
*/
|
||||
function mapTicket(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
company_id: data.companyID,
|
||||
ticket_number: data.ticketNumber,
|
||||
title: data.title,
|
||||
description: data.description,
|
||||
status: data.status,
|
||||
priority: data.priority,
|
||||
queue_id: data.queueID,
|
||||
issue_type: data.issueType,
|
||||
sub_issue_type: data.subIssueType,
|
||||
source: data.source,
|
||||
assigned_resource_id: data.assignedResourceID,
|
||||
assigned_resource_role_id: data.assignedResourceRoleID,
|
||||
contact_id: data.contactID,
|
||||
account_physical_location_id: data.companyLocationID,
|
||||
due_date_time: data.dueDateTime,
|
||||
estimated_hours: data.estimatedHours,
|
||||
completed_date: data.completedDate,
|
||||
create_date: data.createDate,
|
||||
created_by_contact_id: data.createdByContactID,
|
||||
last_activity_date: data.lastActivityDate,
|
||||
last_customer_notification_date_time: data.lastCustomerNotificationDateTime,
|
||||
last_customer_visible_activity_date_time: data.lastCustomerVisibleActivityDateTime,
|
||||
first_response_date_time: data.first_response_date_time,
|
||||
resolution_plan_date_time: data.resolution_plan_date_time,
|
||||
resolved_date_time: data.resolved_date_time,
|
||||
first_response_assigned_resource_id: data.first_response_assigned_resource_id,
|
||||
first_response_initiating_resource_id: data.first_response_initiating_resource_id,
|
||||
project_id: data.project_id,
|
||||
opportunity_id: data.opportunity_id,
|
||||
change_approval_board: data.change_approval_board,
|
||||
change_approval_status: data.change_approval_status,
|
||||
change_approval_type: data.change_approval_type,
|
||||
change_info_field1: data.change_info_field1,
|
||||
change_info_field2: data.change_info_field2,
|
||||
change_info_field3: data.change_info_field3,
|
||||
change_info_field4: data.change_info_field4,
|
||||
change_info_field5: data.change_info_field5,
|
||||
contract_id: data.contract_id,
|
||||
monitor_id: data.monitor_id,
|
||||
monitor_type_id: data.monitor_type_id,
|
||||
ticket_type: data.ticket_type,
|
||||
ticket_category: data.ticket_category,
|
||||
service_level_agreement_id: data.service_level_agreement_id,
|
||||
resolution: data.resolution,
|
||||
purchase_order_number: data.purchase_order_number,
|
||||
ticket_completion_date: data.ticket_completion_date,
|
||||
last_activity_person_type: data.last_activity_person_type,
|
||||
last_activity_resource_id: data.last_activity_resource_id,
|
||||
current_service_thermometer_rating: data.current_service_thermometer_rating,
|
||||
previous_service_thermometer_rating: data.previous_service_thermometer_rating,
|
||||
service_thermometer_temperature: data.service_thermometer_temperature,
|
||||
api_vendor_id: data.api_vendor_id,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Task entity
|
||||
*/
|
||||
function mapTask(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
title: data.title,
|
||||
description: data.description,
|
||||
status: data.status,
|
||||
priority: data.priority,
|
||||
assigned_resource_id: data.assignedResourceID,
|
||||
assigned_resource_role_id: data.assignedResourceRoleID,
|
||||
department_id: data.departmentID,
|
||||
estimated_hours: data.estimatedHours,
|
||||
remaining_hours: data.remainingHours,
|
||||
hours_to_be_scheduled: data.hoursToBeScheduled,
|
||||
start_date_time: data.startDateTime,
|
||||
end_date_time: data.endDateTime,
|
||||
completed_date_time: data.completedDateTime,
|
||||
create_date_time: data.createDateTime,
|
||||
creator_resource_id: data.creatorResourceID,
|
||||
completed_by_resource_id: data.completedByResourceID,
|
||||
last_activity_date_time: data.lastActivityDateTime,
|
||||
project_id: data.projectID,
|
||||
ticket_id: data.ticketID,
|
||||
phase_id: data.phaseID,
|
||||
allocation_code_id: data.allocationCodeID,
|
||||
task_type: data.taskType,
|
||||
task_is_billable: data.taskIsBillable !== undefined ? data.taskIsBillable : true,
|
||||
task_number: data.taskNumber,
|
||||
purchase_order_number: data.purchaseOrderNumber,
|
||||
can_client_portal_user_complete_task: data.canClientPortalUserCompleteTask || false,
|
||||
creator_type: data.creatorType,
|
||||
task_category_id: data.taskCategoryID,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Project entity
|
||||
*/
|
||||
function mapProject(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
company_id: data.companyID,
|
||||
project_name: data.projectName,
|
||||
project_number: data.projectNumber,
|
||||
description: data.description,
|
||||
start_date_time: data.start_date_time,
|
||||
end_date_time: data.end_date_time,
|
||||
estimated_time: data.estimated_time,
|
||||
actual_hours: data.actual_hours,
|
||||
estimated_sale_cost: data.estimated_sale_cost,
|
||||
labor_estimated_costs: data.labor_estimated_costs,
|
||||
labor_estimated_revenue: data.labor_estimated_revenue,
|
||||
project_cost_estimated_margin_percentage: data.project_cost_estimated_margin_percentage,
|
||||
status: data.status,
|
||||
type: data.type,
|
||||
project_lead_resource_id: data.project_lead_resource_id,
|
||||
account_executive_resource_id: data.account_executive_resource_id,
|
||||
owner_resource_id: data.owner_resource_id,
|
||||
creator_resource_id: data.creator_resource_id,
|
||||
completed_percentage: data.completed_percentage,
|
||||
completed_date_time: data.completed_date_time,
|
||||
duration: data.duration,
|
||||
original_estimated_revenue: data.original_estimated_revenue,
|
||||
estimated_time_cost: data.estimated_time_cost,
|
||||
purchase_order_number: data.purchase_order_number,
|
||||
business_division_subdivision_id: data.business_division_subdivision_id,
|
||||
line_of_business_id: data.line_of_business_id,
|
||||
department: data.department,
|
||||
last_activity_date_time: data.last_activity_date_time,
|
||||
last_activity_person_type: data.last_activity_person_type,
|
||||
last_activity_resource_id: data.last_activity_resource_id,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Resource entity
|
||||
*/
|
||||
function mapResource(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
first_name: data.first_name,
|
||||
last_name: data.last_name,
|
||||
email: data.email || data.email_address,
|
||||
user_name: data.user_name || data.username,
|
||||
title: data.title,
|
||||
office_phone: data.office_phone,
|
||||
mobile_phone: data.mobile_phone,
|
||||
office_extension: data.office_extension,
|
||||
is_active: data.is_active !== undefined ? data.is_active : true,
|
||||
location_id: data.location_id,
|
||||
resource_type: data.resource_type,
|
||||
pay_roll_identifier: data.pay_roll_identifier,
|
||||
hire_date: data.hire_date,
|
||||
travel_availability_pct: data.travel_availability_pct,
|
||||
survey_resource_rating: data.survey_resource_rating,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Configuration Item entity
|
||||
*/
|
||||
function mapConfigurationItem(data: any): Record<string, any> {
|
||||
// Configuration items have many fields, map all of them
|
||||
const mapped: Record<string, any> = {
|
||||
id: data.id,
|
||||
company_id: data.companyID,
|
||||
product_id: data.productID,
|
||||
reference_title: data.referenceTitle,
|
||||
reference_number: data.referenceNumber,
|
||||
serial_number: data.serialNumber,
|
||||
install_date: data.installDate,
|
||||
warranty_expiration_date: data.warrantyExpirationDate,
|
||||
is_active: data.isActive !== undefined ? data.isActive : true,
|
||||
contact_id: data.contactID,
|
||||
configuration_item_type: data.type,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
|
||||
// Add all other fields dynamically
|
||||
const fieldsToInclude = [
|
||||
'daily_cost', 'hourly_cost', 'monthly_cost', 'per_use_cost', 'setup_fee',
|
||||
'location_id', 'vendor_id', 'installed_by_id', 'installed_by_contact_id',
|
||||
'parent_configuration_item_id', 'notes', 'create_date', 'created_by_person_id',
|
||||
'last_modified_time', 'last_activity_person_type', 'impersonator_creator_resource_id',
|
||||
'configuration_item_category_id', 'configuration_item_type', 'device_type',
|
||||
'rmm_device_uid', 'api_vendor_id',
|
||||
];
|
||||
|
||||
// Add Datto RMM fields
|
||||
const dattoFields = Object.keys(data).filter(key => key.startsWith('datto_') || key.startsWith('rmm_'));
|
||||
fieldsToInclude.push(...dattoFields);
|
||||
|
||||
fieldsToInclude.forEach(field => {
|
||||
if (data[field] !== undefined) {
|
||||
mapped[field] = data[field];
|
||||
}
|
||||
});
|
||||
|
||||
return mapped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Contact entity
|
||||
*/
|
||||
function mapContact(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
company_id: data.companyID,
|
||||
first_name: data.firstName,
|
||||
last_name: data.lastName,
|
||||
title: data.title,
|
||||
email_address: data.emailAddress || data.email_address || data.email,
|
||||
email_address2: data.emailAddress2 || data.email_address2,
|
||||
email_address3: data.emailAddress3 || data.email_address3,
|
||||
phone: data.phone,
|
||||
extension: data.extension,
|
||||
alternate_phone: data.alternate_phone,
|
||||
mobile_phone: data.mobile_phone,
|
||||
fax: data.faxNumber || data.fax,
|
||||
address_line: data.addressLine || data.address_line,
|
||||
address_line1: data.addressLine1 || data.address_line1 || data.address_line_1,
|
||||
city: data.city,
|
||||
state: data.state,
|
||||
zip_code: data.zipCode || data.zip_code || data.postal_code,
|
||||
country: data.country,
|
||||
is_active: data.is_active !== undefined ? data.is_active : true,
|
||||
name_prefix: data.name_prefix,
|
||||
name_suffix: data.name_suffix,
|
||||
facebook_url: data.facebook_url,
|
||||
twitter_url: data.twitter_url,
|
||||
linked_in_url: data.linked_in_url,
|
||||
primary_contact: data.primary_contact || false,
|
||||
account_physical_location_id: data.account_physical_location_id,
|
||||
solicitation_opt_out: data.solicitation_opt_out || false,
|
||||
room_number: data.room_number,
|
||||
last_activity_date: data.last_activity_date,
|
||||
last_modified_date: data.last_modified_date,
|
||||
api_vendor_id: data.api_vendor_id,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Contract entity
|
||||
*/
|
||||
function mapContract(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
company_id: data.companyID,
|
||||
contract_name: data.contractName,
|
||||
contract_number: data.contractNumber,
|
||||
description: data.description,
|
||||
start_date: data.start_date,
|
||||
end_date: data.end_date,
|
||||
time_reporting_requires_start_and_stop_times: data.time_reporting_requires_start_and_stop_times,
|
||||
service_level_agreement_id: data.service_level_agreement_id,
|
||||
contract_type: data.contract_type,
|
||||
contract_category: data.contract_category,
|
||||
status: data.status,
|
||||
business_division_subdivision_id: data.business_division_subdivision_id,
|
||||
contact_id: data.contact_id,
|
||||
contact_name: data.contact_name,
|
||||
billing_preference: data.billing_preference,
|
||||
purchase_order_number: data.purchase_order_number,
|
||||
setup_fee: data.setup_fee,
|
||||
setup_fee_allocation_code_id: data.setup_fee_allocation_code_id,
|
||||
estimated_cost: data.estimated_cost,
|
||||
estimated_hours: data.estimated_hours,
|
||||
estimated_revenue: data.estimated_revenue,
|
||||
over_budget_dollar_amount: data.over_budget_dollar_amount,
|
||||
over_budget_hours: data.over_budget_hours,
|
||||
contract_period_type: data.contract_period_type,
|
||||
opportunity_id: data.opportunity_id,
|
||||
renewed_contract_id: data.renewed_contract_id,
|
||||
is_default_contract: data.is_default_contract || false,
|
||||
internal_currency_setup_fee: data.internal_currency_setup_fee,
|
||||
internal_currency_over_budget_dollar_amount: data.internal_currency_over_budget_dollar_amount,
|
||||
internal_currency_estimated_cost: data.internal_currency_estimated_cost,
|
||||
internal_currency_estimated_revenue: data.internal_currency_estimated_revenue,
|
||||
exclusion_contract_id: data.exclusion_contract_id,
|
||||
internal_currency_monthly_revenue: data.internal_currency_monthly_revenue,
|
||||
internal_currency_quarterly_revenue: data.internal_currency_quarterly_revenue,
|
||||
internal_currency_semi_annual_revenue: data.internal_currency_semi_annual_revenue,
|
||||
internal_currency_yearly_revenue: data.internal_currency_yearly_revenue,
|
||||
internal_currency_one_time_revenue: data.internal_currency_one_time_revenue,
|
||||
compliance: data.compliance,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Billing Item entity
|
||||
*/
|
||||
function mapBillingItem(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
company_id: data.companyID,
|
||||
product_id: data.productID,
|
||||
description: data.description,
|
||||
quantity: data.quantity,
|
||||
rate: data.rate,
|
||||
total_amount: data.total_amount,
|
||||
line_discount_dollars: data.line_discount_dollars,
|
||||
line_discount_percent: data.line_discount_percent,
|
||||
tax_category_id: data.tax_category_id,
|
||||
internal_currency_line_discount_dollars: data.internal_currency_line_discount_dollars,
|
||||
allocation_code_id: data.allocation_code_id,
|
||||
invoice_id: data.invoice_id,
|
||||
vendor_id: data.vendor_id,
|
||||
expense_item: data.expense_item || false,
|
||||
task_id: data.task_id,
|
||||
ticket_id: data.ticket_id,
|
||||
project_id: data.project_id,
|
||||
our_cost: data.our_cost,
|
||||
list_price: data.list_price,
|
||||
unit_cost: data.unit_cost,
|
||||
unit_price: data.unit_price,
|
||||
extended_price: data.extended_price,
|
||||
tax_dollars: data.tax_dollars,
|
||||
internal_currency_unit_price: data.internal_currency_unit_price,
|
||||
internal_currency_total_amount: data.internal_currency_total_amount,
|
||||
synced_at: data.synced_at,
|
||||
is_deleted: data.is_deleted || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Time Entry entity
|
||||
*/
|
||||
function mapTimeEntry(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
resource_id: data.resourceID,
|
||||
ticket_id: data.ticketID,
|
||||
task_id: data.taskID,
|
||||
project_id: data.projectID,
|
||||
company_id: data.companyID,
|
||||
entry_date: data.dateWorked, // Autotask API returns dateWorked
|
||||
hours_worked: data.hoursWorked, // Autotask API returns hoursWorked
|
||||
notes: data.summaryNotes, // Autotask uses summaryNotes
|
||||
internal_notes: data.internalNotes,
|
||||
title: data.title,
|
||||
type: data.type,
|
||||
start_date_time: data.startDateTime,
|
||||
end_date_time: data.endDateTime,
|
||||
billable: data.billable, // Autotask returns billable
|
||||
billing_rate: data.billingRate,
|
||||
billing_rate_currency_id: data.billingRateCurrencyID,
|
||||
cost_rate: data.costRate,
|
||||
cost_rate_currency_id: data.costRateCurrencyID,
|
||||
cost: data.cost,
|
||||
cost_currency_id: data.costCurrencyID,
|
||||
revenue: data.revenue,
|
||||
revenue_currency_id: data.revenueCurrencyID,
|
||||
margin: data.margin,
|
||||
margin_currency_id: data.marginCurrencyID,
|
||||
approved: data.approved,
|
||||
approved_by_resource_id: data.approvedByResourceID,
|
||||
approved_date_time: data.approvedDateTime,
|
||||
non_billable: data.nonBillable,
|
||||
contract_service_id: data.contractServiceID,
|
||||
contract_service_bundle_id: data.contractServiceBundleID,
|
||||
role_id: data.roleID,
|
||||
department_id: data.departmentID,
|
||||
location_id: data.locationID,
|
||||
allocation_code_id: data.allocationCodeID,
|
||||
imp_project_schedule_id: data.impProjectScheduleID,
|
||||
imp_project_schedule_task_id: data.impProjectScheduleTaskID,
|
||||
api_vendor_id: data.apiVendorID,
|
||||
synced_at: new Date(),
|
||||
is_deleted: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Picklist entity (statuses, issue types, etc.)
|
||||
*/
|
||||
function mapPicklist(data: any): Record<string, any> {
|
||||
return {
|
||||
value: data.value,
|
||||
label: data.label || data.name,
|
||||
is_active: data.isActive !== undefined ? data.isActive : true,
|
||||
is_system: data.isSystem || false,
|
||||
sort_order: data.sortOrder,
|
||||
parent_value: data.parentValue,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch map multiple entities
|
||||
*/
|
||||
export function mapAutotaskBatch(
|
||||
entity: EntityType,
|
||||
items: any[]
|
||||
): Record<string, any>[] {
|
||||
return items.map(item => mapAutotaskToDatabase(entity, item));
|
||||
}
|
||||
186
lib/utils/issue-type-helper.ts
Normal file
186
lib/utils/issue-type-helper.ts
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
/**
|
||||
* Issue Type Helper Utilities
|
||||
* Client-side utilities for working with parent/child issue type relationships
|
||||
*/
|
||||
|
||||
export interface IssueType {
|
||||
value: number;
|
||||
label: string;
|
||||
is_active: boolean;
|
||||
}
|
||||
|
||||
export interface SubIssueType {
|
||||
value: number;
|
||||
label: string;
|
||||
is_active: boolean;
|
||||
parent_value?: number | null;
|
||||
parent_issue_type_label?: string;
|
||||
}
|
||||
|
||||
export interface TicketWithIssueTypes {
|
||||
id: number;
|
||||
ticket_number: string;
|
||||
title: string;
|
||||
issue_type?: number;
|
||||
sub_issue_type?: number;
|
||||
issue_type_label?: string;
|
||||
sub_issue_type_label?: string;
|
||||
parent_issue_type_label?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parent issue type for a sub-issue type from cached data
|
||||
*/
|
||||
export function getParentIssueType(
|
||||
subIssueTypeValue: number,
|
||||
subIssueTypes: SubIssueType[]
|
||||
): string | null {
|
||||
const subIssueType = subIssueTypes.find(sit => sit.value === subIssueTypeValue);
|
||||
return subIssueType?.parent_issue_type_label || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter sub-issue types by parent issue type
|
||||
*/
|
||||
export function getSubIssueTypesByParent(
|
||||
parentIssueTypeValue: number,
|
||||
subIssueTypes: SubIssueType[]
|
||||
): SubIssueType[] {
|
||||
return subIssueTypes.filter(sit => sit.parent_value === parentIssueTypeValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a hierarchical tree of issue types and sub-issue types
|
||||
*/
|
||||
export function createIssueTypeTree(
|
||||
issueTypes: IssueType[],
|
||||
subIssueTypes: SubIssueType[]
|
||||
): Array<IssueType & { subIssues: SubIssueType[] }> {
|
||||
return issueTypes.map(issueType => ({
|
||||
...issueType,
|
||||
subIssues: getSubIssueTypesByParent(issueType.value, subIssueTypes)
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Group tickets by parent issue type
|
||||
*/
|
||||
export function groupTicketsByParentIssueType(
|
||||
tickets: TicketWithIssueTypes[],
|
||||
subIssueTypes: SubIssueType[]
|
||||
): Record<string, TicketWithIssueTypes[]> {
|
||||
const groups: Record<string, TicketWithIssueTypes[]> = {};
|
||||
|
||||
tickets.forEach(ticket => {
|
||||
const parentType = getParentIssueType(ticket.sub_issue_type!, subIssueTypes) || 'Uncategorized';
|
||||
|
||||
if (!groups[parentType]) {
|
||||
groups[parentType] = [];
|
||||
}
|
||||
|
||||
groups[parentType].push(ticket);
|
||||
});
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics for issue types and sub-issue types
|
||||
*/
|
||||
export function getIssueTypeStatistics(
|
||||
tickets: TicketWithIssueTypes[],
|
||||
subIssueTypes: SubIssueType[]
|
||||
): {
|
||||
totalTickets: number;
|
||||
ticketsByParentType: Record<string, number>;
|
||||
ticketsBySubType: Record<string, number>;
|
||||
unassignedTickets: number;
|
||||
} {
|
||||
const stats = {
|
||||
totalTickets: tickets.length,
|
||||
ticketsByParentType: {} as Record<string, number>,
|
||||
ticketsBySubType: {} as Record<string, number>,
|
||||
unassignedTickets: 0
|
||||
};
|
||||
|
||||
tickets.forEach(ticket => {
|
||||
if (!ticket.sub_issue_type) {
|
||||
stats.unassignedTickets++;
|
||||
return;
|
||||
}
|
||||
|
||||
const parentType = getParentIssueType(ticket.sub_issue_type, subIssueTypes);
|
||||
if (parentType) {
|
||||
stats.ticketsByParentType[parentType] = (stats.ticketsByParentType[parentType] || 0) + 1;
|
||||
}
|
||||
|
||||
const subType = subIssueTypes.find(sit => sit.value === ticket.sub_issue_type);
|
||||
if (subType) {
|
||||
stats.ticketsBySubType[subType.label] = (stats.ticketsBySubType[subType.label] || 0) + 1;
|
||||
}
|
||||
});
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format ticket data for display with parent issue type information
|
||||
*/
|
||||
export function formatTicketWithParentIssueType(
|
||||
ticket: TicketWithIssueTypes,
|
||||
subIssueTypes: SubIssueType[]
|
||||
): string {
|
||||
const parentType = getParentIssueType(ticket.sub_issue_type!, subIssueTypes);
|
||||
|
||||
if (parentType) {
|
||||
return `${ticket.ticket_number}: ${ticket.title} (${parentType} > ${ticket.sub_issue_type_label})`;
|
||||
}
|
||||
|
||||
return `${ticket.ticket_number}: ${ticket.title} (${ticket.sub_issue_type_label || 'No sub-issue type'})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that a sub-issue type belongs to a specific parent issue type
|
||||
*/
|
||||
export function validateSubIssueTypeParent(
|
||||
subIssueTypeValue: number,
|
||||
expectedParentValue: number,
|
||||
subIssueTypes: SubIssueType[]
|
||||
): boolean {
|
||||
const subIssueType = subIssueTypes.find(sit => sit.value === subIssueTypeValue);
|
||||
return subIssueType?.parent_value === expectedParentValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all valid parent-child combinations
|
||||
*/
|
||||
export function getValidParentChildCombinations(
|
||||
issueTypes: IssueType[],
|
||||
subIssueTypes: SubIssueType[]
|
||||
): Array<{
|
||||
parentValue: number;
|
||||
parentLabel: string;
|
||||
childValue: number;
|
||||
childLabel: string;
|
||||
}> {
|
||||
const combinations: Array<{
|
||||
parentValue: number;
|
||||
parentLabel: string;
|
||||
childValue: number;
|
||||
childLabel: string;
|
||||
}> = [];
|
||||
|
||||
issueTypes.forEach(issueType => {
|
||||
const children = getSubIssueTypesByParent(issueType.value, subIssueTypes);
|
||||
children.forEach(child => {
|
||||
combinations.push({
|
||||
parentValue: issueType.value,
|
||||
parentLabel: issueType.label,
|
||||
childValue: child.value,
|
||||
childLabel: child.label,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
return combinations;
|
||||
}
|
||||
140
lib/utils/logger.ts
Normal file
140
lib/utils/logger.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/**
|
||||
* Logger Utility
|
||||
* Provides structured logging for sync operations
|
||||
*/
|
||||
|
||||
export enum LogLevel {
|
||||
DEBUG = 'DEBUG',
|
||||
INFO = 'INFO',
|
||||
WARN = 'WARN',
|
||||
ERROR = 'ERROR',
|
||||
}
|
||||
|
||||
export interface LogContext {
|
||||
syncId?: string;
|
||||
entity?: string;
|
||||
operation?: string;
|
||||
duration?: number;
|
||||
recordCount?: number;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
/**
|
||||
* Logger class for structured logging
|
||||
*/
|
||||
export class Logger {
|
||||
private context: LogContext;
|
||||
private minLevel: LogLevel;
|
||||
|
||||
constructor(context: LogContext = {}, minLevel: LogLevel = LogLevel.INFO) {
|
||||
this.context = context;
|
||||
this.minLevel = minLevel;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a child logger with additional context
|
||||
*/
|
||||
child(additionalContext: LogContext): Logger {
|
||||
return new Logger({ ...this.context, ...additionalContext }, this.minLevel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log debug message
|
||||
*/
|
||||
debug(message: string, data?: any): void {
|
||||
this.log(LogLevel.DEBUG, message, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log info message
|
||||
*/
|
||||
info(message: string, data?: any): void {
|
||||
this.log(LogLevel.INFO, message, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log warning message
|
||||
*/
|
||||
warn(message: string, data?: any): void {
|
||||
this.log(LogLevel.WARN, message, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log error message
|
||||
*/
|
||||
error(message: string, error?: Error | any, data?: any): void {
|
||||
const errorData = {
|
||||
...data,
|
||||
error: error instanceof Error ? {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name,
|
||||
} : error,
|
||||
};
|
||||
this.log(LogLevel.ERROR, message, errorData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal log method
|
||||
*/
|
||||
private log(level: LogLevel, message: string, data?: any): void {
|
||||
if (!this.shouldLog(level)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
const contextStr = Object.keys(this.context).length > 0
|
||||
? ` [${this.formatContext()}]`
|
||||
: '';
|
||||
|
||||
const logMessage = `[${timestamp}] [${level}]${contextStr} ${message}`;
|
||||
|
||||
switch (level) {
|
||||
case LogLevel.DEBUG:
|
||||
console.debug(logMessage, data || '');
|
||||
break;
|
||||
case LogLevel.INFO:
|
||||
console.log(logMessage, data || '');
|
||||
break;
|
||||
case LogLevel.WARN:
|
||||
console.warn(logMessage, data || '');
|
||||
break;
|
||||
case LogLevel.ERROR:
|
||||
console.error(logMessage, data || '');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if log level should be logged
|
||||
*/
|
||||
private shouldLog(level: LogLevel): boolean {
|
||||
const levels = [LogLevel.DEBUG, LogLevel.INFO, LogLevel.WARN, LogLevel.ERROR];
|
||||
const currentIndex = levels.indexOf(this.minLevel);
|
||||
const messageIndex = levels.indexOf(level);
|
||||
return messageIndex >= currentIndex;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format context for logging
|
||||
*/
|
||||
private formatContext(): string {
|
||||
return Object.entries(this.context)
|
||||
.map(([key, value]) => `${key}=${value}`)
|
||||
.join(', ');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a logger instance
|
||||
*/
|
||||
export function createLogger(context?: LogContext, minLevel?: LogLevel): Logger {
|
||||
return new Logger(context, minLevel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Default logger instance
|
||||
*/
|
||||
export const logger = new Logger();
|
||||
|
||||
export default logger;
|
||||
414
lib/utils/sync-helpers.ts
Normal file
414
lib/utils/sync-helpers.ts
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
/**
|
||||
* Sync Helper Functions
|
||||
* Utility functions for sync operations including dependency ordering
|
||||
*/
|
||||
|
||||
import { EntityType, ENTITY_DEPENDENCIES } from '../types/sync';
|
||||
|
||||
/**
|
||||
* Get entities in dependency order (parents before children)
|
||||
* @param entities List of entities to sync
|
||||
* @returns Ordered list of entities respecting dependencies
|
||||
*/
|
||||
export function getEntitySyncOrder(entities: EntityType[]): EntityType[] {
|
||||
const ordered: EntityType[] = [];
|
||||
const visited = new Set<EntityType>();
|
||||
const visiting = new Set<EntityType>();
|
||||
|
||||
function visit(entity: EntityType) {
|
||||
if (visited.has(entity)) return;
|
||||
if (visiting.has(entity)) {
|
||||
throw new Error(`Circular dependency detected for entity: ${entity}`);
|
||||
}
|
||||
|
||||
visiting.add(entity);
|
||||
|
||||
// Visit dependencies first
|
||||
const dependencies = ENTITY_DEPENDENCIES[entity] || [];
|
||||
for (const dep of dependencies) {
|
||||
if (entities.includes(dep)) {
|
||||
visit(dep);
|
||||
}
|
||||
}
|
||||
|
||||
visiting.delete(entity);
|
||||
visited.add(entity);
|
||||
ordered.push(entity);
|
||||
}
|
||||
|
||||
// Visit all entities
|
||||
for (const entity of entities) {
|
||||
visit(entity);
|
||||
}
|
||||
|
||||
return ordered;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all entities in default sync order
|
||||
* @returns All entities in dependency order
|
||||
*/
|
||||
export function getAllEntitiesInOrder(): EntityType[] {
|
||||
return getEntitySyncOrder([
|
||||
EntityType.COMPANIES,
|
||||
EntityType.RESOURCES,
|
||||
EntityType.STATUSES,
|
||||
EntityType.ISSUE_TYPES,
|
||||
EntityType.SUB_ISSUE_TYPES,
|
||||
EntityType.WORK_TYPES,
|
||||
EntityType.CONTACTS,
|
||||
EntityType.PROJECTS,
|
||||
EntityType.TICKETS,
|
||||
EntityType.TASKS,
|
||||
EntityType.CONFIGURATION_ITEMS,
|
||||
EntityType.CONTRACTS,
|
||||
EntityType.BILLING_ITEMS,
|
||||
EntityType.TIME_ENTRIES,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table name for entity type
|
||||
* @param entity Entity type
|
||||
* @returns PostgreSQL table name
|
||||
*/
|
||||
export function getTableName(entity: EntityType): string {
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Autotask API entity name
|
||||
* @param entity Entity type
|
||||
* @returns Autotask API entity name (PascalCase)
|
||||
*/
|
||||
export function getAutotaskEntityName(entity: EntityType): string {
|
||||
const mapping: Record<EntityType, string> = {
|
||||
[EntityType.COMPANIES]: 'Companies',
|
||||
[EntityType.TICKETS]: 'Tickets',
|
||||
[EntityType.TASKS]: 'Tasks',
|
||||
[EntityType.PROJECTS]: 'Projects',
|
||||
[EntityType.RESOURCES]: 'Resources',
|
||||
[EntityType.STATUSES]: 'Statuses',
|
||||
[EntityType.ISSUE_TYPES]: 'IssueTypes',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'SubIssueTypes',
|
||||
[EntityType.WORK_TYPES]: 'WorkTypes',
|
||||
[EntityType.BILLING_ITEMS]: 'BillingItems',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'ConfigurationItems',
|
||||
[EntityType.CONTACTS]: 'Contacts',
|
||||
[EntityType.CONTRACTS]: 'Contracts',
|
||||
[EntityType.TIME_ENTRIES]: 'TimeEntries',
|
||||
};
|
||||
|
||||
return mapping[entity] || entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if entity is a picklist type
|
||||
* @param entity Entity type
|
||||
* @returns True if entity is a picklist
|
||||
*/
|
||||
export function isPicklistEntity(entity: EntityType): boolean {
|
||||
return [
|
||||
EntityType.STATUSES,
|
||||
EntityType.ISSUE_TYPES,
|
||||
EntityType.SUB_ISSUE_TYPES,
|
||||
EntityType.WORK_TYPES,
|
||||
].includes(entity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get field name for last modified date in Autotask
|
||||
* @param entity Entity type
|
||||
* @returns Field name for filtering by last modified date
|
||||
*/
|
||||
export function getLastModifiedField(entity: EntityType): string {
|
||||
const mapping: Record<EntityType, string> = {
|
||||
[EntityType.COMPANIES]: 'lastTrackedModificationDateTime',
|
||||
[EntityType.TICKETS]: 'lastActivityDate',
|
||||
[EntityType.TASKS]: 'lastActivityDateTime',
|
||||
[EntityType.PROJECTS]: 'lastActivityDateTime',
|
||||
[EntityType.RESOURCES]: 'lastModifiedDate',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'lastModifiedTime',
|
||||
[EntityType.CONTACTS]: 'lastModifiedDate',
|
||||
[EntityType.CONTRACTS]: 'lastModifiedDateTime',
|
||||
[EntityType.BILLING_ITEMS]: 'createDate',
|
||||
[EntityType.TIME_ENTRIES]: 'lastModifiedDate',
|
||||
[EntityType.STATUSES]: 'lastModifiedDate',
|
||||
[EntityType.ISSUE_TYPES]: 'lastModifiedDate',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'lastModifiedDate',
|
||||
[EntityType.WORK_TYPES]: 'lastModifiedDate',
|
||||
};
|
||||
|
||||
return mapping[entity] || 'lastModifiedDate';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get active status field name for entity
|
||||
* @param entity Entity type
|
||||
* @returns Field name for active status
|
||||
*/
|
||||
export function getActiveField(entity: EntityType): string | null {
|
||||
const mapping: Record<EntityType, string | null> = {
|
||||
[EntityType.COMPANIES]: 'isActive',
|
||||
[EntityType.TICKETS]: null, // Use status field instead
|
||||
[EntityType.TASKS]: null, // Use status field instead
|
||||
[EntityType.PROJECTS]: null, // Use status field instead
|
||||
[EntityType.RESOURCES]: 'isActive',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'isActive',
|
||||
[EntityType.CONTACTS]: 'isActive',
|
||||
[EntityType.CONTRACTS]: null, // Use status field instead
|
||||
[EntityType.BILLING_ITEMS]: null,
|
||||
[EntityType.TIME_ENTRIES]: null, // Time entries don't have active status
|
||||
[EntityType.STATUSES]: 'isActive',
|
||||
[EntityType.ISSUE_TYPES]: 'isActive',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'isActive',
|
||||
[EntityType.WORK_TYPES]: 'isActive',
|
||||
};
|
||||
|
||||
return mapping[entity] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Autotask query filter for incremental sync
|
||||
* @param entity Entity type
|
||||
* @param lastSyncTime Last successful sync timestamp
|
||||
* @returns Query filter array
|
||||
*/
|
||||
export function buildIncrementalFilter(
|
||||
entity: EntityType,
|
||||
lastSyncTime: Date
|
||||
): Array<{ field: string; op: string; value: any }> {
|
||||
const lastModifiedField = getLastModifiedField(entity);
|
||||
|
||||
return [
|
||||
{
|
||||
field: lastModifiedField,
|
||||
op: 'gte',
|
||||
value: lastSyncTime.toISOString(),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Autotask query filter for active records only
|
||||
* @param entity Entity type
|
||||
* @returns Query filter array or null if no active field
|
||||
*/
|
||||
export function buildActiveFilter(
|
||||
entity: EntityType
|
||||
): Array<{ field: string; op: string; value: any }> | null {
|
||||
const activeField = getActiveField(entity);
|
||||
|
||||
if (!activeField) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
field: activeField,
|
||||
op: 'eq',
|
||||
value: true,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build date range filter for entities to limit sync to recent records
|
||||
* @param entity Entity type
|
||||
* @param yearsBack Number of years to look back (default: 2)
|
||||
* @returns Query filter array or null if entity doesn't support date filtering
|
||||
*/
|
||||
export function buildDateRangeFilter(
|
||||
entity: EntityType,
|
||||
yearsBack: number = 2
|
||||
): Array<{ field: string; op: string; value: any }> | null {
|
||||
// Only apply date range filters to time-based entities
|
||||
// Note: TIME_ENTRIES removed because Autotask API doesn't support date filtering on TimeEntry
|
||||
const timeBasedEntities = [
|
||||
EntityType.TICKETS,
|
||||
EntityType.TASKS,
|
||||
];
|
||||
|
||||
if (!timeBasedEntities.includes(entity)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Define which entities should have date range filters and which field to use
|
||||
const dateFieldMapping: Record<EntityType, string | null> = {
|
||||
[EntityType.TICKETS]: 'createDate',
|
||||
[EntityType.TASKS]: 'createDateTime',
|
||||
[EntityType.TIME_ENTRIES]: 'createDate', // TimeEntry uses createDate for filtering
|
||||
[EntityType.PROJECTS]: 'startDateTime',
|
||||
[EntityType.BILLING_ITEMS]: 'itemDate',
|
||||
[EntityType.CONTRACTS]: 'startDate', // Contracts use startDate
|
||||
[EntityType.COMPANIES]: null,
|
||||
[EntityType.RESOURCES]: null,
|
||||
[EntityType.CONTACTS]: null,
|
||||
[EntityType.CONFIGURATION_ITEMS]: null,
|
||||
[EntityType.STATUSES]: null,
|
||||
[EntityType.ISSUE_TYPES]: null,
|
||||
[EntityType.SUB_ISSUE_TYPES]: null,
|
||||
[EntityType.WORK_TYPES]: null,
|
||||
};
|
||||
|
||||
const dateField = dateFieldMapping[entity];
|
||||
|
||||
if (!dateField) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculate date from X years ago
|
||||
// Convert years to milliseconds for accurate calculation (including fractional years)
|
||||
const cutoffDate = new Date();
|
||||
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000; // Account for leap years
|
||||
const millisecondsBack = yearsBack * millisecondsPerYear;
|
||||
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
||||
|
||||
console.log(`Date range filter: ${dateField} >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
||||
|
||||
return [
|
||||
{
|
||||
field: dateField,
|
||||
op: 'gte',
|
||||
value: cutoffDate.toISOString(),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build special filter for contracts (requires status filter)
|
||||
* @returns Query filter array for active contracts
|
||||
*/
|
||||
export function buildContractsFilter(): Array<{ field: string; op: string; value: any }> {
|
||||
// Contracts API requires a filter. Use status = 1 for Active contracts
|
||||
// Status values: 1 = Active, others are inactive/expired
|
||||
return [
|
||||
{
|
||||
field: 'status',
|
||||
op: 'eq',
|
||||
value: 1,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build special filter for projects (requires status filter)
|
||||
* @returns Query filter array for active projects
|
||||
*/
|
||||
export function buildProjectsFilter(): Array<{ field: string; op: string; value: any }> {
|
||||
// Projects API requires a filter. Use status = 1 for New/Active projects
|
||||
// Status values: 1 = New, others include Complete, Cancelled, etc.
|
||||
// To get all active projects, we should filter for status NOT equal to Complete (5)
|
||||
return [
|
||||
{
|
||||
field: 'status',
|
||||
op: 'noteq',
|
||||
value: 5, // 5 = Complete
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build special filter for time entries (requires filter)
|
||||
* @param yearsBack Number of years to look back (default: 2)
|
||||
* @returns Query filter array for time entries
|
||||
*/
|
||||
export function buildTimeEntriesFilter(yearsBack: number = 2): Array<{ field: string; op: string; value: any }> {
|
||||
// TimeEntries API requires a filter. Use dateWorked to limit the range
|
||||
// Calculate date from X years ago
|
||||
const cutoffDate = new Date();
|
||||
const millisecondsPerYear = 365.25 * 24 * 60 * 60 * 1000;
|
||||
const millisecondsBack = yearsBack * millisecondsPerYear;
|
||||
cutoffDate.setTime(cutoffDate.getTime() - millisecondsBack);
|
||||
|
||||
console.log(`TimeEntries filter: dateWorked >= ${cutoffDate.toISOString()} (${yearsBack} years back)`);
|
||||
|
||||
return [
|
||||
{
|
||||
field: 'dateWorked',
|
||||
op: 'gte',
|
||||
value: cutoffDate.toISOString(),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate estimated sync duration based on record count
|
||||
* @param recordCount Number of records to sync
|
||||
* @param rateLimit Requests per second
|
||||
* @param pageSize Records per page
|
||||
* @returns Estimated duration in milliseconds
|
||||
*/
|
||||
export function estimateSyncDuration(
|
||||
recordCount: number,
|
||||
rateLimit: number = 10,
|
||||
pageSize: number = 500
|
||||
): number {
|
||||
const totalPages = Math.ceil(recordCount / pageSize);
|
||||
const secondsNeeded = totalPages / rateLimit;
|
||||
const processingOverhead = recordCount * 0.001; // 1ms per record for processing
|
||||
|
||||
return (secondsNeeded * 1000) + processingOverhead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format sync duration for display
|
||||
* @param milliseconds Duration in milliseconds
|
||||
* @returns Formatted duration string
|
||||
*/
|
||||
export function formatDuration(milliseconds: number): string {
|
||||
const seconds = Math.floor(milliseconds / 1000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const hours = Math.floor(minutes / 60);
|
||||
|
||||
if (hours > 0) {
|
||||
return `${hours}h ${minutes % 60}m`;
|
||||
} else if (minutes > 0) {
|
||||
return `${minutes}m ${seconds % 60}s`;
|
||||
} else {
|
||||
return `${seconds}s`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate unique sync ID
|
||||
* @returns Unique sync identifier
|
||||
*/
|
||||
export function generateSyncId(): string {
|
||||
return `sync_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate entity type
|
||||
* @param entity Entity type string
|
||||
* @returns True if valid entity type
|
||||
*/
|
||||
export function isValidEntityType(entity: string): entity is EntityType {
|
||||
return Object.values(EntityType).includes(entity as EntityType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get entity display name
|
||||
* @param entity Entity type
|
||||
* @returns Human-readable entity name
|
||||
*/
|
||||
export function getEntityDisplayName(entity: EntityType): string {
|
||||
const mapping: Record<EntityType, string> = {
|
||||
[EntityType.COMPANIES]: 'Companies',
|
||||
[EntityType.TICKETS]: 'Tickets',
|
||||
[EntityType.TASKS]: 'Tasks',
|
||||
[EntityType.PROJECTS]: 'Projects',
|
||||
[EntityType.RESOURCES]: 'Resources',
|
||||
[EntityType.STATUSES]: 'Statuses',
|
||||
[EntityType.ISSUE_TYPES]: 'Issue Types',
|
||||
[EntityType.SUB_ISSUE_TYPES]: 'Sub-Issue Types',
|
||||
[EntityType.WORK_TYPES]: 'Work Types',
|
||||
[EntityType.BILLING_ITEMS]: 'Billing Items',
|
||||
[EntityType.CONFIGURATION_ITEMS]: 'Configuration Items',
|
||||
[EntityType.CONTACTS]: 'Contacts',
|
||||
[EntityType.CONTRACTS]: 'Contracts',
|
||||
[EntityType.TIME_ENTRIES]: 'Time Entries',
|
||||
};
|
||||
|
||||
return mapping[entity] || entity;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue