- Add comprehensive structured logging utility (lib/utils/sync-logger.ts) - Log levels: DEBUG, INFO, WARN, ERROR - Automatic error categorization (NETWORK_ERROR, AUTH_ERROR, DATABASE_ERROR, etc.) - Phase tracking (INITIALIZING, FETCHING, MAPPING, VALIDATING, UPSERTING, DELETING, COMPLETING) - Contextual metadata (syncId, entityType, phase, duration, record counts) - Timing helpers for operations - Update sync-service.ts with structured logging - Replace console.log/error with structured logger - Consistent error categorization across all error paths - Fix swallowed errors in sync history updates - Add child loggers for entity-specific operations - Update entity-sync.ts with structured logging - Complete phase tracking throughout sync lifecycle - Detailed context for warnings and errors - Update chunked sync methods with structured logging - Update picklist sync methods (issue types, sub-issue types) - Better debugging info with timing and sample data - Add sync failure analysis script (scripts/analyze-sync-failures.ts) - Query sync history by date range, entity type, or status - Generate failure summaries by entity - Automatic error categorization - Calculate success rates and statistics - Display detailed sync records with timing - Add comprehensive documentation (docs/SYNC_LOGGING_IMPROVEMENTS.md) - Usage guide and examples - Migration guide for developers - Before/after comparisons This addresses inconsistent logging and improves debugging capabilities for sync operations.
257 lines
7 KiB
TypeScript
257 lines
7 KiB
TypeScript
/**
|
|
* Structured Logging Utility for Sync Operations
|
|
* Provides consistent, categorized logging with context
|
|
*/
|
|
|
|
export enum LogLevel {
|
|
DEBUG = 'DEBUG',
|
|
INFO = 'INFO',
|
|
WARN = 'WARN',
|
|
ERROR = 'ERROR',
|
|
}
|
|
|
|
export enum ErrorCategory {
|
|
NETWORK_ERROR = 'NETWORK_ERROR',
|
|
AUTH_ERROR = 'AUTH_ERROR',
|
|
RATE_LIMIT_ERROR = 'RATE_LIMIT_ERROR',
|
|
DATABASE_CONSTRAINT_ERROR = 'DATABASE_CONSTRAINT_ERROR',
|
|
DATABASE_ERROR = 'DATABASE_ERROR',
|
|
API_ERROR = 'API_ERROR',
|
|
MAPPING_ERROR = 'MAPPING_ERROR',
|
|
VALIDATION_ERROR = 'VALIDATION_ERROR',
|
|
UNKNOWN = 'UNKNOWN',
|
|
}
|
|
|
|
export enum SyncPhase {
|
|
INITIALIZING = 'initializing',
|
|
FETCHING = 'fetching',
|
|
MAPPING = 'mapping',
|
|
VALIDATING = 'validating',
|
|
UPSERTING = 'upserting',
|
|
DELETING = 'deleting',
|
|
COMPLETING = 'completing',
|
|
}
|
|
|
|
export interface LogContext {
|
|
syncId?: string;
|
|
entityType?: string;
|
|
phase?: SyncPhase;
|
|
recordCount?: number;
|
|
duration?: number;
|
|
errorCategory?: ErrorCategory;
|
|
[key: string]: any;
|
|
}
|
|
|
|
export interface LogEntry {
|
|
timestamp: string;
|
|
level: LogLevel;
|
|
message: string;
|
|
context?: LogContext;
|
|
error?: {
|
|
message: string;
|
|
stack?: string;
|
|
category?: ErrorCategory;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Categorize error based on error message
|
|
*/
|
|
export function categorizeError(error: Error | string): ErrorCategory {
|
|
const errorMessage = typeof error === 'string' ? error : error.message;
|
|
|
|
if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('ETIMEDOUT') || errorMessage.includes('ENOTFOUND')) {
|
|
return ErrorCategory.NETWORK_ERROR;
|
|
} else if (errorMessage.includes('401') || errorMessage.includes('403') || errorMessage.includes('Unauthorized')) {
|
|
return ErrorCategory.AUTH_ERROR;
|
|
} else if (errorMessage.includes('429') || errorMessage.includes('rate limit')) {
|
|
return ErrorCategory.RATE_LIMIT_ERROR;
|
|
} else if (errorMessage.includes('constraint') || errorMessage.includes('duplicate') || errorMessage.includes('foreign key')) {
|
|
return ErrorCategory.DATABASE_CONSTRAINT_ERROR;
|
|
} else if (errorMessage.includes('query') || errorMessage.includes('SQL') || errorMessage.includes('database')) {
|
|
return ErrorCategory.DATABASE_ERROR;
|
|
} else if (errorMessage.includes('API') || errorMessage.includes('endpoint')) {
|
|
return ErrorCategory.API_ERROR;
|
|
} else if (errorMessage.includes('mapping') || errorMessage.includes('transform')) {
|
|
return ErrorCategory.MAPPING_ERROR;
|
|
} else if (errorMessage.includes('validation') || errorMessage.includes('invalid')) {
|
|
return ErrorCategory.VALIDATION_ERROR;
|
|
}
|
|
|
|
return ErrorCategory.UNKNOWN;
|
|
}
|
|
|
|
/**
|
|
* Structured Logger Class
|
|
*/
|
|
export class SyncLogger {
|
|
private context: LogContext;
|
|
|
|
constructor(context: LogContext = {}) {
|
|
this.context = context;
|
|
}
|
|
|
|
/**
|
|
* Create a child logger with additional context
|
|
*/
|
|
child(additionalContext: LogContext): SyncLogger {
|
|
return new SyncLogger({ ...this.context, ...additionalContext });
|
|
}
|
|
|
|
/**
|
|
* Update logger context
|
|
*/
|
|
updateContext(updates: LogContext): void {
|
|
this.context = { ...this.context, ...updates };
|
|
}
|
|
|
|
/**
|
|
* Format log entry
|
|
*/
|
|
private formatLog(level: LogLevel, message: string, context?: LogContext, error?: Error): LogEntry {
|
|
const entry: LogEntry = {
|
|
timestamp: new Date().toISOString(),
|
|
level,
|
|
message,
|
|
context: { ...this.context, ...context },
|
|
};
|
|
|
|
if (error) {
|
|
entry.error = {
|
|
message: error.message,
|
|
stack: error.stack,
|
|
category: categorizeError(error),
|
|
};
|
|
}
|
|
|
|
return entry;
|
|
}
|
|
|
|
/**
|
|
* Output log entry
|
|
*/
|
|
private output(entry: LogEntry): void {
|
|
const prefix = `[${entry.timestamp}] [${entry.level}]`;
|
|
const contextStr = entry.context ? ` [${this.formatContext(entry.context)}]` : '';
|
|
const fullMessage = `${prefix}${contextStr} ${entry.message}`;
|
|
|
|
switch (entry.level) {
|
|
case LogLevel.DEBUG:
|
|
console.log(fullMessage);
|
|
break;
|
|
case LogLevel.INFO:
|
|
console.log(fullMessage);
|
|
break;
|
|
case LogLevel.WARN:
|
|
console.warn(fullMessage);
|
|
if (entry.error) {
|
|
console.warn(` Error: ${entry.error.message}`);
|
|
console.warn(` Category: ${entry.error.category}`);
|
|
}
|
|
break;
|
|
case LogLevel.ERROR:
|
|
console.error(fullMessage);
|
|
if (entry.error) {
|
|
console.error(` Error: ${entry.error.message}`);
|
|
console.error(` Category: ${entry.error.category}`);
|
|
if (entry.error.stack) {
|
|
console.error(` Stack: ${entry.error.stack}`);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Format context for display
|
|
*/
|
|
private formatContext(context: LogContext): string {
|
|
const parts: string[] = [];
|
|
|
|
if (context.syncId) parts.push(`syncId=${context.syncId}`);
|
|
if (context.entityType) parts.push(`entity=${context.entityType}`);
|
|
if (context.phase) parts.push(`phase=${context.phase}`);
|
|
if (context.recordCount !== undefined) parts.push(`records=${context.recordCount}`);
|
|
if (context.duration !== undefined) parts.push(`duration=${context.duration}ms`);
|
|
if (context.errorCategory) parts.push(`errorCategory=${context.errorCategory}`);
|
|
|
|
return parts.join(', ');
|
|
}
|
|
|
|
/**
|
|
* Debug level logging
|
|
*/
|
|
debug(message: string, context?: LogContext): void {
|
|
const entry = this.formatLog(LogLevel.DEBUG, message, context);
|
|
this.output(entry);
|
|
}
|
|
|
|
/**
|
|
* Info level logging
|
|
*/
|
|
info(message: string, context?: LogContext): void {
|
|
const entry = this.formatLog(LogLevel.INFO, message, context);
|
|
this.output(entry);
|
|
}
|
|
|
|
/**
|
|
* Warning level logging
|
|
*/
|
|
warn(message: string, context?: LogContext, error?: Error): void {
|
|
const entry = this.formatLog(LogLevel.WARN, message, context, error);
|
|
this.output(entry);
|
|
}
|
|
|
|
/**
|
|
* Error level logging
|
|
*/
|
|
error(message: string, context?: LogContext, error?: Error): void {
|
|
const entry = this.formatLog(LogLevel.ERROR, message, context, error);
|
|
this.output(entry);
|
|
}
|
|
|
|
/**
|
|
* Log phase transition
|
|
*/
|
|
phase(phase: SyncPhase, message?: string): void {
|
|
this.updateContext({ phase });
|
|
this.info(message || `Entering phase: ${phase}`, { phase });
|
|
}
|
|
|
|
/**
|
|
* Log operation start
|
|
*/
|
|
start(operation: string, context?: LogContext): number {
|
|
this.info(`Starting ${operation}`, context);
|
|
return Date.now();
|
|
}
|
|
|
|
/**
|
|
* Log operation completion
|
|
*/
|
|
complete(operation: string, startTime: number, context?: LogContext): void {
|
|
const duration = Date.now() - startTime;
|
|
this.info(`Completed ${operation}`, { ...context, duration });
|
|
}
|
|
|
|
/**
|
|
* Log operation failure
|
|
*/
|
|
fail(operation: string, startTime: number, error: Error, context?: LogContext): void {
|
|
const duration = Date.now() - startTime;
|
|
const errorCategory = categorizeError(error);
|
|
this.error(`Failed ${operation}`, { ...context, duration, errorCategory }, error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a new sync logger instance
|
|
*/
|
|
export function createSyncLogger(context?: LogContext): SyncLogger {
|
|
return new SyncLogger(context);
|
|
}
|
|
|
|
/**
|
|
* Default logger instance
|
|
*/
|
|
export const defaultLogger = new SyncLogger();
|