From ff9e34cafe05c6b3525a02ab92b1c728c136ed4a Mon Sep 17 00:00:00 2001 From: lorentz Date: Fri, 20 Mar 2026 09:22:40 -0400 Subject: [PATCH] feat: add Autotask tags sync (tag groups, tags, ticket tag associations) - Migration 057: autotask_tag_groups, autotask_tags, and junction tables (ticket_tags, company_tags, configuration_item_tags, contact_tags) - Add TAG_GROUPS and TAGS to EntityType enum and dependency map - Add mapTagGroup() and mapTag() entity mapper functions - Add syncTagGroups(), syncTags(), syncTicketTagAssociations() methods - Wire TicketTagAssociations bulk sync into full/incremental sync flow - Add 'exist' operator to QueryFilter type - No FK on ticket_id (tagged tickets may be outside 2yr sync window) Synced: 26 tag groups, 7299 tags, 10141 ticket-tag associations --- lib/auth.ts | 8 ++ lib/services/entity-sync.ts | 92 ++++++++++++++++++- lib/services/sync-service.ts | 12 +++ lib/types/autotask.ts | 4 +- lib/types/sync.ts | 4 + lib/utils/entity-mapper.ts | 35 +++++++ lib/utils/sync-helpers.ts | 12 +++ .../057_create_autotask_tags_tables.sql | 74 +++++++++++++++ 8 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 migrations/057_create_autotask_tags_tables.sql diff --git a/lib/auth.ts b/lib/auth.ts index af6d694..12c17e5 100644 --- a/lib/auth.ts +++ b/lib/auth.ts @@ -73,6 +73,14 @@ export const auth = betterAuth({ nextCookies(), ], + // Allow Microsoft OAuth to link to existing accounts created by admins + account: { + accountLinking: { + enabled: true, + trustedProviders: ["microsoft"], + }, + }, + // User configuration user: { additionalFields: { diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 7e2490c..5c3da00 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -587,7 +587,7 @@ export class EntitySyncService { if (mappedRecords.length > 0) { const upsertedCount = await bulkUpsertRecords(entity, mappedRecords, 100); - + // Estimate added vs updated const recordsAdded = Math.floor(upsertedCount * 0.1); const recordsUpdated = upsertedCount - recordsAdded; @@ -1149,6 +1149,96 @@ export class EntitySyncService { async syncTimeEntries(isIncremental: boolean = false): Promise { return await this.syncEntity(EntityType.TIME_ENTRIES, isIncremental); } + + /** + * Sync Tag Groups + */ + async syncTagGroups(isIncremental: boolean = false): Promise { + return await this.syncEntity(EntityType.TAG_GROUPS, isIncremental); + } + + /** + * Sync Tags + */ + async syncTags(isIncremental: boolean = false): Promise { + return await this.syncEntity(EntityType.TAGS, isIncremental); + } + + /** + * Sync TicketTagAssociations from Autotask API into the ticket_tags junction table. + * Autotask exposes TicketTagAssociations as a bulk-queryable entity with fields: + * id, tagID, ticketID + */ + async syncTicketTagAssociations(): Promise { + const tagLogger = this.logger.child({ entityType: 'ticket_tag_associations', syncType: 'full' }); + const syncStartTime = tagLogger.start('TicketTagAssociations sync'); + + try { + // Fetch all TicketTagAssociations from Autotask + tagLogger.info('Fetching TicketTagAssociations from Autotask'); + const associations = await this.autotaskClient.queryEntityPaginated<{ + id: number; + tagID: number; + ticketID: number; + }>('TicketTagAssociations', { filter: [{ op: 'exist', field: 'id' }] }, 500); + + tagLogger.info('Fetched TicketTagAssociations', { count: associations.length }); + + if (associations.length === 0) { + tagLogger.info('No ticket tag associations found'); + tagLogger.complete('TicketTagAssociations sync', syncStartTime, { recordsAdded: 0 }); + return { recordsAdded: 0, recordsUpdated: 0, recordsDeleted: 0 }; + } + + // Validate tag IDs exist in our DB to avoid FK violations on autotask_tags + // (ticket_id has no FK constraint since tagged tickets may be outside sync window) + const validTagResult = await postgresClient.query<{ id: number }>( + 'SELECT id FROM autotask_tags' + ); + const validTagIds = new Set(validTagResult.rows.map(r => Number(r.id))); + + const validAssociations = associations.filter(a => validTagIds.has(Number(a.tagID))); + + tagLogger.info('Validated associations', { + total: associations.length, + valid: validAssociations.length, + skippedInvalidTag: associations.length - validAssociations.length, + }); + + // Replace all ticket_tags: truncate and re-insert for a clean full sync + await postgresClient.query('DELETE FROM ticket_tags'); + + const CHUNK = 500; + let inserted = 0; + for (let i = 0; i < validAssociations.length; i += CHUNK) { + const chunk = validAssociations.slice(i, i + CHUNK); + const values = chunk + .map((_, idx) => `($${idx * 2 + 1}::bigint, $${idx * 2 + 2}::bigint, NOW())`) + .join(', '); + const params = chunk.flatMap(a => [a.ticketID, a.tagID]); + await postgresClient.query( + `INSERT INTO ticket_tags (ticket_id, tag_id, synced_at) + VALUES ${values} + ON CONFLICT (ticket_id, tag_id) DO UPDATE SET synced_at = NOW()`, + params + ); + inserted += chunk.length; + } + + const stats: EntitySyncStats = { + recordsAdded: inserted, + recordsUpdated: 0, + recordsDeleted: 0, + }; + + tagLogger.complete('TicketTagAssociations sync', syncStartTime, stats); + return stats; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + tagLogger.fail('TicketTagAssociations sync', syncStartTime, err); + throw error; + } + } } /** diff --git a/lib/services/sync-service.ts b/lib/services/sync-service.ts index b5d3bc4..bcad600 100644 --- a/lib/services/sync-service.ts +++ b/lib/services/sync-service.ts @@ -225,6 +225,18 @@ export class SyncService { } } + // After all entities are synced, sync TicketTagAssociations (junction table) + try { + const tagAssocLogger = syncLogger.child({ entityType: 'ticket_tag_associations' }); + tagAssocLogger.info('Syncing TicketTagAssociations'); + const tagStats = await this.entitySyncService.syncTicketTagAssociations(); + tagAssocLogger.info('TicketTagAssociations sync complete', { recordsAdded: tagStats.recordsAdded }); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + syncLogger.warn('TicketTagAssociations sync failed, continuing', {}, err); + errors.push(`TicketTagAssociations: ${err.message}`); + } + const endTime = new Date(); const totalDuration = endTime.getTime() - startTime.getTime(); diff --git a/lib/types/autotask.ts b/lib/types/autotask.ts index 24671ed..c44d9de 100644 --- a/lib/types/autotask.ts +++ b/lib/types/autotask.ts @@ -16,9 +16,9 @@ export type AutotaskHeaders = { } export interface QueryFilter { - op: 'eq' | 'noteq' | 'gt' | 'lt' | 'gte' | 'lte' | 'contains' | 'beginsWith' | 'endsWith'; + op: 'eq' | 'noteq' | 'gt' | 'lt' | 'gte' | 'lte' | 'contains' | 'beginsWith' | 'endsWith' | 'exist'; field: string; - value: string | number | boolean; + value?: string | number | boolean; } export interface QueryParams { diff --git a/lib/types/sync.ts b/lib/types/sync.ts index 82bf75a..dc2dbc5 100644 --- a/lib/types/sync.ts +++ b/lib/types/sync.ts @@ -25,6 +25,8 @@ export enum EntityType { AUTOTASK_SERVICES = 'autotask_services', TIME_ENTRIES = 'time_entries', TICKET_NOTES = 'ticket_notes', + TAG_GROUPS = 'autotask_tag_groups', + TAGS = 'autotask_tags', } // Sync operation types @@ -171,6 +173,8 @@ export const ENTITY_DEPENDENCIES: Record = { [EntityType.BILLING_ITEMS]: [EntityType.COMPANIES, EntityType.TASKS, EntityType.TICKETS, EntityType.PROJECTS], // Depends on multiple entities [EntityType.TIME_ENTRIES]: [EntityType.COMPANIES, EntityType.RESOURCES, EntityType.CONTACTS, EntityType.PROJECTS, EntityType.TASKS, EntityType.TICKETS], // Depends on many entities [EntityType.TICKET_NOTES]: [EntityType.TICKETS], // Depends on tickets + [EntityType.TAG_GROUPS]: [], // No dependencies — standalone lookup + [EntityType.TAGS]: [EntityType.TAG_GROUPS], // Depends on tag groups (FK) }; // Autotask API field names (for incremental sync) diff --git a/lib/utils/entity-mapper.ts b/lib/utils/entity-mapper.ts index c51a500..bea80f7 100644 --- a/lib/utils/entity-mapper.ts +++ b/lib/utils/entity-mapper.ts @@ -72,6 +72,12 @@ export function mapAutotaskToDatabase( case EntityType.WORK_TYPES: mapped = mapPicklist(data); break; + case EntityType.TAG_GROUPS: + mapped = mapTagGroup(data); + break; + case EntityType.TAGS: + mapped = mapTag(data); + break; default: // Fallback: auto-convert camelCase to snake_case mapped = {}; @@ -724,6 +730,35 @@ function mapTimeEntry(data: any): Record { }; } +/** + * Map TagGroup entity + */ +function mapTagGroup(data: any): Record { + return { + id: data.id, + label: data.label, + display_color: data.displayColor, + is_active: data.isActive !== undefined ? data.isActive : true, + is_system: data.isSystem || false, + }; +} + +/** + * Map Tag entity + */ +function mapTag(data: any): Record { + return { + id: data.id, + label: data.label, + tag_group_id: data.tagGroupID, + is_active: data.isActive !== undefined ? data.isActive : true, + is_system: data.isSystem || false, + is_excluded_from_automatic_tagging: data.isExcludedFromAutomaticTagging || false, + create_date_time: data.createDateTime, + last_modified_date_time: data.lastModifiedDateTime, + }; +} + /** * Map Picklist entity (statuses, issue types, etc.) */ diff --git a/lib/utils/sync-helpers.ts b/lib/utils/sync-helpers.ts index 6af2ece..1f37015 100644 --- a/lib/utils/sync-helpers.ts +++ b/lib/utils/sync-helpers.ts @@ -69,6 +69,8 @@ export function getAllEntitiesInOrder(): EntityType[] { EntityType.AUTOTASK_SERVICES, EntityType.BILLING_ITEMS, EntityType.TIME_ENTRIES, + EntityType.TAG_GROUPS, + EntityType.TAGS, ]); } @@ -122,6 +124,8 @@ export function getAutotaskEntityName(entity: EntityType): string { [EntityType.AUTOTASK_SERVICES]: 'Services', [EntityType.TIME_ENTRIES]: 'TimeEntries', [EntityType.TICKET_NOTES]: 'TicketNotes', + [EntityType.TAG_GROUPS]: 'TagGroups', + [EntityType.TAGS]: 'Tags', }; return mapping[entity] || entity; @@ -171,6 +175,8 @@ export function getLastModifiedField(entity: EntityType): string { [EntityType.QUEUES]: 'lastModifiedDate', [EntityType.PRIORITIES]: 'lastModifiedDate', [EntityType.TICKET_CATEGORIES]: 'lastModifiedDate', + [EntityType.TAG_GROUPS]: 'lastModifiedDate', + [EntityType.TAGS]: 'lastModifiedDateTime', }; return mapping[entity] || 'lastModifiedDate'; @@ -203,6 +209,8 @@ export function getActiveField(entity: EntityType): string | null { [EntityType.QUEUES]: 'isActive', [EntityType.PRIORITIES]: 'isActive', [EntityType.TICKET_CATEGORIES]: 'isActive', + [EntityType.TAG_GROUPS]: 'isActive', + [EntityType.TAGS]: 'isActive', }; return mapping[entity] || null; @@ -288,6 +296,8 @@ export function buildDateRangeFilter( [EntityType.CONTACTS]: null, [EntityType.CONFIGURATION_ITEMS]: null, [EntityType.TICKET_NOTES]: null, + [EntityType.TAG_GROUPS]: null, + [EntityType.TAGS]: null, [EntityType.STATUSES]: null, [EntityType.ISSUE_TYPES]: null, [EntityType.SUB_ISSUE_TYPES]: null, @@ -486,6 +496,8 @@ export function getEntityDisplayName(entity: EntityType): string { [EntityType.AUTOTASK_SERVICES]: 'Autotask Services', [EntityType.TIME_ENTRIES]: 'Time Entries', [EntityType.TICKET_NOTES]: 'Ticket Notes', + [EntityType.TAG_GROUPS]: 'Tag Groups', + [EntityType.TAGS]: 'Tags', }; return mapping[entity] || entity; diff --git a/migrations/057_create_autotask_tags_tables.sql b/migrations/057_create_autotask_tags_tables.sql new file mode 100644 index 0000000..0c87a7e --- /dev/null +++ b/migrations/057_create_autotask_tags_tables.sql @@ -0,0 +1,74 @@ +-- ============================================================================ +-- Autotask Tags & Tag Groups +-- Standalone tag catalog + junction tables for tickets, companies, +-- configuration items, and contacts. +-- ============================================================================ + +-- Tag groups (categories that tags belong to) +CREATE TABLE IF NOT EXISTS autotask_tag_groups ( + id BIGINT PRIMARY KEY, + label VARCHAR(100) NOT NULL, + display_color INTEGER, + is_active BOOLEAN DEFAULT TRUE, + is_system BOOLEAN DEFAULT FALSE, + synced_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + is_deleted BOOLEAN DEFAULT FALSE, + deleted_at TIMESTAMPTZ +); + +-- Tags catalog +CREATE TABLE IF NOT EXISTS autotask_tags ( + id BIGINT PRIMARY KEY, + label VARCHAR(100) NOT NULL, + tag_group_id BIGINT REFERENCES autotask_tag_groups(id) ON DELETE SET NULL, + is_active BOOLEAN DEFAULT TRUE, + is_system BOOLEAN DEFAULT FALSE, + is_excluded_from_automatic_tagging BOOLEAN DEFAULT FALSE, + create_date_time TIMESTAMPTZ, + last_modified_date_time TIMESTAMPTZ, + synced_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + is_deleted BOOLEAN DEFAULT FALSE, + deleted_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_autotask_tags_group ON autotask_tags (tag_group_id); +CREATE INDEX IF NOT EXISTS idx_autotask_tags_label ON autotask_tags (label); +CREATE INDEX IF NOT EXISTS idx_autotask_tags_active ON autotask_tags (is_active) WHERE is_active = true; + +-- Junction: ticket ↔ tag +CREATE TABLE IF NOT EXISTS ticket_tags ( + ticket_id BIGINT NOT NULL, + tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE, + synced_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (ticket_id, tag_id) +); +CREATE INDEX IF NOT EXISTS idx_ticket_tags_tag ON ticket_tags (tag_id); + +-- Junction: company ↔ tag +CREATE TABLE IF NOT EXISTS company_tags ( + company_id BIGINT NOT NULL, + tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE, + synced_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (company_id, tag_id) +); +CREATE INDEX IF NOT EXISTS idx_company_tags_tag ON company_tags (tag_id); + +-- Junction: configuration_item ↔ tag +CREATE TABLE IF NOT EXISTS configuration_item_tags ( + configuration_item_id BIGINT NOT NULL, + tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE, + synced_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (configuration_item_id, tag_id) +); +CREATE INDEX IF NOT EXISTS idx_config_item_tags_tag ON configuration_item_tags (tag_id); + +-- Junction: contact ↔ tag +CREATE TABLE IF NOT EXISTS contact_tags ( + contact_id BIGINT NOT NULL, + tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE, + synced_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (contact_id, tag_id) +); +CREATE INDEX IF NOT EXISTS idx_contact_tags_tag ON contact_tags (tag_id);