feat: add Autotask tags sync (tag groups, tags, ticket tag associations)
- Migration 057: autotask_tag_groups, autotask_tags, and junction tables (ticket_tags, company_tags, configuration_item_tags, contact_tags) - Add TAG_GROUPS and TAGS to EntityType enum and dependency map - Add mapTagGroup() and mapTag() entity mapper functions - Add syncTagGroups(), syncTags(), syncTicketTagAssociations() methods - Wire TicketTagAssociations bulk sync into full/incremental sync flow - Add 'exist' operator to QueryFilter type - No FK on ticket_id (tagged tickets may be outside 2yr sync window) Synced: 26 tag groups, 7299 tags, 10141 ticket-tag associations
This commit is contained in:
parent
fe9f806c50
commit
ff9e34cafe
8 changed files with 238 additions and 3 deletions
|
|
@ -73,6 +73,14 @@ export const auth = betterAuth({
|
|||
nextCookies(),
|
||||
],
|
||||
|
||||
// Allow Microsoft OAuth to link to existing accounts created by admins
|
||||
account: {
|
||||
accountLinking: {
|
||||
enabled: true,
|
||||
trustedProviders: ["microsoft"],
|
||||
},
|
||||
},
|
||||
|
||||
// User configuration
|
||||
user: {
|
||||
additionalFields: {
|
||||
|
|
|
|||
|
|
@ -587,7 +587,7 @@ export class EntitySyncService {
|
|||
|
||||
if (mappedRecords.length > 0) {
|
||||
const upsertedCount = await bulkUpsertRecords(entity, mappedRecords, 100);
|
||||
|
||||
|
||||
// Estimate added vs updated
|
||||
const recordsAdded = Math.floor(upsertedCount * 0.1);
|
||||
const recordsUpdated = upsertedCount - recordsAdded;
|
||||
|
|
@ -1149,6 +1149,96 @@ export class EntitySyncService {
|
|||
async syncTimeEntries(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.TIME_ENTRIES, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Tag Groups
|
||||
*/
|
||||
async syncTagGroups(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.TAG_GROUPS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync Tags
|
||||
*/
|
||||
async syncTags(isIncremental: boolean = false): Promise<EntitySyncStats> {
|
||||
return await this.syncEntity(EntityType.TAGS, isIncremental);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync TicketTagAssociations from Autotask API into the ticket_tags junction table.
|
||||
* Autotask exposes TicketTagAssociations as a bulk-queryable entity with fields:
|
||||
* id, tagID, ticketID
|
||||
*/
|
||||
async syncTicketTagAssociations(): Promise<EntitySyncStats> {
|
||||
const tagLogger = this.logger.child({ entityType: 'ticket_tag_associations', syncType: 'full' });
|
||||
const syncStartTime = tagLogger.start('TicketTagAssociations sync');
|
||||
|
||||
try {
|
||||
// Fetch all TicketTagAssociations from Autotask
|
||||
tagLogger.info('Fetching TicketTagAssociations from Autotask');
|
||||
const associations = await this.autotaskClient.queryEntityPaginated<{
|
||||
id: number;
|
||||
tagID: number;
|
||||
ticketID: number;
|
||||
}>('TicketTagAssociations', { filter: [{ op: 'exist', field: 'id' }] }, 500);
|
||||
|
||||
tagLogger.info('Fetched TicketTagAssociations', { count: associations.length });
|
||||
|
||||
if (associations.length === 0) {
|
||||
tagLogger.info('No ticket tag associations found');
|
||||
tagLogger.complete('TicketTagAssociations sync', syncStartTime, { recordsAdded: 0 });
|
||||
return { recordsAdded: 0, recordsUpdated: 0, recordsDeleted: 0 };
|
||||
}
|
||||
|
||||
// Validate tag IDs exist in our DB to avoid FK violations on autotask_tags
|
||||
// (ticket_id has no FK constraint since tagged tickets may be outside sync window)
|
||||
const validTagResult = await postgresClient.query<{ id: number }>(
|
||||
'SELECT id FROM autotask_tags'
|
||||
);
|
||||
const validTagIds = new Set(validTagResult.rows.map(r => Number(r.id)));
|
||||
|
||||
const validAssociations = associations.filter(a => validTagIds.has(Number(a.tagID)));
|
||||
|
||||
tagLogger.info('Validated associations', {
|
||||
total: associations.length,
|
||||
valid: validAssociations.length,
|
||||
skippedInvalidTag: associations.length - validAssociations.length,
|
||||
});
|
||||
|
||||
// Replace all ticket_tags: truncate and re-insert for a clean full sync
|
||||
await postgresClient.query('DELETE FROM ticket_tags');
|
||||
|
||||
const CHUNK = 500;
|
||||
let inserted = 0;
|
||||
for (let i = 0; i < validAssociations.length; i += CHUNK) {
|
||||
const chunk = validAssociations.slice(i, i + CHUNK);
|
||||
const values = chunk
|
||||
.map((_, idx) => `($${idx * 2 + 1}::bigint, $${idx * 2 + 2}::bigint, NOW())`)
|
||||
.join(', ');
|
||||
const params = chunk.flatMap(a => [a.ticketID, a.tagID]);
|
||||
await postgresClient.query(
|
||||
`INSERT INTO ticket_tags (ticket_id, tag_id, synced_at)
|
||||
VALUES ${values}
|
||||
ON CONFLICT (ticket_id, tag_id) DO UPDATE SET synced_at = NOW()`,
|
||||
params
|
||||
);
|
||||
inserted += chunk.length;
|
||||
}
|
||||
|
||||
const stats: EntitySyncStats = {
|
||||
recordsAdded: inserted,
|
||||
recordsUpdated: 0,
|
||||
recordsDeleted: 0,
|
||||
};
|
||||
|
||||
tagLogger.complete('TicketTagAssociations sync', syncStartTime, stats);
|
||||
return stats;
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
tagLogger.fail('TicketTagAssociations sync', syncStartTime, err);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -225,6 +225,18 @@ export class SyncService {
|
|||
}
|
||||
}
|
||||
|
||||
// After all entities are synced, sync TicketTagAssociations (junction table)
|
||||
try {
|
||||
const tagAssocLogger = syncLogger.child({ entityType: 'ticket_tag_associations' });
|
||||
tagAssocLogger.info('Syncing TicketTagAssociations');
|
||||
const tagStats = await this.entitySyncService.syncTicketTagAssociations();
|
||||
tagAssocLogger.info('TicketTagAssociations sync complete', { recordsAdded: tagStats.recordsAdded });
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
syncLogger.warn('TicketTagAssociations sync failed, continuing', {}, err);
|
||||
errors.push(`TicketTagAssociations: ${err.message}`);
|
||||
}
|
||||
|
||||
const endTime = new Date();
|
||||
const totalDuration = endTime.getTime() - startTime.getTime();
|
||||
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ export type AutotaskHeaders = {
|
|||
}
|
||||
|
||||
export interface QueryFilter {
|
||||
op: 'eq' | 'noteq' | 'gt' | 'lt' | 'gte' | 'lte' | 'contains' | 'beginsWith' | 'endsWith';
|
||||
op: 'eq' | 'noteq' | 'gt' | 'lt' | 'gte' | 'lte' | 'contains' | 'beginsWith' | 'endsWith' | 'exist';
|
||||
field: string;
|
||||
value: string | number | boolean;
|
||||
value?: string | number | boolean;
|
||||
}
|
||||
|
||||
export interface QueryParams {
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ export enum EntityType {
|
|||
AUTOTASK_SERVICES = 'autotask_services',
|
||||
TIME_ENTRIES = 'time_entries',
|
||||
TICKET_NOTES = 'ticket_notes',
|
||||
TAG_GROUPS = 'autotask_tag_groups',
|
||||
TAGS = 'autotask_tags',
|
||||
}
|
||||
|
||||
// Sync operation types
|
||||
|
|
@ -171,6 +173,8 @@ export const ENTITY_DEPENDENCIES: Record<EntityType, EntityType[]> = {
|
|||
[EntityType.BILLING_ITEMS]: [EntityType.COMPANIES, EntityType.TASKS, EntityType.TICKETS, EntityType.PROJECTS], // Depends on multiple entities
|
||||
[EntityType.TIME_ENTRIES]: [EntityType.COMPANIES, EntityType.RESOURCES, EntityType.CONTACTS, EntityType.PROJECTS, EntityType.TASKS, EntityType.TICKETS], // Depends on many entities
|
||||
[EntityType.TICKET_NOTES]: [EntityType.TICKETS], // Depends on tickets
|
||||
[EntityType.TAG_GROUPS]: [], // No dependencies — standalone lookup
|
||||
[EntityType.TAGS]: [EntityType.TAG_GROUPS], // Depends on tag groups (FK)
|
||||
};
|
||||
|
||||
// Autotask API field names (for incremental sync)
|
||||
|
|
|
|||
|
|
@ -72,6 +72,12 @@ export function mapAutotaskToDatabase(
|
|||
case EntityType.WORK_TYPES:
|
||||
mapped = mapPicklist(data);
|
||||
break;
|
||||
case EntityType.TAG_GROUPS:
|
||||
mapped = mapTagGroup(data);
|
||||
break;
|
||||
case EntityType.TAGS:
|
||||
mapped = mapTag(data);
|
||||
break;
|
||||
default:
|
||||
// Fallback: auto-convert camelCase to snake_case
|
||||
mapped = {};
|
||||
|
|
@ -724,6 +730,35 @@ function mapTimeEntry(data: any): Record<string, any> {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map TagGroup entity
|
||||
*/
|
||||
function mapTagGroup(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
label: data.label,
|
||||
display_color: data.displayColor,
|
||||
is_active: data.isActive !== undefined ? data.isActive : true,
|
||||
is_system: data.isSystem || false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Tag entity
|
||||
*/
|
||||
function mapTag(data: any): Record<string, any> {
|
||||
return {
|
||||
id: data.id,
|
||||
label: data.label,
|
||||
tag_group_id: data.tagGroupID,
|
||||
is_active: data.isActive !== undefined ? data.isActive : true,
|
||||
is_system: data.isSystem || false,
|
||||
is_excluded_from_automatic_tagging: data.isExcludedFromAutomaticTagging || false,
|
||||
create_date_time: data.createDateTime,
|
||||
last_modified_date_time: data.lastModifiedDateTime,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Picklist entity (statuses, issue types, etc.)
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ export function getAllEntitiesInOrder(): EntityType[] {
|
|||
EntityType.AUTOTASK_SERVICES,
|
||||
EntityType.BILLING_ITEMS,
|
||||
EntityType.TIME_ENTRIES,
|
||||
EntityType.TAG_GROUPS,
|
||||
EntityType.TAGS,
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
@ -122,6 +124,8 @@ export function getAutotaskEntityName(entity: EntityType): string {
|
|||
[EntityType.AUTOTASK_SERVICES]: 'Services',
|
||||
[EntityType.TIME_ENTRIES]: 'TimeEntries',
|
||||
[EntityType.TICKET_NOTES]: 'TicketNotes',
|
||||
[EntityType.TAG_GROUPS]: 'TagGroups',
|
||||
[EntityType.TAGS]: 'Tags',
|
||||
};
|
||||
|
||||
return mapping[entity] || entity;
|
||||
|
|
@ -171,6 +175,8 @@ export function getLastModifiedField(entity: EntityType): string {
|
|||
[EntityType.QUEUES]: 'lastModifiedDate',
|
||||
[EntityType.PRIORITIES]: 'lastModifiedDate',
|
||||
[EntityType.TICKET_CATEGORIES]: 'lastModifiedDate',
|
||||
[EntityType.TAG_GROUPS]: 'lastModifiedDate',
|
||||
[EntityType.TAGS]: 'lastModifiedDateTime',
|
||||
};
|
||||
|
||||
return mapping[entity] || 'lastModifiedDate';
|
||||
|
|
@ -203,6 +209,8 @@ export function getActiveField(entity: EntityType): string | null {
|
|||
[EntityType.QUEUES]: 'isActive',
|
||||
[EntityType.PRIORITIES]: 'isActive',
|
||||
[EntityType.TICKET_CATEGORIES]: 'isActive',
|
||||
[EntityType.TAG_GROUPS]: 'isActive',
|
||||
[EntityType.TAGS]: 'isActive',
|
||||
};
|
||||
|
||||
return mapping[entity] || null;
|
||||
|
|
@ -288,6 +296,8 @@ export function buildDateRangeFilter(
|
|||
[EntityType.CONTACTS]: null,
|
||||
[EntityType.CONFIGURATION_ITEMS]: null,
|
||||
[EntityType.TICKET_NOTES]: null,
|
||||
[EntityType.TAG_GROUPS]: null,
|
||||
[EntityType.TAGS]: null,
|
||||
[EntityType.STATUSES]: null,
|
||||
[EntityType.ISSUE_TYPES]: null,
|
||||
[EntityType.SUB_ISSUE_TYPES]: null,
|
||||
|
|
@ -486,6 +496,8 @@ export function getEntityDisplayName(entity: EntityType): string {
|
|||
[EntityType.AUTOTASK_SERVICES]: 'Autotask Services',
|
||||
[EntityType.TIME_ENTRIES]: 'Time Entries',
|
||||
[EntityType.TICKET_NOTES]: 'Ticket Notes',
|
||||
[EntityType.TAG_GROUPS]: 'Tag Groups',
|
||||
[EntityType.TAGS]: 'Tags',
|
||||
};
|
||||
|
||||
return mapping[entity] || entity;
|
||||
|
|
|
|||
74
migrations/057_create_autotask_tags_tables.sql
Normal file
74
migrations/057_create_autotask_tags_tables.sql
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
-- ============================================================================
|
||||
-- Autotask Tags & Tag Groups
|
||||
-- Standalone tag catalog + junction tables for tickets, companies,
|
||||
-- configuration items, and contacts.
|
||||
-- ============================================================================
|
||||
|
||||
-- Tag groups (categories that tags belong to)
|
||||
CREATE TABLE IF NOT EXISTS autotask_tag_groups (
|
||||
id BIGINT PRIMARY KEY,
|
||||
label VARCHAR(100) NOT NULL,
|
||||
display_color INTEGER,
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
is_system BOOLEAN DEFAULT FALSE,
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Tags catalog
|
||||
CREATE TABLE IF NOT EXISTS autotask_tags (
|
||||
id BIGINT PRIMARY KEY,
|
||||
label VARCHAR(100) NOT NULL,
|
||||
tag_group_id BIGINT REFERENCES autotask_tag_groups(id) ON DELETE SET NULL,
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
is_system BOOLEAN DEFAULT FALSE,
|
||||
is_excluded_from_automatic_tagging BOOLEAN DEFAULT FALSE,
|
||||
create_date_time TIMESTAMPTZ,
|
||||
last_modified_date_time TIMESTAMPTZ,
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_autotask_tags_group ON autotask_tags (tag_group_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_autotask_tags_label ON autotask_tags (label);
|
||||
CREATE INDEX IF NOT EXISTS idx_autotask_tags_active ON autotask_tags (is_active) WHERE is_active = true;
|
||||
|
||||
-- Junction: ticket ↔ tag
|
||||
CREATE TABLE IF NOT EXISTS ticket_tags (
|
||||
ticket_id BIGINT NOT NULL,
|
||||
tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE,
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
PRIMARY KEY (ticket_id, tag_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ticket_tags_tag ON ticket_tags (tag_id);
|
||||
|
||||
-- Junction: company ↔ tag
|
||||
CREATE TABLE IF NOT EXISTS company_tags (
|
||||
company_id BIGINT NOT NULL,
|
||||
tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE,
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
PRIMARY KEY (company_id, tag_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_company_tags_tag ON company_tags (tag_id);
|
||||
|
||||
-- Junction: configuration_item ↔ tag
|
||||
CREATE TABLE IF NOT EXISTS configuration_item_tags (
|
||||
configuration_item_id BIGINT NOT NULL,
|
||||
tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE,
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
PRIMARY KEY (configuration_item_id, tag_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_config_item_tags_tag ON configuration_item_tags (tag_id);
|
||||
|
||||
-- Junction: contact ↔ tag
|
||||
CREATE TABLE IF NOT EXISTS contact_tags (
|
||||
contact_id BIGINT NOT NULL,
|
||||
tag_id BIGINT NOT NULL REFERENCES autotask_tags(id) ON DELETE CASCADE,
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
PRIMARY KEY (contact_id, tag_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_contact_tags_tag ON contact_tags (tag_id);
|
||||
Loading…
Add table
Add a link
Reference in a new issue