feat: Duo Security integration — full data sync from Accounts + Admin API
Duo API Client (lib/services/duo-client.ts): - HMAC-SHA1 request signing, GET/POST, automatic pagination - Rate-limit handling (429 + Retry-After), configurable timeout - Accounts API: listAccounts() via POST /accounts/v1/account/list - Admin API: getUsers, getPhones, getGroups, getIntegrations, getAuthLogs - Child account access: parent creds signed against child api_hostname + account_id - Factory helpers: getDuoAccountsClient(), getDuoAdminClient() Database (migration 058): - 6 tables: duo_accounts, duo_users, duo_phones, duo_auth_logs, duo_groups, duo_integrations - All with proper FKs, indexes, JSONB fields for capabilities/location/groups Sync Service (lib/services/duo-sync-service.ts): - syncAll(): accounts → per-child data + auth logs → parent account → company matching - Sequential child processing to respect rate limits - Incremental auth logs (mintime = last synced timestamp, default 30 days) - Company matching: exact → case-insensitive containment (30/32 = 94% matched) - Non-blocking with sync ID tracking API Routes: - POST/GET /api/duo/sync — trigger sync / check status - GET /api/duo/accounts — list all accounts with stats + matched company - GET /api/duo/accounts/[id]/users — users for a specific account - POST /api/openclaw/sync/duo — OpenClaw trigger with API key auth Results: 33 accounts, 832 users, 925 phones, 5927 auth logs, 46 groups, 78 integrations
This commit is contained in:
parent
9a448d111c
commit
a4242b81be
10 changed files with 1565 additions and 0 deletions
334
lib/services/duo-client.ts
Normal file
334
lib/services/duo-client.ts
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
/**
|
||||
* Duo Security API Client
|
||||
* Handles HMAC-SHA1 request signing, pagination, rate-limit handling.
|
||||
* Supports both the Accounts API (parent) and Admin API (parent + child accounts).
|
||||
*/
|
||||
|
||||
import crypto from 'crypto';
|
||||
import https from 'https';
|
||||
|
||||
export interface DuoAccount {
|
||||
account_id: string;
|
||||
api_hostname: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface DuoClientOptions {
|
||||
ikey: string;
|
||||
skey: string;
|
||||
host: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface DuoApiResponse<T = any> {
|
||||
stat: string;
|
||||
response: T;
|
||||
metadata?: { total_objects?: number; next_offset?: number; };
|
||||
}
|
||||
|
||||
export interface DuoUserSummary {
|
||||
user_count: number;
|
||||
integration_count: number;
|
||||
telephony_credits_remaining?: number;
|
||||
}
|
||||
|
||||
export interface DuoUser {
|
||||
user_id: string;
|
||||
username: string;
|
||||
email: string;
|
||||
realname: string;
|
||||
status: string;
|
||||
is_enrolled: boolean;
|
||||
last_login: number | null;
|
||||
last_directory_sync: number | null;
|
||||
created: number;
|
||||
notes: string;
|
||||
phones: any[];
|
||||
groups: any[];
|
||||
aliases: Record<string, string | null>;
|
||||
enable_auto_prompt: boolean;
|
||||
firstname: string;
|
||||
lastname: string;
|
||||
}
|
||||
|
||||
export interface DuoPhone {
|
||||
phone_id: string;
|
||||
name: string;
|
||||
number: string;
|
||||
type: string;
|
||||
platform: string;
|
||||
model: string;
|
||||
os_version: string;
|
||||
app_version: string;
|
||||
activated: boolean;
|
||||
last_seen: string;
|
||||
capabilities: string[];
|
||||
users: any[];
|
||||
}
|
||||
|
||||
export interface DuoGroup {
|
||||
group_id: string;
|
||||
name: string;
|
||||
desc: string;
|
||||
status: string;
|
||||
mobile_otp_enabled: boolean;
|
||||
push_enabled: boolean;
|
||||
sms_enabled: boolean;
|
||||
voice_enabled: boolean;
|
||||
}
|
||||
|
||||
export interface DuoIntegration {
|
||||
integration_key: string;
|
||||
name: string;
|
||||
type: string;
|
||||
notes: string;
|
||||
}
|
||||
|
||||
export interface DuoAuthLog {
|
||||
txid: string;
|
||||
timestamp: number;
|
||||
user: { key: string; name: string; };
|
||||
factor: string;
|
||||
result: string;
|
||||
reason: string;
|
||||
application: { key: string; name: string; };
|
||||
access_device: { ip: string; hostname: string | null; location: { city: string; state: string; country: string; }; };
|
||||
auth_device: { ip: string | null; name: string; };
|
||||
event_type: string;
|
||||
}
|
||||
|
||||
export class DuoClient {
|
||||
private readonly ikey: string;
|
||||
private readonly skey: string;
|
||||
private readonly host: string;
|
||||
private readonly timeoutMs: number;
|
||||
|
||||
constructor(options: DuoClientOptions) {
|
||||
if (!options.ikey || !options.skey || !options.host) {
|
||||
throw new Error('DuoClient requires ikey, skey, and host');
|
||||
}
|
||||
this.ikey = options.ikey;
|
||||
this.skey = options.skey;
|
||||
this.host = options.host;
|
||||
this.timeoutMs = options.timeoutMs ?? 30_000;
|
||||
}
|
||||
|
||||
// ── 1.2 HMAC-SHA1 request signing ────────────────────────────────────────
|
||||
|
||||
private sign(method: string, host: string, path: string, params: Record<string, string>, date: string): string {
|
||||
const sortedParams = Object.keys(params).sort()
|
||||
.map(k => `${encodeURIComponent(k)}=${encodeURIComponent(params[k])}`)
|
||||
.join('&');
|
||||
const canon = [date, method.toUpperCase(), host.toLowerCase(), path, sortedParams].join('\n');
|
||||
const sig = crypto.createHmac('sha1', this.skey).update(canon).digest('hex');
|
||||
return `Basic ${Buffer.from(`${this.ikey}:${sig}`).toString('base64')}`;
|
||||
}
|
||||
|
||||
// ── 1.3 Core request methods (with 1.5 rate-limit + 1.6 timeout) ────────
|
||||
|
||||
private request<T = any>(
|
||||
method: string,
|
||||
path: string,
|
||||
params: Record<string, string> = {},
|
||||
hostOverride?: string,
|
||||
): Promise<DuoApiResponse<T>> {
|
||||
const host = hostOverride ?? this.host;
|
||||
const date = new Date().toUTCString();
|
||||
const auth = this.sign(method, host, path, params, date);
|
||||
|
||||
const sortedParams = Object.keys(params).sort()
|
||||
.map(k => `${encodeURIComponent(k)}=${encodeURIComponent(params[k])}`)
|
||||
.join('&');
|
||||
|
||||
const isGet = method.toUpperCase() === 'GET';
|
||||
const urlPath = isGet ? `${path}${sortedParams ? '?' + sortedParams : ''}` : path;
|
||||
const body = isGet ? '' : sortedParams;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({
|
||||
hostname: host,
|
||||
path: urlPath,
|
||||
method: method.toUpperCase(),
|
||||
headers: {
|
||||
Authorization: auth,
|
||||
Date: date,
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
...(body ? { 'Content-Length': String(Buffer.byteLength(body)) } : {}),
|
||||
},
|
||||
}, res => {
|
||||
let data = '';
|
||||
res.on('data', (chunk: Buffer) => data += chunk);
|
||||
res.on('end', async () => {
|
||||
// 1.5 rate-limit handling
|
||||
if (res.statusCode === 429) {
|
||||
const retryAfter = parseInt(res.headers['retry-after'] as string || '5', 10);
|
||||
console.log(`[DuoClient] Rate limited on ${path}, retrying in ${retryAfter}s`);
|
||||
await new Promise(r => setTimeout(r, retryAfter * 1000));
|
||||
return resolve(this.request<T>(method, path, params, hostOverride));
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(data) as DuoApiResponse<T>;
|
||||
if (parsed.stat !== 'OK') {
|
||||
reject(new Error(`Duo API error on ${path}: ${(parsed as any).message || JSON.stringify(parsed)}`));
|
||||
} else {
|
||||
resolve(parsed);
|
||||
}
|
||||
} catch {
|
||||
reject(new Error(`Duo API non-JSON response on ${path}: HTTP ${res.statusCode} — ${data.slice(0, 200)}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 1.6 timeout
|
||||
req.setTimeout(this.timeoutMs, () => {
|
||||
req.destroy();
|
||||
reject(new Error(`Duo API timeout on ${path} after ${this.timeoutMs}ms`));
|
||||
});
|
||||
req.on('error', reject);
|
||||
if (body) req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async get<T = any>(path: string, params: Record<string, string> = {}, hostOverride?: string): Promise<DuoApiResponse<T>> {
|
||||
return this.request<T>('GET', path, params, hostOverride);
|
||||
}
|
||||
|
||||
async post<T = any>(path: string, params: Record<string, string> = {}, hostOverride?: string): Promise<DuoApiResponse<T>> {
|
||||
return this.request<T>('POST', path, params, hostOverride);
|
||||
}
|
||||
|
||||
// ── 1.4 Paginated GET — follows metadata.next_offset ─────────────────────
|
||||
|
||||
async getPaginated<T = any>(path: string, params: Record<string, string> = {}, hostOverride?: string): Promise<T[]> {
|
||||
const all: T[] = [];
|
||||
let offset = 0;
|
||||
const limit = '300';
|
||||
|
||||
while (true) {
|
||||
const pageParams = { ...params, limit, offset: String(offset) };
|
||||
const res = await this.get<T[]>(path, pageParams, hostOverride);
|
||||
const items = res.response;
|
||||
if (!Array.isArray(items)) break;
|
||||
all.push(...items);
|
||||
|
||||
if (res.metadata?.next_offset !== undefined && res.metadata.next_offset !== null) {
|
||||
offset = res.metadata.next_offset;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
// ── 1.7 Accounts API ─────────────────────────────────────────────────────
|
||||
|
||||
async listAccounts(): Promise<DuoAccount[]> {
|
||||
const res = await this.post<DuoAccount[]>('/accounts/v1/account/list');
|
||||
return res.response;
|
||||
}
|
||||
|
||||
// ── 1.8 + 1.9 Admin API methods (with child account support) ─────────────
|
||||
// When `child` is provided, requests are signed against child.api_hostname
|
||||
// and account_id is passed as a parameter.
|
||||
|
||||
private childParams(child?: DuoAccount): Record<string, string> {
|
||||
return child ? { account_id: child.account_id } : {};
|
||||
}
|
||||
|
||||
private childHost(child?: DuoAccount): string | undefined {
|
||||
return child?.api_hostname;
|
||||
}
|
||||
|
||||
async getAccountSummary(child?: DuoAccount): Promise<DuoUserSummary> {
|
||||
const res = await this.get<DuoUserSummary>(
|
||||
'/admin/v1/info/summary',
|
||||
this.childParams(child),
|
||||
this.childHost(child),
|
||||
);
|
||||
return res.response;
|
||||
}
|
||||
|
||||
async getUsers(child?: DuoAccount): Promise<DuoUser[]> {
|
||||
return this.getPaginated<DuoUser>(
|
||||
'/admin/v1/users',
|
||||
this.childParams(child),
|
||||
this.childHost(child),
|
||||
);
|
||||
}
|
||||
|
||||
async getPhones(child?: DuoAccount): Promise<DuoPhone[]> {
|
||||
return this.getPaginated<DuoPhone>(
|
||||
'/admin/v1/phones',
|
||||
this.childParams(child),
|
||||
this.childHost(child),
|
||||
);
|
||||
}
|
||||
|
||||
async getGroups(child?: DuoAccount): Promise<DuoGroup[]> {
|
||||
return this.getPaginated<DuoGroup>(
|
||||
'/admin/v1/groups',
|
||||
this.childParams(child),
|
||||
this.childHost(child),
|
||||
);
|
||||
}
|
||||
|
||||
async getIntegrations(child?: DuoAccount): Promise<DuoIntegration[]> {
|
||||
return this.getPaginated<DuoIntegration>(
|
||||
'/admin/v1/integrations',
|
||||
this.childParams(child),
|
||||
this.childHost(child),
|
||||
);
|
||||
}
|
||||
|
||||
async getAuthLogs(mintime: number, child?: DuoAccount): Promise<DuoAuthLog[]> {
|
||||
// Auth logs v2 has a different response structure:
|
||||
// { stat: "OK", response: { authlogs: [...], metadata: { next_offset: [...] } } }
|
||||
// So we can't use getPaginated — custom pagination here.
|
||||
const all: DuoAuthLog[] = [];
|
||||
let nextOffset: string[] | undefined;
|
||||
|
||||
while (true) {
|
||||
const params: Record<string, string> = {
|
||||
...this.childParams(child),
|
||||
mintime: String(mintime),
|
||||
maxtime: String(Date.now()),
|
||||
limit: '1000',
|
||||
};
|
||||
if (nextOffset) {
|
||||
params.next_offset = JSON.stringify(nextOffset);
|
||||
}
|
||||
|
||||
const res = await this.get<{ authlogs: DuoAuthLog[]; metadata: { next_offset?: string[] } }>(
|
||||
'/admin/v2/logs/authentication',
|
||||
params,
|
||||
this.childHost(child),
|
||||
);
|
||||
|
||||
const logs = res.response?.authlogs ?? [];
|
||||
all.push(...logs);
|
||||
|
||||
nextOffset = res.response?.metadata?.next_offset;
|
||||
if (!nextOffset || logs.length === 0) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Factory helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
export function getDuoAccountsClient(): DuoClient {
|
||||
return new DuoClient({
|
||||
ikey: process.env.DUOACCOUNTS_INTEGRATION_KEY!,
|
||||
skey: process.env.DUOACCOUNTS_SECRET_KEY!,
|
||||
host: process.env.DUOACCOUNTS_API_HOSTNAME!,
|
||||
});
|
||||
}
|
||||
|
||||
export function getDuoAdminClient(): DuoClient {
|
||||
return new DuoClient({
|
||||
ikey: process.env.DUOADMIN_INTEGRATION_KEY!,
|
||||
skey: process.env.DUOADMIN_SECRET_KEY!,
|
||||
host: process.env.DUOADMIN_API_HOSTNAME!,
|
||||
});
|
||||
}
|
||||
601
lib/services/duo-sync-service.ts
Normal file
601
lib/services/duo-sync-service.ts
Normal file
|
|
@ -0,0 +1,601 @@
|
|||
/**
|
||||
* Duo Security Sync Service
|
||||
* Orchestrates data pull from all Duo accounts (Accounts API + Admin API) into PostgreSQL.
|
||||
*/
|
||||
|
||||
import { postgresClient } from './postgres-client';
|
||||
import {
|
||||
getDuoAccountsClient,
|
||||
getDuoAdminClient,
|
||||
DuoClient,
|
||||
DuoAccount,
|
||||
} from './duo-client';
|
||||
|
||||
export interface DuoSyncEntityResult {
|
||||
entity: string;
|
||||
account: string;
|
||||
success: boolean;
|
||||
recordsUpserted: number;
|
||||
duration: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface DuoSyncResult {
|
||||
syncId: string;
|
||||
status: 'completed' | 'failed';
|
||||
startedAt: Date;
|
||||
completedAt: Date;
|
||||
duration: number;
|
||||
entities: DuoSyncEntityResult[];
|
||||
totalUpserted: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
let currentSyncId: string | null = null;
|
||||
let isSyncing = false;
|
||||
|
||||
export class DuoSyncService {
|
||||
|
||||
isSyncInProgress(): boolean { return isSyncing; }
|
||||
getCurrentSyncId(): string | null { return currentSyncId; }
|
||||
|
||||
// ── 3.4 syncAll — full orchestration ──────────────────────────────────────
|
||||
|
||||
async syncAll(triggeredBy = 'system'): Promise<DuoSyncResult> {
|
||||
if (isSyncing) throw new Error('Duo sync already in progress');
|
||||
isSyncing = true;
|
||||
currentSyncId = `duo_sync_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||
|
||||
const startedAt = new Date();
|
||||
const entities: DuoSyncEntityResult[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
console.log(`[DuoSync] Starting full sync (${currentSyncId}) triggered by ${triggeredBy}`);
|
||||
|
||||
try {
|
||||
const accountsClient = getDuoAccountsClient();
|
||||
const adminClient = getDuoAdminClient();
|
||||
|
||||
// Step 1: sync child account list
|
||||
const accounts = await this.syncAccounts(accountsClient, entities, errors);
|
||||
|
||||
// Step 2: for each child, sync all data sequentially
|
||||
for (const account of accounts) {
|
||||
console.log(`[DuoSync] Syncing child: ${account.name} (${account.account_id})`);
|
||||
await this.syncAccountData(accountsClient, account, entities, errors);
|
||||
await this.syncAuthLogs(accountsClient, account, entities, errors);
|
||||
}
|
||||
|
||||
// Step 3: sync parent account using Admin API creds
|
||||
console.log(`[DuoSync] Syncing parent account`);
|
||||
await this.syncParentAccount(adminClient, entities, errors);
|
||||
|
||||
// Step 4: company matching
|
||||
await this.matchCompanies(entities, errors);
|
||||
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Fatal error:`, err);
|
||||
errors.push(err.message);
|
||||
} finally {
|
||||
isSyncing = false;
|
||||
}
|
||||
|
||||
const completedAt = new Date();
|
||||
const result: DuoSyncResult = {
|
||||
syncId: currentSyncId,
|
||||
status: errors.length > 0 ? 'failed' : 'completed',
|
||||
startedAt,
|
||||
completedAt,
|
||||
duration: completedAt.getTime() - startedAt.getTime(),
|
||||
entities,
|
||||
totalUpserted: entities.reduce((sum, e) => sum + e.recordsUpserted, 0),
|
||||
errors,
|
||||
};
|
||||
|
||||
console.log(`[DuoSync] Completed in ${result.duration}ms — ${result.totalUpserted} total records, ${errors.length} error(s)`);
|
||||
currentSyncId = null;
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── 3.1 syncAccounts ──────────────────────────────────────────────────────
|
||||
|
||||
private async syncAccounts(
|
||||
client: DuoClient,
|
||||
entities: DuoSyncEntityResult[],
|
||||
errors: string[],
|
||||
): Promise<DuoAccount[]> {
|
||||
const start = Date.now();
|
||||
const entityName = 'duo_accounts';
|
||||
try {
|
||||
const accounts = await client.listAccounts();
|
||||
let upserted = 0;
|
||||
|
||||
for (const acct of accounts) {
|
||||
// Get summary for each child
|
||||
let userCount = 0;
|
||||
let integrationCount = 0;
|
||||
try {
|
||||
const summary = await client.getAccountSummary(acct);
|
||||
userCount = summary.user_count ?? 0;
|
||||
integrationCount = summary.integration_count ?? 0;
|
||||
} catch (err: any) {
|
||||
console.warn(`[DuoSync] Could not get summary for ${acct.name}: ${err.message}`);
|
||||
}
|
||||
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_accounts (account_id, name, api_hostname, user_count, integration_count, is_parent, synced_at)
|
||||
VALUES ($1, $2, $3, $4, $5, false, NOW())
|
||||
ON CONFLICT (account_id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
api_hostname = EXCLUDED.api_hostname,
|
||||
user_count = EXCLUDED.user_count,
|
||||
integration_count = EXCLUDED.integration_count,
|
||||
synced_at = NOW()
|
||||
`, [acct.account_id, acct.name, acct.api_hostname, userCount, integrationCount]);
|
||||
upserted++;
|
||||
}
|
||||
|
||||
entities.push({ entity: entityName, account: 'all', success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
console.log(`[DuoSync] Synced ${upserted} child accounts`);
|
||||
return accounts;
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing accounts:`, err);
|
||||
errors.push(`accounts: ${err.message}`);
|
||||
entities.push({ entity: entityName, account: 'all', success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ── 3.2 syncAccountData — users, phones, groups, integrations ─────────────
|
||||
|
||||
private async syncAccountData(
|
||||
client: DuoClient,
|
||||
account: DuoAccount,
|
||||
entities: DuoSyncEntityResult[],
|
||||
errors: string[],
|
||||
): Promise<void> {
|
||||
await this.syncUsers(client, account, entities, errors);
|
||||
await this.syncPhones(client, account, entities, errors);
|
||||
await this.syncGroups(client, account, entities, errors);
|
||||
await this.syncIntegrations(client, account, entities, errors);
|
||||
}
|
||||
|
||||
private async syncUsers(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const users = await client.getUsers(account);
|
||||
let upserted = 0;
|
||||
for (const u of users) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_users (user_id, duo_account_id, username, email, realname, status, is_enrolled, last_login, last_directory_sync, created, notes, phones_count, groups, aliases, enable_auto_prompt, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW())
|
||||
ON CONFLICT (user_id) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id,
|
||||
username = EXCLUDED.username,
|
||||
email = EXCLUDED.email,
|
||||
realname = EXCLUDED.realname,
|
||||
status = EXCLUDED.status,
|
||||
is_enrolled = EXCLUDED.is_enrolled,
|
||||
last_login = EXCLUDED.last_login,
|
||||
last_directory_sync = EXCLUDED.last_directory_sync,
|
||||
notes = EXCLUDED.notes,
|
||||
phones_count = EXCLUDED.phones_count,
|
||||
groups = EXCLUDED.groups,
|
||||
aliases = EXCLUDED.aliases,
|
||||
enable_auto_prompt = EXCLUDED.enable_auto_prompt,
|
||||
synced_at = NOW()
|
||||
`, [
|
||||
u.user_id,
|
||||
account.account_id,
|
||||
u.username,
|
||||
u.email || null,
|
||||
u.realname || null,
|
||||
u.status,
|
||||
u.is_enrolled,
|
||||
u.last_login ? new Date(u.last_login * 1000) : null,
|
||||
u.last_directory_sync ? new Date(u.last_directory_sync * 1000) : null,
|
||||
u.created ? new Date(u.created * 1000) : null,
|
||||
u.notes || null,
|
||||
u.phones?.length ?? 0,
|
||||
JSON.stringify(u.groups?.map((g: any) => ({ group_id: g.group_id, name: g.name })) ?? []),
|
||||
JSON.stringify(u.aliases ?? {}),
|
||||
u.enable_auto_prompt ?? true,
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
entities.push({ entity: 'duo_users', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing users for ${account.name}:`, err.message);
|
||||
errors.push(`users/${account.name}: ${err.message}`);
|
||||
entities.push({ entity: 'duo_users', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
private async syncPhones(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const phones = await client.getPhones(account);
|
||||
let upserted = 0;
|
||||
for (const p of phones) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_phones (phone_id, duo_account_id, name, number, type, platform, model, os_version, app_version, activated, last_seen, capabilities, users, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NOW())
|
||||
ON CONFLICT (phone_id) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id,
|
||||
name = EXCLUDED.name,
|
||||
number = EXCLUDED.number,
|
||||
type = EXCLUDED.type,
|
||||
platform = EXCLUDED.platform,
|
||||
model = EXCLUDED.model,
|
||||
os_version = EXCLUDED.os_version,
|
||||
app_version = EXCLUDED.app_version,
|
||||
activated = EXCLUDED.activated,
|
||||
last_seen = EXCLUDED.last_seen,
|
||||
capabilities = EXCLUDED.capabilities,
|
||||
users = EXCLUDED.users,
|
||||
synced_at = NOW()
|
||||
`, [
|
||||
p.phone_id,
|
||||
account.account_id,
|
||||
p.name || null,
|
||||
p.number || null,
|
||||
p.type || null,
|
||||
p.platform || null,
|
||||
p.model || null,
|
||||
p.os_version || null,
|
||||
p.app_version || null,
|
||||
p.activated ?? false,
|
||||
p.last_seen ? new Date(p.last_seen) : null,
|
||||
JSON.stringify(p.capabilities ?? []),
|
||||
JSON.stringify(p.users?.map((u: any) => ({ user_id: u.user_id, username: u.username })) ?? []),
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
entities.push({ entity: 'duo_phones', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing phones for ${account.name}:`, err.message);
|
||||
errors.push(`phones/${account.name}: ${err.message}`);
|
||||
entities.push({ entity: 'duo_phones', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
private async syncGroups(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const groups = await client.getGroups(account);
|
||||
let upserted = 0;
|
||||
for (const g of groups) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_groups (group_id, duo_account_id, name, description, status, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW())
|
||||
ON CONFLICT (group_id) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id,
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
status = EXCLUDED.status,
|
||||
synced_at = NOW()
|
||||
`, [
|
||||
g.group_id,
|
||||
account.account_id,
|
||||
g.name || null,
|
||||
g.desc || null,
|
||||
g.status || null,
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
entities.push({ entity: 'duo_groups', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing groups for ${account.name}:`, err.message);
|
||||
errors.push(`groups/${account.name}: ${err.message}`);
|
||||
entities.push({ entity: 'duo_groups', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
private async syncIntegrations(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const integrations = await client.getIntegrations(account);
|
||||
let upserted = 0;
|
||||
for (const i of integrations) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_integrations (integration_key, duo_account_id, name, type, notes, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW())
|
||||
ON CONFLICT (integration_key) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id,
|
||||
name = EXCLUDED.name,
|
||||
type = EXCLUDED.type,
|
||||
notes = EXCLUDED.notes,
|
||||
synced_at = NOW()
|
||||
`, [
|
||||
i.integration_key,
|
||||
account.account_id,
|
||||
i.name || null,
|
||||
i.type || null,
|
||||
i.notes || null,
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
entities.push({ entity: 'duo_integrations', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing integrations for ${account.name}:`, err.message);
|
||||
errors.push(`integrations/${account.name}: ${err.message}`);
|
||||
entities.push({ entity: 'duo_integrations', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ── 3.3 syncAuthLogs — incremental ────────────────────────────────────────
|
||||
|
||||
private async syncAuthLogs(
|
||||
client: DuoClient,
|
||||
account: DuoAccount,
|
||||
entities: DuoSyncEntityResult[],
|
||||
errors: string[],
|
||||
since?: number,
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
// Default: last synced timestamp or 30 days ago
|
||||
if (!since) {
|
||||
const lastRow = await postgresClient.query(
|
||||
`SELECT MAX(timestamp) as last_ts FROM duo_auth_logs WHERE duo_account_id = $1`,
|
||||
[account.account_id],
|
||||
);
|
||||
const lastTs = lastRow.rows[0]?.last_ts;
|
||||
since = lastTs ? new Date(lastTs).getTime() : Date.now() - 30 * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
const logs = await client.getAuthLogs(since, account);
|
||||
let upserted = 0;
|
||||
for (const l of logs) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_auth_logs (txid, duo_account_id, timestamp, user_name, user_id, factor, result, reason, application_name, application_key, access_device_ip, access_device_location, auth_device_ip, auth_device_name, event_type, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW())
|
||||
ON CONFLICT (txid) DO NOTHING
|
||||
`, [
|
||||
l.txid,
|
||||
account.account_id,
|
||||
new Date(l.timestamp * 1000),
|
||||
l.user?.name || null,
|
||||
l.user?.key || null,
|
||||
l.factor || null,
|
||||
l.result || null,
|
||||
l.reason || null,
|
||||
l.application?.name || null,
|
||||
l.application?.key || null,
|
||||
l.access_device?.ip || null,
|
||||
l.access_device?.location ? JSON.stringify(l.access_device.location) : null,
|
||||
l.auth_device?.ip || null,
|
||||
l.auth_device?.name || null,
|
||||
l.event_type || null,
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
entities.push({ entity: 'duo_auth_logs', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing auth logs for ${account.name}:`, err.message);
|
||||
errors.push(`auth_logs/${account.name}: ${err.message}`);
|
||||
entities.push({ entity: 'duo_auth_logs', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ── Parent account sync using DUOADMIN creds ───────────────────────────────
|
||||
|
||||
private async syncParentAccount(
|
||||
adminClient: DuoClient,
|
||||
entities: DuoSyncEntityResult[],
|
||||
errors: string[],
|
||||
): Promise<void> {
|
||||
const parentAccountId = 'PARENT_WULF';
|
||||
const parentHost = process.env.DUOADMIN_API_HOSTNAME!;
|
||||
|
||||
// Upsert parent account row
|
||||
try {
|
||||
const summary = await adminClient.getAccountSummary();
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_accounts (account_id, name, api_hostname, user_count, integration_count, is_parent, synced_at)
|
||||
VALUES ($1, 'Wulf Consulting (Parent)', $2, $3, $4, true, NOW())
|
||||
ON CONFLICT (account_id) DO UPDATE SET
|
||||
user_count = EXCLUDED.user_count,
|
||||
integration_count = EXCLUDED.integration_count,
|
||||
synced_at = NOW()
|
||||
`, [parentAccountId, parentHost, summary.user_count, summary.integration_count]);
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing parent summary:`, err.message);
|
||||
errors.push(`parent/summary: ${err.message}`);
|
||||
}
|
||||
|
||||
// Create a pseudo DuoAccount for the parent so we can reuse syncAccountData/syncAuthLogs
|
||||
const parentAccount: DuoAccount = { account_id: parentAccountId, api_hostname: parentHost, name: 'Wulf Consulting (Parent)' };
|
||||
|
||||
// Sync users, phones, groups, integrations using adminClient (no child override needed)
|
||||
await this.syncParentEntity('users', async () => {
|
||||
const users = await adminClient.getUsers();
|
||||
let upserted = 0;
|
||||
for (const u of users) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_users (user_id, duo_account_id, username, email, realname, status, is_enrolled, last_login, last_directory_sync, created, notes, phones_count, groups, aliases, enable_auto_prompt, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW())
|
||||
ON CONFLICT (user_id) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id, username = EXCLUDED.username, email = EXCLUDED.email,
|
||||
realname = EXCLUDED.realname, status = EXCLUDED.status, is_enrolled = EXCLUDED.is_enrolled,
|
||||
last_login = EXCLUDED.last_login, last_directory_sync = EXCLUDED.last_directory_sync,
|
||||
notes = EXCLUDED.notes, phones_count = EXCLUDED.phones_count, groups = EXCLUDED.groups,
|
||||
aliases = EXCLUDED.aliases, enable_auto_prompt = EXCLUDED.enable_auto_prompt, synced_at = NOW()
|
||||
`, [
|
||||
u.user_id, parentAccountId, u.username, u.email || null, u.realname || null,
|
||||
u.status, u.is_enrolled,
|
||||
u.last_login ? new Date(u.last_login * 1000) : null,
|
||||
u.last_directory_sync ? new Date(u.last_directory_sync * 1000) : null,
|
||||
u.created ? new Date(u.created * 1000) : null,
|
||||
u.notes || null, u.phones?.length ?? 0,
|
||||
JSON.stringify(u.groups?.map((g: any) => ({ group_id: g.group_id, name: g.name })) ?? []),
|
||||
JSON.stringify(u.aliases ?? {}), u.enable_auto_prompt ?? true,
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
return upserted;
|
||||
}, parentAccount.name, entities, errors);
|
||||
|
||||
await this.syncParentEntity('phones', async () => {
|
||||
const phones = await adminClient.getPhones();
|
||||
let upserted = 0;
|
||||
for (const p of phones) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_phones (phone_id, duo_account_id, name, number, type, platform, model, os_version, app_version, activated, last_seen, capabilities, users, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NOW())
|
||||
ON CONFLICT (phone_id) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id, name = EXCLUDED.name, number = EXCLUDED.number,
|
||||
type = EXCLUDED.type, platform = EXCLUDED.platform, model = EXCLUDED.model,
|
||||
os_version = EXCLUDED.os_version, app_version = EXCLUDED.app_version,
|
||||
activated = EXCLUDED.activated, last_seen = EXCLUDED.last_seen,
|
||||
capabilities = EXCLUDED.capabilities, users = EXCLUDED.users, synced_at = NOW()
|
||||
`, [
|
||||
p.phone_id, parentAccountId, p.name || null, p.number || null, p.type || null,
|
||||
p.platform || null, p.model || null, p.os_version || null, p.app_version || null,
|
||||
p.activated ?? false, p.last_seen ? new Date(p.last_seen) : null,
|
||||
JSON.stringify(p.capabilities ?? []),
|
||||
JSON.stringify(p.users?.map((u: any) => ({ user_id: u.user_id, username: u.username })) ?? []),
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
return upserted;
|
||||
}, parentAccount.name, entities, errors);
|
||||
|
||||
await this.syncParentEntity('groups', async () => {
|
||||
const groups = await adminClient.getGroups();
|
||||
let upserted = 0;
|
||||
for (const g of groups) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_groups (group_id, duo_account_id, name, description, status, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW())
|
||||
ON CONFLICT (group_id) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id, name = EXCLUDED.name,
|
||||
description = EXCLUDED.description, status = EXCLUDED.status, synced_at = NOW()
|
||||
`, [g.group_id, parentAccountId, g.name || null, g.desc || null, g.status || null]);
|
||||
upserted++;
|
||||
}
|
||||
return upserted;
|
||||
}, parentAccount.name, entities, errors);
|
||||
|
||||
await this.syncParentEntity('integrations', async () => {
|
||||
const integrations = await adminClient.getIntegrations();
|
||||
let upserted = 0;
|
||||
for (const i of integrations) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_integrations (integration_key, duo_account_id, name, type, notes, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,NOW())
|
||||
ON CONFLICT (integration_key) DO UPDATE SET
|
||||
duo_account_id = EXCLUDED.duo_account_id, name = EXCLUDED.name,
|
||||
type = EXCLUDED.type, notes = EXCLUDED.notes, synced_at = NOW()
|
||||
`, [i.integration_key, parentAccountId, i.name || null, i.type || null, i.notes || null]);
|
||||
upserted++;
|
||||
}
|
||||
return upserted;
|
||||
}, parentAccount.name, entities, errors);
|
||||
|
||||
// Auth logs for parent
|
||||
const start = Date.now();
|
||||
try {
|
||||
const lastRow = await postgresClient.query(
|
||||
`SELECT MAX(timestamp) as last_ts FROM duo_auth_logs WHERE duo_account_id = $1`,
|
||||
[parentAccountId],
|
||||
);
|
||||
const lastTs = lastRow.rows[0]?.last_ts;
|
||||
const since = lastTs ? new Date(lastTs).getTime() : Date.now() - 30 * 24 * 60 * 60 * 1000;
|
||||
const logs = await adminClient.getAuthLogs(since);
|
||||
let upserted = 0;
|
||||
for (const l of logs) {
|
||||
await postgresClient.query(`
|
||||
INSERT INTO duo_auth_logs (txid, duo_account_id, timestamp, user_name, user_id, factor, result, reason, application_name, application_key, access_device_ip, access_device_location, auth_device_ip, auth_device_name, event_type, synced_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW())
|
||||
ON CONFLICT (txid) DO NOTHING
|
||||
`, [
|
||||
l.txid, parentAccountId, new Date(l.timestamp * 1000),
|
||||
l.user?.name || null, l.user?.key || null, l.factor || null,
|
||||
l.result || null, l.reason || null,
|
||||
l.application?.name || null, l.application?.key || null,
|
||||
l.access_device?.ip || null,
|
||||
l.access_device?.location ? JSON.stringify(l.access_device.location) : null,
|
||||
l.auth_device?.ip || null, l.auth_device?.name || null, l.event_type || null,
|
||||
]);
|
||||
upserted++;
|
||||
}
|
||||
entities.push({ entity: 'duo_auth_logs', account: parentAccount.name, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing parent auth logs:`, err.message);
|
||||
errors.push(`auth_logs/parent: ${err.message}`);
|
||||
entities.push({ entity: 'duo_auth_logs', account: parentAccount.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
private async syncParentEntity(
|
||||
entityName: string,
|
||||
fn: () => Promise<number>,
|
||||
accountName: string,
|
||||
entities: DuoSyncEntityResult[],
|
||||
errors: string[],
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const upserted = await fn();
|
||||
entities.push({ entity: `duo_${entityName}`, account: accountName, success: true, recordsUpserted: upserted, duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error syncing parent ${entityName}:`, err.message);
|
||||
errors.push(`${entityName}/parent: ${err.message}`);
|
||||
entities.push({ entity: `duo_${entityName}`, account: accountName, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ── 3.5 Company matching ──────────────────────────────────────────────────
|
||||
|
||||
private async matchCompanies(
|
||||
entities: DuoSyncEntityResult[],
|
||||
errors: string[],
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
// Exact match first
|
||||
await postgresClient.query(`
|
||||
UPDATE duo_accounts da
|
||||
SET autotask_company_id = c.id
|
||||
FROM companies c
|
||||
WHERE da.autotask_company_id IS NULL
|
||||
AND da.is_parent = false
|
||||
AND LOWER(TRIM(da.name)) = LOWER(TRIM(c.company_name))
|
||||
`);
|
||||
|
||||
// Case-insensitive containment (company name contains duo account name or vice versa)
|
||||
await postgresClient.query(`
|
||||
UPDATE duo_accounts da
|
||||
SET autotask_company_id = sub.company_id
|
||||
FROM (
|
||||
SELECT DISTINCT ON (da2.account_id) da2.account_id, c.id as company_id
|
||||
FROM duo_accounts da2
|
||||
JOIN companies c ON (
|
||||
LOWER(c.company_name) LIKE '%' || LOWER(TRIM(da2.name)) || '%'
|
||||
OR LOWER(TRIM(da2.name)) LIKE '%' || LOWER(TRIM(c.company_name)) || '%'
|
||||
)
|
||||
WHERE da2.autotask_company_id IS NULL AND da2.is_parent = false
|
||||
ORDER BY da2.account_id, LENGTH(c.company_name) ASC
|
||||
) sub
|
||||
WHERE da.account_id = sub.account_id
|
||||
`);
|
||||
|
||||
const matched = await postgresClient.query(`
|
||||
SELECT COUNT(*) as matched FROM duo_accounts WHERE autotask_company_id IS NOT NULL AND is_parent = false
|
||||
`);
|
||||
const total = await postgresClient.query(`
|
||||
SELECT COUNT(*) as total FROM duo_accounts WHERE is_parent = false
|
||||
`);
|
||||
|
||||
console.log(`[DuoSync] Company matching: ${matched.rows[0].matched}/${total.rows[0].total} accounts matched`);
|
||||
entities.push({ entity: 'company_matching', account: 'all', success: true, recordsUpserted: parseInt(matched.rows[0].matched), duration: Date.now() - start });
|
||||
} catch (err: any) {
|
||||
console.error(`[DuoSync] Error matching companies:`, err.message);
|
||||
errors.push(`company_matching: ${err.message}`);
|
||||
entities.push({ entity: 'company_matching', account: 'all', success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const duoSyncService = new DuoSyncService();
|
||||
Loading…
Add table
Add a link
Reference in a new issue