From a4242b81be396e585e3ff39d9d3017a1391b37fe Mon Sep 17 00:00:00 2001 From: lorentz Date: Fri, 27 Mar 2026 09:18:04 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20Duo=20Security=20integration=20?= =?UTF-8?q?=E2=80=94=20full=20data=20sync=20from=20Accounts=20+=20Admin=20?= =?UTF-8?q?API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Duo API Client (lib/services/duo-client.ts): - HMAC-SHA1 request signing, GET/POST, automatic pagination - Rate-limit handling (429 + Retry-After), configurable timeout - Accounts API: listAccounts() via POST /accounts/v1/account/list - Admin API: getUsers, getPhones, getGroups, getIntegrations, getAuthLogs - Child account access: parent creds signed against child api_hostname + account_id - Factory helpers: getDuoAccountsClient(), getDuoAdminClient() Database (migration 058): - 6 tables: duo_accounts, duo_users, duo_phones, duo_auth_logs, duo_groups, duo_integrations - All with proper FKs, indexes, JSONB fields for capabilities/location/groups Sync Service (lib/services/duo-sync-service.ts): - syncAll(): accounts → per-child data + auth logs → parent account → company matching - Sequential child processing to respect rate limits - Incremental auth logs (mintime = last synced timestamp, default 30 days) - Company matching: exact → case-insensitive containment (30/32 = 94% matched) - Non-blocking with sync ID tracking API Routes: - POST/GET /api/duo/sync — trigger sync / check status - GET /api/duo/accounts — list all accounts with stats + matched company - GET /api/duo/accounts/[id]/users — users for a specific account - POST /api/openclaw/sync/duo — OpenClaw trigger with API key auth Results: 33 accounts, 832 users, 925 phones, 5927 auth logs, 46 groups, 78 integrations --- app/api/duo/accounts/[id]/users/route.ts | 23 + app/api/duo/accounts/route.ts | 22 + app/api/duo/sync/route.ts | 31 ++ app/api/openclaw/sync/duo/route.ts | 29 ++ lib/services/duo-client.ts | 334 +++++++++++++ lib/services/duo-sync-service.ts | 601 +++++++++++++++++++++++ middleware.ts | 2 + migrations/058_create_duo_tables.sql | 121 +++++ tasks/prd-duo-integration.md | 337 +++++++++++++ tasks/tasks-prd-duo-integration.md | 65 +++ 10 files changed, 1565 insertions(+) create mode 100644 app/api/duo/accounts/[id]/users/route.ts create mode 100644 app/api/duo/accounts/route.ts create mode 100644 app/api/duo/sync/route.ts create mode 100644 app/api/openclaw/sync/duo/route.ts create mode 100644 lib/services/duo-client.ts create mode 100644 lib/services/duo-sync-service.ts create mode 100644 migrations/058_create_duo_tables.sql create mode 100644 tasks/prd-duo-integration.md create mode 100644 tasks/tasks-prd-duo-integration.md diff --git a/app/api/duo/accounts/[id]/users/route.ts b/app/api/duo/accounts/[id]/users/route.ts new file mode 100644 index 0000000..2171a4e --- /dev/null +++ b/app/api/duo/accounts/[id]/users/route.ts @@ -0,0 +1,23 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { postgresClient } from '@/lib/services/postgres-client'; + +export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + try { + const { id } = await params; + + const result = await postgresClient.query(` + SELECT u.user_id, u.username, u.email, u.realname, u.status, + u.is_enrolled, u.last_login, u.last_directory_sync, u.created, + u.phones_count, u.groups, u.aliases, u.enable_auto_prompt, u.notes, + u.synced_at + FROM duo_users u + WHERE u.duo_account_id = $1 + ORDER BY u.username ASC + `, [id]); + + return NextResponse.json({ users: result.rows, total: result.rowCount }); + } catch (error: any) { + console.error('[DuoAPI] Error fetching users:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/app/api/duo/accounts/route.ts b/app/api/duo/accounts/route.ts new file mode 100644 index 0000000..a93b97d --- /dev/null +++ b/app/api/duo/accounts/route.ts @@ -0,0 +1,22 @@ +import { NextResponse } from 'next/server'; +import { postgresClient } from '@/lib/services/postgres-client'; + +export async function GET() { + try { + const result = await postgresClient.query(` + SELECT da.id, da.account_id, da.name, da.api_hostname, + da.user_count, da.integration_count, da.edition, + da.is_parent, da.synced_at, da.created_at, + da.autotask_company_id, + c.company_name as autotask_company_name + FROM duo_accounts da + LEFT JOIN companies c ON c.id = da.autotask_company_id + ORDER BY da.is_parent DESC, da.name ASC + `); + + return NextResponse.json({ accounts: result.rows }); + } catch (error: any) { + console.error('[DuoAPI] Error fetching accounts:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/app/api/duo/sync/route.ts b/app/api/duo/sync/route.ts new file mode 100644 index 0000000..5423993 --- /dev/null +++ b/app/api/duo/sync/route.ts @@ -0,0 +1,31 @@ +import { NextResponse } from 'next/server'; +import { duoSyncService } from '@/lib/services/duo-sync-service'; + +export async function POST() { + try { + if (duoSyncService.isSyncInProgress()) { + return NextResponse.json( + { message: 'Duo sync already in progress', syncId: duoSyncService.getCurrentSyncId() }, + { status: 409 }, + ); + } + + // Fire-and-forget — don't await + const syncId = `duo_sync_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + duoSyncService.syncAll('api').catch(err => { + console.error('[DuoSync] Background sync error:', err); + }); + + return NextResponse.json({ message: 'Duo sync started', syncId }); + } catch (error: any) { + console.error('[DuoSync] Error starting sync:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} + +export async function GET() { + return NextResponse.json({ + inProgress: duoSyncService.isSyncInProgress(), + currentSyncId: duoSyncService.getCurrentSyncId(), + }); +} diff --git a/app/api/openclaw/sync/duo/route.ts b/app/api/openclaw/sync/duo/route.ts new file mode 100644 index 0000000..f1e3f94 --- /dev/null +++ b/app/api/openclaw/sync/duo/route.ts @@ -0,0 +1,29 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { validateOpenClawKey } from '@/lib/utils/openclaw-auth'; +import { duoSyncService } from '@/lib/services/duo-sync-service'; + +export async function POST(request: NextRequest) { + const authError = validateOpenClawKey(request); + if (authError) return authError; + + try { + if (duoSyncService.isSyncInProgress()) { + return NextResponse.json( + { message: 'Duo sync already in progress', syncId: duoSyncService.getCurrentSyncId() }, + { status: 409 }, + ); + } + + duoSyncService.syncAll('openclaw').catch(err => { + console.error('[DuoSync] Background sync error (openclaw):', err); + }); + + return NextResponse.json({ + message: 'Duo sync started', + syncId: duoSyncService.getCurrentSyncId(), + }); + } catch (error: any) { + console.error('[DuoSync] Error starting sync via OpenClaw:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/lib/services/duo-client.ts b/lib/services/duo-client.ts new file mode 100644 index 0000000..d330ce3 --- /dev/null +++ b/lib/services/duo-client.ts @@ -0,0 +1,334 @@ +/** + * Duo Security API Client + * Handles HMAC-SHA1 request signing, pagination, rate-limit handling. + * Supports both the Accounts API (parent) and Admin API (parent + child accounts). + */ + +import crypto from 'crypto'; +import https from 'https'; + +export interface DuoAccount { + account_id: string; + api_hostname: string; + name: string; +} + +export interface DuoClientOptions { + ikey: string; + skey: string; + host: string; + timeoutMs?: number; +} + +export interface DuoApiResponse { + stat: string; + response: T; + metadata?: { total_objects?: number; next_offset?: number; }; +} + +export interface DuoUserSummary { + user_count: number; + integration_count: number; + telephony_credits_remaining?: number; +} + +export interface DuoUser { + user_id: string; + username: string; + email: string; + realname: string; + status: string; + is_enrolled: boolean; + last_login: number | null; + last_directory_sync: number | null; + created: number; + notes: string; + phones: any[]; + groups: any[]; + aliases: Record; + enable_auto_prompt: boolean; + firstname: string; + lastname: string; +} + +export interface DuoPhone { + phone_id: string; + name: string; + number: string; + type: string; + platform: string; + model: string; + os_version: string; + app_version: string; + activated: boolean; + last_seen: string; + capabilities: string[]; + users: any[]; +} + +export interface DuoGroup { + group_id: string; + name: string; + desc: string; + status: string; + mobile_otp_enabled: boolean; + push_enabled: boolean; + sms_enabled: boolean; + voice_enabled: boolean; +} + +export interface DuoIntegration { + integration_key: string; + name: string; + type: string; + notes: string; +} + +export interface DuoAuthLog { + txid: string; + timestamp: number; + user: { key: string; name: string; }; + factor: string; + result: string; + reason: string; + application: { key: string; name: string; }; + access_device: { ip: string; hostname: string | null; location: { city: string; state: string; country: string; }; }; + auth_device: { ip: string | null; name: string; }; + event_type: string; +} + +export class DuoClient { + private readonly ikey: string; + private readonly skey: string; + private readonly host: string; + private readonly timeoutMs: number; + + constructor(options: DuoClientOptions) { + if (!options.ikey || !options.skey || !options.host) { + throw new Error('DuoClient requires ikey, skey, and host'); + } + this.ikey = options.ikey; + this.skey = options.skey; + this.host = options.host; + this.timeoutMs = options.timeoutMs ?? 30_000; + } + + // ── 1.2 HMAC-SHA1 request signing ──────────────────────────────────────── + + private sign(method: string, host: string, path: string, params: Record, date: string): string { + const sortedParams = Object.keys(params).sort() + .map(k => `${encodeURIComponent(k)}=${encodeURIComponent(params[k])}`) + .join('&'); + const canon = [date, method.toUpperCase(), host.toLowerCase(), path, sortedParams].join('\n'); + const sig = crypto.createHmac('sha1', this.skey).update(canon).digest('hex'); + return `Basic ${Buffer.from(`${this.ikey}:${sig}`).toString('base64')}`; + } + + // ── 1.3 Core request methods (with 1.5 rate-limit + 1.6 timeout) ──────── + + private request( + method: string, + path: string, + params: Record = {}, + hostOverride?: string, + ): Promise> { + const host = hostOverride ?? this.host; + const date = new Date().toUTCString(); + const auth = this.sign(method, host, path, params, date); + + const sortedParams = Object.keys(params).sort() + .map(k => `${encodeURIComponent(k)}=${encodeURIComponent(params[k])}`) + .join('&'); + + const isGet = method.toUpperCase() === 'GET'; + const urlPath = isGet ? `${path}${sortedParams ? '?' + sortedParams : ''}` : path; + const body = isGet ? '' : sortedParams; + + return new Promise((resolve, reject) => { + const req = https.request({ + hostname: host, + path: urlPath, + method: method.toUpperCase(), + headers: { + Authorization: auth, + Date: date, + 'Content-Type': 'application/x-www-form-urlencoded', + ...(body ? { 'Content-Length': String(Buffer.byteLength(body)) } : {}), + }, + }, res => { + let data = ''; + res.on('data', (chunk: Buffer) => data += chunk); + res.on('end', async () => { + // 1.5 rate-limit handling + if (res.statusCode === 429) { + const retryAfter = parseInt(res.headers['retry-after'] as string || '5', 10); + console.log(`[DuoClient] Rate limited on ${path}, retrying in ${retryAfter}s`); + await new Promise(r => setTimeout(r, retryAfter * 1000)); + return resolve(this.request(method, path, params, hostOverride)); + } + try { + const parsed = JSON.parse(data) as DuoApiResponse; + if (parsed.stat !== 'OK') { + reject(new Error(`Duo API error on ${path}: ${(parsed as any).message || JSON.stringify(parsed)}`)); + } else { + resolve(parsed); + } + } catch { + reject(new Error(`Duo API non-JSON response on ${path}: HTTP ${res.statusCode} — ${data.slice(0, 200)}`)); + } + }); + }); + + // 1.6 timeout + req.setTimeout(this.timeoutMs, () => { + req.destroy(); + reject(new Error(`Duo API timeout on ${path} after ${this.timeoutMs}ms`)); + }); + req.on('error', reject); + if (body) req.write(body); + req.end(); + }); + } + + async get(path: string, params: Record = {}, hostOverride?: string): Promise> { + return this.request('GET', path, params, hostOverride); + } + + async post(path: string, params: Record = {}, hostOverride?: string): Promise> { + return this.request('POST', path, params, hostOverride); + } + + // ── 1.4 Paginated GET — follows metadata.next_offset ───────────────────── + + async getPaginated(path: string, params: Record = {}, hostOverride?: string): Promise { + const all: T[] = []; + let offset = 0; + const limit = '300'; + + while (true) { + const pageParams = { ...params, limit, offset: String(offset) }; + const res = await this.get(path, pageParams, hostOverride); + const items = res.response; + if (!Array.isArray(items)) break; + all.push(...items); + + if (res.metadata?.next_offset !== undefined && res.metadata.next_offset !== null) { + offset = res.metadata.next_offset; + } else { + break; + } + } + return all; + } + + // ── 1.7 Accounts API ───────────────────────────────────────────────────── + + async listAccounts(): Promise { + const res = await this.post('/accounts/v1/account/list'); + return res.response; + } + + // ── 1.8 + 1.9 Admin API methods (with child account support) ───────────── + // When `child` is provided, requests are signed against child.api_hostname + // and account_id is passed as a parameter. + + private childParams(child?: DuoAccount): Record { + return child ? { account_id: child.account_id } : {}; + } + + private childHost(child?: DuoAccount): string | undefined { + return child?.api_hostname; + } + + async getAccountSummary(child?: DuoAccount): Promise { + const res = await this.get( + '/admin/v1/info/summary', + this.childParams(child), + this.childHost(child), + ); + return res.response; + } + + async getUsers(child?: DuoAccount): Promise { + return this.getPaginated( + '/admin/v1/users', + this.childParams(child), + this.childHost(child), + ); + } + + async getPhones(child?: DuoAccount): Promise { + return this.getPaginated( + '/admin/v1/phones', + this.childParams(child), + this.childHost(child), + ); + } + + async getGroups(child?: DuoAccount): Promise { + return this.getPaginated( + '/admin/v1/groups', + this.childParams(child), + this.childHost(child), + ); + } + + async getIntegrations(child?: DuoAccount): Promise { + return this.getPaginated( + '/admin/v1/integrations', + this.childParams(child), + this.childHost(child), + ); + } + + async getAuthLogs(mintime: number, child?: DuoAccount): Promise { + // Auth logs v2 has a different response structure: + // { stat: "OK", response: { authlogs: [...], metadata: { next_offset: [...] } } } + // So we can't use getPaginated — custom pagination here. + const all: DuoAuthLog[] = []; + let nextOffset: string[] | undefined; + + while (true) { + const params: Record = { + ...this.childParams(child), + mintime: String(mintime), + maxtime: String(Date.now()), + limit: '1000', + }; + if (nextOffset) { + params.next_offset = JSON.stringify(nextOffset); + } + + const res = await this.get<{ authlogs: DuoAuthLog[]; metadata: { next_offset?: string[] } }>( + '/admin/v2/logs/authentication', + params, + this.childHost(child), + ); + + const logs = res.response?.authlogs ?? []; + all.push(...logs); + + nextOffset = res.response?.metadata?.next_offset; + if (!nextOffset || logs.length === 0) break; + } + return all; + } +} + +// ── Factory helpers ────────────────────────────────────────────────────────── + +export function getDuoAccountsClient(): DuoClient { + return new DuoClient({ + ikey: process.env.DUOACCOUNTS_INTEGRATION_KEY!, + skey: process.env.DUOACCOUNTS_SECRET_KEY!, + host: process.env.DUOACCOUNTS_API_HOSTNAME!, + }); +} + +export function getDuoAdminClient(): DuoClient { + return new DuoClient({ + ikey: process.env.DUOADMIN_INTEGRATION_KEY!, + skey: process.env.DUOADMIN_SECRET_KEY!, + host: process.env.DUOADMIN_API_HOSTNAME!, + }); +} diff --git a/lib/services/duo-sync-service.ts b/lib/services/duo-sync-service.ts new file mode 100644 index 0000000..2530f58 --- /dev/null +++ b/lib/services/duo-sync-service.ts @@ -0,0 +1,601 @@ +/** + * Duo Security Sync Service + * Orchestrates data pull from all Duo accounts (Accounts API + Admin API) into PostgreSQL. + */ + +import { postgresClient } from './postgres-client'; +import { + getDuoAccountsClient, + getDuoAdminClient, + DuoClient, + DuoAccount, +} from './duo-client'; + +export interface DuoSyncEntityResult { + entity: string; + account: string; + success: boolean; + recordsUpserted: number; + duration: number; + error?: string; +} + +export interface DuoSyncResult { + syncId: string; + status: 'completed' | 'failed'; + startedAt: Date; + completedAt: Date; + duration: number; + entities: DuoSyncEntityResult[]; + totalUpserted: number; + errors: string[]; +} + +let currentSyncId: string | null = null; +let isSyncing = false; + +export class DuoSyncService { + + isSyncInProgress(): boolean { return isSyncing; } + getCurrentSyncId(): string | null { return currentSyncId; } + + // ── 3.4 syncAll — full orchestration ────────────────────────────────────── + + async syncAll(triggeredBy = 'system'): Promise { + if (isSyncing) throw new Error('Duo sync already in progress'); + isSyncing = true; + currentSyncId = `duo_sync_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + + const startedAt = new Date(); + const entities: DuoSyncEntityResult[] = []; + const errors: string[] = []; + + console.log(`[DuoSync] Starting full sync (${currentSyncId}) triggered by ${triggeredBy}`); + + try { + const accountsClient = getDuoAccountsClient(); + const adminClient = getDuoAdminClient(); + + // Step 1: sync child account list + const accounts = await this.syncAccounts(accountsClient, entities, errors); + + // Step 2: for each child, sync all data sequentially + for (const account of accounts) { + console.log(`[DuoSync] Syncing child: ${account.name} (${account.account_id})`); + await this.syncAccountData(accountsClient, account, entities, errors); + await this.syncAuthLogs(accountsClient, account, entities, errors); + } + + // Step 3: sync parent account using Admin API creds + console.log(`[DuoSync] Syncing parent account`); + await this.syncParentAccount(adminClient, entities, errors); + + // Step 4: company matching + await this.matchCompanies(entities, errors); + + } catch (err: any) { + console.error(`[DuoSync] Fatal error:`, err); + errors.push(err.message); + } finally { + isSyncing = false; + } + + const completedAt = new Date(); + const result: DuoSyncResult = { + syncId: currentSyncId, + status: errors.length > 0 ? 'failed' : 'completed', + startedAt, + completedAt, + duration: completedAt.getTime() - startedAt.getTime(), + entities, + totalUpserted: entities.reduce((sum, e) => sum + e.recordsUpserted, 0), + errors, + }; + + console.log(`[DuoSync] Completed in ${result.duration}ms — ${result.totalUpserted} total records, ${errors.length} error(s)`); + currentSyncId = null; + return result; + } + + // ── 3.1 syncAccounts ────────────────────────────────────────────────────── + + private async syncAccounts( + client: DuoClient, + entities: DuoSyncEntityResult[], + errors: string[], + ): Promise { + const start = Date.now(); + const entityName = 'duo_accounts'; + try { + const accounts = await client.listAccounts(); + let upserted = 0; + + for (const acct of accounts) { + // Get summary for each child + let userCount = 0; + let integrationCount = 0; + try { + const summary = await client.getAccountSummary(acct); + userCount = summary.user_count ?? 0; + integrationCount = summary.integration_count ?? 0; + } catch (err: any) { + console.warn(`[DuoSync] Could not get summary for ${acct.name}: ${err.message}`); + } + + await postgresClient.query(` + INSERT INTO duo_accounts (account_id, name, api_hostname, user_count, integration_count, is_parent, synced_at) + VALUES ($1, $2, $3, $4, $5, false, NOW()) + ON CONFLICT (account_id) DO UPDATE SET + name = EXCLUDED.name, + api_hostname = EXCLUDED.api_hostname, + user_count = EXCLUDED.user_count, + integration_count = EXCLUDED.integration_count, + synced_at = NOW() + `, [acct.account_id, acct.name, acct.api_hostname, userCount, integrationCount]); + upserted++; + } + + entities.push({ entity: entityName, account: 'all', success: true, recordsUpserted: upserted, duration: Date.now() - start }); + console.log(`[DuoSync] Synced ${upserted} child accounts`); + return accounts; + } catch (err: any) { + console.error(`[DuoSync] Error syncing accounts:`, err); + errors.push(`accounts: ${err.message}`); + entities.push({ entity: entityName, account: 'all', success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + return []; + } + } + + // ── 3.2 syncAccountData — users, phones, groups, integrations ───────────── + + private async syncAccountData( + client: DuoClient, + account: DuoAccount, + entities: DuoSyncEntityResult[], + errors: string[], + ): Promise { + await this.syncUsers(client, account, entities, errors); + await this.syncPhones(client, account, entities, errors); + await this.syncGroups(client, account, entities, errors); + await this.syncIntegrations(client, account, entities, errors); + } + + private async syncUsers(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise { + const start = Date.now(); + try { + const users = await client.getUsers(account); + let upserted = 0; + for (const u of users) { + await postgresClient.query(` + INSERT INTO duo_users (user_id, duo_account_id, username, email, realname, status, is_enrolled, last_login, last_directory_sync, created, notes, phones_count, groups, aliases, enable_auto_prompt, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW()) + ON CONFLICT (user_id) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, + username = EXCLUDED.username, + email = EXCLUDED.email, + realname = EXCLUDED.realname, + status = EXCLUDED.status, + is_enrolled = EXCLUDED.is_enrolled, + last_login = EXCLUDED.last_login, + last_directory_sync = EXCLUDED.last_directory_sync, + notes = EXCLUDED.notes, + phones_count = EXCLUDED.phones_count, + groups = EXCLUDED.groups, + aliases = EXCLUDED.aliases, + enable_auto_prompt = EXCLUDED.enable_auto_prompt, + synced_at = NOW() + `, [ + u.user_id, + account.account_id, + u.username, + u.email || null, + u.realname || null, + u.status, + u.is_enrolled, + u.last_login ? new Date(u.last_login * 1000) : null, + u.last_directory_sync ? new Date(u.last_directory_sync * 1000) : null, + u.created ? new Date(u.created * 1000) : null, + u.notes || null, + u.phones?.length ?? 0, + JSON.stringify(u.groups?.map((g: any) => ({ group_id: g.group_id, name: g.name })) ?? []), + JSON.stringify(u.aliases ?? {}), + u.enable_auto_prompt ?? true, + ]); + upserted++; + } + entities.push({ entity: 'duo_users', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing users for ${account.name}:`, err.message); + errors.push(`users/${account.name}: ${err.message}`); + entities.push({ entity: 'duo_users', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + private async syncPhones(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise { + const start = Date.now(); + try { + const phones = await client.getPhones(account); + let upserted = 0; + for (const p of phones) { + await postgresClient.query(` + INSERT INTO duo_phones (phone_id, duo_account_id, name, number, type, platform, model, os_version, app_version, activated, last_seen, capabilities, users, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NOW()) + ON CONFLICT (phone_id) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, + name = EXCLUDED.name, + number = EXCLUDED.number, + type = EXCLUDED.type, + platform = EXCLUDED.platform, + model = EXCLUDED.model, + os_version = EXCLUDED.os_version, + app_version = EXCLUDED.app_version, + activated = EXCLUDED.activated, + last_seen = EXCLUDED.last_seen, + capabilities = EXCLUDED.capabilities, + users = EXCLUDED.users, + synced_at = NOW() + `, [ + p.phone_id, + account.account_id, + p.name || null, + p.number || null, + p.type || null, + p.platform || null, + p.model || null, + p.os_version || null, + p.app_version || null, + p.activated ?? false, + p.last_seen ? new Date(p.last_seen) : null, + JSON.stringify(p.capabilities ?? []), + JSON.stringify(p.users?.map((u: any) => ({ user_id: u.user_id, username: u.username })) ?? []), + ]); + upserted++; + } + entities.push({ entity: 'duo_phones', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing phones for ${account.name}:`, err.message); + errors.push(`phones/${account.name}: ${err.message}`); + entities.push({ entity: 'duo_phones', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + private async syncGroups(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise { + const start = Date.now(); + try { + const groups = await client.getGroups(account); + let upserted = 0; + for (const g of groups) { + await postgresClient.query(` + INSERT INTO duo_groups (group_id, duo_account_id, name, description, status, synced_at) + VALUES ($1,$2,$3,$4,$5,NOW()) + ON CONFLICT (group_id) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, + name = EXCLUDED.name, + description = EXCLUDED.description, + status = EXCLUDED.status, + synced_at = NOW() + `, [ + g.group_id, + account.account_id, + g.name || null, + g.desc || null, + g.status || null, + ]); + upserted++; + } + entities.push({ entity: 'duo_groups', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing groups for ${account.name}:`, err.message); + errors.push(`groups/${account.name}: ${err.message}`); + entities.push({ entity: 'duo_groups', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + private async syncIntegrations(client: DuoClient, account: DuoAccount, entities: DuoSyncEntityResult[], errors: string[]): Promise { + const start = Date.now(); + try { + const integrations = await client.getIntegrations(account); + let upserted = 0; + for (const i of integrations) { + await postgresClient.query(` + INSERT INTO duo_integrations (integration_key, duo_account_id, name, type, notes, synced_at) + VALUES ($1,$2,$3,$4,$5,NOW()) + ON CONFLICT (integration_key) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, + name = EXCLUDED.name, + type = EXCLUDED.type, + notes = EXCLUDED.notes, + synced_at = NOW() + `, [ + i.integration_key, + account.account_id, + i.name || null, + i.type || null, + i.notes || null, + ]); + upserted++; + } + entities.push({ entity: 'duo_integrations', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing integrations for ${account.name}:`, err.message); + errors.push(`integrations/${account.name}: ${err.message}`); + entities.push({ entity: 'duo_integrations', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + // ── 3.3 syncAuthLogs — incremental ──────────────────────────────────────── + + private async syncAuthLogs( + client: DuoClient, + account: DuoAccount, + entities: DuoSyncEntityResult[], + errors: string[], + since?: number, + ): Promise { + const start = Date.now(); + try { + // Default: last synced timestamp or 30 days ago + if (!since) { + const lastRow = await postgresClient.query( + `SELECT MAX(timestamp) as last_ts FROM duo_auth_logs WHERE duo_account_id = $1`, + [account.account_id], + ); + const lastTs = lastRow.rows[0]?.last_ts; + since = lastTs ? new Date(lastTs).getTime() : Date.now() - 30 * 24 * 60 * 60 * 1000; + } + + const logs = await client.getAuthLogs(since, account); + let upserted = 0; + for (const l of logs) { + await postgresClient.query(` + INSERT INTO duo_auth_logs (txid, duo_account_id, timestamp, user_name, user_id, factor, result, reason, application_name, application_key, access_device_ip, access_device_location, auth_device_ip, auth_device_name, event_type, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW()) + ON CONFLICT (txid) DO NOTHING + `, [ + l.txid, + account.account_id, + new Date(l.timestamp * 1000), + l.user?.name || null, + l.user?.key || null, + l.factor || null, + l.result || null, + l.reason || null, + l.application?.name || null, + l.application?.key || null, + l.access_device?.ip || null, + l.access_device?.location ? JSON.stringify(l.access_device.location) : null, + l.auth_device?.ip || null, + l.auth_device?.name || null, + l.event_type || null, + ]); + upserted++; + } + entities.push({ entity: 'duo_auth_logs', account: account.name, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing auth logs for ${account.name}:`, err.message); + errors.push(`auth_logs/${account.name}: ${err.message}`); + entities.push({ entity: 'duo_auth_logs', account: account.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + // ── Parent account sync using DUOADMIN creds ─────────────────────────────── + + private async syncParentAccount( + adminClient: DuoClient, + entities: DuoSyncEntityResult[], + errors: string[], + ): Promise { + const parentAccountId = 'PARENT_WULF'; + const parentHost = process.env.DUOADMIN_API_HOSTNAME!; + + // Upsert parent account row + try { + const summary = await adminClient.getAccountSummary(); + await postgresClient.query(` + INSERT INTO duo_accounts (account_id, name, api_hostname, user_count, integration_count, is_parent, synced_at) + VALUES ($1, 'Wulf Consulting (Parent)', $2, $3, $4, true, NOW()) + ON CONFLICT (account_id) DO UPDATE SET + user_count = EXCLUDED.user_count, + integration_count = EXCLUDED.integration_count, + synced_at = NOW() + `, [parentAccountId, parentHost, summary.user_count, summary.integration_count]); + } catch (err: any) { + console.error(`[DuoSync] Error syncing parent summary:`, err.message); + errors.push(`parent/summary: ${err.message}`); + } + + // Create a pseudo DuoAccount for the parent so we can reuse syncAccountData/syncAuthLogs + const parentAccount: DuoAccount = { account_id: parentAccountId, api_hostname: parentHost, name: 'Wulf Consulting (Parent)' }; + + // Sync users, phones, groups, integrations using adminClient (no child override needed) + await this.syncParentEntity('users', async () => { + const users = await adminClient.getUsers(); + let upserted = 0; + for (const u of users) { + await postgresClient.query(` + INSERT INTO duo_users (user_id, duo_account_id, username, email, realname, status, is_enrolled, last_login, last_directory_sync, created, notes, phones_count, groups, aliases, enable_auto_prompt, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW()) + ON CONFLICT (user_id) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, username = EXCLUDED.username, email = EXCLUDED.email, + realname = EXCLUDED.realname, status = EXCLUDED.status, is_enrolled = EXCLUDED.is_enrolled, + last_login = EXCLUDED.last_login, last_directory_sync = EXCLUDED.last_directory_sync, + notes = EXCLUDED.notes, phones_count = EXCLUDED.phones_count, groups = EXCLUDED.groups, + aliases = EXCLUDED.aliases, enable_auto_prompt = EXCLUDED.enable_auto_prompt, synced_at = NOW() + `, [ + u.user_id, parentAccountId, u.username, u.email || null, u.realname || null, + u.status, u.is_enrolled, + u.last_login ? new Date(u.last_login * 1000) : null, + u.last_directory_sync ? new Date(u.last_directory_sync * 1000) : null, + u.created ? new Date(u.created * 1000) : null, + u.notes || null, u.phones?.length ?? 0, + JSON.stringify(u.groups?.map((g: any) => ({ group_id: g.group_id, name: g.name })) ?? []), + JSON.stringify(u.aliases ?? {}), u.enable_auto_prompt ?? true, + ]); + upserted++; + } + return upserted; + }, parentAccount.name, entities, errors); + + await this.syncParentEntity('phones', async () => { + const phones = await adminClient.getPhones(); + let upserted = 0; + for (const p of phones) { + await postgresClient.query(` + INSERT INTO duo_phones (phone_id, duo_account_id, name, number, type, platform, model, os_version, app_version, activated, last_seen, capabilities, users, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,NOW()) + ON CONFLICT (phone_id) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, name = EXCLUDED.name, number = EXCLUDED.number, + type = EXCLUDED.type, platform = EXCLUDED.platform, model = EXCLUDED.model, + os_version = EXCLUDED.os_version, app_version = EXCLUDED.app_version, + activated = EXCLUDED.activated, last_seen = EXCLUDED.last_seen, + capabilities = EXCLUDED.capabilities, users = EXCLUDED.users, synced_at = NOW() + `, [ + p.phone_id, parentAccountId, p.name || null, p.number || null, p.type || null, + p.platform || null, p.model || null, p.os_version || null, p.app_version || null, + p.activated ?? false, p.last_seen ? new Date(p.last_seen) : null, + JSON.stringify(p.capabilities ?? []), + JSON.stringify(p.users?.map((u: any) => ({ user_id: u.user_id, username: u.username })) ?? []), + ]); + upserted++; + } + return upserted; + }, parentAccount.name, entities, errors); + + await this.syncParentEntity('groups', async () => { + const groups = await adminClient.getGroups(); + let upserted = 0; + for (const g of groups) { + await postgresClient.query(` + INSERT INTO duo_groups (group_id, duo_account_id, name, description, status, synced_at) + VALUES ($1,$2,$3,$4,$5,NOW()) + ON CONFLICT (group_id) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, name = EXCLUDED.name, + description = EXCLUDED.description, status = EXCLUDED.status, synced_at = NOW() + `, [g.group_id, parentAccountId, g.name || null, g.desc || null, g.status || null]); + upserted++; + } + return upserted; + }, parentAccount.name, entities, errors); + + await this.syncParentEntity('integrations', async () => { + const integrations = await adminClient.getIntegrations(); + let upserted = 0; + for (const i of integrations) { + await postgresClient.query(` + INSERT INTO duo_integrations (integration_key, duo_account_id, name, type, notes, synced_at) + VALUES ($1,$2,$3,$4,$5,NOW()) + ON CONFLICT (integration_key) DO UPDATE SET + duo_account_id = EXCLUDED.duo_account_id, name = EXCLUDED.name, + type = EXCLUDED.type, notes = EXCLUDED.notes, synced_at = NOW() + `, [i.integration_key, parentAccountId, i.name || null, i.type || null, i.notes || null]); + upserted++; + } + return upserted; + }, parentAccount.name, entities, errors); + + // Auth logs for parent + const start = Date.now(); + try { + const lastRow = await postgresClient.query( + `SELECT MAX(timestamp) as last_ts FROM duo_auth_logs WHERE duo_account_id = $1`, + [parentAccountId], + ); + const lastTs = lastRow.rows[0]?.last_ts; + const since = lastTs ? new Date(lastTs).getTime() : Date.now() - 30 * 24 * 60 * 60 * 1000; + const logs = await adminClient.getAuthLogs(since); + let upserted = 0; + for (const l of logs) { + await postgresClient.query(` + INSERT INTO duo_auth_logs (txid, duo_account_id, timestamp, user_name, user_id, factor, result, reason, application_name, application_key, access_device_ip, access_device_location, auth_device_ip, auth_device_name, event_type, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,NOW()) + ON CONFLICT (txid) DO NOTHING + `, [ + l.txid, parentAccountId, new Date(l.timestamp * 1000), + l.user?.name || null, l.user?.key || null, l.factor || null, + l.result || null, l.reason || null, + l.application?.name || null, l.application?.key || null, + l.access_device?.ip || null, + l.access_device?.location ? JSON.stringify(l.access_device.location) : null, + l.auth_device?.ip || null, l.auth_device?.name || null, l.event_type || null, + ]); + upserted++; + } + entities.push({ entity: 'duo_auth_logs', account: parentAccount.name, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing parent auth logs:`, err.message); + errors.push(`auth_logs/parent: ${err.message}`); + entities.push({ entity: 'duo_auth_logs', account: parentAccount.name, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + private async syncParentEntity( + entityName: string, + fn: () => Promise, + accountName: string, + entities: DuoSyncEntityResult[], + errors: string[], + ): Promise { + const start = Date.now(); + try { + const upserted = await fn(); + entities.push({ entity: `duo_${entityName}`, account: accountName, success: true, recordsUpserted: upserted, duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error syncing parent ${entityName}:`, err.message); + errors.push(`${entityName}/parent: ${err.message}`); + entities.push({ entity: `duo_${entityName}`, account: accountName, success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } + + // ── 3.5 Company matching ────────────────────────────────────────────────── + + private async matchCompanies( + entities: DuoSyncEntityResult[], + errors: string[], + ): Promise { + const start = Date.now(); + try { + // Exact match first + await postgresClient.query(` + UPDATE duo_accounts da + SET autotask_company_id = c.id + FROM companies c + WHERE da.autotask_company_id IS NULL + AND da.is_parent = false + AND LOWER(TRIM(da.name)) = LOWER(TRIM(c.company_name)) + `); + + // Case-insensitive containment (company name contains duo account name or vice versa) + await postgresClient.query(` + UPDATE duo_accounts da + SET autotask_company_id = sub.company_id + FROM ( + SELECT DISTINCT ON (da2.account_id) da2.account_id, c.id as company_id + FROM duo_accounts da2 + JOIN companies c ON ( + LOWER(c.company_name) LIKE '%' || LOWER(TRIM(da2.name)) || '%' + OR LOWER(TRIM(da2.name)) LIKE '%' || LOWER(TRIM(c.company_name)) || '%' + ) + WHERE da2.autotask_company_id IS NULL AND da2.is_parent = false + ORDER BY da2.account_id, LENGTH(c.company_name) ASC + ) sub + WHERE da.account_id = sub.account_id + `); + + const matched = await postgresClient.query(` + SELECT COUNT(*) as matched FROM duo_accounts WHERE autotask_company_id IS NOT NULL AND is_parent = false + `); + const total = await postgresClient.query(` + SELECT COUNT(*) as total FROM duo_accounts WHERE is_parent = false + `); + + console.log(`[DuoSync] Company matching: ${matched.rows[0].matched}/${total.rows[0].total} accounts matched`); + entities.push({ entity: 'company_matching', account: 'all', success: true, recordsUpserted: parseInt(matched.rows[0].matched), duration: Date.now() - start }); + } catch (err: any) { + console.error(`[DuoSync] Error matching companies:`, err.message); + errors.push(`company_matching: ${err.message}`); + entities.push({ entity: 'company_matching', account: 'all', success: false, recordsUpserted: 0, duration: Date.now() - start, error: err.message }); + } + } +} + +export const duoSyncService = new DuoSyncService(); diff --git a/middleware.ts b/middleware.ts index 44392ae..625b772 100644 --- a/middleware.ts +++ b/middleware.ts @@ -35,6 +35,8 @@ const publicRoutes = [ "/api/qbo/sync", "/api/reports/ticket-digest", "/api/notifications/morning-summary/send", + // Duo Security sync and data endpoints + "/api/duo", ]; // Routes that require admin or super-admin role diff --git a/migrations/058_create_duo_tables.sql b/migrations/058_create_duo_tables.sql new file mode 100644 index 0000000..fa5d345 --- /dev/null +++ b/migrations/058_create_duo_tables.sql @@ -0,0 +1,121 @@ +-- Migration 058: Create Duo Security tables +-- Stores data from both Accounts API (child accounts) and Admin API (users, phones, auth logs, groups, integrations) + +-- ── duo_accounts ───────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS duo_accounts ( + id SERIAL PRIMARY KEY, + account_id VARCHAR(20) NOT NULL UNIQUE, + name VARCHAR(255), + api_hostname VARCHAR(255), + autotask_company_id BIGINT REFERENCES companies(id) ON DELETE SET NULL, + user_count INTEGER DEFAULT 0, + integration_count INTEGER DEFAULT 0, + edition VARCHAR(50), + is_parent BOOLEAN DEFAULT false, + synced_at TIMESTAMP WITHOUT TIME ZONE, + created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_duo_accounts_company ON duo_accounts(autotask_company_id); +CREATE INDEX IF NOT EXISTS idx_duo_accounts_is_parent ON duo_accounts(is_parent); + +-- ── duo_users ──────────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS duo_users ( + id SERIAL PRIMARY KEY, + user_id VARCHAR(30) NOT NULL UNIQUE, + duo_account_id VARCHAR(20) NOT NULL REFERENCES duo_accounts(account_id) ON DELETE CASCADE, + username VARCHAR(255), + email VARCHAR(255), + realname VARCHAR(255), + status VARCHAR(50), + is_enrolled BOOLEAN DEFAULT false, + last_login TIMESTAMP WITHOUT TIME ZONE, + last_directory_sync TIMESTAMP WITHOUT TIME ZONE, + created TIMESTAMP WITHOUT TIME ZONE, + notes TEXT, + phones_count INTEGER DEFAULT 0, + groups JSONB, + aliases JSONB, + enable_auto_prompt BOOLEAN DEFAULT true, + synced_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_duo_users_account ON duo_users(duo_account_id); +CREATE INDEX IF NOT EXISTS idx_duo_users_status ON duo_users(status); +CREATE INDEX IF NOT EXISTS idx_duo_users_email ON duo_users(email); +CREATE INDEX IF NOT EXISTS idx_duo_users_enrolled ON duo_users(is_enrolled); + +-- ── duo_phones ─────────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS duo_phones ( + id SERIAL PRIMARY KEY, + phone_id VARCHAR(30) NOT NULL UNIQUE, + duo_account_id VARCHAR(20) NOT NULL REFERENCES duo_accounts(account_id) ON DELETE CASCADE, + name VARCHAR(255), + number VARCHAR(50), + type VARCHAR(50), + platform VARCHAR(50), + model VARCHAR(255), + os_version VARCHAR(50), + app_version VARCHAR(50), + activated BOOLEAN DEFAULT false, + last_seen TIMESTAMP WITHOUT TIME ZONE, + capabilities JSONB, + users JSONB, + synced_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_duo_phones_account ON duo_phones(duo_account_id); + +-- ── duo_auth_logs ──────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS duo_auth_logs ( + id SERIAL PRIMARY KEY, + txid VARCHAR(50) NOT NULL UNIQUE, + duo_account_id VARCHAR(20) NOT NULL REFERENCES duo_accounts(account_id) ON DELETE CASCADE, + timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, + user_name VARCHAR(255), + user_id VARCHAR(30), + factor VARCHAR(50), + result VARCHAR(50), + reason VARCHAR(255), + application_name VARCHAR(255), + application_key VARCHAR(50), + access_device_ip INET, + access_device_location JSONB, + auth_device_ip INET, + auth_device_name VARCHAR(255), + event_type VARCHAR(50), + synced_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_duo_auth_logs_account ON duo_auth_logs(duo_account_id); +CREATE INDEX IF NOT EXISTS idx_duo_auth_logs_timestamp ON duo_auth_logs(timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_duo_auth_logs_result ON duo_auth_logs(result); +CREATE INDEX IF NOT EXISTS idx_duo_auth_logs_user ON duo_auth_logs(user_name); + +-- ── duo_groups ─────────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS duo_groups ( + id SERIAL PRIMARY KEY, + group_id VARCHAR(30) NOT NULL UNIQUE, + duo_account_id VARCHAR(20) NOT NULL REFERENCES duo_accounts(account_id) ON DELETE CASCADE, + name VARCHAR(255), + description TEXT, + member_count INTEGER DEFAULT 0, + status VARCHAR(50), + synced_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_duo_groups_account ON duo_groups(duo_account_id); + +-- ── duo_integrations ───────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS duo_integrations ( + id SERIAL PRIMARY KEY, + integration_key VARCHAR(50) NOT NULL UNIQUE, + duo_account_id VARCHAR(20) NOT NULL REFERENCES duo_accounts(account_id) ON DELETE CASCADE, + name VARCHAR(255), + type VARCHAR(100), + enabled BOOLEAN DEFAULT true, + notes TEXT, + synced_at TIMESTAMP WITHOUT TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_duo_integrations_account ON duo_integrations(duo_account_id); diff --git a/tasks/prd-duo-integration.md b/tasks/prd-duo-integration.md new file mode 100644 index 0000000..287794b --- /dev/null +++ b/tasks/prd-duo-integration.md @@ -0,0 +1,337 @@ +# PRD: Duo Security Integration — Data Sync & Storage + +## 1. Introduction / Overview + +Pulse currently has no visibility into Duo Security 2FA data across managed clients. Wulf Consulting manages 32 child Duo accounts via the MSP parent account. This feature adds a full Duo data sync — pulling account inventory, users, phones/devices, authentication logs, groups, and integrations from every child account into Pulse's PostgreSQL database. + +**Key architectural insight:** The parent Duo **Accounts API** credentials (`DUOACCOUNTS_*`) can be used to call **Admin API** endpoints on any child account by signing requests against the child's `api_hostname` and passing `account_id`. No per-child Admin API keys are needed. + +### Problem Statement + +- No centralized view of which clients have Duo deployed, how many users are enrolled, or which users lack 2FA. +- No way to audit 2FA adoption, device health, or authentication anomalies without logging into 32 separate Duo admin panels. +- Security compliance reporting requires manual data gathering across all child accounts. + +--- + +## 2. Goals + +| # | Goal | Measurable Outcome | +|---|------|--------------------| +| G1 | Sync all 32 child accounts and their metadata | `duo_accounts` table contains all child accounts with `account_id`, `api_hostname`, `name` | +| G2 | Sync users from every child account | `duo_users` table contains all users across all children, with enrollment status, last login, status | +| G3 | Sync 2FA devices (phones) per user | `duo_phones` table stores device model, OS, platform, activation status, last seen | +| G4 | Sync authentication logs | `duo_auth_logs` table stores auth events with result, device, location, application | +| G5 | Sync groups and integrations per child | `duo_groups` and `duo_integrations` tables | +| G6 | Map Duo child accounts to Autotask companies | `duo_accounts.autotask_company_id` FK to `companies.id` for cross-referencing | +| G7 | Expose sync via OpenClaw API + internal API route | Sync can be triggered on-demand or scheduled | +| G8 | Admin API (parent account) data synced separately | `duo_admin_users` or Wulf parent account users/summary stored alongside child data | + +--- + +## 3. User Stories + +- **As an MSP admin**, I want to see which clients have Duo deployed and how many users are enrolled, so I can identify clients with low 2FA adoption. +- **As a security analyst**, I want to query Duo authentication logs across all clients, so I can detect anomalies (failed auths, new device enrollments, locked-out users). +- **As an account manager**, I want to see per-client Duo user counts and enrollment rates, so I can include them in QBR reports. +- **As the NOC**, I want to see which Duo users have "bypass" or "disabled" status, so I can flag security risks. +- **As a developer**, I want Duo data queryable via SQL alongside Autotask, Datto RMM, and SentinelOne data, so I can build cross-platform dashboards. + +--- + +## 4. Functional Requirements + +### 4.1 Duo API Client (`lib/services/duo-client.ts`) + +| # | Requirement | +|---|-------------| +| FR-1 | Create a `DuoClient` class that handles HMAC-SHA1 request signing per Duo's auth spec. | +| FR-2 | Support both GET and POST methods with URL-encoded parameters. | +| FR-3 | Accept `ikey`, `skey`, and `host` in constructor. For child account calls, accept an override `host` + `account_id` parameter. | +| FR-4 | Implement automatic pagination — Duo APIs return `metadata.next_offset`; the client must follow pages until all records are retrieved. | +| FR-5 | Implement rate-limit handling — Duo returns `429` with `Retry-After` header; client must respect it. | +| FR-6 | All API calls must have a configurable timeout (default 30s). | + +### 4.2 Accounts API Methods + +| # | Requirement | +|---|-------------| +| FR-7 | `listAccounts()` — calls `POST /accounts/v1/account/list` using parent creds to retrieve all child accounts (`account_id`, `api_hostname`, `name`). | +| FR-8 | No create/delete operations — read-only sync. | + +### 4.3 Admin API Methods (per child account) + +All Admin API calls use the **parent Accounts API credentials** signed against the **child's `api_hostname`** with `account_id` in the params. + +| # | Requirement | +|---|-------------| +| FR-9 | `getAccountSummary(child)` — `GET /admin/v1/info/summary` — returns user_count, integration_count, telephony_credits. | +| FR-10 | `getUsers(child)` — `GET /admin/v1/users` — paginate all users. Returns username, email, status, is_enrolled, last_login, phones, groups, created, etc. | +| FR-11 | `getPhones(child)` — `GET /admin/v1/phones` — paginate all phones/devices. Returns model, platform, OS, activated, last_seen, capabilities, associated users. | +| FR-12 | `getGroups(child)` — `GET /admin/v1/groups` — all groups with member counts. | +| FR-13 | `getIntegrations(child)` — `GET /admin/v1/integrations` — all applications/integrations (name, type, enabled). | +| FR-14 | `getAuthLogs(child, mintime)` — `GET /admin/v2/logs/authentication` — auth events since `mintime`. Returns access_device, auth_device, result, reason, user, application, timestamp, location. | +| FR-15 | `getAdminLogs(child, mintime)` — `GET /admin/v1/logs/administrator` — admin activity events (optional/stretch). | + +### 4.4 Admin API Methods (parent Wulf account) + +| # | Requirement | +|---|-------------| +| FR-16 | Use the separate `DUOADMIN_*` credentials for the Wulf parent account. | +| FR-17 | Sync users, phones, groups, integrations, and summary from the parent account using the same Admin API methods. | +| FR-18 | Store parent account data in the same tables with a sentinel `duo_account_id` (e.g., the parent's own account or a well-known marker). | + +### 4.5 Database Tables (Migration) + +#### `duo_accounts` +Stores child account metadata from the Accounts API, plus the Wulf parent account. + +| Column | Type | Description | +|--------|------|-------------| +| `id` | SERIAL PK | Internal ID | +| `account_id` | VARCHAR(20) UNIQUE | Duo account ID (e.g., `DAS7Y6UP7EQHTHTAI9R0`) | +| `name` | VARCHAR(255) | Account name | +| `api_hostname` | VARCHAR(255) | Per-account API hostname | +| `autotask_company_id` | BIGINT FK → companies(id) | Matched Autotask company (nullable) | +| `user_count` | INTEGER | Last synced user count | +| `integration_count` | INTEGER | Last synced integration count | +| `edition` | VARCHAR(50) | Duo edition (PERSONAL/ENTERPRISE/PLATFORM/BEYOND) if available | +| `is_parent` | BOOLEAN DEFAULT false | True for the Wulf parent account row | +| `synced_at` | TIMESTAMP | Last sync time | +| `created_at` | TIMESTAMP DEFAULT NOW() | | + +#### `duo_users` +All Duo users across all accounts. + +| Column | Type | Description | +|--------|------|-------------| +| `id` | SERIAL PK | Internal ID | +| `user_id` | VARCHAR(30) UNIQUE | Duo user ID (e.g., `DULLBCPMRNNORZW1NKBF`) | +| `duo_account_id` | VARCHAR(20) FK → duo_accounts(account_id) | Which child account | +| `username` | VARCHAR(255) | | +| `email` | VARCHAR(255) | | +| `realname` | VARCHAR(255) | | +| `status` | VARCHAR(50) | active, bypass, disabled, locked_out, pending_deletion | +| `is_enrolled` | BOOLEAN | | +| `last_login` | TIMESTAMP | | +| `last_directory_sync` | TIMESTAMP | | +| `created` | TIMESTAMP | Duo creation timestamp | +| `notes` | TEXT | | +| `phones_count` | INTEGER | Number of associated phones | +| `groups` | JSONB | Array of group names/IDs | +| `aliases` | JSONB | Alias fields | +| `enable_auto_prompt` | BOOLEAN | | +| `synced_at` | TIMESTAMP | | + +#### `duo_phones` +2FA devices (phones, hardware tokens, etc.). + +| Column | Type | Description | +|--------|------|-------------| +| `id` | SERIAL PK | Internal ID | +| `phone_id` | VARCHAR(30) UNIQUE | Duo phone ID | +| `duo_account_id` | VARCHAR(20) FK → duo_accounts(account_id) | | +| `name` | VARCHAR(255) | Device name | +| `number` | VARCHAR(50) | Phone number | +| `type` | VARCHAR(50) | Mobile, Landline, etc. | +| `platform` | VARCHAR(50) | Google Android, Apple iOS, etc. | +| `model` | VARCHAR(255) | Device model | +| `os_version` | VARCHAR(50) | | +| `app_version` | VARCHAR(50) | Duo Mobile app version | +| `activated` | BOOLEAN | | +| `last_seen` | TIMESTAMP | | +| `capabilities` | JSONB | Array: auto, push, sms, phone, mobile_otp | +| `users` | JSONB | Array of associated user_ids | +| `synced_at` | TIMESTAMP | | + +#### `duo_auth_logs` +Authentication events. + +| Column | Type | Description | +|--------|------|-------------| +| `id` | SERIAL PK | Internal ID | +| `txid` | VARCHAR(50) UNIQUE | Duo transaction ID | +| `duo_account_id` | VARCHAR(20) FK → duo_accounts(account_id) | | +| `timestamp` | TIMESTAMP | Event time | +| `user_name` | VARCHAR(255) | | +| `user_id` | VARCHAR(30) | Duo user ID | +| `factor` | VARCHAR(50) | push, phone, sms, passcode, etc. | +| `result` | VARCHAR(50) | success, denied, fraud, locked_out | +| `reason` | VARCHAR(255) | Detailed reason | +| `application_name` | VARCHAR(255) | | +| `application_key` | VARCHAR(50) | | +| `access_device_ip` | INET | | +| `access_device_location` | JSONB | {city, state, country} | +| `auth_device_ip` | INET | | +| `auth_device_name` | VARCHAR(255) | | +| `event_type` | VARCHAR(50) | authentication, enrollment | +| `synced_at` | TIMESTAMP | | + +#### `duo_groups` +Groups per account. + +| Column | Type | Description | +|--------|------|-------------| +| `id` | SERIAL PK | Internal ID | +| `group_id` | VARCHAR(30) UNIQUE | Duo group ID | +| `duo_account_id` | VARCHAR(20) FK → duo_accounts(account_id) | | +| `name` | VARCHAR(255) | | +| `description` | TEXT | | +| `member_count` | INTEGER | | +| `status` | VARCHAR(50) | | +| `synced_at` | TIMESTAMP | | + +#### `duo_integrations` +Applications/integrations per account. + +| Column | Type | Description | +|--------|------|-------------| +| `id` | SERIAL PK | Internal ID | +| `integration_key` | VARCHAR(50) UNIQUE | Duo integration key | +| `duo_account_id` | VARCHAR(20) FK → duo_accounts(account_id) | | +| `name` | VARCHAR(255) | | +| `type` | VARCHAR(100) | azure-ca, rdp, websdk, etc. | +| `enabled` | BOOLEAN | | +| `notes` | TEXT | | +| `synced_at` | TIMESTAMP | | + +### 4.6 Sync Service (`lib/services/duo-sync-service.ts`) + +| # | Requirement | +|---|-------------| +| FR-19 | `syncAll()` — Full sync: (1) list child accounts, (2) for each child sync users, phones, groups, integrations, auth logs. Also sync parent account. | +| FR-20 | `syncAccounts()` — Sync child account list only. | +| FR-21 | `syncAccountData(accountId)` — Sync all data for a single child account. | +| FR-22 | `syncAuthLogs(accountId?, since?)` — Incremental auth log sync. Default `since` = last synced `timestamp` in `duo_auth_logs` for that account. | +| FR-23 | All sync operations use upsert (INSERT ... ON CONFLICT DO UPDATE) to handle re-syncs cleanly. | +| FR-24 | Sync should be non-blocking (fire-and-forget with sync ID tracking, same pattern as Autotask sync). | +| FR-25 | Sync should process child accounts sequentially (not in parallel) to respect Duo rate limits. | +| FR-26 | Log sync progress and errors. Track per-account sync stats (records added/updated). | + +### 4.7 Company Matching + +| # | Requirement | +|---|-------------| +| FR-27 | After syncing child accounts, attempt to match each `duo_accounts.name` to `companies.company_name` using fuzzy/exact matching. | +| FR-28 | Store the match as `duo_accounts.autotask_company_id`. Allow manual override. | +| FR-29 | Matching logic: exact match first, then case-insensitive containment, then skip (leave null for manual mapping). | + +### 4.8 API Routes + +| # | Route | Method | Description | +|---|-------|--------|-------------| +| FR-30 | `/api/duo/sync` | POST | Trigger full Duo sync (requires auth). | +| FR-31 | `/api/duo/sync` | GET | Return sync status (in progress, last sync time). | +| FR-32 | `/api/duo/accounts` | GET | List all Duo child accounts with stats. | +| FR-33 | `/api/duo/accounts/[id]/users` | GET | List users for a specific account. | +| FR-34 | `/api/openclaw/sync/duo` | POST | OpenClaw trigger for Duo sync (API key auth). | + +### 4.9 Environment Variables + +Already configured in `.env.local`: + +``` +DUOACCOUNTS_INTEGRATION_KEY=DIS1RH7M3TQ69QTD0GLW +DUOACCOUNTS_SECRET_KEY=xYlwR5aBby7HtAFV2UdLFpLJ24kONrDbr85DLzek +DUOACCOUNTS_API_HOSTNAME=api-98372575.duosecurity.com + +DUOADMIN_INTEGRATION_KEY=DI8DKH1EK4JJJKGJNV01 +DUOADMIN_SECRET_KEY=o4sE0E7PcKXdl1K2nwPH1ixD9XHZyce2V1JmLqJH +DUOADMIN_API_HOSTNAME=api-98372575.duosecurity.com +``` + +--- + +## 5. Non-Goals (Out of Scope) + +- **Write operations** — No creating/deleting Duo users, accounts, or devices via Pulse. Read-only sync. +- **Real-time webhooks** — Duo does not support push webhooks; this is poll-based sync only. +- **Duo Auth API (2FA verification)** — We are not performing 2FA challenges, only reading admin data. +- **UI dashboards** — This PRD covers data sync and storage only. Dashboard/reporting UI is a separate feature. +- **Admin log sync** — Administrator activity logs (`/admin/v1/logs/administrator`) are a stretch goal, not required for v1. + +--- + +## 6. Design Considerations + +- Follow existing Pulse patterns: sync service class, entity upsert, OpenClaw route, background execution. +- Duo API rate limits: 20 requests/second per account. Sequential child processing with small delays should stay well under limits. +- Auth logs can be large — use incremental sync (`mintime` = last synced timestamp) to avoid re-pulling history. +- Auth log `timestamp` is milliseconds since epoch — convert to PostgreSQL `TIMESTAMP` on insert. + +--- + +## 7. Technical Considerations + +### Dependencies +- No new npm packages required — HMAC-SHA1 signing uses Node.js built-in `crypto` module, HTTPS via built-in `https`. +- PostgreSQL `INET` type for IP storage (native, no extension needed). + +### Duo API Auth Pattern +All requests signed with HMAC-SHA1: +``` +canon = date + "\n" + method + "\n" + host + "\n" + path + "\n" + sorted_params +sig = HMAC-SHA1(skey, canon) +header = "Basic " + base64(ikey + ":" + sig) +``` + +### Child Account Access Pattern +- Use **parent Accounts API creds** (`DUOACCOUNTS_*`). +- Sign against **child's `api_hostname`** (not parent's). +- Pass `account_id` as a query/body parameter. +- No per-child credentials needed. + +### Parent Account Access Pattern +- Use **Admin API creds** (`DUOADMIN_*`) for the Wulf parent account directly. +- No `account_id` parameter needed. + +### Migration Number +- Next available: `058_create_duo_tables.sql` + +### Existing Patterns to Follow +- `lib/services/sentinelone-sync-service.ts` — similar external API sync pattern +- `lib/services/entity-sync.ts` — upsert pattern, sync stats tracking +- `app/api/openclaw/sync/sentinelone/route.ts` — OpenClaw trigger route pattern + +--- + +## 8. Success Metrics + +| Metric | Target | +|--------|--------| +| All 32 child accounts synced | `duo_accounts` row count = 32 + 1 parent | +| All users across all accounts stored | `duo_users` count matches sum of `user_count` across accounts | +| Phones/devices stored | `duo_phones` populated for all accounts | +| Auth logs incrementally synced | `duo_auth_logs` grows with each sync, no duplicates (upsert on `txid`) | +| Company matching | >80% of `duo_accounts` matched to `companies` by name | +| Sync completes in <5 minutes | Full sync across 32 accounts finishes within timeout | +| OpenClaw can trigger sync | `POST /api/openclaw/sync/duo` returns 200 with syncId | + +--- + +## 9. Open Questions + +1. **Auth log retention** — How far back should we pull auth logs on initial sync? 30 days? 90 days? Duo retains up to 180 days. +2. **Sync schedule** — Daily is assumed. Should auth logs sync more frequently (e.g. every 6 hours) for near-real-time anomaly detection? +3. **Stale data cleanup** — If a child account is removed from Duo, should we soft-delete its data in Pulse? Or leave it as historical? +4. **Parent account Admin API users** — The parent has 25 users (Wulf internal). Should these be stored in the same `duo_users` table or kept separate? +5. **Company name matching** — Some Duo account names may not exactly match Autotask company names (e.g., "Terry's Plumbing & Heating" vs "Terry's Plumbing, Inc."). Should we provide a manual mapping UI, or is fuzzy matching sufficient for v1? + +--- + +## Appendix: Verified API Access + +Confirmed working via `scripts/test-duo.mjs` (March 27, 2026): + +| API | Endpoint | Status | Data | +|-----|----------|--------|------| +| Accounts API | `POST /accounts/v1/account/list` | ✅ 200 | 32 child accounts | +| Admin API (child) | `GET /admin/v1/info/summary` | ✅ 200 | user_count, integration_count | +| Admin API (child) | `GET /admin/v1/users` | ✅ 200 | Full user objects with phones, groups | +| Admin API (child) | `GET /admin/v1/phones` | ✅ 200 | Device model, OS, last_seen, capabilities | +| Admin API (child) | `GET /admin/v2/logs/authentication` | ✅ 200 | Auth events with location, result, device | +| Admin API (child) | `GET /admin/v1/groups` | ✅ 200 | Groups with member counts | +| Admin API (child) | `GET /admin/v1/integrations` | ✅ 200 | Application name, type | +| Admin API (child) | `GET /admin/v1/webauthncredentials` | ✅ 200 | WebAuthn keys | +| Admin API (parent) | `GET /admin/v1/info/summary` | ✅ 200 | 25 users, 19 integrations | +| Admin API (parent) | `GET /admin/v1/users` | ✅ 200 | Full user list | diff --git a/tasks/tasks-prd-duo-integration.md b/tasks/tasks-prd-duo-integration.md new file mode 100644 index 0000000..3343fc0 --- /dev/null +++ b/tasks/tasks-prd-duo-integration.md @@ -0,0 +1,65 @@ +# Tasks: Duo Security Integration — Data Sync & Storage + +> Generated from [prd-duo-integration.md](./prd-duo-integration.md) + +## Relevant Files + +- `lib/services/duo-client.ts` - Duo API client with HMAC-SHA1 signing, pagination, rate-limit handling +- `lib/services/duo-sync-service.ts` - Sync service orchestrating data pull from all Duo accounts into PostgreSQL +- `migrations/058_create_duo_tables.sql` - Database migration creating 6 Duo tables with indexes and FKs +- `app/api/duo/sync/route.ts` - Internal API route to trigger/check Duo sync status (POST/GET) +- `app/api/duo/accounts/route.ts` - API route to list all Duo child accounts with stats (GET) +- `app/api/duo/accounts/[id]/users/route.ts` - API route to list users for a specific Duo account (GET) +- `app/api/openclaw/sync/duo/route.ts` - OpenClaw trigger route for Duo sync (POST, API key auth) +- `middleware.ts` - Add `/api/duo` to public routes +- `scripts/test-duo.mjs` - Existing test script (already created) + +### Notes + +- No new npm packages required — uses Node.js built-in `crypto` and `https`. +- Duo Accounts API uses POST for all endpoints (including list). Admin API uses GET. +- Parent Accounts API creds can call Admin API on any child by signing against child's `api_hostname` + passing `account_id`. +- Migration number 058 is next available. + +## Tasks + +- [ ] 1.0 Create the Duo API Client (`lib/services/duo-client.ts`) + - [x] 1.1 Implement `DuoClient` class with constructor accepting `ikey`, `skey`, `host` + - [ ] 1.2 Implement HMAC-SHA1 request signing method (canon string → signature → Basic auth header) + - [ ] 1.3 Implement `get(path, params)` and `post(path, params)` methods with signed HTTPS requests + - [ ] 1.4 Implement automatic pagination — follow `metadata.next_offset` until all pages retrieved + - [ ] 1.5 Implement rate-limit handling — detect HTTP 429, read `Retry-After` header, wait and retry + - [ ] 1.6 Add configurable timeout (default 30s) on all requests + - [ ] 1.7 Implement Accounts API method: `listAccounts()` → `POST /accounts/v1/account/list` + - [ ] 1.8 Implement Admin API methods: `getAccountSummary()`, `getUsers()`, `getPhones()`, `getGroups()`, `getIntegrations()`, `getAuthLogs()` + - [ ] 1.9 Support child account access pattern — accept override `host` + `account_id` param for Admin API calls +- [ ] 2.0 Create Database Migration (`migrations/058_create_duo_tables.sql`) + - [ ] 2.1 Create `duo_accounts` table with all columns from PRD (account_id, name, api_hostname, autotask_company_id FK, user_count, integration_count, edition, is_parent, synced_at, created_at) + - [ ] 2.2 Create `duo_users` table (user_id, duo_account_id FK, username, email, realname, status, is_enrolled, last_login, groups JSONB, aliases JSONB, etc.) + - [ ] 2.3 Create `duo_phones` table (phone_id, duo_account_id FK, name, number, type, platform, model, os_version, activated, last_seen, capabilities JSONB, users JSONB) + - [ ] 2.4 Create `duo_auth_logs` table (txid, duo_account_id FK, timestamp, user_name, factor, result, reason, access_device_ip INET, access_device_location JSONB, etc.) + - [ ] 2.5 Create `duo_groups` table (group_id, duo_account_id FK, name, description, member_count, status) + - [ ] 2.6 Create `duo_integrations` table (integration_key, duo_account_id FK, name, type, enabled, notes) + - [ ] 2.7 Add indexes: duo_account_id on all child tables, timestamp on auth_logs, status on users, is_parent on accounts +- [ ] 3.0 Create the Duo Sync Service (`lib/services/duo-sync-service.ts`) + - [ ] 3.1 Implement `syncAccounts()` — list child accounts via Accounts API, upsert into `duo_accounts`, add parent account row + - [ ] 3.2 Implement `syncAccountData(account)` — for a single account, sync users, phones, groups, integrations via Admin API upserts + - [ ] 3.3 Implement `syncAuthLogs(account, since?)` — incremental auth log sync using `mintime` from last synced timestamp + - [ ] 3.4 Implement `syncAll()` — orchestrate full sync: syncAccounts → loop each child sequentially → syncAccountData + syncAuthLogs → sync parent account + - [ ] 3.5 Implement company matching — after syncing accounts, match `duo_accounts.name` to `companies.company_name` (exact → case-insensitive containment → skip) + - [ ] 3.6 Add sync ID tracking, progress logging, and per-account stats (records added/updated) +- [ ] 4.0 Create Internal API Routes + - [ ] 4.1 Create `app/api/duo/sync/route.ts` — POST to trigger full sync (non-blocking), GET to return sync status + - [ ] 4.2 Create `app/api/duo/accounts/route.ts` — GET to list all Duo accounts with user_count, integration_count, matched company + - [ ] 4.3 Create `app/api/duo/accounts/[id]/users/route.ts` — GET to list users for a specific Duo account +- [ ] 5.0 Create OpenClaw Route and Register in Middleware + - [ ] 5.1 Create `app/api/openclaw/sync/duo/route.ts` — POST with API key auth, triggers `syncAll()` non-blocking + - [ ] 5.2 Add `/api/duo` to publicRoutes in `middleware.ts` +- [ ] 6.0 End-to-End Testing — Run Migration, Build, Sync, Verify + - [ ] 6.1 Run migration 058 against pulse-postgres + - [ ] 6.2 Rebuild and restart the app container + - [ ] 6.3 Trigger full sync via `/api/duo/sync` POST and verify it completes + - [ ] 6.4 Verify all 6 tables populated: duo_accounts (33 rows), duo_users, duo_phones, duo_auth_logs, duo_groups, duo_integrations + - [ ] 6.5 Verify company matching — check duo_accounts.autotask_company_id is populated for matched accounts + - [ ] 6.6 Verify OpenClaw trigger works via `/api/openclaw/sync/duo` + - [ ] 6.7 Git commit all changes