import { registerStepExecutor } from '../pipeline-engine'; import { PipelineStep, PipelineContext, StepExecutorResult } from '../../types/pipeline'; import { postgresClient } from '../postgres-client'; import { ZabbixClient } from '../zabbix-client'; import { ZabbixProblem } from '../../types/zabbix'; import { promises as dns } from 'dns'; /** * Ping Flap Suppression Step * * Detects flapping PING alerts from Datto RMM and suppresses the corresponding * Zabbix problem when a threshold is exceeded. * * Flap = same ping_target triggers >= FLAP_THRESHOLD times within FLAP_WINDOW_HOURS. * When detected: * - Suppresses the open Zabbix problem for the matching host for SUPPRESS_HOURS * - Records the suppression in ping_flap_suppressions to avoid re-firing */ const FLAP_THRESHOLD = 5; const FLAP_WINDOW_HOURS = 2; const SUPPRESS_HOURS = 4; function makeZabbix(): ZabbixClient { if (!process.env.ZABBIX_API_URL || !process.env.ZABBIX_API_TOKEN) { throw new Error('Zabbix not configured'); } return new ZabbixClient({ apiUrl: process.env.ZABBIX_API_URL, apiToken: process.env.ZABBIX_API_TOKEN }); } async function executePingFlapSuppress( _step: PipelineStep, context: PipelineContext, _executionId: number ): Promise { const payload = context.triggerData; if (payload?.alert_type !== 'PING') { return { success: true, output: { skipped: true } }; } const pingTarget: string | null = payload?.ping_target ?? null; if (!pingTarget) { return { success: true, output: { skipped: true, reason: 'no ping_target' } }; } // Check if already suppressed const suppCheck = await postgresClient.query( `SELECT suppressed_until FROM ping_flap_suppressions WHERE ping_target = $1 AND suppressed_until > NOW()`, [pingTarget] ); if (suppCheck.rows.length > 0) { return { success: true, output: { flap_suppressed: true, already_active: true } }; } // Count triggers in the flap window const countResult = await postgresClient.query( `SELECT COUNT(*) as cnt FROM datto_rmm_alerts WHERE alert_type = 'PING' AND ping_target = $1 AND triggered = 'True' AND timestamp >= NOW() - INTERVAL '${FLAP_WINDOW_HOURS} hours'`, [pingTarget] ); const triggerCount = parseInt(countResult.rows[0]?.cnt ?? '0', 10); if (triggerCount < FLAP_THRESHOLD) { return { success: true, output: { flap_detected: false, trigger_count: triggerCount } }; } const suppressUntil = new Date(Date.now() + SUPPRESS_HOURS * 60 * 60 * 1000); const message = `Auto-suppressed: flapping detected (${triggerCount} triggers in ${FLAP_WINDOW_HOURS}h). Awaiting manual resolution.`; // Try to find and suppress the Zabbix problem let zabbixSuppressed = false; let zabbixHost: string | null = null; try { const zabbix = makeZabbix(); // Resolve DNS name to IP — Zabbix stores IPs in interfaces, not DNS names const lookupTargets = [pingTarget]; try { const resolved = await dns.lookup(pingTarget); if (resolved.address && resolved.address !== pingTarget) { lookupTargets.unshift(resolved.address); } } catch { /* not a resolvable hostname — may already be an IP */ } let hosts: Array<{ hostid: string; name: string }> = []; for (const target of lookupTargets) { hosts = await zabbix.findHostsByInterface(target); if (hosts.length > 0) break; } if (hosts.length > 0) { const host = hosts[0]; zabbixHost = host.name; const problems = await zabbix.getOpenProblemsForHost(host.hostid); const unreachable = problems.find((p: ZabbixProblem) => p.name.toLowerCase().includes('unreachable') || p.name.toLowerCase().includes('unavailable') ); if (unreachable) { await zabbix.suppressProblem(unreachable.eventid, suppressUntil, message); zabbixSuppressed = true; } } } catch (err) { console.error(`[ping-flap-suppress] Zabbix error for ${pingTarget}:`, err); } // Record suppression to prevent repeated attempts within the window await postgresClient.query( `INSERT INTO ping_flap_suppressions (ping_target, trigger_count, suppressed_until, zabbix_suppressed, notes) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (ping_target) DO UPDATE SET trigger_count = EXCLUDED.trigger_count, suppressed_until = EXCLUDED.suppressed_until, zabbix_suppressed = EXCLUDED.zabbix_suppressed, notes = EXCLUDED.notes, updated_at = NOW()`, [pingTarget, triggerCount, suppressUntil, zabbixSuppressed, message] ); console.log(`[ping-flap-suppress] Flap detected: ${pingTarget} (${triggerCount} triggers) — zabbix_suppressed=${zabbixSuppressed} host=${zabbixHost}`); return { success: true, output: { flap_detected: true, flap_suppressed: true, trigger_count: triggerCount, zabbix_suppressed: zabbixSuppressed, zabbix_host: zabbixHost, suppressed_until: suppressUntil.toISOString(), }, }; } registerStepExecutor('ping_flap_suppress', executePingFlapSuppress);