wulf-pulse/lib/services/pipeline-steps/ping-flap-suppress.ts

import { registerStepExecutor } from '../pipeline-engine';
import { PipelineStep, PipelineContext, StepExecutorResult } from '../../types/pipeline';
import { postgresClient } from '../postgres-client';
import { ZabbixClient } from '../zabbix-client';
import { ZabbixProblem } from '../../types/zabbix';
import { promises as dns } from 'dns';

/**
 * Ping Flap Suppression Step
 *
 * Detects flapping PING alerts from Datto RMM and suppresses the corresponding
 * Zabbix problem when a threshold is exceeded.
 *
 * Flap = same ping_target triggers >= FLAP_THRESHOLD times within FLAP_WINDOW_HOURS.
 * When detected:
 *   - Suppresses the open Zabbix problem for the matching host for SUPPRESS_HOURS
 *   - Records the suppression in ping_flap_suppressions to avoid re-firing
 */

const FLAP_THRESHOLD = 5;
const FLAP_WINDOW_HOURS = 2;
const SUPPRESS_HOURS = 4;

function makeZabbix(): ZabbixClient {
  if (!process.env.ZABBIX_API_URL || !process.env.ZABBIX_API_TOKEN) {
    throw new Error('Zabbix not configured');
  }
  return new ZabbixClient({ apiUrl: process.env.ZABBIX_API_URL, apiToken: process.env.ZABBIX_API_TOKEN });
}

async function executePingFlapSuppress(
  _step: PipelineStep,
  context: PipelineContext,
  _executionId: number
): Promise<StepExecutorResult> {
  const payload = context.triggerData;

  if (payload?.alert_type !== 'PING') {
    return { success: true, output: { skipped: true } };
  }

  const pingTarget: string | null = payload?.ping_target ?? null;
  if (!pingTarget) {
    return { success: true, output: { skipped: true, reason: 'no ping_target' } };
  }

  // Check if already suppressed
  const suppCheck = await postgresClient.query(
    `SELECT suppressed_until FROM ping_flap_suppressions
     WHERE ping_target = $1 AND suppressed_until > NOW()`,
    [pingTarget]
  );
  if (suppCheck.rows.length > 0) {
    return { success: true, output: { flap_suppressed: true, already_active: true } };
  }

  // Count triggers in the flap window
  const countResult = await postgresClient.query(
    `SELECT COUNT(*) as cnt FROM datto_rmm_alerts
     WHERE alert_type = 'PING'
       AND ping_target = $1
       AND triggered = 'True'
       AND timestamp >= NOW() - INTERVAL '${FLAP_WINDOW_HOURS} hours'`,
    [pingTarget]
  );
  const triggerCount = parseInt(countResult.rows[0]?.cnt ?? '0', 10);

  if (triggerCount < FLAP_THRESHOLD) {
    return { success: true, output: { flap_detected: false, trigger_count: triggerCount } };
  }

  const suppressUntil = new Date(Date.now() + SUPPRESS_HOURS * 60 * 60 * 1000);
  const message = `Auto-suppressed: flapping detected (${triggerCount} triggers in ${FLAP_WINDOW_HOURS}h). Awaiting manual resolution.`;

  // Try to find and suppress the Zabbix problem
  let zabbixSuppressed = false;
  let zabbixHost: string | null = null;
  try {
    const zabbix = makeZabbix();

    // Resolve DNS name to IP — Zabbix stores IPs in interfaces, not DNS names
    const lookupTargets = [pingTarget];
    try {
      const resolved = await dns.lookup(pingTarget);
      if (resolved.address && resolved.address !== pingTarget) {
        lookupTargets.unshift(resolved.address);
      }
    } catch { /* not a resolvable hostname — may already be an IP */ }

    let hosts: Array<{ hostid: string; name: string }> = [];
    for (const target of lookupTargets) {
      hosts = await zabbix.findHostsByInterface(target);
      if (hosts.length > 0) break;
    }

    if (hosts.length > 0) {
      const host = hosts[0];
      zabbixHost = host.name;
      const problems = await zabbix.getOpenProblemsForHost(host.hostid);
      const unreachable = problems.find((p: ZabbixProblem) =>
        p.name.toLowerCase().includes('unreachable') ||
        p.name.toLowerCase().includes('unavailable')
      );

      if (unreachable) {
        await zabbix.suppressProblem(unreachable.eventid, suppressUntil, message);
        zabbixSuppressed = true;
      }
    }
  } catch (err) {
    console.error(`[ping-flap-suppress] Zabbix error for ${pingTarget}:`, err);
  }

  // Record suppression to prevent repeated attempts within the window
  await postgresClient.query(
    `INSERT INTO ping_flap_suppressions (ping_target, trigger_count, suppressed_until, zabbix_suppressed, notes)
     VALUES ($1, $2, $3, $4, $5)
     ON CONFLICT (ping_target) DO UPDATE SET
       trigger_count = EXCLUDED.trigger_count,
       suppressed_until = EXCLUDED.suppressed_until,
       zabbix_suppressed = EXCLUDED.zabbix_suppressed,
       notes = EXCLUDED.notes,
       updated_at = NOW()`,
    [pingTarget, triggerCount, suppressUntil, zabbixSuppressed, message]
  );

  console.log(`[ping-flap-suppress] Flap detected: ${pingTarget} (${triggerCount} triggers) — zabbix_suppressed=${zabbixSuppressed} host=${zabbixHost}`);

  return {
    success: true,
    output: {
      flap_detected: true,
      flap_suppressed: true,
      trigger_count: triggerCount,
      zabbix_suppressed: zabbixSuppressed,
      zabbix_host: zabbixHost,
      suppressed_until: suppressUntil.toISOString(),
    },
  };
}

registerStepExecutor('ping_flap_suppress', executePingFlapSuppress);