wulf-pulse/lib/services/integration-health.ts

/**
 * Integration auth health + token expiry checks.
 *
 * Lives outside any specific integration's client because the goal is to
 * surface "did anything just break silently" without forcing the dashboard
 * to depend on every per-tool client. Each check is a minimal authenticated
 * call against a cheap endpoint of the target API; results are cached
 * in-process for a few minutes so concurrent dashboard hits don't fan out
 * into a wave of API calls.
 *
 * Usage:
 *   const results = await checkIntegrationHealth();
 *
 * Tools covered live: S1, Datto RMM, IT Glue, Autotask, AWS Route 53. Others
 * report configured / not_configured only — extending to live checks is
 * mechanical.
 */

import { ListHostedZonesCommand } from '@aws-sdk/client-route-53';
import { isRoute53Configured, getRoute53Client } from '@/lib/services/route53-factory';
import { checkAllZoneDelegations } from '@/lib/services/route53-dns-delegation';
import { sanitizeAwsError } from '@/lib/services/route53-record-validation';

export type HealthStatus =
  | 'ok'              // configured, auth succeeded
  | 'degraded'        // configured, auth succeeded, but a secondary check found a problem
                      // (e.g. Route 53 D-12 NS-delegation mismatch) — reachable, not fully healthy
  | 'auth_failed'     // configured, server returned 401/403
  | 'unreachable'     // configured, network/DNS/TLS error
  | 'not_configured'  // env vars missing
  | 'unknown'         // configured, no live check implemented
  | 'disabled';       // operator-suppressed (see INTEGRATIONS_DISABLED env)

export interface TokenExpiry {
  envVar: string;
  expiresAt: string;          // ISO
  daysRemaining: number;      // negative when already expired
  subject?: string | null;
}

export interface IntegrationHealth {
  key: string;
  name: string;
  category: 'psa' | 'rmm' | 'docs' | 'security' | 'backup' | 'network' | 'identity' | 'mdm' | 'mail' | 'finance' | 'productivity' | 'llm';
  status: HealthStatus;
  configured: boolean;
  latencyMs?: number;
  error?: string | null;
  tokenExpiry?: TokenExpiry | null;
  checkedAt: string;
  /** D-12: zone names whose live NS answer mismatches Route 53's authoritative NS list. */
  nsDelegationMismatches?: string[] | null;
  /** D-12: zone names whose live NS lookup failed (infrastructure problem, not a mismatch). */
  nsDelegationErrors?: string[] | null;
}

interface CacheEntry {
  expiresAt: number;
  data: IntegrationHealth[];
}

const CACHE_TTL_MS = 5 * 60 * 1000;
let cache: CacheEntry | null = null;

function decodeJwt(token: string, envVar: string): TokenExpiry | null {
  if (!token || !token.startsWith('eyJ')) return null;
  const parts = token.split('.');
  if (parts.length < 2) return null;
  try {
    const b64 = parts[1].replace(/-/g, '+').replace(/_/g, '/');
    const pad = b64.length % 4 ? '='.repeat(4 - (b64.length % 4)) : '';
    const json = Buffer.from(b64 + pad, 'base64').toString('utf8');
    const claims = JSON.parse(json) as { exp?: number; sub?: string };
    if (!claims.exp) return null;
    const expiresMs = claims.exp * 1000;
    return {
      envVar,
      expiresAt: new Date(expiresMs).toISOString(),
      daysRemaining: (expiresMs - Date.now()) / 86400_000,
      subject: claims.sub ?? null,
    };
  } catch {
    return null;
  }
}

async function timed<T>(fn: () => Promise<T>): Promise<{ result: T; latencyMs: number }> {
  const start = Date.now();
  const result = await fn();
  return { result, latencyMs: Date.now() - start };
}

async function liveCheck(opts: {
  url: string;
  headers: Record<string, string>;
  timeoutMs?: number;
}): Promise<{ status: HealthStatus; error: string | null; latencyMs: number; httpStatus: number | null }> {
  const ctrl = new AbortController();
  const timeout = setTimeout(() => ctrl.abort(), opts.timeoutMs ?? 8000);
  try {
    const { result, latencyMs } = await timed(() =>
      fetch(opts.url, { headers: { accept: 'application/json', ...opts.headers }, signal: ctrl.signal })
    );
    clearTimeout(timeout);
    if (result.ok) return { status: 'ok', error: null, latencyMs, httpStatus: result.status };
    if (result.status === 401 || result.status === 403) {
      const body = await result.text().catch(() => '');
      return {
        status: 'auth_failed',
        error: `${result.status}: ${body.slice(0, 200)}`,
        latencyMs,
        httpStatus: result.status,
      };
    }
    return {
      status: 'unknown',
      error: `${result.status} ${result.statusText}`,
      latencyMs,
      httpStatus: result.status,
    };
  } catch (err) {
    clearTimeout(timeout);
    return {
      status: 'unreachable',
      error: err instanceof Error ? err.message : String(err),
      latencyMs: opts.timeoutMs ?? 8000,
      httpStatus: null,
    };
  }
}

async function checkS1(): Promise<IntegrationHealth> {
  const url = process.env.S1_API_URL?.replace(/\/$/, '');
  const token = process.env.S1_API_TOKEN;
  const checkedAt = new Date().toISOString();
  if (!url || !token) {
    return { key: 's1', name: 'SentinelOne', category: 'security', status: 'not_configured', configured: false, checkedAt };
  }
  const tokenExpiry = decodeJwt(token, 'S1_API_TOKEN');
  const live = await liveCheck({
    url: `${url}/web/api/v2.1/system/info`,
    headers: { Authorization: `ApiToken ${token}` },
  });
  return {
    key: 's1',
    name: 'SentinelOne',
    category: 'security',
    status: live.status,
    configured: true,
    latencyMs: live.latencyMs,
    error: live.error,
    tokenExpiry,
    checkedAt,
  };
}

async function checkDattoRmm(): Promise<IntegrationHealth> {
  const url = process.env.DATTO_RMM_API_URL?.replace(/\/$/, '');
  const key = process.env.DATTO_RMM_API_KEY;
  const secret = process.env.DATTO_RMM_API_SECRET;
  const checkedAt = new Date().toISOString();
  if (!url || !key || !secret) {
    return { key: 'datto_rmm', name: 'Datto RMM', category: 'rmm', status: 'not_configured', configured: false, checkedAt };
  }
  // OAuth password grant — same flow the client uses internally.
  const start = Date.now();
  try {
    const ctrl = new AbortController();
    const t = setTimeout(() => ctrl.abort(), 8000);
    const tokRes = await fetch(`${url}/auth/oauth/token`, {
      method: 'POST',
      headers: {
        'content-type': 'application/x-www-form-urlencoded',
        authorization: 'Basic ' + Buffer.from('public-client:public').toString('base64'),
      },
      body: `grant_type=password&username=${encodeURIComponent(key)}&password=${encodeURIComponent(secret)}`,
      signal: ctrl.signal,
    });
    clearTimeout(t);
    if (tokRes.ok) {
      return { key: 'datto_rmm', name: 'Datto RMM', category: 'rmm', status: 'ok', configured: true, latencyMs: Date.now() - start, checkedAt };
    }
    if (tokRes.status === 401 || tokRes.status === 403) {
      const body = await tokRes.text().catch(() => '');
      return {
        key: 'datto_rmm', name: 'Datto RMM', category: 'rmm', status: 'auth_failed',
        configured: true, latencyMs: Date.now() - start,
        error: `${tokRes.status}: ${body.slice(0, 200)}`, checkedAt,
      };
    }
    return {
      key: 'datto_rmm', name: 'Datto RMM', category: 'rmm', status: 'unknown',
      configured: true, latencyMs: Date.now() - start,
      error: `${tokRes.status} ${tokRes.statusText}`, checkedAt,
    };
  } catch (err) {
    return {
      key: 'datto_rmm', name: 'Datto RMM', category: 'rmm', status: 'unreachable',
      configured: true, latencyMs: Date.now() - start,
      error: err instanceof Error ? err.message : String(err), checkedAt,
    };
  }
}

const AWS_AUTH_ERROR_NAMES = new Set([
  'InvalidClientTokenId',
  'SignatureDoesNotMatch',
  'AccessDenied',
  'UnrecognizedClientException',
]);

function isAwsAuthError(err: unknown): boolean {
  if (!(err instanceof Error)) return false;
  const name = (err as Error & { name?: string }).name;
  if (name && AWS_AUTH_ERROR_NAMES.has(name)) return true;
  const httpStatusCode = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
  return httpStatusCode === 401 || httpStatusCode === 403;
}

const ROUTE53_ZONE_CHECK_LIMIT = 50;
const ROUTE53_AUTH_PROBE_TIMEOUT_MS = 8000;

async function checkRoute53(): Promise<IntegrationHealth> {
  const checkedAt = new Date().toISOString();
  if (!isRoute53Configured()) {
    return {
      key: 'route53', name: 'AWS Route 53', category: 'network',
      status: 'not_configured', configured: false, checkedAt,
    };
  }

  const start = Date.now();
  let status: HealthStatus;
  let error: string | null = null;
  try {
    // Bounded the same way liveCheck() bounds every other integration's
    // fetch() (8s) — the AWS SDK's own retry policy has no caller-supplied
    // deadline, and checkIntegrationHealth() fans out via Promise.all, so an
    // unbounded call here would extend the whole aggregate's latency.
    const ctrl = new AbortController();
    const timeout = setTimeout(() => ctrl.abort(), ROUTE53_AUTH_PROBE_TIMEOUT_MS);
    try {
      await getRoute53Client().send(new ListHostedZonesCommand({ MaxItems: 1 }), {
        abortSignal: ctrl.signal,
      });
    } finally {
      clearTimeout(timeout);
    }
    status = 'ok';
  } catch (err) {
    status = isAwsAuthError(err) ? 'auth_failed' : 'unreachable';
    error = sanitizeAwsError(err);
  }
  const latencyMs = Date.now() - start;

  // D-12: live NS-delegation check. Wrapped in its own try/catch — a
  // Postgres failure or a blocked resolver must degrade to
  // nsDelegationErrors, never throw out of checkIntegrationHealth()'s
  // Promise.all (T-24-16).
  let nsDelegationMismatches: string[] | null = null;
  let nsDelegationErrors: string[] | null = null;
  try {
    // Lazy import to avoid pulling postgres-client into edge runtimes.
    const { default: postgresClient } = await import('@/lib/services/postgres-client');
    const res = await postgresClient.query<{ id: string; name: string; authoritative_name_servers: unknown }>(
      `SELECT id, name, authoritative_name_servers
       FROM route53_zones
       WHERE is_deleted = false
       ORDER BY name
       LIMIT ${ROUTE53_ZONE_CHECK_LIMIT}`,
    );
    const zones = res.rows.map((r) => ({
      id: r.id,
      name: r.name,
      authoritativeNameServers: r.authoritative_name_servers,
    }));
    const delegationResults = await checkAllZoneDelegations(zones);
    const mismatches = delegationResults.filter((r) => r.mismatch).map((r) => r.zoneName);
    const errors = delegationResults.filter((r) => r.error).map((r) => r.zoneName);
    if (mismatches.length > 0) nsDelegationMismatches = mismatches;
    if (errors.length > 0) nsDelegationErrors = errors;

    if (mismatches.length > 0 && status === 'ok') {
      status = 'degraded';
      const truncationNote = res.rowCount === ROUTE53_ZONE_CHECK_LIMIT
        ? ` (checked first ${ROUTE53_ZONE_CHECK_LIMIT} zones by name)`
        : '';
      error = `NS delegation mismatch for ${mismatches.length} zone(s): ${mismatches.join(', ')}${truncationNote}`;
    }
  } catch (err) {
    // Postgres unreachable, migration not applied yet, or resolver blocked —
    // an infrastructure problem, not evidence of delegation drift. Preserve
    // the auth-probe status; just note the delegation check itself failed.
    nsDelegationErrors = [err instanceof Error ? err.message : String(err)];
  }

  return {
    key: 'route53', name: 'AWS Route 53', category: 'network',
    status, configured: true, latencyMs, error, checkedAt,
    nsDelegationMismatches, nsDelegationErrors,
  };
}

async function checkItglue(): Promise<IntegrationHealth> {
  const apiKey = process.env.ITGLUE_API_KEY;
  const checkedAt = new Date().toISOString();
  if (!apiKey) {
    return { key: 'itglue', name: 'IT Glue', category: 'docs', status: 'not_configured', configured: false, checkedAt };
  }
  const live = await liveCheck({
    url: 'https://api.itglue.com/organizations?page[size]=1',
    headers: { 'x-api-key': apiKey },
  });
  return {
    key: 'itglue', name: 'IT Glue', category: 'docs',
    status: live.status, configured: true,
    latencyMs: live.latencyMs, error: live.error,
    checkedAt,
  };
}

async function checkAutotask(): Promise<IntegrationHealth> {
  const url = process.env.AUTOTASK_API_URL?.replace(/\/$/, '');
  const user = process.env.AUTOTASK_USERNAME;
  const secret = process.env.AUTOTASK_SECRET;
  const code = process.env.AUTOTASK_API_INTEGRATION_CODE;
  const checkedAt = new Date().toISOString();
  if (!url || !user || !secret || !code) {
    return { key: 'autotask', name: 'Autotask', category: 'psa', status: 'not_configured', configured: false, checkedAt };
  }
  // Cheapest authenticated call — version endpoint (not behind auth at all
  // tenants, but failing here usually means URL/credential mismatch).
  const live = await liveCheck({
    url: `${url}/v1.0/Version`,
    headers: {
      ApiIntegrationCode: code,
      UserName: user,
      Secret: secret,
    },
  });
  return {
    key: 'autotask', name: 'Autotask', category: 'psa',
    status: live.status, configured: true,
    latencyMs: live.latencyMs, error: live.error,
    checkedAt,
  };
}

function checkConfigOnly(
  key: string,
  name: string,
  category: IntegrationHealth['category'],
  envVars: string[]
): IntegrationHealth {
  const checkedAt = new Date().toISOString();
  const allSet = envVars.every((v) => !!process.env[v]);
  return {
    key, name, category,
    status: allSet ? 'unknown' : 'not_configured',
    configured: allSet,
    checkedAt,
  };
}

/**
 * Operator-side disable list.  Two sources, merged:
 *
 *   1. INTEGRATIONS_DISABLED env var (legacy / bootstrap fallback) —
 *      comma- or space-separated keys with aliases.
 *   2. integration_settings table (DB-backed, admin-toggleable at
 *      /admin/integrations) — takes effect within the 5-minute health
 *      cache without requiring a container restart.
 *
 * Aliases (env only — DB rows store canonical keys):
 *   sentinelone, s1   → s1
 *   datto, datto-rmm  → datto_rmm
 *   itglue, it-glue   → itglue
 *   msgraph, ms-graph → msgraph
 */
const KEY_ALIASES: Record<string, string> = {
  sentinelone: 's1',
  's1': 's1',
  datto: 'datto_rmm',
  'datto-rmm': 'datto_rmm',
  'datto_rmm': 'datto_rmm',
  itglue: 'itglue',
  'it-glue': 'itglue',
  'it_glue': 'itglue',
  msgraph: 'msgraph',
  'ms-graph': 'msgraph',
  'ms_graph': 'msgraph',
};

function getEnvDisabledKeys(): Set<string> {
  const raw = process.env.INTEGRATIONS_DISABLED;
  if (!raw) return new Set();
  return new Set(
    raw
      .split(/[\s,]+/)
      .map((s) => s.trim().toLowerCase())
      .filter(Boolean)
      .map((s) => KEY_ALIASES[s] ?? s),
  );
}

async function getDbDisabledKeys(): Promise<Set<string>> {
  // Lazy import to avoid pulling postgres-client into edge runtimes.
  const { default: postgresClient } = await import('@/lib/services/postgres-client');
  try {
    const res = await postgresClient.query<{ key: string }>(
      `SELECT key FROM integration_settings WHERE disabled = true`,
    );
    return new Set(res.rows.map((r) => r.key));
  } catch {
    // Migration not applied yet, or DB unreachable.  Don't break health
    // checks — fall back to env-only behavior.
    return new Set();
  }
}

async function applyDisableOverlay(items: IntegrationHealth[]): Promise<IntegrationHealth[]> {
  const [envSet, dbSet] = [getEnvDisabledKeys(), await getDbDisabledKeys()];
  if (envSet.size === 0 && dbSet.size === 0) return items;
  const disabled = new Set([...envSet, ...dbSet]);
  return items.map((item) =>
    disabled.has(item.key)
      ? { ...item, status: 'disabled', error: null, configured: false }
      : item,
  );
}

export async function checkIntegrationHealth(opts?: { skipCache?: boolean }): Promise<IntegrationHealth[]> {
  if (!opts?.skipCache && cache && cache.expiresAt > Date.now()) {
    return cache.data;
  }
  const results = await Promise.all([
    checkAutotask(),
    checkDattoRmm(),
    checkItglue(),
    checkS1(),
    checkRoute53(),
    Promise.resolve(checkConfigOnly('veeam', 'Veeam VSPC', 'backup',
      ['VEEAM_VSPC_URL', 'VEEAM_VSPC_API_KEY'])),
    Promise.resolve(checkConfigOnly('msgraph', 'Microsoft Graph', 'productivity',
      ['MSGRAPH_CLIENT_ID', 'MSGRAPH_CLIENT_SECRET', 'MSGRAPH_TENANT_ID'])),
    Promise.resolve(checkConfigOnly('auvik', 'Auvik', 'network',
      ['AUVIK_API_URL', 'AUVIK_API_USER', 'AUVIK_API_KEY'])),
    Promise.resolve(checkConfigOnly('addigy', 'Addigy', 'mdm',
      ['ADDIGY_API_URL', 'ADDIGY_API_TOKEN', 'ADDIGY_ORG_ID'])),
    Promise.resolve(checkConfigOnly('mimecast', 'Mimecast', 'mail',
      ['MIMECAST_CLIENT_ID', 'MIMECAST_CLIENT_SECRET'])),
    Promise.resolve(checkConfigOnly('duo', 'Duo', 'identity',
      ['DUO_API_HOST', 'DUO_INTEGRATION_KEY', 'DUO_SECRET_KEY'])),
    Promise.resolve(checkConfigOnly('zabbix', 'Zabbix', 'network',
      ['ZABBIX_API_URL', 'ZABBIX_API_TOKEN'])),
    Promise.resolve(checkConfigOnly('qbo', 'QuickBooks Online', 'finance',
      ['QBO_CLIENT_ID', 'QBO_CLIENT_SECRET'])),
    Promise.resolve(checkConfigOnly('appgate', 'AppGate SDP', 'security',
      ['APPGATE_URL', 'APPGATE_USERNAME', 'APPGATE_PASSWORD', 'APPGATE_DEVICE_ID'])),
    Promise.resolve(checkConfigOnly('pax8', 'PAX8', 'finance',
      ['PAX8_CLIENT_ID', 'PAX8_CLIENT_SECRET'])),
    Promise.resolve(checkConfigOnly('anthropic', 'Anthropic', 'llm',
      ['ANTHROPIC_API_KEY'])),
  ]);
  const overlaid = await applyDisableOverlay(results);
  cache = { expiresAt: Date.now() + CACHE_TTL_MS, data: overlaid };
  return overlaid;
}

export function clearIntegrationHealthCache(): void {
  cache = null;
}

export interface HealthSummary {
  total: number;
  ok: number;
  failed: number;
  notConfigured: number;
  disabled: number;
  expiringWithin14Days: number;
  expired: number;
  hasIssues: boolean;
}

export function summarize(items: IntegrationHealth[]): HealthSummary {
  let ok = 0, failed = 0, notConfigured = 0, disabled = 0;
  let expiringWithin14Days = 0, expired = 0;
  for (const i of items) {
    if (i.status === 'disabled') {
      disabled += 1;
      continue;
    }
    if (i.status === 'ok' || i.status === 'unknown') ok += 1;
    else if (i.status === 'auth_failed' || i.status === 'unreachable' || i.status === 'degraded') failed += 1;
    else if (i.status === 'not_configured') notConfigured += 1;
    if (i.tokenExpiry) {
      if (i.tokenExpiry.daysRemaining <= 0) expired += 1;
      else if (i.tokenExpiry.daysRemaining <= 14) expiringWithin14Days += 1;
    }
  }
  return {
    total: items.length,
    ok, failed, notConfigured, disabled,
    expiringWithin14Days, expired,
    hasIssues: failed > 0 || expired > 0 || expiringWithin14Days > 0,
  };
}