wulf-pulse/migrations/034_seed_veeam_backup_failure_pipeline.sql

-- Migration 034: Seed Veeam Backup Failure Diagnostic Pipeline
-- A comprehensive pipeline that enriches, diagnoses, and creates smart tickets
-- for Veeam backup failure alerts from Datto RMM.

INSERT INTO webhook_pipelines (name, description, is_active, trigger_source, trigger_conditions, sort_order)
VALUES (
  'Veeam Backup Failure → Smart Diagnostic Ticket',
  'When RMM detects a Veeam backup failure: enrich from VSPC + DB, run diagnostics via quick job, AI-analyze all findings, create rich Autotask ticket, notify Teams.',
  false,
  'datto_rmm',
  '[
    {"field": "triggered", "operator": "equals", "value": "True"},
    {"field": "alert_message_en", "operator": "contains", "value": "Veeam"}
  ]'::jsonb,
  10
)
ON CONFLICT DO NOTHING;

-- Get the pipeline ID
DO $$
DECLARE
  pid INTEGER;
BEGIN
  SELECT id INTO pid FROM webhook_pipelines WHERE name = 'Veeam Backup Failure → Smart Diagnostic Ticket' LIMIT 1;

  IF pid IS NULL THEN
    RAISE NOTICE 'Pipeline not found, skipping step insertion';
    RETURN;
  END IF;

  -- Step 1: Extract alert fields from RMM payload
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 1, 'transform', 'Extract alert fields', '{
    "mappings": {
      "device_hostname": "{{trigger.device_hostname}}",
      "device_uid": "{{trigger.device_uid}}",
      "site_name": "{{trigger.site_name}}",
      "site_uid": "{{trigger.site_uid}}",
      "alert_type": "{{trigger.alert_type}}",
      "alert_message": "{{trigger.alert_message_en}}",
      "alert_uid": "{{trigger.alert_uid}}",
      "alert_priority": "{{trigger.alert_priority}}",
      "device_ip": "{{trigger.device_ip}}",
      "device_os": "{{trigger.device_os}}",
      "last_user": "{{trigger.last_user}}"
    }
  }'::jsonb, 'stop');

  -- Step 2: Enrich device from local RMM DB
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 2, 'enrich_device', 'Lookup device details', '{
    "lookup_by": "device_uid",
    "source_field": "{{context.device_uid}}"
  }'::jsonb, 'continue');

  -- Step 3: Enrich company from site name
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 3, 'enrich_company', 'Lookup company from site', '{
    "lookup_by": "site_name",
    "source_field": "{{context.site_name}}"
  }'::jsonb, 'continue');

  -- Step 4: Query VSPC for backup status
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 4, 'enrich_vspc', 'VSPC backup status lookup', '{
    "lookup_by": "device_name",
    "source_field": "{{context.device_hostname}}"
  }'::jsonb, 'continue');

  -- Step 5: DB query — backup failure trend (last 7 days)
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 5, 'db_query', 'Backup failure trend (7 days)', '{
    "query": "SELECT status, COUNT(*) as count, MAX(last_run) as latest FROM veeam_backup_agent_jobs WHERE LOWER(name) LIKE LOWER($1) AND last_run > NOW() - INTERVAL ''7 days'' GROUP BY status ORDER BY count DESC",
    "params": ["%{{context.device_hostname}}%"],
    "output_key": "backup_trend",
    "single_row": false
  }'::jsonb, 'continue');

  -- Step 6: DB query — recent RMM alerts for this device (pattern detection)
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 6, 'db_query', 'Recent alerts for device (7 days)', '{
    "query": "SELECT alert_type, priority, message, resolved, created_at FROM datto_rmm_alerts WHERE device_uid = $1 AND created_at > NOW() - INTERVAL ''7 days'' ORDER BY created_at DESC LIMIT 20",
    "params": ["{{context.device_uid}}"],
    "output_key": "recent_alerts",
    "single_row": false
  }'::jsonb, 'continue');

  -- Step 7: Run diagnostic script on device via RMM Quick Job
  -- NOTE: component_uid must be set after uploading the script to Datto RMM
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 7, 'rmm_quick_job', 'Run Veeam diagnostic script', '{
    "device_uid": "{{context.device_uid}}",
    "component_uid": "REPLACE_WITH_COMPONENT_UID",
    "job_name": "Veeam Backup Diagnostic - {{context.device_hostname}}"
  }'::jsonb, 'continue');

  -- Step 8: Wait for job results
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 8, 'delay', 'Wait for diagnostic script', '{
    "seconds": 60
  }'::jsonb, 'continue');

  -- Step 9: Get job results
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 9, 'rmm_get_job_results', 'Retrieve diagnostic results', '{
    "job_uid": "{{context.job_uid}}",
    "device_uid": "{{context.device_uid}}"
  }'::jsonb, 'continue');

  -- Step 10: AI analysis of all collected data
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 10, 'ai_analyze', 'AI root cause analysis', '{
    "system_prompt": "You are a senior systems engineer specializing in Veeam Backup & Replication and Windows Server infrastructure. Analyze the provided diagnostic data and give a clear, actionable assessment.",
    "prompt": "A Veeam backup failure alert was triggered for device {{context.device_hostname}} at site {{context.site_name}}.\n\n## Alert Details\n- Type: {{context.alert_type}}\n- Message: {{context.alert_message}}\n- Priority: {{context.alert_priority}}\n- Device OS: {{context.device_os}}\n- Last User: {{context.last_user}}\n\n## VSPC Backup Status\n{{context.vspc_summary}}\n\n## Backup Trend (Last 7 Days)\n{{context.backup_trend}}\n\n## Recent RMM Alerts for This Device\n{{context.recent_alerts}}\n\n## On-Device Diagnostic Script Results\n{{context.job_results}}\n\nBased on ALL of this data:\n1. What is the most likely ROOT CAUSE of the backup failure?\n2. Is this a recurring issue or a one-time failure?\n3. What are the specific REMEDIATION STEPS (in order of priority)?\n4. Is this CRITICAL (needs immediate attention) or can it wait?\n5. Are there any related issues that should be addressed?",
    "max_tokens": 1500
  }'::jsonb, 'continue');

  -- Step 11: Create rich Autotask ticket
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 11, 'create_ticket', 'Create diagnostic ticket', '{
    "template": {
      "title": "[Veeam Backup Failure] {{context.device_hostname}} - {{context.site_name}}",
      "description": "## Automated Veeam Backup Failure Diagnostic\n\n**Device:** {{context.device_hostname}} ({{context.device_ip}})\n**Site:** {{context.site_name}}\n**Alert:** {{context.alert_message}}\n**OS:** {{context.device_os}}\n**Last User:** {{context.last_user}}\n\n---\n\n## VSPC Backup Status\n{{context.vspc_summary}}\n\n**Last Successful Backup:** {{context.vspc_last_success}} ({{context.vspc_hours_since_success}}h ago)\n**Failed Jobs:** {{context.vspc_failed_job_count}}\n**Active Alarms:** {{context.vspc_alarm_count}}\n**Restore Points:** {{context.vspc_restore_points}}\n\n---\n\n## AI Root Cause Analysis\n{{context.ai_response}}\n\n---\n\n## On-Device Diagnostics\n{{context.job_results}}\n\n---\n\n## Backup Trend (7 Days)\n{{context.backup_trend}}\n\n## Recent Device Alerts\n{{context.recent_alerts}}\n\n---\n*This ticket was automatically generated by Pulse Pipeline Engine with full diagnostic enrichment.*",
      "companyID": "{{context.company_id}}",
      "ticketType": 2,
      "priority": 2,
      "status": 1,
      "queueID": 29682833
    }
  }'::jsonb, 'stop');

  -- Step 12: Add AI analysis as internal note
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 12, 'create_note', 'Add AI analysis note', '{
    "ticket_id": "{{context.ticket_id}}",
    "title": "AI Root Cause Analysis",
    "body": "{{context.ai_response}}",
    "note_type": 1,
    "publish": 1
  }'::jsonb, 'continue');

  -- Step 13: Notify Teams
  -- NOTE: channel_id must be set after creating a notification channel
  INSERT INTO pipeline_steps (pipeline_id, step_order, step_type, name, config, on_failure)
  VALUES (pid, 13, 'notify', 'Notify Teams channel', '{
    "channel_id": 1,
    "title": "Veeam Backup Failure: {{context.device_hostname}}",
    "message": "**Device:** {{context.device_hostname}} @ {{context.site_name}}\n**Alert:** {{context.alert_message}}\n**Last Success:** {{context.vspc_last_success}} ({{context.vspc_hours_since_success}}h ago)\n**Failed Jobs:** {{context.vspc_failed_job_count}}\n**Ticket:** #{{context.ticket_number}}\n\n**AI Assessment:**\n{{context.ai_response}}"
  }'::jsonb, 'continue');

  RAISE NOTICE 'Veeam Backup Failure pipeline seeded with 13 steps (pipeline_id=%)', pid;
END $$;