<#
.SYNOPSIS
    Veeam Backup Diagnostic Script — Run via Datto RMM Quick Job
.DESCRIPTION
    Checks Veeam services, backup job status, disk space, event logs,
    and network connectivity. Returns structured JSON for pipeline consumption.
.NOTES
    Deploy as a Datto RMM component. Output via Write-Host for StdOut capture.
    Compatible with PowerShell 5.1+.
#>

try {

$ErrorActionPreference = 'SilentlyContinue'

$result = @{
    timestamp       = ([DateTime]::UtcNow.ToString('yyyy-MM-dd HH:mm:ss UTC'))
    hostname        = $env:COMPUTERNAME
    checks          = @{}
    issues_found    = @()
    recommendations = @()
}

# ============================================================================
# 1. Veeam Services Status
# ============================================================================
$veeamServices = @(
    'VeeamBackupSvc',
    'VeeamBrokerSvc',
    'VeeamCatalogSvc',
    'VeeamCloudSvc',
    'VeeamDeploySvc',
    'VeeamDistributionSvc',
    'VeeamMountSvc',
    'VeeamNFSSvc',
    'VeeamTransportSvc',
    'VeeamEndpointBackupSvc',
    'VeeamFilesysVssSvc'
)

$serviceResults = @()
$stoppedCritical = @()

foreach ($svcName in $veeamServices) {
    $svc = Get-Service -Name $svcName -ErrorAction SilentlyContinue
    if ($svc) {
        $serviceResults += @{
            name        = $svc.Name
            display     = $svc.DisplayName
            status      = $svc.Status.ToString()
            start_type  = $svc.StartType.ToString()
        }
        if ($svc.Status -ne 'Running' -and $svc.StartType -ne 'Disabled') {
            $stoppedCritical += $svc.DisplayName
        }
    }
}

$result.checks.services = @{
    total_found    = $serviceResults.Count
    services       = $serviceResults
    stopped_critical = $stoppedCritical
}

if ($stoppedCritical.Count -gt 0) {
    $result.issues_found += "Veeam services not running: $($stoppedCritical -join ', ')"
    $result.recommendations += "Restart stopped Veeam services: $($stoppedCritical -join ', ')"
}

# ============================================================================
# 2. Veeam Backup Job Status (via PowerShell Snap-in if available)
# ============================================================================
$jobResults = @()
$vbrSnapinLoaded = $false

try {
    if (Get-PSSnapin -Registered -Name VeeamPSSnapin -ErrorAction SilentlyContinue) {
        Add-PSSnapin VeeamPSSnapin -ErrorAction Stop
        $vbrSnapinLoaded = $true
    }
    elseif (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell -ErrorAction SilentlyContinue) {
        Import-Module Veeam.Backup.PowerShell -ErrorAction Stop
        $vbrSnapinLoaded = $true
    }
} catch {
    # Snap-in not available — skip VBR-specific checks
}

if ($vbrSnapinLoaded) {
    try {
        $jobs = Get-VBRJob -ErrorAction SilentlyContinue
        foreach ($job in $jobs) {
            $lastSession = $job.FindLastSession()
            $jobResults += @{
                name        = $job.Name
                type        = $job.TypeToString
                is_enabled  = $job.IsScheduleEnabled
                status      = if ($lastSession) { $lastSession.Result.ToString() } else { 'NoSession' }
                last_run    = if ($lastSession) { $lastSession.CreationTime.ToString('yyyy-MM-dd HH:mm:ss') } else { $null }
                end_time    = if ($lastSession) { $lastSession.EndTime.ToString('yyyy-MM-dd HH:mm:ss') } else { $null }
                duration_min = if ($lastSession -and $lastSession.EndTime -gt $lastSession.CreationTime) {
                    [math]::Round(($lastSession.EndTime - $lastSession.CreationTime).TotalMinutes, 1)
                } else { $null }
                failure_msg = if ($lastSession -and $lastSession.Result -eq 'Failed') {
                    ($lastSession.GetTaskSessions() | Where-Object { $_.Status -eq 'Failed' } |
                     Select-Object -First 1 -ExpandProperty Details -ErrorAction SilentlyContinue)
                } else { $null }
            }
        }

        $failedJobs = $jobResults | Where-Object { $_.status -eq 'Failed' }
        if ($failedJobs.Count -gt 0) {
            $result.issues_found += "Failed backup jobs: $(($failedJobs | ForEach-Object { $_.name }) -join ', ')"
            $result.recommendations += "Investigate failed jobs and check task session logs in Veeam console"
        }

        # Check for stuck/running jobs > 24h
        $stuckJobs = $jobResults | Where-Object {
            $_.status -eq 'Working' -and $_.last_run -and
            ((Get-Date) - [datetime]$_.last_run).TotalHours -gt 24
        }
        if ($stuckJobs.Count -gt 0) {
            $result.issues_found += "Stuck jobs running >24h: $(($stuckJobs | ForEach-Object { $_.name }) -join ', ')"
            $result.recommendations += "Consider stopping and restarting stuck backup jobs"
        }
    } catch {
        $jobResults = @(@{ error = $_.Exception.Message })
    }
}

$result.checks.backup_jobs = @{
    vbr_available = $vbrSnapinLoaded
    total_jobs    = $jobResults.Count
    jobs          = $jobResults
}

# ============================================================================
# 3. Disk Space Check (all fixed drives)
# ============================================================================
$diskResults = @()
$lowDiskDrives = @()

$drives = Get-WmiObject Win32_LogicalDisk -Filter "DriveType=3" -ErrorAction SilentlyContinue
foreach ($drive in $drives) {
    $freeGB = [math]::Round($drive.FreeSpace / 1GB, 2)
    $totalGB = [math]::Round($drive.Size / 1GB, 2)
    $usedPct = if ($totalGB -gt 0) { [math]::Round((($totalGB - $freeGB) / $totalGB) * 100, 1) } else { 0 }

    $diskResults += @{
        drive     = $drive.DeviceID
        label     = $drive.VolumeName
        total_gb  = $totalGB
        free_gb   = $freeGB
        used_pct  = $usedPct
    }

    if ($usedPct -gt 90) {
        $lowDiskDrives += "$($drive.DeviceID) ($usedPct% used, $freeGB GB free)"
    }
}

$result.checks.disk_space = @{
    drives        = $diskResults
    low_disk      = $lowDiskDrives
}

if ($lowDiskDrives.Count -gt 0) {
    $result.issues_found += "Low disk space: $($lowDiskDrives -join ', ')"
    $result.recommendations += "Free disk space or expand storage on affected drives"
}

# ============================================================================
# 4. Windows Event Log — Veeam errors (last 48 hours)
# ============================================================================
$eventResults = @()
$cutoff = (Get-Date).AddHours(-48)

# Veeam Backup log
$veeamEvents = Get-WinEvent -FilterHashtable @{
    LogName   = 'Veeam Backup'
    Level     = @(1, 2)  # Critical, Error
    StartTime = $cutoff
} -MaxEvents 20 -ErrorAction SilentlyContinue

foreach ($evt in $veeamEvents) {
    $eventResults += @{
        source    = 'Veeam Backup'
        level     = $evt.LevelDisplayName
        id        = $evt.Id
        time      = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
        message   = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
    }
}

# Veeam Agent log
$agentEvents = Get-WinEvent -FilterHashtable @{
    LogName   = 'Veeam Agent'
    Level     = @(1, 2)
    StartTime = $cutoff
} -MaxEvents 10 -ErrorAction SilentlyContinue

foreach ($evt in $agentEvents) {
    $eventResults += @{
        source    = 'Veeam Agent'
        level     = $evt.LevelDisplayName
        id        = $evt.Id
        time      = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
        message   = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
    }
}

# Application log — Veeam source
$appEvents = Get-WinEvent -FilterHashtable @{
    LogName      = 'Application'
    ProviderName = @('Veeam*')
    Level        = @(1, 2)
    StartTime    = $cutoff
} -MaxEvents 10 -ErrorAction SilentlyContinue

foreach ($evt in $appEvents) {
    $eventResults += @{
        source    = "Application/$($evt.ProviderName)"
        level     = $evt.LevelDisplayName
        id        = $evt.Id
        time      = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
        message   = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
    }
}

$result.checks.event_logs = @{
    total_errors = $eventResults.Count
    events       = $eventResults
}

if ($eventResults.Count -gt 0) {
    $result.issues_found += "$($eventResults.Count) Veeam error events in last 48h"
}

# ============================================================================
# 5. Veeam Process Check — is anything stuck?
# ============================================================================
$veeamProcesses = Get-Process -Name "Veeam*" -ErrorAction SilentlyContinue |
    Select-Object Name, Id, CPU,
        @{N='MemoryMB';E={[math]::Round($_.WorkingSet64/1MB,1)}},
        @{N='RunningHours';E={[math]::Round(((Get-Date) - $_.StartTime).TotalHours, 1)}}

$stuckProcesses = $veeamProcesses | Where-Object { $_.RunningHours -gt 48 }

$result.checks.processes = @{
    running   = @($veeamProcesses | ForEach-Object {
        @{ name = $_.Name; pid = $_.Id; memory_mb = $_.MemoryMB; running_hours = $_.RunningHours }
    })
    stuck     = @($stuckProcesses | ForEach-Object { $_.Name })
}

if ($stuckProcesses.Count -gt 0) {
    $result.issues_found += "Potentially stuck Veeam processes (>48h): $(($stuckProcesses | ForEach-Object { $_.Name }) -join ', ')"
    $result.recommendations += "Review and potentially restart long-running Veeam processes"
}

# ============================================================================
# 6. Network Connectivity to Backup Targets
# ============================================================================
$networkResults = @()

# Try to find backup repository paths from registry
$repoKeys = Get-ItemProperty -Path "HKLM:\SOFTWARE\Veeam\Veeam Backup and Replication" -ErrorAction SilentlyContinue
$sqlServer = $repoKeys.SqlServerName

if ($sqlServer) {
    $testSql = Test-NetConnection -ComputerName $sqlServer -Port 1433 -WarningAction SilentlyContinue -ErrorAction SilentlyContinue
    $networkResults += @{
        target  = "SQL: $sqlServer"
        port    = 1433
        success = $testSql.TcpTestSucceeded
    }
    if (-not $testSql.TcpTestSucceeded) {
        $result.issues_found += "Cannot reach Veeam SQL server: $sqlServer"
        $result.recommendations += "Check network connectivity and SQL Server service on $sqlServer"
    }
}

# Test common backup infrastructure ports
$vbrServer = $repoKeys.SqlDatabaseName  # Often same host
$localPorts = @(
    @{ Name = "Veeam Backup Service"; Port = 9392 },
    @{ Name = "Veeam REST API"; Port = 9419 },
    @{ Name = "Veeam Cloud Connect"; Port = 6180 }
)

foreach ($p in $localPorts) {
    $test = Test-NetConnection -ComputerName 'localhost' -Port $p.Port -WarningAction SilentlyContinue -ErrorAction SilentlyContinue
    $networkResults += @{
        target  = $p.Name
        port    = $p.Port
        success = $test.TcpTestSucceeded
    }
}

$result.checks.network = @{
    tests = $networkResults
}

# ============================================================================
# 7. Summary
# ============================================================================
$result.total_issues = $result.issues_found.Count
$result.severity = if ($result.issues_found.Count -eq 0) { 'OK' }
    elseif ($result.issues_found.Count -le 2) { 'WARNING' }
    else { 'CRITICAL' }

# ============================================================================
# 8. Upload to B2 (S3-compatible) and output object key
# ============================================================================
$jsonOutput = $result | ConvertTo-Json -Depth 5 -Compress

# B2 credentials — set these as Datto RMM component variables or site variables
$b2KeyId     = if ($env:B2_KEY_ID)   { $env:B2_KEY_ID }   else { $env:usrB2KeyId }
$b2AppKey    = if ($env:B2_APP_KEY)  { $env:B2_APP_KEY }  else { $env:usrB2AppKey }
$b2Bucket    = if ($env:B2_BUCKET)   { $env:B2_BUCKET }   else { if ($env:usrB2Bucket) { $env:usrB2Bucket } else { 'wulf-audits' } }
$b2Region    = if ($env:B2_REGION)   { $env:B2_REGION }   else { if ($env:usrB2Region) { $env:usrB2Region } else { 'us-west-002' } }
$b2Endpoint  = if ($env:B2_ENDPOINT) { $env:B2_ENDPOINT } else { "s3.$b2Region.backblazeb2.com" }

$datePrefix  = (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd')
$timeStamp   = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
$objectKey   = "diagnostics/$($env:COMPUTERNAME)/$datePrefix/$timeStamp.json"

if ($b2KeyId -and $b2AppKey) {
    try {
        # S3v4 presigned PUT
        $method        = 'PUT'
        $host_          = $b2Endpoint
        $canonicalUri  = "/$b2Bucket/$objectKey"
        $algorithm     = 'AWS4-HMAC-SHA256'
        $amzDate       = $timeStamp
        $dateStamp     = $amzDate.Substring(0, 8)
        $credScope     = "$dateStamp/$b2Region/s3/aws4_request"
        $contentHash   = [System.BitConverter]::ToString(
            [System.Security.Cryptography.SHA256]::Create().ComputeHash(
                [System.Text.Encoding]::UTF8.GetBytes($jsonOutput)
            )
        ).Replace('-','').ToLower()

        $canonicalHeaders = "content-type:application/json`nhost:$host_`nx-amz-content-sha256:$contentHash`nx-amz-date:$amzDate`n"
        $signedHeaders    = 'content-type;host;x-amz-content-sha256;x-amz-date'

        $canonicalRequest = "$method`n$canonicalUri`n`n$canonicalHeaders`n$signedHeaders`n$contentHash"
        $crHash = [System.BitConverter]::ToString(
            [System.Security.Cryptography.SHA256]::Create().ComputeHash(
                [System.Text.Encoding]::UTF8.GetBytes($canonicalRequest)
            )
        ).Replace('-','').ToLower()

        $stringToSign = "$algorithm`n$amzDate`n$credScope`n$crHash"

        # Derive signing key
        function HmacSHA256($key, $data) {
            $hmac = New-Object System.Security.Cryptography.HMACSHA256
            $hmac.Key = if ($key -is [byte[]]) { $key } else { [System.Text.Encoding]::UTF8.GetBytes($key) }
            return $hmac.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($data))
        }

        $kDate    = HmacSHA256 "AWS4$b2AppKey" $dateStamp
        $kRegion  = HmacSHA256 $kDate $b2Region
        $kService = HmacSHA256 $kRegion 's3'
        $kSigning = HmacSHA256 $kService 'aws4_request'

        $signature = [System.BitConverter]::ToString(
            (HmacSHA256 $kSigning $stringToSign)
        ).Replace('-','').ToLower()

        $authHeader = "$algorithm Credential=$b2KeyId/$credScope, SignedHeaders=$signedHeaders, Signature=$signature"

        $headers = @{
            'Authorization'        = $authHeader
            'x-amz-date'          = $amzDate
            'x-amz-content-sha256' = $contentHash
            'Content-Type'         = 'application/json'
        }

        $uri = "https://$host_$canonicalUri"
        $bodyBytes = [System.Text.Encoding]::UTF8.GetBytes($jsonOutput)

        # Use .NET WebRequest for PS 5.1 compatibility
        $webRequest = [System.Net.HttpWebRequest]::Create($uri)
        $webRequest.Method = 'PUT'
        $webRequest.ContentType = 'application/json'
        $webRequest.ContentLength = $bodyBytes.Length
        foreach ($h in $headers.GetEnumerator()) {
            if ($h.Key -notin @('Content-Type')) {
                $webRequest.Headers.Add($h.Key, $h.Value)
            }
        }

        $stream = $webRequest.GetRequestStream()
        $stream.Write($bodyBytes, 0, $bodyBytes.Length)
        $stream.Close()

        $response = $webRequest.GetResponse()
        $statusCode = [int]$response.StatusCode
        $response.Close()

        if ($statusCode -eq 200) {
            # Success — output object key for pipeline to fetch
            Write-Host $objectKey
        } else {
            # Upload failed — fall back to inline JSON
            Write-Host "UPLOAD_FAILED:$statusCode"
            Write-Host $jsonOutput
        }
    } catch {
        # Upload error — fall back to inline JSON
        Write-Host "UPLOAD_ERROR:$($_.Exception.Message)"
        Write-Host $jsonOutput
    }
} else {
    # No B2 credentials — output JSON directly (fallback)
    Write-Host $jsonOutput
}

} catch {
    # Ensure errors are visible in RMM StdErr/StdOut
    $errorResult = @{
        hostname = $env:COMPUTERNAME
        error    = $_.Exception.Message
        line     = $_.InvocationInfo.ScriptLineNumber
        severity = 'SCRIPT_ERROR'
    } | ConvertTo-Json -Compress
    Write-Host $errorResult
    exit 1
}