431 lines
16 KiB
PowerShell
431 lines
16 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Veeam Backup Diagnostic Script — Run via Datto RMM Quick Job
|
|
.DESCRIPTION
|
|
Checks Veeam services, backup job status, disk space, event logs,
|
|
and network connectivity. Returns structured JSON for pipeline consumption.
|
|
.NOTES
|
|
Deploy as a Datto RMM component. Output via Write-Host for StdOut capture.
|
|
Compatible with PowerShell 5.1+.
|
|
#>
|
|
|
|
try {
|
|
|
|
$ErrorActionPreference = 'SilentlyContinue'
|
|
|
|
$result = @{
|
|
timestamp = ([DateTime]::UtcNow.ToString('yyyy-MM-dd HH:mm:ss UTC'))
|
|
hostname = $env:COMPUTERNAME
|
|
checks = @{}
|
|
issues_found = @()
|
|
recommendations = @()
|
|
}
|
|
|
|
# ============================================================================
|
|
# 1. Veeam Services Status
|
|
# ============================================================================
|
|
$veeamServices = @(
|
|
'VeeamBackupSvc',
|
|
'VeeamBrokerSvc',
|
|
'VeeamCatalogSvc',
|
|
'VeeamCloudSvc',
|
|
'VeeamDeploySvc',
|
|
'VeeamDistributionSvc',
|
|
'VeeamMountSvc',
|
|
'VeeamNFSSvc',
|
|
'VeeamTransportSvc',
|
|
'VeeamEndpointBackupSvc',
|
|
'VeeamFilesysVssSvc'
|
|
)
|
|
|
|
$serviceResults = @()
|
|
$stoppedCritical = @()
|
|
|
|
foreach ($svcName in $veeamServices) {
|
|
$svc = Get-Service -Name $svcName -ErrorAction SilentlyContinue
|
|
if ($svc) {
|
|
$serviceResults += @{
|
|
name = $svc.Name
|
|
display = $svc.DisplayName
|
|
status = $svc.Status.ToString()
|
|
start_type = $svc.StartType.ToString()
|
|
}
|
|
if ($svc.Status -ne 'Running' -and $svc.StartType -ne 'Disabled') {
|
|
$stoppedCritical += $svc.DisplayName
|
|
}
|
|
}
|
|
}
|
|
|
|
$result.checks.services = @{
|
|
total_found = $serviceResults.Count
|
|
services = $serviceResults
|
|
stopped_critical = $stoppedCritical
|
|
}
|
|
|
|
if ($stoppedCritical.Count -gt 0) {
|
|
$result.issues_found += "Veeam services not running: $($stoppedCritical -join ', ')"
|
|
$result.recommendations += "Restart stopped Veeam services: $($stoppedCritical -join ', ')"
|
|
}
|
|
|
|
# ============================================================================
|
|
# 2. Veeam Backup Job Status (via PowerShell Snap-in if available)
|
|
# ============================================================================
|
|
$jobResults = @()
|
|
$vbrSnapinLoaded = $false
|
|
|
|
try {
|
|
if (Get-PSSnapin -Registered -Name VeeamPSSnapin -ErrorAction SilentlyContinue) {
|
|
Add-PSSnapin VeeamPSSnapin -ErrorAction Stop
|
|
$vbrSnapinLoaded = $true
|
|
}
|
|
elseif (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell -ErrorAction SilentlyContinue) {
|
|
Import-Module Veeam.Backup.PowerShell -ErrorAction Stop
|
|
$vbrSnapinLoaded = $true
|
|
}
|
|
} catch {
|
|
# Snap-in not available — skip VBR-specific checks
|
|
}
|
|
|
|
if ($vbrSnapinLoaded) {
|
|
try {
|
|
$jobs = Get-VBRJob -ErrorAction SilentlyContinue
|
|
foreach ($job in $jobs) {
|
|
$lastSession = $job.FindLastSession()
|
|
$jobResults += @{
|
|
name = $job.Name
|
|
type = $job.TypeToString
|
|
is_enabled = $job.IsScheduleEnabled
|
|
status = if ($lastSession) { $lastSession.Result.ToString() } else { 'NoSession' }
|
|
last_run = if ($lastSession) { $lastSession.CreationTime.ToString('yyyy-MM-dd HH:mm:ss') } else { $null }
|
|
end_time = if ($lastSession) { $lastSession.EndTime.ToString('yyyy-MM-dd HH:mm:ss') } else { $null }
|
|
duration_min = if ($lastSession -and $lastSession.EndTime -gt $lastSession.CreationTime) {
|
|
[math]::Round(($lastSession.EndTime - $lastSession.CreationTime).TotalMinutes, 1)
|
|
} else { $null }
|
|
failure_msg = if ($lastSession -and $lastSession.Result -eq 'Failed') {
|
|
($lastSession.GetTaskSessions() | Where-Object { $_.Status -eq 'Failed' } |
|
|
Select-Object -First 1 -ExpandProperty Details -ErrorAction SilentlyContinue)
|
|
} else { $null }
|
|
}
|
|
}
|
|
|
|
$failedJobs = $jobResults | Where-Object { $_.status -eq 'Failed' }
|
|
if ($failedJobs.Count -gt 0) {
|
|
$result.issues_found += "Failed backup jobs: $(($failedJobs | ForEach-Object { $_.name }) -join ', ')"
|
|
$result.recommendations += "Investigate failed jobs and check task session logs in Veeam console"
|
|
}
|
|
|
|
# Check for stuck/running jobs > 24h
|
|
$stuckJobs = $jobResults | Where-Object {
|
|
$_.status -eq 'Working' -and $_.last_run -and
|
|
((Get-Date) - [datetime]$_.last_run).TotalHours -gt 24
|
|
}
|
|
if ($stuckJobs.Count -gt 0) {
|
|
$result.issues_found += "Stuck jobs running >24h: $(($stuckJobs | ForEach-Object { $_.name }) -join ', ')"
|
|
$result.recommendations += "Consider stopping and restarting stuck backup jobs"
|
|
}
|
|
} catch {
|
|
$jobResults = @(@{ error = $_.Exception.Message })
|
|
}
|
|
}
|
|
|
|
$result.checks.backup_jobs = @{
|
|
vbr_available = $vbrSnapinLoaded
|
|
total_jobs = $jobResults.Count
|
|
jobs = $jobResults
|
|
}
|
|
|
|
# ============================================================================
|
|
# 3. Disk Space Check (all fixed drives)
|
|
# ============================================================================
|
|
$diskResults = @()
|
|
$lowDiskDrives = @()
|
|
|
|
$drives = Get-WmiObject Win32_LogicalDisk -Filter "DriveType=3" -ErrorAction SilentlyContinue
|
|
foreach ($drive in $drives) {
|
|
$freeGB = [math]::Round($drive.FreeSpace / 1GB, 2)
|
|
$totalGB = [math]::Round($drive.Size / 1GB, 2)
|
|
$usedPct = if ($totalGB -gt 0) { [math]::Round((($totalGB - $freeGB) / $totalGB) * 100, 1) } else { 0 }
|
|
|
|
$diskResults += @{
|
|
drive = $drive.DeviceID
|
|
label = $drive.VolumeName
|
|
total_gb = $totalGB
|
|
free_gb = $freeGB
|
|
used_pct = $usedPct
|
|
}
|
|
|
|
if ($usedPct -gt 90) {
|
|
$lowDiskDrives += "$($drive.DeviceID) ($usedPct% used, $freeGB GB free)"
|
|
}
|
|
}
|
|
|
|
$result.checks.disk_space = @{
|
|
drives = $diskResults
|
|
low_disk = $lowDiskDrives
|
|
}
|
|
|
|
if ($lowDiskDrives.Count -gt 0) {
|
|
$result.issues_found += "Low disk space: $($lowDiskDrives -join ', ')"
|
|
$result.recommendations += "Free disk space or expand storage on affected drives"
|
|
}
|
|
|
|
# ============================================================================
|
|
# 4. Windows Event Log — Veeam errors (last 48 hours)
|
|
# ============================================================================
|
|
$eventResults = @()
|
|
$cutoff = (Get-Date).AddHours(-48)
|
|
|
|
# Veeam Backup log
|
|
$veeamEvents = Get-WinEvent -FilterHashtable @{
|
|
LogName = 'Veeam Backup'
|
|
Level = @(1, 2) # Critical, Error
|
|
StartTime = $cutoff
|
|
} -MaxEvents 20 -ErrorAction SilentlyContinue
|
|
|
|
foreach ($evt in $veeamEvents) {
|
|
$eventResults += @{
|
|
source = 'Veeam Backup'
|
|
level = $evt.LevelDisplayName
|
|
id = $evt.Id
|
|
time = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
|
message = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
|
|
}
|
|
}
|
|
|
|
# Veeam Agent log
|
|
$agentEvents = Get-WinEvent -FilterHashtable @{
|
|
LogName = 'Veeam Agent'
|
|
Level = @(1, 2)
|
|
StartTime = $cutoff
|
|
} -MaxEvents 10 -ErrorAction SilentlyContinue
|
|
|
|
foreach ($evt in $agentEvents) {
|
|
$eventResults += @{
|
|
source = 'Veeam Agent'
|
|
level = $evt.LevelDisplayName
|
|
id = $evt.Id
|
|
time = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
|
message = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
|
|
}
|
|
}
|
|
|
|
# Application log — Veeam source
|
|
$appEvents = Get-WinEvent -FilterHashtable @{
|
|
LogName = 'Application'
|
|
ProviderName = @('Veeam*')
|
|
Level = @(1, 2)
|
|
StartTime = $cutoff
|
|
} -MaxEvents 10 -ErrorAction SilentlyContinue
|
|
|
|
foreach ($evt in $appEvents) {
|
|
$eventResults += @{
|
|
source = "Application/$($evt.ProviderName)"
|
|
level = $evt.LevelDisplayName
|
|
id = $evt.Id
|
|
time = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
|
message = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
|
|
}
|
|
}
|
|
|
|
$result.checks.event_logs = @{
|
|
total_errors = $eventResults.Count
|
|
events = $eventResults
|
|
}
|
|
|
|
if ($eventResults.Count -gt 0) {
|
|
$result.issues_found += "$($eventResults.Count) Veeam error events in last 48h"
|
|
}
|
|
|
|
# ============================================================================
|
|
# 5. Veeam Process Check — is anything stuck?
|
|
# ============================================================================
|
|
$veeamProcesses = Get-Process -Name "Veeam*" -ErrorAction SilentlyContinue |
|
|
Select-Object Name, Id, CPU,
|
|
@{N='MemoryMB';E={[math]::Round($_.WorkingSet64/1MB,1)}},
|
|
@{N='RunningHours';E={[math]::Round(((Get-Date) - $_.StartTime).TotalHours, 1)}}
|
|
|
|
$stuckProcesses = $veeamProcesses | Where-Object { $_.RunningHours -gt 48 }
|
|
|
|
$result.checks.processes = @{
|
|
running = @($veeamProcesses | ForEach-Object {
|
|
@{ name = $_.Name; pid = $_.Id; memory_mb = $_.MemoryMB; running_hours = $_.RunningHours }
|
|
})
|
|
stuck = @($stuckProcesses | ForEach-Object { $_.Name })
|
|
}
|
|
|
|
if ($stuckProcesses.Count -gt 0) {
|
|
$result.issues_found += "Potentially stuck Veeam processes (>48h): $(($stuckProcesses | ForEach-Object { $_.Name }) -join ', ')"
|
|
$result.recommendations += "Review and potentially restart long-running Veeam processes"
|
|
}
|
|
|
|
# ============================================================================
|
|
# 6. Network Connectivity to Backup Targets
|
|
# ============================================================================
|
|
$networkResults = @()
|
|
|
|
# Try to find backup repository paths from registry
|
|
$repoKeys = Get-ItemProperty -Path "HKLM:\SOFTWARE\Veeam\Veeam Backup and Replication" -ErrorAction SilentlyContinue
|
|
$sqlServer = $repoKeys.SqlServerName
|
|
|
|
if ($sqlServer) {
|
|
$testSql = Test-NetConnection -ComputerName $sqlServer -Port 1433 -WarningAction SilentlyContinue -ErrorAction SilentlyContinue
|
|
$networkResults += @{
|
|
target = "SQL: $sqlServer"
|
|
port = 1433
|
|
success = $testSql.TcpTestSucceeded
|
|
}
|
|
if (-not $testSql.TcpTestSucceeded) {
|
|
$result.issues_found += "Cannot reach Veeam SQL server: $sqlServer"
|
|
$result.recommendations += "Check network connectivity and SQL Server service on $sqlServer"
|
|
}
|
|
}
|
|
|
|
# Test common backup infrastructure ports
|
|
$vbrServer = $repoKeys.SqlDatabaseName # Often same host
|
|
$localPorts = @(
|
|
@{ Name = "Veeam Backup Service"; Port = 9392 },
|
|
@{ Name = "Veeam REST API"; Port = 9419 },
|
|
@{ Name = "Veeam Cloud Connect"; Port = 6180 }
|
|
)
|
|
|
|
foreach ($p in $localPorts) {
|
|
$test = Test-NetConnection -ComputerName 'localhost' -Port $p.Port -WarningAction SilentlyContinue -ErrorAction SilentlyContinue
|
|
$networkResults += @{
|
|
target = $p.Name
|
|
port = $p.Port
|
|
success = $test.TcpTestSucceeded
|
|
}
|
|
}
|
|
|
|
$result.checks.network = @{
|
|
tests = $networkResults
|
|
}
|
|
|
|
# ============================================================================
|
|
# 7. Summary
|
|
# ============================================================================
|
|
$result.total_issues = $result.issues_found.Count
|
|
$result.severity = if ($result.issues_found.Count -eq 0) { 'OK' }
|
|
elseif ($result.issues_found.Count -le 2) { 'WARNING' }
|
|
else { 'CRITICAL' }
|
|
|
|
# ============================================================================
|
|
# 8. Upload to B2 (S3-compatible) and output object key
|
|
# ============================================================================
|
|
$jsonOutput = $result | ConvertTo-Json -Depth 5 -Compress
|
|
|
|
# B2 credentials — set these as Datto RMM component variables or site variables
|
|
$b2KeyId = if ($env:B2_KEY_ID) { $env:B2_KEY_ID } else { $env:usrB2KeyId }
|
|
$b2AppKey = if ($env:B2_APP_KEY) { $env:B2_APP_KEY } else { $env:usrB2AppKey }
|
|
$b2Bucket = if ($env:B2_BUCKET) { $env:B2_BUCKET } else { if ($env:usrB2Bucket) { $env:usrB2Bucket } else { 'wulf-audits' } }
|
|
$b2Region = if ($env:B2_REGION) { $env:B2_REGION } else { if ($env:usrB2Region) { $env:usrB2Region } else { 'us-west-002' } }
|
|
$b2Endpoint = if ($env:B2_ENDPOINT) { $env:B2_ENDPOINT } else { "s3.$b2Region.backblazeb2.com" }
|
|
|
|
$datePrefix = (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd')
|
|
$timeStamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
|
|
$objectKey = "diagnostics/$($env:COMPUTERNAME)/$datePrefix/$timeStamp.json"
|
|
|
|
if ($b2KeyId -and $b2AppKey) {
|
|
try {
|
|
# S3v4 presigned PUT
|
|
$method = 'PUT'
|
|
$host_ = $b2Endpoint
|
|
$canonicalUri = "/$b2Bucket/$objectKey"
|
|
$algorithm = 'AWS4-HMAC-SHA256'
|
|
$amzDate = $timeStamp
|
|
$dateStamp = $amzDate.Substring(0, 8)
|
|
$credScope = "$dateStamp/$b2Region/s3/aws4_request"
|
|
$contentHash = [System.BitConverter]::ToString(
|
|
[System.Security.Cryptography.SHA256]::Create().ComputeHash(
|
|
[System.Text.Encoding]::UTF8.GetBytes($jsonOutput)
|
|
)
|
|
).Replace('-','').ToLower()
|
|
|
|
$canonicalHeaders = "content-type:application/json`nhost:$host_`nx-amz-content-sha256:$contentHash`nx-amz-date:$amzDate`n"
|
|
$signedHeaders = 'content-type;host;x-amz-content-sha256;x-amz-date'
|
|
|
|
$canonicalRequest = "$method`n$canonicalUri`n`n$canonicalHeaders`n$signedHeaders`n$contentHash"
|
|
$crHash = [System.BitConverter]::ToString(
|
|
[System.Security.Cryptography.SHA256]::Create().ComputeHash(
|
|
[System.Text.Encoding]::UTF8.GetBytes($canonicalRequest)
|
|
)
|
|
).Replace('-','').ToLower()
|
|
|
|
$stringToSign = "$algorithm`n$amzDate`n$credScope`n$crHash"
|
|
|
|
# Derive signing key
|
|
function HmacSHA256($key, $data) {
|
|
$hmac = New-Object System.Security.Cryptography.HMACSHA256
|
|
$hmac.Key = if ($key -is [byte[]]) { $key } else { [System.Text.Encoding]::UTF8.GetBytes($key) }
|
|
return $hmac.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($data))
|
|
}
|
|
|
|
$kDate = HmacSHA256 "AWS4$b2AppKey" $dateStamp
|
|
$kRegion = HmacSHA256 $kDate $b2Region
|
|
$kService = HmacSHA256 $kRegion 's3'
|
|
$kSigning = HmacSHA256 $kService 'aws4_request'
|
|
|
|
$signature = [System.BitConverter]::ToString(
|
|
(HmacSHA256 $kSigning $stringToSign)
|
|
).Replace('-','').ToLower()
|
|
|
|
$authHeader = "$algorithm Credential=$b2KeyId/$credScope, SignedHeaders=$signedHeaders, Signature=$signature"
|
|
|
|
$headers = @{
|
|
'Authorization' = $authHeader
|
|
'x-amz-date' = $amzDate
|
|
'x-amz-content-sha256' = $contentHash
|
|
'Content-Type' = 'application/json'
|
|
}
|
|
|
|
$uri = "https://$host_$canonicalUri"
|
|
$bodyBytes = [System.Text.Encoding]::UTF8.GetBytes($jsonOutput)
|
|
|
|
# Use .NET WebRequest for PS 5.1 compatibility
|
|
$webRequest = [System.Net.HttpWebRequest]::Create($uri)
|
|
$webRequest.Method = 'PUT'
|
|
$webRequest.ContentType = 'application/json'
|
|
$webRequest.ContentLength = $bodyBytes.Length
|
|
foreach ($h in $headers.GetEnumerator()) {
|
|
if ($h.Key -notin @('Content-Type')) {
|
|
$webRequest.Headers.Add($h.Key, $h.Value)
|
|
}
|
|
}
|
|
|
|
$stream = $webRequest.GetRequestStream()
|
|
$stream.Write($bodyBytes, 0, $bodyBytes.Length)
|
|
$stream.Close()
|
|
|
|
$response = $webRequest.GetResponse()
|
|
$statusCode = [int]$response.StatusCode
|
|
$response.Close()
|
|
|
|
if ($statusCode -eq 200) {
|
|
# Success — output object key for pipeline to fetch
|
|
Write-Host $objectKey
|
|
} else {
|
|
# Upload failed — fall back to inline JSON
|
|
Write-Host "UPLOAD_FAILED:$statusCode"
|
|
Write-Host $jsonOutput
|
|
}
|
|
} catch {
|
|
# Upload error — fall back to inline JSON
|
|
Write-Host "UPLOAD_ERROR:$($_.Exception.Message)"
|
|
Write-Host $jsonOutput
|
|
}
|
|
} else {
|
|
# No B2 credentials — output JSON directly (fallback)
|
|
Write-Host $jsonOutput
|
|
}
|
|
|
|
} catch {
|
|
# Ensure errors are visible in RMM StdErr/StdOut
|
|
$errorResult = @{
|
|
hostname = $env:COMPUTERNAME
|
|
error = $_.Exception.Message
|
|
line = $_.InvocationInfo.ScriptLineNumber
|
|
severity = 'SCRIPT_ERROR'
|
|
} | ConvertTo-Json -Compress
|
|
Write-Host $errorResult
|
|
exit 1
|
|
}
|