feat: IT Glue integration, workflow engine, pipelines, Zabbix WAN, notification channels, backup status UI improvements, nav alignment fixes
This commit is contained in:
parent
ed6c4a8b65
commit
19605f82aa
97 changed files with 17080 additions and 304 deletions
431
scripts/rmm-diagnostics/veeam-backup-diagnostic.ps1
Normal file
431
scripts/rmm-diagnostics/veeam-backup-diagnostic.ps1
Normal file
|
|
@ -0,0 +1,431 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Veeam Backup Diagnostic Script — Run via Datto RMM Quick Job
|
||||
.DESCRIPTION
|
||||
Checks Veeam services, backup job status, disk space, event logs,
|
||||
and network connectivity. Returns structured JSON for pipeline consumption.
|
||||
.NOTES
|
||||
Deploy as a Datto RMM component. Output via Write-Host for StdOut capture.
|
||||
Compatible with PowerShell 5.1+.
|
||||
#>
|
||||
|
||||
try {
|
||||
|
||||
$ErrorActionPreference = 'SilentlyContinue'
|
||||
|
||||
$result = @{
|
||||
timestamp = ([DateTime]::UtcNow.ToString('yyyy-MM-dd HH:mm:ss UTC'))
|
||||
hostname = $env:COMPUTERNAME
|
||||
checks = @{}
|
||||
issues_found = @()
|
||||
recommendations = @()
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 1. Veeam Services Status
|
||||
# ============================================================================
|
||||
$veeamServices = @(
|
||||
'VeeamBackupSvc',
|
||||
'VeeamBrokerSvc',
|
||||
'VeeamCatalogSvc',
|
||||
'VeeamCloudSvc',
|
||||
'VeeamDeploySvc',
|
||||
'VeeamDistributionSvc',
|
||||
'VeeamMountSvc',
|
||||
'VeeamNFSSvc',
|
||||
'VeeamTransportSvc',
|
||||
'VeeamEndpointBackupSvc',
|
||||
'VeeamFilesysVssSvc'
|
||||
)
|
||||
|
||||
$serviceResults = @()
|
||||
$stoppedCritical = @()
|
||||
|
||||
foreach ($svcName in $veeamServices) {
|
||||
$svc = Get-Service -Name $svcName -ErrorAction SilentlyContinue
|
||||
if ($svc) {
|
||||
$serviceResults += @{
|
||||
name = $svc.Name
|
||||
display = $svc.DisplayName
|
||||
status = $svc.Status.ToString()
|
||||
start_type = $svc.StartType.ToString()
|
||||
}
|
||||
if ($svc.Status -ne 'Running' -and $svc.StartType -ne 'Disabled') {
|
||||
$stoppedCritical += $svc.DisplayName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$result.checks.services = @{
|
||||
total_found = $serviceResults.Count
|
||||
services = $serviceResults
|
||||
stopped_critical = $stoppedCritical
|
||||
}
|
||||
|
||||
if ($stoppedCritical.Count -gt 0) {
|
||||
$result.issues_found += "Veeam services not running: $($stoppedCritical -join ', ')"
|
||||
$result.recommendations += "Restart stopped Veeam services: $($stoppedCritical -join ', ')"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 2. Veeam Backup Job Status (via PowerShell Snap-in if available)
|
||||
# ============================================================================
|
||||
$jobResults = @()
|
||||
$vbrSnapinLoaded = $false
|
||||
|
||||
try {
|
||||
if (Get-PSSnapin -Registered -Name VeeamPSSnapin -ErrorAction SilentlyContinue) {
|
||||
Add-PSSnapin VeeamPSSnapin -ErrorAction Stop
|
||||
$vbrSnapinLoaded = $true
|
||||
}
|
||||
elseif (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell -ErrorAction SilentlyContinue) {
|
||||
Import-Module Veeam.Backup.PowerShell -ErrorAction Stop
|
||||
$vbrSnapinLoaded = $true
|
||||
}
|
||||
} catch {
|
||||
# Snap-in not available — skip VBR-specific checks
|
||||
}
|
||||
|
||||
if ($vbrSnapinLoaded) {
|
||||
try {
|
||||
$jobs = Get-VBRJob -ErrorAction SilentlyContinue
|
||||
foreach ($job in $jobs) {
|
||||
$lastSession = $job.FindLastSession()
|
||||
$jobResults += @{
|
||||
name = $job.Name
|
||||
type = $job.TypeToString
|
||||
is_enabled = $job.IsScheduleEnabled
|
||||
status = if ($lastSession) { $lastSession.Result.ToString() } else { 'NoSession' }
|
||||
last_run = if ($lastSession) { $lastSession.CreationTime.ToString('yyyy-MM-dd HH:mm:ss') } else { $null }
|
||||
end_time = if ($lastSession) { $lastSession.EndTime.ToString('yyyy-MM-dd HH:mm:ss') } else { $null }
|
||||
duration_min = if ($lastSession -and $lastSession.EndTime -gt $lastSession.CreationTime) {
|
||||
[math]::Round(($lastSession.EndTime - $lastSession.CreationTime).TotalMinutes, 1)
|
||||
} else { $null }
|
||||
failure_msg = if ($lastSession -and $lastSession.Result -eq 'Failed') {
|
||||
($lastSession.GetTaskSessions() | Where-Object { $_.Status -eq 'Failed' } |
|
||||
Select-Object -First 1 -ExpandProperty Details -ErrorAction SilentlyContinue)
|
||||
} else { $null }
|
||||
}
|
||||
}
|
||||
|
||||
$failedJobs = $jobResults | Where-Object { $_.status -eq 'Failed' }
|
||||
if ($failedJobs.Count -gt 0) {
|
||||
$result.issues_found += "Failed backup jobs: $(($failedJobs | ForEach-Object { $_.name }) -join ', ')"
|
||||
$result.recommendations += "Investigate failed jobs and check task session logs in Veeam console"
|
||||
}
|
||||
|
||||
# Check for stuck/running jobs > 24h
|
||||
$stuckJobs = $jobResults | Where-Object {
|
||||
$_.status -eq 'Working' -and $_.last_run -and
|
||||
((Get-Date) - [datetime]$_.last_run).TotalHours -gt 24
|
||||
}
|
||||
if ($stuckJobs.Count -gt 0) {
|
||||
$result.issues_found += "Stuck jobs running >24h: $(($stuckJobs | ForEach-Object { $_.name }) -join ', ')"
|
||||
$result.recommendations += "Consider stopping and restarting stuck backup jobs"
|
||||
}
|
||||
} catch {
|
||||
$jobResults = @(@{ error = $_.Exception.Message })
|
||||
}
|
||||
}
|
||||
|
||||
$result.checks.backup_jobs = @{
|
||||
vbr_available = $vbrSnapinLoaded
|
||||
total_jobs = $jobResults.Count
|
||||
jobs = $jobResults
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 3. Disk Space Check (all fixed drives)
|
||||
# ============================================================================
|
||||
$diskResults = @()
|
||||
$lowDiskDrives = @()
|
||||
|
||||
$drives = Get-WmiObject Win32_LogicalDisk -Filter "DriveType=3" -ErrorAction SilentlyContinue
|
||||
foreach ($drive in $drives) {
|
||||
$freeGB = [math]::Round($drive.FreeSpace / 1GB, 2)
|
||||
$totalGB = [math]::Round($drive.Size / 1GB, 2)
|
||||
$usedPct = if ($totalGB -gt 0) { [math]::Round((($totalGB - $freeGB) / $totalGB) * 100, 1) } else { 0 }
|
||||
|
||||
$diskResults += @{
|
||||
drive = $drive.DeviceID
|
||||
label = $drive.VolumeName
|
||||
total_gb = $totalGB
|
||||
free_gb = $freeGB
|
||||
used_pct = $usedPct
|
||||
}
|
||||
|
||||
if ($usedPct -gt 90) {
|
||||
$lowDiskDrives += "$($drive.DeviceID) ($usedPct% used, $freeGB GB free)"
|
||||
}
|
||||
}
|
||||
|
||||
$result.checks.disk_space = @{
|
||||
drives = $diskResults
|
||||
low_disk = $lowDiskDrives
|
||||
}
|
||||
|
||||
if ($lowDiskDrives.Count -gt 0) {
|
||||
$result.issues_found += "Low disk space: $($lowDiskDrives -join ', ')"
|
||||
$result.recommendations += "Free disk space or expand storage on affected drives"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 4. Windows Event Log — Veeam errors (last 48 hours)
|
||||
# ============================================================================
|
||||
$eventResults = @()
|
||||
$cutoff = (Get-Date).AddHours(-48)
|
||||
|
||||
# Veeam Backup log
|
||||
$veeamEvents = Get-WinEvent -FilterHashtable @{
|
||||
LogName = 'Veeam Backup'
|
||||
Level = @(1, 2) # Critical, Error
|
||||
StartTime = $cutoff
|
||||
} -MaxEvents 20 -ErrorAction SilentlyContinue
|
||||
|
||||
foreach ($evt in $veeamEvents) {
|
||||
$eventResults += @{
|
||||
source = 'Veeam Backup'
|
||||
level = $evt.LevelDisplayName
|
||||
id = $evt.Id
|
||||
time = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
||||
message = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
|
||||
}
|
||||
}
|
||||
|
||||
# Veeam Agent log
|
||||
$agentEvents = Get-WinEvent -FilterHashtable @{
|
||||
LogName = 'Veeam Agent'
|
||||
Level = @(1, 2)
|
||||
StartTime = $cutoff
|
||||
} -MaxEvents 10 -ErrorAction SilentlyContinue
|
||||
|
||||
foreach ($evt in $agentEvents) {
|
||||
$eventResults += @{
|
||||
source = 'Veeam Agent'
|
||||
level = $evt.LevelDisplayName
|
||||
id = $evt.Id
|
||||
time = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
||||
message = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
|
||||
}
|
||||
}
|
||||
|
||||
# Application log — Veeam source
|
||||
$appEvents = Get-WinEvent -FilterHashtable @{
|
||||
LogName = 'Application'
|
||||
ProviderName = @('Veeam*')
|
||||
Level = @(1, 2)
|
||||
StartTime = $cutoff
|
||||
} -MaxEvents 10 -ErrorAction SilentlyContinue
|
||||
|
||||
foreach ($evt in $appEvents) {
|
||||
$eventResults += @{
|
||||
source = "Application/$($evt.ProviderName)"
|
||||
level = $evt.LevelDisplayName
|
||||
id = $evt.Id
|
||||
time = $evt.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
||||
message = $evt.Message.Substring(0, [Math]::Min(500, $evt.Message.Length))
|
||||
}
|
||||
}
|
||||
|
||||
$result.checks.event_logs = @{
|
||||
total_errors = $eventResults.Count
|
||||
events = $eventResults
|
||||
}
|
||||
|
||||
if ($eventResults.Count -gt 0) {
|
||||
$result.issues_found += "$($eventResults.Count) Veeam error events in last 48h"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 5. Veeam Process Check — is anything stuck?
|
||||
# ============================================================================
|
||||
$veeamProcesses = Get-Process -Name "Veeam*" -ErrorAction SilentlyContinue |
|
||||
Select-Object Name, Id, CPU,
|
||||
@{N='MemoryMB';E={[math]::Round($_.WorkingSet64/1MB,1)}},
|
||||
@{N='RunningHours';E={[math]::Round(((Get-Date) - $_.StartTime).TotalHours, 1)}}
|
||||
|
||||
$stuckProcesses = $veeamProcesses | Where-Object { $_.RunningHours -gt 48 }
|
||||
|
||||
$result.checks.processes = @{
|
||||
running = @($veeamProcesses | ForEach-Object {
|
||||
@{ name = $_.Name; pid = $_.Id; memory_mb = $_.MemoryMB; running_hours = $_.RunningHours }
|
||||
})
|
||||
stuck = @($stuckProcesses | ForEach-Object { $_.Name })
|
||||
}
|
||||
|
||||
if ($stuckProcesses.Count -gt 0) {
|
||||
$result.issues_found += "Potentially stuck Veeam processes (>48h): $(($stuckProcesses | ForEach-Object { $_.Name }) -join ', ')"
|
||||
$result.recommendations += "Review and potentially restart long-running Veeam processes"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 6. Network Connectivity to Backup Targets
|
||||
# ============================================================================
|
||||
$networkResults = @()
|
||||
|
||||
# Try to find backup repository paths from registry
|
||||
$repoKeys = Get-ItemProperty -Path "HKLM:\SOFTWARE\Veeam\Veeam Backup and Replication" -ErrorAction SilentlyContinue
|
||||
$sqlServer = $repoKeys.SqlServerName
|
||||
|
||||
if ($sqlServer) {
|
||||
$testSql = Test-NetConnection -ComputerName $sqlServer -Port 1433 -WarningAction SilentlyContinue -ErrorAction SilentlyContinue
|
||||
$networkResults += @{
|
||||
target = "SQL: $sqlServer"
|
||||
port = 1433
|
||||
success = $testSql.TcpTestSucceeded
|
||||
}
|
||||
if (-not $testSql.TcpTestSucceeded) {
|
||||
$result.issues_found += "Cannot reach Veeam SQL server: $sqlServer"
|
||||
$result.recommendations += "Check network connectivity and SQL Server service on $sqlServer"
|
||||
}
|
||||
}
|
||||
|
||||
# Test common backup infrastructure ports
|
||||
$vbrServer = $repoKeys.SqlDatabaseName # Often same host
|
||||
$localPorts = @(
|
||||
@{ Name = "Veeam Backup Service"; Port = 9392 },
|
||||
@{ Name = "Veeam REST API"; Port = 9419 },
|
||||
@{ Name = "Veeam Cloud Connect"; Port = 6180 }
|
||||
)
|
||||
|
||||
foreach ($p in $localPorts) {
|
||||
$test = Test-NetConnection -ComputerName 'localhost' -Port $p.Port -WarningAction SilentlyContinue -ErrorAction SilentlyContinue
|
||||
$networkResults += @{
|
||||
target = $p.Name
|
||||
port = $p.Port
|
||||
success = $test.TcpTestSucceeded
|
||||
}
|
||||
}
|
||||
|
||||
$result.checks.network = @{
|
||||
tests = $networkResults
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# 7. Summary
|
||||
# ============================================================================
|
||||
$result.total_issues = $result.issues_found.Count
|
||||
$result.severity = if ($result.issues_found.Count -eq 0) { 'OK' }
|
||||
elseif ($result.issues_found.Count -le 2) { 'WARNING' }
|
||||
else { 'CRITICAL' }
|
||||
|
||||
# ============================================================================
|
||||
# 8. Upload to B2 (S3-compatible) and output object key
|
||||
# ============================================================================
|
||||
$jsonOutput = $result | ConvertTo-Json -Depth 5 -Compress
|
||||
|
||||
# B2 credentials — set these as Datto RMM component variables or site variables
|
||||
$b2KeyId = if ($env:B2_KEY_ID) { $env:B2_KEY_ID } else { $env:usrB2KeyId }
|
||||
$b2AppKey = if ($env:B2_APP_KEY) { $env:B2_APP_KEY } else { $env:usrB2AppKey }
|
||||
$b2Bucket = if ($env:B2_BUCKET) { $env:B2_BUCKET } else { if ($env:usrB2Bucket) { $env:usrB2Bucket } else { 'wulf-audits' } }
|
||||
$b2Region = if ($env:B2_REGION) { $env:B2_REGION } else { if ($env:usrB2Region) { $env:usrB2Region } else { 'us-west-002' } }
|
||||
$b2Endpoint = if ($env:B2_ENDPOINT) { $env:B2_ENDPOINT } else { "s3.$b2Region.backblazeb2.com" }
|
||||
|
||||
$datePrefix = (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd')
|
||||
$timeStamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssZ')
|
||||
$objectKey = "diagnostics/$($env:COMPUTERNAME)/$datePrefix/$timeStamp.json"
|
||||
|
||||
if ($b2KeyId -and $b2AppKey) {
|
||||
try {
|
||||
# S3v4 presigned PUT
|
||||
$method = 'PUT'
|
||||
$host_ = $b2Endpoint
|
||||
$canonicalUri = "/$b2Bucket/$objectKey"
|
||||
$algorithm = 'AWS4-HMAC-SHA256'
|
||||
$amzDate = $timeStamp
|
||||
$dateStamp = $amzDate.Substring(0, 8)
|
||||
$credScope = "$dateStamp/$b2Region/s3/aws4_request"
|
||||
$contentHash = [System.BitConverter]::ToString(
|
||||
[System.Security.Cryptography.SHA256]::Create().ComputeHash(
|
||||
[System.Text.Encoding]::UTF8.GetBytes($jsonOutput)
|
||||
)
|
||||
).Replace('-','').ToLower()
|
||||
|
||||
$canonicalHeaders = "content-type:application/json`nhost:$host_`nx-amz-content-sha256:$contentHash`nx-amz-date:$amzDate`n"
|
||||
$signedHeaders = 'content-type;host;x-amz-content-sha256;x-amz-date'
|
||||
|
||||
$canonicalRequest = "$method`n$canonicalUri`n`n$canonicalHeaders`n$signedHeaders`n$contentHash"
|
||||
$crHash = [System.BitConverter]::ToString(
|
||||
[System.Security.Cryptography.SHA256]::Create().ComputeHash(
|
||||
[System.Text.Encoding]::UTF8.GetBytes($canonicalRequest)
|
||||
)
|
||||
).Replace('-','').ToLower()
|
||||
|
||||
$stringToSign = "$algorithm`n$amzDate`n$credScope`n$crHash"
|
||||
|
||||
# Derive signing key
|
||||
function HmacSHA256($key, $data) {
|
||||
$hmac = New-Object System.Security.Cryptography.HMACSHA256
|
||||
$hmac.Key = if ($key -is [byte[]]) { $key } else { [System.Text.Encoding]::UTF8.GetBytes($key) }
|
||||
return $hmac.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($data))
|
||||
}
|
||||
|
||||
$kDate = HmacSHA256 "AWS4$b2AppKey" $dateStamp
|
||||
$kRegion = HmacSHA256 $kDate $b2Region
|
||||
$kService = HmacSHA256 $kRegion 's3'
|
||||
$kSigning = HmacSHA256 $kService 'aws4_request'
|
||||
|
||||
$signature = [System.BitConverter]::ToString(
|
||||
(HmacSHA256 $kSigning $stringToSign)
|
||||
).Replace('-','').ToLower()
|
||||
|
||||
$authHeader = "$algorithm Credential=$b2KeyId/$credScope, SignedHeaders=$signedHeaders, Signature=$signature"
|
||||
|
||||
$headers = @{
|
||||
'Authorization' = $authHeader
|
||||
'x-amz-date' = $amzDate
|
||||
'x-amz-content-sha256' = $contentHash
|
||||
'Content-Type' = 'application/json'
|
||||
}
|
||||
|
||||
$uri = "https://$host_$canonicalUri"
|
||||
$bodyBytes = [System.Text.Encoding]::UTF8.GetBytes($jsonOutput)
|
||||
|
||||
# Use .NET WebRequest for PS 5.1 compatibility
|
||||
$webRequest = [System.Net.HttpWebRequest]::Create($uri)
|
||||
$webRequest.Method = 'PUT'
|
||||
$webRequest.ContentType = 'application/json'
|
||||
$webRequest.ContentLength = $bodyBytes.Length
|
||||
foreach ($h in $headers.GetEnumerator()) {
|
||||
if ($h.Key -notin @('Content-Type')) {
|
||||
$webRequest.Headers.Add($h.Key, $h.Value)
|
||||
}
|
||||
}
|
||||
|
||||
$stream = $webRequest.GetRequestStream()
|
||||
$stream.Write($bodyBytes, 0, $bodyBytes.Length)
|
||||
$stream.Close()
|
||||
|
||||
$response = $webRequest.GetResponse()
|
||||
$statusCode = [int]$response.StatusCode
|
||||
$response.Close()
|
||||
|
||||
if ($statusCode -eq 200) {
|
||||
# Success — output object key for pipeline to fetch
|
||||
Write-Host $objectKey
|
||||
} else {
|
||||
# Upload failed — fall back to inline JSON
|
||||
Write-Host "UPLOAD_FAILED:$statusCode"
|
||||
Write-Host $jsonOutput
|
||||
}
|
||||
} catch {
|
||||
# Upload error — fall back to inline JSON
|
||||
Write-Host "UPLOAD_ERROR:$($_.Exception.Message)"
|
||||
Write-Host $jsonOutput
|
||||
}
|
||||
} else {
|
||||
# No B2 credentials — output JSON directly (fallback)
|
||||
Write-Host $jsonOutput
|
||||
}
|
||||
|
||||
} catch {
|
||||
# Ensure errors are visible in RMM StdErr/StdOut
|
||||
$errorResult = @{
|
||||
hostname = $env:COMPUTERNAME
|
||||
error = $_.Exception.Message
|
||||
line = $_.InvocationInfo.ScriptLineNumber
|
||||
severity = 'SCRIPT_ERROR'
|
||||
} | ConvertTo-Json -Compress
|
||||
Write-Host $errorResult
|
||||
exit 1
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue