feat: RMM Overshell, IT Glue audit/write-back, LogLift, link-aware bundles, dashboard overhaul
- RMM Overshell (migration 077): admin page, dispatch UI, executor/worker, target resolver, script registry (AD/DHCP/DNS/event-log/services/software/network/loglift) - LogLift evidence pipeline (migration 078): upload webhook, B2 storage client, receiver/matcher, EventLogCollector PowerShell script - IT Glue audit + write-back (migrations 075, 076): asset-audit runner, ticket xrefs, applications/configurations browse pages + apply/revert/audit endpoints - Link-aware analyzer bundles (migration 073) + provider toggle (migration 074): link-discovery service, OpenRouter LLM provider, related-tickets/itglue-suggestion panels, analyze-bundle endpoint - Endpoint data model + device-link reconciliation (migrations 079, 080): conflicts admin page, reconciler service, resolve endpoints - Dashboard overhaul: integration-health service + alerts, overview/health endpoints - Permissions: add itglue + rmm scopes; middleware: public /api/rmm/loglift route Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
378e68ad8a
commit
1112a06afe
132 changed files with 21352 additions and 743 deletions
960
scripts/loglift/EventLogCollector-DattoRMM.ps1
Normal file
960
scripts/loglift/EventLogCollector-DattoRMM.ps1
Normal file
|
|
@ -0,0 +1,960 @@
|
|||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Windows Event Log Collector for LLM Analysis (Datto RMM Component).
|
||||
|
||||
.DESCRIPTION
|
||||
Collects Windows event logs + system context, gzips the JSON, uploads to
|
||||
Backblaze B2, and POSTs metadata to a webhook for LLM analysis.
|
||||
|
||||
Two transport modes — auto-selected by env vars supplied at job dispatch:
|
||||
|
||||
1. Pulse-driven dispatch (preferred). Pulse passes these as Datto
|
||||
component variables (delivered as env vars):
|
||||
RunId — correlation token; the webhook handler uses it to
|
||||
find the dispatched execution row.
|
||||
WebhookUrl — Pulse's /api/rmm/loglift/upload metadata endpoint.
|
||||
WebhookSecret — OPENCLAW_API_KEY; sent as x-openclaw-key.
|
||||
UploadUrl — pre-presigned B2 PUT URL (30-min TTL).
|
||||
ObjectKey — full B2 object key.
|
||||
ClientId — Datto site uuid (informational; matches CS_PROFILE_UID).
|
||||
The script uploads to UploadUrl directly (no presign-fetch round-trip)
|
||||
and POSTs to WebhookUrl with x-openclaw-key.
|
||||
|
||||
2. Legacy n8n flow. With none of the Pulse env vars set, the script falls
|
||||
back to the original behavior: POST to $WebhookUrl to fetch a
|
||||
presigned URL, upload, then POST to $NotifyWebhookUrl.
|
||||
|
||||
.NOTES
|
||||
Version: 3.0.1 (Pulse + n8n dual-mode)
|
||||
Author : Wulf Consulting / MSP Automation
|
||||
Requires: PowerShell 5.1+, Windows 10/11 or Server 2016+
|
||||
Platform: Datto RMM Component
|
||||
#>
|
||||
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter()]
|
||||
[string]$ClientId,
|
||||
|
||||
# Legacy n8n presign-fetch URL. Ignored when $env:WebhookUrl is set.
|
||||
[Parameter()]
|
||||
[string]$WebhookUrl = "https://n8n.wulfnetwork.com/webhook/audit/get-upload-url/7uI1aKMcF",
|
||||
|
||||
# Legacy n8n metadata-notify URL. Ignored when $env:WebhookUrl is set.
|
||||
[Parameter()]
|
||||
[string]$NotifyWebhookUrl = "https://n8n.wulfnetwork.com/webhook/dd9e1afc-a494-4688-9542-010d388c7bd9",
|
||||
|
||||
[Parameter()]
|
||||
[int]$HoursBack = 24,
|
||||
|
||||
[Parameter()]
|
||||
[string[]]$IncludeLogs = @('System', 'Application', 'Security'),
|
||||
|
||||
[Parameter()]
|
||||
[ValidateSet('Critical', 'Error', 'Warning', 'Information', 'Verbose')]
|
||||
[string]$MinLevel = 'Warning',
|
||||
|
||||
[Parameter()]
|
||||
[string]$IssueDescription = "",
|
||||
|
||||
[Parameter()]
|
||||
[string]$TicketNumber = "",
|
||||
|
||||
[Parameter()]
|
||||
[switch]$DryRun,
|
||||
|
||||
[Parameter()]
|
||||
[switch]$IncludeInstalledSoftware,
|
||||
|
||||
[Parameter()]
|
||||
[switch]$IncludeRunningProcesses,
|
||||
|
||||
[Parameter()]
|
||||
[switch]$IncludeNetworkConfig,
|
||||
|
||||
[Parameter()]
|
||||
[int]$MaxEventsPerLog = 1000,
|
||||
|
||||
[Parameter()]
|
||||
[string]$OutputPath = "C:\wulf\results"
|
||||
)
|
||||
|
||||
# ============================================================================
|
||||
# LOGGING HELPER
|
||||
# ============================================================================
|
||||
|
||||
function Write-Log {
|
||||
param(
|
||||
[string]$Message,
|
||||
[ValidateSet('Info', 'Warning', 'Error', 'Success')]
|
||||
[string]$Level = 'Info'
|
||||
)
|
||||
|
||||
$colors = @{
|
||||
'Info' = 'Cyan'
|
||||
'Warning' = 'Yellow'
|
||||
'Error' = 'Red'
|
||||
'Success' = 'Green'
|
||||
}
|
||||
|
||||
$timestamp = Get-Date -Format 'HH:mm:ss'
|
||||
Write-Host "[$timestamp] " -NoNewline -ForegroundColor Gray
|
||||
Write-Host $Message -ForegroundColor $colors[$Level]
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# DATTO RMM INTEGRATION
|
||||
# ============================================================================
|
||||
|
||||
function Get-DattoDeviceUid {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Retrieves the unique Device UID from Datto RMM agent. Falls back to
|
||||
hostname when the agent isn't detected.
|
||||
#>
|
||||
|
||||
$regPaths = @(
|
||||
'HKLM:\SOFTWARE\CentraStage',
|
||||
'HKLM:\SOFTWARE\WOW6432Node\CentraStage'
|
||||
)
|
||||
|
||||
foreach ($path in $regPaths) {
|
||||
if (Test-Path $path) {
|
||||
try {
|
||||
$deviceUid = (Get-ItemProperty -Path $path -Name 'DeviceID' -ErrorAction SilentlyContinue).DeviceID
|
||||
if ($deviceUid) {
|
||||
Write-Log "Found Datto RMM Device UID: $deviceUid" -Level Success
|
||||
return $deviceUid
|
||||
}
|
||||
} catch {
|
||||
# Continue to next path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($env:CS_DEVICE_UID) {
|
||||
Write-Log "Found Device UID from environment: $env:CS_DEVICE_UID" -Level Success
|
||||
return $env:CS_DEVICE_UID
|
||||
}
|
||||
|
||||
Write-Log "Datto RMM agent not detected, using hostname: $env:COMPUTERNAME" -Level Warning
|
||||
return $env:COMPUTERNAME
|
||||
}
|
||||
|
||||
function Get-DattoRmmContext {
|
||||
$rmmContext = @{
|
||||
AccountUid = $env:CS_ACCOUNT_UID
|
||||
SiteUid = $env:CS_PROFILE_UID
|
||||
SiteName = $env:CS_PROFILE_NAME
|
||||
SiteDesc = $env:CS_PROFILE_DESC
|
||||
Domain = $env:CS_DOMAIN
|
||||
CcHost = $env:CS_CC_HOST
|
||||
WsAddress = $env:CS_WS_ADDRESS
|
||||
CsmAddress = $env:CS_CSM_ADDRESS
|
||||
ProxyEnabled = ($env:CS_PROFILE_PROXY_TYPE -eq '1')
|
||||
IsRmmManaged = [bool]$env:CS_ACCOUNT_UID
|
||||
}
|
||||
|
||||
$udfs = @{}
|
||||
for ($i = 1; $i -le 30; $i++) {
|
||||
$udfValue = [Environment]::GetEnvironmentVariable("UDF_$i")
|
||||
if ($udfValue) {
|
||||
$udfs["UDF_$i"] = $udfValue
|
||||
}
|
||||
}
|
||||
if ($udfs.Count -gt 0) {
|
||||
$rmmContext.UserDefinedFields = $udfs
|
||||
}
|
||||
|
||||
return $rmmContext
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# HELPERS
|
||||
# ============================================================================
|
||||
|
||||
function Get-SystemContext {
|
||||
Write-Log "Collecting system context..."
|
||||
|
||||
$os = Get-CimInstance -ClassName Win32_OperatingSystem
|
||||
$cs = Get-CimInstance -ClassName Win32_ComputerSystem
|
||||
$bios = Get-CimInstance -ClassName Win32_BIOS
|
||||
|
||||
$disks = Get-CimInstance -ClassName Win32_LogicalDisk -Filter "DriveType=3" | ForEach-Object {
|
||||
@{
|
||||
Drive = $_.DeviceID
|
||||
SizeGB = [math]::Round($_.Size / 1GB, 2)
|
||||
FreeGB = [math]::Round($_.FreeSpace / 1GB, 2)
|
||||
PercentFree = [math]::Round(($_.FreeSpace / $_.Size) * 100, 1)
|
||||
}
|
||||
}
|
||||
|
||||
$memory = @{
|
||||
TotalGB = [math]::Round($cs.TotalPhysicalMemory / 1GB, 2)
|
||||
AvailableGB = [math]::Round(($os.FreePhysicalMemory * 1KB) / 1GB, 2)
|
||||
}
|
||||
$memory.UsedPercent = [math]::Round((1 - ($memory.AvailableGB / $memory.TotalGB)) * 100, 1)
|
||||
|
||||
$uptime = (Get-Date) - $os.LastBootUpTime
|
||||
|
||||
$pendingReboot = $false
|
||||
$rebootReasons = @()
|
||||
if (Test-Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired') {
|
||||
$pendingReboot = $true
|
||||
$rebootReasons += 'Windows Update'
|
||||
}
|
||||
if (Test-Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending') {
|
||||
$pendingReboot = $true
|
||||
$rebootReasons += 'Component Servicing'
|
||||
}
|
||||
if (Test-Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\PendingFileRenameOperations') {
|
||||
$pendingReboot = $true
|
||||
$rebootReasons += 'File Rename Operations'
|
||||
}
|
||||
|
||||
$recentUpdates = @()
|
||||
try {
|
||||
$session = New-Object -ComObject Microsoft.Update.Session
|
||||
$searcher = $session.CreateUpdateSearcher()
|
||||
$history = $searcher.GetTotalHistoryCount()
|
||||
if ($history -gt 0) {
|
||||
$recentUpdates = $searcher.QueryHistory(0, [Math]::Min(10, $history)) |
|
||||
Where-Object { $_.Date -gt (Get-Date).AddDays(-7) } |
|
||||
ForEach-Object {
|
||||
@{
|
||||
Title = $_.Title
|
||||
Date = $_.Date.ToString('yyyy-MM-dd HH:mm:ss')
|
||||
Result = switch ($_.ResultCode) {
|
||||
1 { 'In Progress' }
|
||||
2 { 'Succeeded' }
|
||||
3 { 'Succeeded With Errors' }
|
||||
4 { 'Failed' }
|
||||
5 { 'Aborted' }
|
||||
default { 'Unknown' }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
Write-Log "Could not retrieve Windows Update history: $_" -Level Warning
|
||||
}
|
||||
|
||||
$context = @{
|
||||
ComputerName = $env:COMPUTERNAME
|
||||
Domain = $cs.Domain
|
||||
Workgroup = if ($cs.PartOfDomain) { $null } else { $cs.Workgroup }
|
||||
IsDomainJoined = $cs.PartOfDomain
|
||||
OS = @{
|
||||
Caption = $os.Caption
|
||||
Version = $os.Version
|
||||
Build = $os.BuildNumber
|
||||
Architecture = $os.OSArchitecture
|
||||
InstallDate = $os.InstallDate.ToString('yyyy-MM-dd')
|
||||
}
|
||||
Hardware = @{
|
||||
Manufacturer = $cs.Manufacturer
|
||||
Model = $cs.Model
|
||||
SerialNumber = $bios.SerialNumber
|
||||
BIOSVersion = $bios.SMBIOSBIOSVersion
|
||||
}
|
||||
Memory = $memory
|
||||
Disks = $disks
|
||||
Uptime = @{
|
||||
Days = [math]::Floor($uptime.TotalDays)
|
||||
Hours = $uptime.Hours
|
||||
Minutes = $uptime.Minutes
|
||||
Total = $uptime.ToString('d\.hh\:mm\:ss')
|
||||
}
|
||||
LastBoot = $os.LastBootUpTime.ToString('yyyy-MM-dd HH:mm:ss')
|
||||
PendingReboot = $pendingReboot
|
||||
RebootReasons = $rebootReasons
|
||||
RecentUpdates = $recentUpdates
|
||||
CurrentUser = $env:USERNAME
|
||||
TimeZone = (Get-TimeZone).Id
|
||||
CollectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss')
|
||||
CollectedAtUtc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ssZ')
|
||||
}
|
||||
|
||||
if ($IncludeInstalledSoftware) {
|
||||
Write-Log "Collecting installed software list..."
|
||||
$software = @()
|
||||
$regPaths = @(
|
||||
'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*',
|
||||
'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*'
|
||||
)
|
||||
foreach ($path in $regPaths) {
|
||||
$software += Get-ItemProperty -Path $path -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.DisplayName -and $_.DisplayName -notmatch '^(Update|Hotfix|Security Update)' } |
|
||||
Select-Object DisplayName, DisplayVersion, Publisher, InstallDate |
|
||||
ForEach-Object {
|
||||
@{
|
||||
Name = $_.DisplayName
|
||||
Version = $_.DisplayVersion
|
||||
Publisher = $_.Publisher
|
||||
InstallDate = $_.InstallDate
|
||||
}
|
||||
}
|
||||
}
|
||||
$context.InstalledSoftware = $software | Sort-Object { $_.Name } -Unique
|
||||
}
|
||||
|
||||
if ($IncludeRunningProcesses) {
|
||||
Write-Log "Collecting running processes..."
|
||||
$context.RunningProcesses = Get-Process |
|
||||
Group-Object ProcessName |
|
||||
Where-Object { $_.Count -gt 0 } |
|
||||
ForEach-Object {
|
||||
@{
|
||||
Name = $_.Name
|
||||
Count = $_.Count
|
||||
TotalMemMB = [math]::Round(($_.Group | Measure-Object WorkingSet64 -Sum).Sum / 1MB, 2)
|
||||
}
|
||||
} |
|
||||
Sort-Object { $_.TotalMemMB } -Descending |
|
||||
Select-Object -First 50
|
||||
}
|
||||
|
||||
if ($IncludeNetworkConfig) {
|
||||
Write-Log "Collecting network configuration..."
|
||||
$context.NetworkAdapters = Get-NetAdapter | Where-Object { $_.Status -eq 'Up' } | ForEach-Object {
|
||||
$ipConfig = Get-NetIPAddress -InterfaceIndex $_.ifIndex -ErrorAction SilentlyContinue
|
||||
@{
|
||||
Name = $_.Name
|
||||
Description = $_.InterfaceDescription
|
||||
MacAddress = $_.MacAddress
|
||||
Speed = "$([math]::Round($_.LinkSpeed / 1000000))Mbps"
|
||||
IPv4 = ($ipConfig | Where-Object { $_.AddressFamily -eq 'IPv4' }).IPAddress
|
||||
IPv6 = ($ipConfig | Where-Object { $_.AddressFamily -eq 'IPv6' }).IPAddress | Select-Object -First 1
|
||||
}
|
||||
}
|
||||
$context.DNSServers = (Get-DnsClientServerAddress -AddressFamily IPv4 |
|
||||
Where-Object { $_.ServerAddresses } |
|
||||
Select-Object -ExpandProperty ServerAddresses -Unique)
|
||||
}
|
||||
|
||||
return $context
|
||||
}
|
||||
|
||||
function Get-FilteredEvents {
|
||||
param(
|
||||
[string]$LogName,
|
||||
[datetime]$StartTime,
|
||||
[int]$MaxEvents
|
||||
)
|
||||
|
||||
Write-Log "Collecting events from '$LogName' log..."
|
||||
|
||||
$events = @()
|
||||
$criticalIds = $CriticalEventIds[$LogName]
|
||||
|
||||
try {
|
||||
$filterHash = @{
|
||||
LogName = $LogName
|
||||
StartTime = $StartTime
|
||||
}
|
||||
|
||||
$rawEvents = Get-WinEvent -FilterHashtable $filterHash -MaxEvents $MaxEvents -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $rawEvents) {
|
||||
Write-Log "No events found in '$LogName' log for the specified time period" -Level Warning
|
||||
return @()
|
||||
}
|
||||
|
||||
Write-Log "Processing $($rawEvents.Count) raw events from '$LogName'..."
|
||||
|
||||
foreach ($event in $rawEvents) {
|
||||
$isNoise = $false
|
||||
if ($NoiseFilter.ContainsKey($event.ProviderName)) {
|
||||
if ($NoiseFilter[$event.ProviderName] -contains $event.Id) {
|
||||
$isNoise = $true
|
||||
}
|
||||
}
|
||||
if ($isNoise) { continue }
|
||||
|
||||
$isCriticalEvent = $criticalIds -and ($criticalIds -contains $event.Id)
|
||||
$levelValue = switch ($event.LevelDisplayName) {
|
||||
'Critical' { 1 }
|
||||
'Error' { 2 }
|
||||
'Warning' { 3 }
|
||||
'Information' { 4 }
|
||||
'Verbose' { 5 }
|
||||
default { 4 }
|
||||
}
|
||||
|
||||
if ($isCriticalEvent -or $levelValue -le $MinLevelValue) {
|
||||
$events += @{
|
||||
TimeCreated = $event.TimeCreated.ToString('yyyy-MM-dd HH:mm:ss')
|
||||
LogName = $LogName
|
||||
EventId = $event.Id
|
||||
Level = $event.LevelDisplayName
|
||||
Source = $event.ProviderName
|
||||
Message = $event.Message
|
||||
TaskCategory = $event.TaskDisplayName
|
||||
IsCritical = $isCriticalEvent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log "Filtered to $($events.Count) relevant events from '$LogName'" -Level Success
|
||||
} catch {
|
||||
Write-Log "Error reading '$LogName' log: $_" -Level Error
|
||||
}
|
||||
|
||||
return $events
|
||||
}
|
||||
|
||||
function Get-EventSummary {
|
||||
param([array]$Events)
|
||||
|
||||
$summary = @{
|
||||
TotalEvents = $Events.Count
|
||||
ByLevel = @{}
|
||||
ByLog = @{}
|
||||
CriticalEvents = ($Events | Where-Object { $_.IsCritical }).Count
|
||||
TopEventIds = @()
|
||||
TimeRange = @{
|
||||
Earliest = $null
|
||||
Latest = $null
|
||||
}
|
||||
}
|
||||
|
||||
$Events | Group-Object Level | ForEach-Object {
|
||||
$summary.ByLevel[$_.Name] = $_.Count
|
||||
}
|
||||
|
||||
$Events | Group-Object LogName | ForEach-Object {
|
||||
$summary.ByLog[$_.Name] = $_.Count
|
||||
}
|
||||
|
||||
$summary.TopEventIds = $Events |
|
||||
Group-Object EventId, LogName, Level, Source |
|
||||
Sort-Object Count -Descending |
|
||||
Select-Object -First 10 |
|
||||
ForEach-Object {
|
||||
$parts = $_.Name -split ', '
|
||||
$sample = $_.Group | Select-Object -First 1
|
||||
@{
|
||||
EventId = [int]$parts[0]
|
||||
LogName = $parts[1]
|
||||
Level = $parts[2]
|
||||
Source = $parts[3]
|
||||
Count = $_.Count
|
||||
Sample = if ($sample.Message.Length -gt 200) {
|
||||
$sample.Message.Substring(0, 200) + '...'
|
||||
} else {
|
||||
$sample.Message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$sortedTimes = $Events | Sort-Object TimeCreated
|
||||
$summary.TimeRange.Earliest = ($sortedTimes | Select-Object -First 1).TimeCreated
|
||||
$summary.TimeRange.Latest = ($sortedTimes | Select-Object -Last 1).TimeCreated
|
||||
|
||||
return $summary
|
||||
}
|
||||
|
||||
function Get-PresignedUrl {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Legacy n8n flow only — fetches a B2 presigned PUT URL from the n8n
|
||||
webhook. Pulse mode skips this and uses $env:UploadUrl directly.
|
||||
#>
|
||||
param(
|
||||
[string]$WebhookUrl,
|
||||
[string]$ClientId,
|
||||
[string]$DeviceUid,
|
||||
[string]$RunId
|
||||
)
|
||||
|
||||
Write-Log "Requesting presigned upload URL (legacy n8n flow)..."
|
||||
|
||||
$body = @{
|
||||
clientId = $ClientId
|
||||
deviceUid = $DeviceUid
|
||||
runId = $RunId
|
||||
pathFormat = "flat"
|
||||
} | ConvertTo-Json
|
||||
|
||||
try {
|
||||
$response = Invoke-RestMethod -Uri $WebhookUrl -Method POST -Body $body `
|
||||
-ContentType 'application/json' -UseBasicParsing -TimeoutSec 30
|
||||
if ($response.presignedUrl) {
|
||||
Write-Log "Received presigned URL for: $($response.objectKey)" -Level Success
|
||||
return $response
|
||||
} else {
|
||||
throw "Response did not contain presignedUrl"
|
||||
}
|
||||
} catch {
|
||||
Write-Log "Failed to get presigned URL: $_" -Level Error
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
function Upload-ToB2 {
|
||||
param(
|
||||
[string]$PresignedUrl,
|
||||
[byte[]]$Data,
|
||||
[string]$ContentType = 'application/gzip'
|
||||
)
|
||||
|
||||
Write-Log "Uploading $([math]::Round($Data.Length / 1KB, 2)) KB to B2..."
|
||||
|
||||
try {
|
||||
$headers = @{ "Content-Type" = $ContentType }
|
||||
$response = Invoke-WebRequest -Uri $PresignedUrl -Method PUT -Body $Data `
|
||||
-Headers $headers -UseBasicParsing -ErrorAction Stop
|
||||
if ($response.StatusCode -in 200, 201) {
|
||||
Write-Log "Upload successful!" -Level Success
|
||||
return $true
|
||||
} else {
|
||||
throw "Upload failed with status $($response.StatusCode)"
|
||||
}
|
||||
} catch {
|
||||
Write-Log "Upload failed: $_" -Level Error
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ProcessingNotification {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
POST the upload's metadata to the receiving service. In Pulse mode
|
||||
the WebhookSecret is sent as the x-openclaw-key header.
|
||||
#>
|
||||
param(
|
||||
[string]$WebhookUrl,
|
||||
[string]$WebhookSecret,
|
||||
[string]$ObjectKey,
|
||||
[string]$ClientId,
|
||||
[string]$DeviceUid,
|
||||
[string]$ComputerName,
|
||||
[string]$RunId,
|
||||
[hashtable]$Summary,
|
||||
[hashtable]$RmmContext,
|
||||
[string]$IssueDescription,
|
||||
[string]$TicketNumber
|
||||
)
|
||||
|
||||
Write-Log "Sending processing notification..."
|
||||
|
||||
$body = @{
|
||||
objectKey = $ObjectKey
|
||||
clientId = $ClientId
|
||||
deviceUid = $DeviceUid
|
||||
computerName = $ComputerName
|
||||
runId = $RunId
|
||||
collectedAt = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
|
||||
summary = @{
|
||||
totalEvents = $Summary.TotalEvents
|
||||
criticalEvents = $Summary.CriticalEvents
|
||||
errorCount = $Summary.ByLevel['Error']
|
||||
warningCount = $Summary.ByLevel['Warning']
|
||||
timeRange = "$($Summary.TimeRange.Earliest) to $($Summary.TimeRange.Latest)"
|
||||
}
|
||||
rmmContext = @{
|
||||
siteName = $RmmContext.SiteName
|
||||
siteUid = $RmmContext.SiteUid
|
||||
accountUid = $RmmContext.AccountUid
|
||||
}
|
||||
issueDescription = $IssueDescription
|
||||
ticketNumber = $TicketNumber
|
||||
} | ConvertTo-Json -Depth 10
|
||||
|
||||
$headers = @{}
|
||||
if ($WebhookSecret) {
|
||||
$headers['x-openclaw-key'] = $WebhookSecret
|
||||
}
|
||||
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method POST -Body $body `
|
||||
-ContentType 'application/json' -Headers $headers `
|
||||
-UseBasicParsing -TimeoutSec 60
|
||||
Write-Log "Processing notification sent" -Level Success
|
||||
return $true
|
||||
} catch {
|
||||
Write-Log "Failed to send processing notification: $_" -Level Warning
|
||||
# Don't throw — upload succeeded; notification is secondary.
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Compress-Data {
|
||||
param([string]$JsonString)
|
||||
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($JsonString)
|
||||
$memoryStream = New-Object System.IO.MemoryStream
|
||||
$gzipStream = New-Object System.IO.Compression.GZipStream(
|
||||
$memoryStream,
|
||||
[System.IO.Compression.CompressionMode]::Compress
|
||||
)
|
||||
|
||||
$gzipStream.Write($bytes, 0, $bytes.Length)
|
||||
$gzipStream.Close()
|
||||
|
||||
$compressedData = $memoryStream.ToArray()
|
||||
$memoryStream.Close()
|
||||
|
||||
$ratio = [math]::Round((1 - ($compressedData.Length / $bytes.Length)) * 100, 1)
|
||||
Write-Log "Compressed from $([math]::Round($bytes.Length / 1KB, 2)) KB to $([math]::Round($compressedData.Length / 1KB, 2)) KB ($ratio% reduction)"
|
||||
|
||||
return $compressedData
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PULSE-MODE OVERRIDES (read Datto component variables)
|
||||
# ============================================================================
|
||||
|
||||
# When the Quick Job is dispatched from Pulse, these arrive as env vars.
|
||||
# Empty in the legacy n8n flow.
|
||||
$PulseRunId = $env:RunId
|
||||
$PulseUploadUrl = $env:UploadUrl
|
||||
$PulseObjectKey = $env:ObjectKey
|
||||
$PulseWebhookUrl = $env:WebhookUrl
|
||||
$PulseWebhookSecret = $env:WebhookSecret
|
||||
$PulseClientId = $env:ClientId
|
||||
|
||||
$IsPulseMode = [bool]($PulseWebhookUrl -and $PulseUploadUrl -and $PulseObjectKey)
|
||||
|
||||
# ============================================================================
|
||||
# RESOLVE PARAMETERS FROM DATTO RMM ENVIRONMENT
|
||||
# ============================================================================
|
||||
|
||||
$DeviceUid = Get-DattoDeviceUid
|
||||
|
||||
# ClientId: explicit param > Pulse env > Datto site UID env > error
|
||||
if (-not $ClientId) {
|
||||
if ($PulseClientId) {
|
||||
$ClientId = $PulseClientId
|
||||
} elseif ($env:CS_PROFILE_UID) {
|
||||
$ClientId = $env:CS_PROFILE_UID
|
||||
} else {
|
||||
Write-Error "ClientId not provided and CS_PROFILE_UID not available. Please provide a ClientId."
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
# Force TLS 1.2 (and 1.3 when available). Without this, Invoke-WebRequest on
|
||||
# older Windows / PS 5.1 falls back to TLS 1.0/1.1, which Backblaze B2's S3
|
||||
# endpoint rejects with "Could not create SSL/TLS secure channel."
|
||||
try {
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
if ([Net.SecurityProtocolType].GetEnumNames() -contains 'Tls13') {
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls13
|
||||
}
|
||||
} catch {
|
||||
Write-Log "Could not set TLS 1.2/1.3 protocol — uploads may fail on older systems: $_" -Level Warning
|
||||
}
|
||||
|
||||
$LevelMap = @{
|
||||
'Critical' = 1
|
||||
'Error' = 2
|
||||
'Warning' = 3
|
||||
'Information' = 4
|
||||
'Verbose' = 5
|
||||
}
|
||||
$MinLevelValue = $LevelMap[$MinLevel]
|
||||
|
||||
$NoiseFilter = @{
|
||||
'Microsoft-Windows-Security-SPP' = @(903, 904, 905)
|
||||
'Microsoft-Windows-Time-Service' = @(35, 37)
|
||||
'VSS' = @(8224)
|
||||
}
|
||||
|
||||
$CriticalEventIds = @{
|
||||
'System' = @(
|
||||
1001, # Bugcheck (BSOD)
|
||||
6008, # Unexpected shutdown
|
||||
7031, # Service crash
|
||||
7034, # Service terminated unexpectedly
|
||||
41, # Kernel power (unexpected restart)
|
||||
1074, # Shutdown initiated
|
||||
6005, # Event log started (boot)
|
||||
6006, # Event log stopped (shutdown)
|
||||
7045, # New service installed
|
||||
10016 # DCOM permission error
|
||||
)
|
||||
'Application' = @(
|
||||
1000, # Application crash
|
||||
1001, # Windows Error Reporting
|
||||
1002, # Application hang
|
||||
1026, # .NET Runtime error
|
||||
11707, # Install completed
|
||||
11708, # Install failed
|
||||
11724 # Uninstall completed
|
||||
)
|
||||
'Security' = @(
|
||||
4624, # Successful logon
|
||||
4625, # Failed logon
|
||||
4648, # Explicit credential logon
|
||||
4672, # Special privileges assigned
|
||||
4720, # User account created
|
||||
4726, # User account deleted
|
||||
4732, # Member added to security group
|
||||
4756, # Member added to universal group
|
||||
1102 # Audit log cleared
|
||||
)
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# MAIN EXECUTION
|
||||
# ============================================================================
|
||||
|
||||
$startTime = Get-Date
|
||||
Write-Host ""
|
||||
Write-Host "=================================================================" -ForegroundColor Cyan
|
||||
Write-Host " Windows Event Log Collector for LLM Analysis" -ForegroundColor Cyan
|
||||
Write-Host " Datto RMM Component" -ForegroundColor Cyan
|
||||
Write-Host "=================================================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
|
||||
if ($IsPulseMode) {
|
||||
Write-Log "Mode: Pulse-driven dispatch" -Level Success
|
||||
} else {
|
||||
Write-Log "Mode: legacy (n8n / standalone) flow" -Level Info
|
||||
}
|
||||
Write-Log "Starting collection on $env:COMPUTERNAME"
|
||||
Write-Log "Client ID: $ClientId"
|
||||
Write-Log "Device UID: $DeviceUid"
|
||||
Write-Log "Time Range: Last $HoursBack hours | Min Level: $MinLevel"
|
||||
|
||||
if ($IssueDescription) {
|
||||
Write-Log "Issue: $IssueDescription"
|
||||
}
|
||||
|
||||
$rmmContext = Get-DattoRmmContext
|
||||
if ($rmmContext.IsRmmManaged) {
|
||||
Write-Log "RMM Site: $($rmmContext.SiteName) ($($rmmContext.SiteUid))" -Level Info
|
||||
}
|
||||
|
||||
# RunId: Pulse-supplied wins so the receiver can correlate to the dispatched
|
||||
# execution row. Otherwise generate a timestamp-shaped one for n8n compat.
|
||||
$runId = if ($PulseRunId) { $PulseRunId } else { "eventlogs_$(Get-Date -Format 'yyyyMMdd_HHmmss')" }
|
||||
$eventStartTime = (Get-Date).AddHours(-$HoursBack)
|
||||
|
||||
$systemContext = Get-SystemContext
|
||||
|
||||
$allEvents = @()
|
||||
foreach ($logName in $IncludeLogs) {
|
||||
$events = Get-FilteredEvents -LogName $logName -StartTime $eventStartTime -MaxEvents $MaxEventsPerLog
|
||||
$allEvents += $events
|
||||
}
|
||||
$allEvents = $allEvents | Sort-Object { [datetime]$_.TimeCreated } -Descending
|
||||
|
||||
$summary = Get-EventSummary -Events $allEvents
|
||||
|
||||
$payload = @{
|
||||
metadata = @{
|
||||
version = "3.0.1"
|
||||
clientId = $ClientId
|
||||
deviceUid = $DeviceUid
|
||||
computerName = $env:COMPUTERNAME
|
||||
runId = $runId
|
||||
collectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss')
|
||||
collectedAtUtc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ssZ')
|
||||
collectionParams = @{
|
||||
hoursBack = $HoursBack
|
||||
minLevel = $MinLevel
|
||||
includedLogs = $IncludeLogs
|
||||
maxEvents = $MaxEventsPerLog
|
||||
}
|
||||
issueDescription = $IssueDescription
|
||||
ticketNumber = $TicketNumber
|
||||
transport = if ($IsPulseMode) { 'pulse' } else { 'n8n_legacy' }
|
||||
}
|
||||
rmmContext = $rmmContext
|
||||
systemContext = $systemContext
|
||||
summary = $summary
|
||||
events = $allEvents
|
||||
analysisHints = @{
|
||||
focus = @(
|
||||
"Look for patterns in Critical and Error events"
|
||||
"Check for service crashes (Event ID 7031, 7034)"
|
||||
"Look for unexpected shutdowns (Event ID 41, 6008)"
|
||||
"Check application crashes (Event ID 1000, 1001)"
|
||||
"Review any security anomalies (failed logins, privilege escalation)"
|
||||
)
|
||||
correlate = @(
|
||||
"Events occurring within seconds of each other may be related"
|
||||
"Service failures often trigger multiple cascading events"
|
||||
"Look for patterns before and after the issue timestamp"
|
||||
)
|
||||
context = @(
|
||||
"System uptime and last boot time"
|
||||
"Pending reboots may indicate incomplete updates"
|
||||
"Recent Windows Updates may correlate with new issues"
|
||||
"Low disk space or memory can cause various failures"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
$jsonPayload = $payload | ConvertTo-Json -Depth 20 -Compress
|
||||
|
||||
# ----- Summary ---------------------------------------------------------------
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "-----------------------------------------------------------------" -ForegroundColor Yellow
|
||||
Write-Host " COLLECTION SUMMARY" -ForegroundColor Yellow
|
||||
Write-Host "-----------------------------------------------------------------" -ForegroundColor Yellow
|
||||
Write-Host ""
|
||||
Write-Host " Device: $env:COMPUTERNAME" -ForegroundColor White
|
||||
Write-Host " Device UID: $DeviceUid" -ForegroundColor Gray
|
||||
Write-Host " Client ID: $ClientId" -ForegroundColor Gray
|
||||
Write-Host " Run ID: $runId" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
Write-Host " Total Events Collected: $($summary.TotalEvents)" -ForegroundColor White
|
||||
Write-Host ""
|
||||
|
||||
Write-Host " By Level:" -ForegroundColor Gray
|
||||
$summary.ByLevel.GetEnumerator() | Sort-Object {
|
||||
switch ($_.Key) { 'Critical' { 1 } 'Error' { 2 } 'Warning' { 3 } 'Information' { 4 } default { 5 } }
|
||||
} | ForEach-Object {
|
||||
$color = switch ($_.Key) {
|
||||
'Critical' { 'Magenta' }
|
||||
'Error' { 'Red' }
|
||||
'Warning' { 'Yellow' }
|
||||
'Information' { 'Cyan' }
|
||||
default { 'Gray' }
|
||||
}
|
||||
Write-Host " $($_.Key): $($_.Value)" -ForegroundColor $color
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host " By Log:" -ForegroundColor Gray
|
||||
$summary.ByLog.GetEnumerator() | ForEach-Object {
|
||||
Write-Host " $($_.Key): $($_.Value)" -ForegroundColor White
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host " Critical Events (always captured): $($summary.CriticalEvents)" -ForegroundColor $(if ($summary.CriticalEvents -gt 0) { 'Red' } else { 'Green' })
|
||||
Write-Host " Time Range: $($summary.TimeRange.Earliest) to $($summary.TimeRange.Latest)" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
|
||||
if ($summary.TopEventIds.Count -gt 0) {
|
||||
Write-Host " Top Event Types:" -ForegroundColor Gray
|
||||
$summary.TopEventIds | Select-Object -First 5 | ForEach-Object {
|
||||
$levelColor = switch ($_.Level) {
|
||||
'Critical' { 'Magenta' }
|
||||
'Error' { 'Red' }
|
||||
'Warning' { 'Yellow' }
|
||||
default { 'White' }
|
||||
}
|
||||
Write-Host " [$($_.LogName)] ID $($_.EventId): $($_.Count) occurrences" -ForegroundColor $levelColor -NoNewline
|
||||
Write-Host " ($($_.Source))" -ForegroundColor Gray
|
||||
}
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
Write-Host " System Health:" -ForegroundColor Gray
|
||||
Write-Host " Memory Used: $($systemContext.Memory.UsedPercent)%" -ForegroundColor $(if ($systemContext.Memory.UsedPercent -gt 90) { 'Red' } elseif ($systemContext.Memory.UsedPercent -gt 75) { 'Yellow' } else { 'Green' })
|
||||
Write-Host " Uptime: $($systemContext.Uptime.Total)" -ForegroundColor White
|
||||
Write-Host " Pending Reboot: $(if ($systemContext.PendingReboot) { 'Yes - ' + ($systemContext.RebootReasons -join ', ') } else { 'No' })" -ForegroundColor $(if ($systemContext.PendingReboot) { 'Yellow' } else { 'Green' })
|
||||
|
||||
foreach ($disk in $systemContext.Disks) {
|
||||
$diskColor = if ($disk.PercentFree -lt 10) { 'Red' } elseif ($disk.PercentFree -lt 20) { 'Yellow' } else { 'Green' }
|
||||
Write-Host " Disk $($disk.Drive): $($disk.FreeGB) GB free ($($disk.PercentFree)%)" -ForegroundColor $diskColor
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "-----------------------------------------------------------------" -ForegroundColor Yellow
|
||||
|
||||
# ----- Upload ----------------------------------------------------------------
|
||||
|
||||
if ($DryRun) {
|
||||
Write-Host ""
|
||||
Write-Log "DRY RUN - Skipping upload" -Level Warning
|
||||
Write-Host ""
|
||||
Write-Host "Payload size: $([math]::Round($jsonPayload.Length / 1KB, 2)) KB (uncompressed)" -ForegroundColor Gray
|
||||
|
||||
if (-not (Test-Path $OutputPath)) {
|
||||
New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null
|
||||
}
|
||||
$localPath = Join-Path $OutputPath "EventLogExport_$runId.json"
|
||||
$jsonPayload | Out-File -FilePath $localPath -Encoding UTF8
|
||||
Write-Log "Saved to: $localPath" -Level Success
|
||||
|
||||
} else {
|
||||
Write-Host ""
|
||||
|
||||
$compressedData = Compress-Data -JsonString $jsonPayload
|
||||
|
||||
try {
|
||||
if ($IsPulseMode) {
|
||||
# Pulse mode: PUT URL + object key were pre-presigned at dispatch.
|
||||
Write-Log "Uploading to Pulse-supplied URL: $PulseObjectKey" -Level Success
|
||||
$urlResponse = @{
|
||||
presignedUrl = $PulseUploadUrl
|
||||
objectKey = $PulseObjectKey
|
||||
}
|
||||
} else {
|
||||
# Legacy n8n flow: fetch a presigned URL from the n8n webhook.
|
||||
$urlResponse = Get-PresignedUrl -WebhookUrl $WebhookUrl `
|
||||
-ClientId $ClientId -DeviceUid $DeviceUid -RunId $runId
|
||||
}
|
||||
|
||||
Upload-ToB2 -PresignedUrl $urlResponse.presignedUrl -Data $compressedData
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "=================================================================" -ForegroundColor Green
|
||||
Write-Host " UPLOAD COMPLETE" -ForegroundColor Green
|
||||
Write-Host "=================================================================" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
Write-Host " Object Key: $($urlResponse.objectKey)" -ForegroundColor White
|
||||
Write-Host " Events: $($summary.TotalEvents)" -ForegroundColor White
|
||||
Write-Host " Size: $([math]::Round($compressedData.Length / 1KB, 2)) KB (compressed)" -ForegroundColor White
|
||||
Write-Host ""
|
||||
|
||||
# Pick the metadata endpoint + auth secret based on mode.
|
||||
$notifyUrl = if ($IsPulseMode) { $PulseWebhookUrl } else { $NotifyWebhookUrl }
|
||||
$notifySecret = if ($IsPulseMode) { $PulseWebhookSecret } else { '' }
|
||||
|
||||
Send-ProcessingNotification `
|
||||
-WebhookUrl $notifyUrl `
|
||||
-WebhookSecret $notifySecret `
|
||||
-ObjectKey $urlResponse.objectKey `
|
||||
-ClientId $ClientId `
|
||||
-DeviceUid $DeviceUid `
|
||||
-ComputerName $env:COMPUTERNAME `
|
||||
-RunId $runId `
|
||||
-Summary $summary `
|
||||
-RmmContext $rmmContext `
|
||||
-IssueDescription $IssueDescription `
|
||||
-TicketNumber $TicketNumber
|
||||
|
||||
} catch {
|
||||
Write-Host ""
|
||||
Write-Log "Failed to upload event logs: $_" -Level Error
|
||||
|
||||
if (-not (Test-Path $OutputPath)) {
|
||||
New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null
|
||||
}
|
||||
$localPath = Join-Path $OutputPath "EventLogExport_$runId.json.gz"
|
||||
[System.IO.File]::WriteAllBytes($localPath, $compressedData)
|
||||
Write-Log "Saved locally as fallback: $localPath" -Level Warning
|
||||
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
$duration = (Get-Date) - $startTime
|
||||
Write-Log "Collection completed in $($duration.TotalSeconds.ToString('F1')) seconds"
|
||||
|
||||
return @{
|
||||
Success = $true
|
||||
Mode = if ($IsPulseMode) { 'pulse' } else { 'n8n_legacy' }
|
||||
RunId = $runId
|
||||
DeviceUid = $DeviceUid
|
||||
ClientId = $ClientId
|
||||
EventCount = $summary.TotalEvents
|
||||
CriticalCount = $summary.CriticalEvents
|
||||
ErrorCount = $summary.ByLevel['Error']
|
||||
WarningCount = $summary.ByLevel['Warning']
|
||||
}
|
||||
57
scripts/reconcile-device-links.ts
Normal file
57
scripts/reconcile-device-links.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* reconcile-device-links.ts
|
||||
*
|
||||
* Walks `device_external_ids` rows where `configuration_item_id IS NULL` and
|
||||
* tries to link each to a configuration_item via cascading match strategies
|
||||
* (serial → MAC → hostname-in-company). Conflicts (multiple CIs match) are
|
||||
* logged and skipped — no auto-merge.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/reconcile-device-links.ts --dry-run
|
||||
* npx tsx scripts/reconcile-device-links.ts --dry-run --limit=10000
|
||||
* npx tsx scripts/reconcile-device-links.ts --limit=5000
|
||||
*/
|
||||
|
||||
import { config } from 'dotenv';
|
||||
config({ path: '.env.local' });
|
||||
|
||||
import { reconcileUnlinkedDevices } from '@/lib/services/device-link-reconciler';
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const limitArg = args.find((a) => a.startsWith('--limit='));
|
||||
const limit = limitArg ? parseInt(limitArg.split('=')[1], 10) : 5000;
|
||||
return { dryRun, limit };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { dryRun, limit } = parseArgs();
|
||||
console.log(
|
||||
`[reconcile-device-links] starting${dryRun ? ' (DRY RUN — no writes)' : ''}; limit=${limit}`
|
||||
);
|
||||
const start = Date.now();
|
||||
const result = await reconcileUnlinkedDevices({ limit, dryRun });
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
|
||||
console.log('');
|
||||
console.log(`scanned: ${result.scanned}`);
|
||||
console.log(`linked: ${result.linked}${dryRun ? ' (would link)' : ''}`);
|
||||
console.log(`conflicts: ${result.conflicts} (multiple CIs matched — skipped)`);
|
||||
console.log(`unmatched: ${result.unmatched}`);
|
||||
console.log('');
|
||||
console.log('by confidence:');
|
||||
for (const [k, v] of Object.entries(result.byConfidence)) {
|
||||
if (v > 0) console.log(` ${k.padEnd(22)} ${v}`);
|
||||
}
|
||||
console.log('');
|
||||
console.log(`done in ${elapsed}s`);
|
||||
|
||||
// pg pool keeps the process alive otherwise.
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('[reconcile-device-links] fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue