wulf-pulse/lib/services/morning-summary-service.ts
lorentz c518eefdb2 feat: Morning NOC Summary adaptive card for Teams
- Add MorningSummaryService with Zabbix aggregation and adaptive card builder
- Add webhook delivery system with Teams incoming webhooks
- Add admin UI at /admin/morning-summary for webhook/config management
- Add API routes: /send, /test, /webhooks, /webhooks/[id], /config, /history
- Register morning-summary cron job in SyncScheduler (Mon-Fri 6:30 AM)
- Add outages_only filter (Unavailable triggers only)
- Fix host resolution: use getTriggerEnabledHosts to exclude disabled hosts
- Fix resolved events: event.get value:1 scoped to window with r_eventid filter
- Remove emojis from fact rows and section headers in card
- Remove Open Zabbix button (duplicate of View Problems)
- Add migrations: morning_summary_config + morning_summaries tables
- Add outages_only column to morning_summary_config
2026-03-11 09:34:51 -04:00

543 lines
19 KiB
TypeScript

import { postgresClient } from './postgres-client';
import { ZabbixClient } from './zabbix-client';
export interface MorningSummaryProblem {
eventid: string;
hostid: string;
hostName: string;
clientName: string;
triggerName: string;
severity: number;
severityLabel: string;
startedAt: Date;
durationLabel: string;
acknowledged: boolean;
needsAttention: boolean;
}
export interface MorningSummaryResolved {
eventid: string;
hostName: string;
clientName: string;
triggerName: string;
startedAt: Date;
resolvedAt: Date;
durationLabel: string;
}
export interface MorningSummary {
generatedAt: Date;
windowFrom: Date;
windowTo: Date;
isWeekendWindow: boolean;
openProblems: MorningSummaryProblem[];
resolvedOvernight: MorningSummaryResolved[];
openCount: number;
resolvedCount: number;
mttrMinutes: number | null;
clientsAffected: string[];
}
export interface WebhookConfig {
id: number;
label: string;
webhook_url: string;
enabled: boolean;
last_delivered_at: string | null;
last_status: string | null;
created_at: string;
}
export interface SummaryConfig {
weekend_suppression: boolean;
monday_extended_window: boolean;
severity_filter: number;
outages_only: boolean;
}
export interface DeliveryResult {
webhookId: number;
label: string;
success: boolean;
httpStatus?: number;
error?: string;
}
const SEVERITY_LABELS: Record<number, string> = {
0: 'Not classified',
1: 'Information',
2: 'Warning',
3: 'Average',
4: 'High',
5: 'Disaster',
};
function formatDuration(ms: number): string {
const totalMinutes = Math.floor(ms / 60000);
if (totalMinutes < 1) return '<1m';
if (totalMinutes < 60) return `${totalMinutes}m`;
const hours = Math.floor(totalMinutes / 60);
const mins = totalMinutes % 60;
return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`;
}
function makeZabbix(): ZabbixClient {
if (!process.env.ZABBIX_API_URL || !process.env.ZABBIX_API_TOKEN) {
throw new Error('Zabbix not configured: ZABBIX_API_URL and ZABBIX_API_TOKEN required');
}
return new ZabbixClient({
apiUrl: process.env.ZABBIX_API_URL,
apiToken: process.env.ZABBIX_API_TOKEN,
});
}
function getWindowStart(now: Date, config: SummaryConfig): { from: Date; isWeekend: boolean } {
const day = now.getDay(); // 0=Sun,1=Mon
if (day === 1 && config.monday_extended_window) {
const friday = new Date(now);
friday.setDate(friday.getDate() - 3);
friday.setHours(18, 0, 0, 0);
return { from: friday, isWeekend: true };
}
const yesterday = new Date(now);
yesterday.setDate(yesterday.getDate() - 1);
yesterday.setHours(18, 0, 0, 0);
return { from: yesterday, isWeekend: false };
}
function buildAdaptiveCard(summary: MorningSummary): object {
const headerText = `☀️ Morning NOC Summary${summary.isWeekendWindow ? ' — Weekend Coverage' : ''}`;
const mttrText = summary.mttrMinutes != null ? formatDuration(summary.mttrMinutes * 60000) : '—';
const statsColumns = {
type: 'ColumnSet',
columns: [
{
type: 'Column', width: 'stretch',
items: [{
type: 'TextBlock',
text: `${summary.openCount} Open`,
weight: 'Bolder', color: summary.openCount > 0 ? 'Attention' : 'Default',
wrap: true,
}],
},
{
type: 'Column', width: 'stretch',
items: [{
type: 'TextBlock',
text: `${summary.resolvedCount} Resolved`,
weight: 'Bolder', color: summary.resolvedCount > 0 ? 'Good' : 'Default',
wrap: true,
}],
},
{
type: 'Column', width: 'stretch',
items: [{
type: 'TextBlock',
text: `${mttrText} MTTR`,
weight: 'Bolder', wrap: true,
}],
},
],
};
const bodyItems: object[] = [
{ type: 'TextBlock', text: headerText, weight: 'Bolder', size: 'Large', wrap: true },
statsColumns,
{ type: 'TextBlock', text: ' ', spacing: 'None' },
];
// Open issues section
if (summary.openProblems.length > 0) {
const facts = summary.openProblems.map(p => ({
title: `${p.clientName} / ${p.hostName}`,
value: `${p.triggerName}${p.durationLabel}`,
}));
bodyItems.push({
type: 'Container',
style: 'attention',
items: [
{ type: 'TextBlock', text: 'Open Issues', weight: 'Bolder', color: 'Attention', wrap: true },
{ type: 'FactSet', facts },
],
});
} else {
bodyItems.push({
type: 'Container',
style: 'good',
items: [{ type: 'TextBlock', text: 'All Clear — No open issues', weight: 'Bolder', color: 'Good', wrap: true }],
});
}
// Resolved overnight section
if (summary.resolvedOvernight.length > 0) {
const shown = summary.resolvedOvernight.slice(0, 5);
const extra = summary.resolvedOvernight.length - 5;
const facts = shown.map(r => ({
title: `${r.clientName} / ${r.hostName}`,
value: `${r.triggerName}${r.durationLabel}`,
}));
if (extra > 0) {
facts.push({ title: '', value: `+${extra} more — all resolved` });
}
bodyItems.push({
type: 'Container',
style: 'good',
items: [
{ type: 'TextBlock', text: 'Resolved', weight: 'Bolder', color: 'Good', wrap: true },
{ type: 'FactSet', facts },
],
});
}
// Window info footer
const windowFrom = summary.windowFrom.toLocaleString('en-US', { month: 'short', day: 'numeric', hour: 'numeric', minute: '2-digit', hour12: true });
const windowTo = summary.windowTo.toLocaleString('en-US', { hour: 'numeric', minute: '2-digit', hour12: true });
bodyItems.push({
type: 'TextBlock',
text: `Window: ${windowFrom}${windowTo}`,
size: 'Small', color: 'Default', isSubtle: true, wrap: true, spacing: 'Small',
});
return {
$schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
type: 'AdaptiveCard',
version: '1.4',
body: bodyItems,
actions: [
{ type: 'Action.OpenUrl', title: 'Open Pulse', url: 'https://pulse.wulfconsulting.cloud' },
{ type: 'Action.OpenUrl', title: 'View Problems', url: 'https://zabbix.wulfconsulting.cloud/zabbix.php?action=problem.view' },
],
};
}
export class MorningSummaryService {
async getConfig(): Promise<SummaryConfig> {
const result = await postgresClient.query('SELECT * FROM morning_summary_config WHERE id = 1');
if (result.rows.length === 0) {
return { weekend_suppression: true, monday_extended_window: true, severity_filter: 2, outages_only: false };
}
return result.rows[0] as SummaryConfig;
}
async updateConfig(updates: Partial<SummaryConfig>): Promise<SummaryConfig> {
const fields: string[] = [];
const values: unknown[] = [];
let idx = 1;
if (updates.weekend_suppression !== undefined) { fields.push(`weekend_suppression = $${idx++}`); values.push(updates.weekend_suppression); }
if (updates.monday_extended_window !== undefined) { fields.push(`monday_extended_window = $${idx++}`); values.push(updates.monday_extended_window); }
if (updates.severity_filter !== undefined) { fields.push(`severity_filter = $${idx++}`); values.push(updates.severity_filter); }
if (updates.outages_only !== undefined) { fields.push(`outages_only = $${idx++}`); values.push(updates.outages_only); }
if (fields.length === 0) return this.getConfig();
fields.push('updated_at = NOW()');
values.push(1);
const result = await postgresClient.query(
`UPDATE morning_summary_config SET ${fields.join(', ')} WHERE id = $${idx} RETURNING *`,
values
);
return result.rows[0] as SummaryConfig;
}
async getWebhooks(): Promise<WebhookConfig[]> {
const result = await postgresClient.query('SELECT * FROM morning_summary_webhooks ORDER BY id');
return result.rows as WebhookConfig[];
}
async createWebhook(label: string, webhookUrl: string): Promise<WebhookConfig> {
const result = await postgresClient.query(
'INSERT INTO morning_summary_webhooks (label, webhook_url, enabled) VALUES ($1, $2, true) RETURNING *',
[label, webhookUrl]
);
return result.rows[0] as WebhookConfig;
}
async updateWebhook(id: number, updates: { label?: string; webhook_url?: string; enabled?: boolean }): Promise<WebhookConfig> {
const fields: string[] = [];
const values: unknown[] = [];
let idx = 1;
if (updates.label !== undefined) { fields.push(`label = $${idx++}`); values.push(updates.label); }
if (updates.webhook_url !== undefined) { fields.push(`webhook_url = $${idx++}`); values.push(updates.webhook_url); }
if (updates.enabled !== undefined) { fields.push(`enabled = $${idx++}`); values.push(updates.enabled); }
if (fields.length === 0) throw new Error('No fields to update');
values.push(id);
const result = await postgresClient.query(
`UPDATE morning_summary_webhooks SET ${fields.join(', ')} WHERE id = $${idx} RETURNING *`,
values
);
return result.rows[0] as WebhookConfig;
}
async deleteWebhook(id: number): Promise<void> {
await postgresClient.query('DELETE FROM morning_summary_webhooks WHERE id = $1', [id]);
}
async getLatestSummaryRow(): Promise<{
id: number;
generated_at: string;
window_from: string;
window_to: string;
open_count: number;
resolved_count: number;
mttr_minutes: number | null;
clients_affected: string[];
is_weekend_window: boolean;
card_payload: object | null;
delivery_status: object;
} | null> {
const result = await postgresClient.query('SELECT * FROM morning_summaries ORDER BY generated_at DESC LIMIT 1');
return result.rows.length > 0 ? result.rows[0] : null;
}
async getSummaryHistory(limit = 10): Promise<Array<{
id: number;
generated_at: string;
open_count: number;
resolved_count: number;
mttr_minutes: number | null;
is_weekend_window: boolean;
delivery_status: object;
}>> {
const result = await postgresClient.query(
'SELECT id, generated_at, open_count, resolved_count, mttr_minutes, is_weekend_window, delivery_status FROM morning_summaries ORDER BY generated_at DESC LIMIT $1',
[limit]
);
return result.rows;
}
/**
* Pull data from Zabbix and build the MorningSummary + Adaptive Card.
*/
async aggregate(overrideWindowFrom?: Date): Promise<{ summary: MorningSummary; card: object }> {
const now = new Date();
const config = await this.getConfig();
const { from: windowFrom, isWeekend } = overrideWindowFrom
? { from: overrideWindowFrom, isWeekend: false }
: getWindowStart(now, config);
const zabbix = makeZabbix();
// Fetch open problems and resolved events in parallel
const [rawProblems, rawResolved] = await Promise.all([
zabbix.getOpenProblems(config.severity_filter),
zabbix.getResolvedEvents(windowFrom, now),
]);
// Build the set of unique triggerids from both result sets
const allTriggerIds = [
...new Set([
...rawProblems.map(p => p.objectid),
...rawResolved.map(e => e.objectid),
]),
];
// Resolve triggerid → { hostid, hostName } for ENABLED hosts only.
// Triggerids tied solely to disabled hosts will be absent from this map.
const triggerHostMap = await zabbix.getTriggerEnabledHosts(allTriggerIds);
// Fetch groups for every enabled hostid so we can derive clientName
const enabledHostIds = [...new Set([...triggerHostMap.values()].map(h => h.hostid))];
const groupMap = await zabbix.getHostGroupMap(enabledHostIds);
function resolveClient(triggerid: string): { hostid: string; hostName: string; clientName: string } {
const host = triggerHostMap.get(triggerid);
if (!host) return { hostid: '', hostName: 'Unknown', clientName: 'Unknown' };
const groups = groupMap.get(host.hostid) ?? [];
const clientGroup = groups.find((g: string) => g.startsWith('Clients/'));
const clientName = clientGroup ? clientGroup.replace('Clients/', '').trim() : host.hostName;
return { hostid: host.hostid, hostName: host.hostName, clientName };
}
// Only include problems whose trigger maps to an enabled host; optionally filter to outages only
const openProblems: MorningSummaryProblem[] = rawProblems
.filter(p => triggerHostMap.has(p.objectid))
.filter(p => !config.outages_only || p.name.toLowerCase().includes('unavailable'))
.map(p => {
const { hostid, hostName, clientName } = resolveClient(p.objectid);
const severity = parseInt(p.severity, 10);
const startedAt = new Date(parseInt(p.clock, 10) * 1000);
const durationMs = now.getTime() - startedAt.getTime();
const ackArray = Array.isArray(p.acknowledges) ? p.acknowledges : [];
const acknowledged = ackArray.length > 0;
const needsAttention = !acknowledged && durationMs > 4 * 60 * 60 * 1000;
return {
eventid: p.eventid,
hostid,
hostName,
clientName,
triggerName: p.name,
severity,
severityLabel: SEVERITY_LABELS[severity] ?? 'Unknown',
startedAt,
durationLabel: formatDuration(durationMs),
acknowledged,
needsAttention,
};
});
// Sort: needs attention first, then by severity desc, then by duration desc
openProblems.sort((a, b) => {
if (a.needsAttention !== b.needsAttention) return a.needsAttention ? -1 : 1;
if (b.severity !== a.severity) return b.severity - a.severity;
return b.startedAt.getTime() - a.startedAt.getTime();
});
// Only include resolved events whose trigger maps to an enabled host; optionally filter to outages only
const resolvedOvernight: MorningSummaryResolved[] = rawResolved
.filter(ev => triggerHostMap.has(ev.objectid))
.filter(ev => !config.outages_only || ev.name.toLowerCase().includes('unavailable'))
.map(ev => {
const { hostName, clientName } = resolveClient(ev.objectid);
const startedAt = new Date(parseInt(ev.clock, 10) * 1000);
const resolvedAt = ev.r_clock ? new Date(parseInt(ev.r_clock, 10) * 1000) : now;
const durationMs = resolvedAt.getTime() - startedAt.getTime();
return {
eventid: ev.eventid,
hostName,
clientName,
triggerName: ev.name,
startedAt,
resolvedAt,
durationLabel: formatDuration(durationMs),
};
});
// Stats
const mttrMinutes = resolvedOvernight.length > 0
? Math.round(resolvedOvernight.reduce((sum, r) => {
return sum + (r.resolvedAt.getTime() - r.startedAt.getTime()) / 60000;
}, 0) / resolvedOvernight.length)
: null;
const clientsAffected = [...new Set(openProblems.map(p => p.clientName))].sort();
const summary: MorningSummary = {
generatedAt: now,
windowFrom,
windowTo: now,
isWeekendWindow: isWeekend,
openProblems,
resolvedOvernight,
openCount: openProblems.length,
resolvedCount: resolvedOvernight.length,
mttrMinutes,
clientsAffected,
};
const card = buildAdaptiveCard(summary);
// Persist
await postgresClient.query(
`INSERT INTO morning_summaries
(generated_at, window_from, window_to, open_count, resolved_count, mttr_minutes,
clients_affected, is_weekend_window, card_payload, delivery_status)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
[
now, windowFrom, now,
summary.openCount, summary.resolvedCount, summary.mttrMinutes,
summary.clientsAffected, summary.isWeekendWindow,
JSON.stringify(card), JSON.stringify({}),
]
);
return { summary, card };
}
/**
* Post an Adaptive Card to one or more webhook URLs.
* Returns per-webhook results and updates DB delivery_status.
*/
async deliver(card: object, webhookIds?: number[]): Promise<DeliveryResult[]> {
const all = await this.getWebhooks();
const targets = webhookIds
? all.filter(w => webhookIds.includes(w.id))
: all.filter(w => w.enabled);
const envelope = {
type: 'message',
attachments: [{
contentType: 'application/vnd.microsoft.card.adaptive',
contentUrl: null,
content: card,
}],
};
const results: DeliveryResult[] = await Promise.all(
targets.map(async (webhook): Promise<DeliveryResult> => {
try {
const res = await fetch(webhook.webhook_url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(envelope),
});
const success = res.ok;
const httpStatus = res.status;
await postgresClient.query(
`UPDATE morning_summary_webhooks
SET last_delivered_at = NOW(), last_status = $1 WHERE id = $2`,
[success ? 'success' : 'failed', webhook.id]
);
return { webhookId: webhook.id, label: webhook.label, success, httpStatus };
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
await postgresClient.query(
`UPDATE morning_summary_webhooks SET last_delivered_at = NOW(), last_status = 'failed' WHERE id = $1`,
[webhook.id]
);
return { webhookId: webhook.id, label: webhook.label, success: false, error };
}
})
);
// Update delivery_status on the latest summary row
const statusMap: Record<number, object> = {};
for (const r of results) {
statusMap[r.webhookId] = { success: r.success, httpStatus: r.httpStatus, error: r.error };
}
await postgresClient.query(
`UPDATE morning_summaries SET delivery_status = $1
WHERE id = (SELECT id FROM morning_summaries ORDER BY generated_at DESC LIMIT 1)`,
[JSON.stringify(statusMap)]
);
return results;
}
/**
* Full run: aggregate + deliver. Used by scheduler and /send endpoint.
*/
async run(webhookIds?: number[]): Promise<{ summary: MorningSummary; results: DeliveryResult[] }> {
const { summary, card } = await this.aggregate();
const results = await this.deliver(card, webhookIds);
console.log(`[MORNING-SUMMARY] Open: ${summary.openCount}, Resolved: ${summary.resolvedCount}, Webhooks: ${results.length}`);
return { summary, results };
}
/**
* Test send: aggregate current data and send to a single webhook only.
*/
async testSend(webhookId: number): Promise<DeliveryResult> {
const { card } = await this.aggregate();
const results = await this.deliver(card, [webhookId]);
return results[0] ?? { webhookId, label: '', success: false, error: 'Webhook not found' };
}
}
let _instance: MorningSummaryService | null = null;
export function getMorningSummaryService(): MorningSummaryService {
if (!_instance) _instance = new MorningSummaryService();
return _instance;
}