- Add MorningSummaryService with Zabbix aggregation and adaptive card builder - Add webhook delivery system with Teams incoming webhooks - Add admin UI at /admin/morning-summary for webhook/config management - Add API routes: /send, /test, /webhooks, /webhooks/[id], /config, /history - Register morning-summary cron job in SyncScheduler (Mon-Fri 6:30 AM) - Add outages_only filter (Unavailable triggers only) - Fix host resolution: use getTriggerEnabledHosts to exclude disabled hosts - Fix resolved events: event.get value:1 scoped to window with r_eventid filter - Remove emojis from fact rows and section headers in card - Remove Open Zabbix button (duplicate of View Problems) - Add migrations: morning_summary_config + morning_summaries tables - Add outages_only column to morning_summary_config
543 lines
19 KiB
TypeScript
543 lines
19 KiB
TypeScript
import { postgresClient } from './postgres-client';
|
|
import { ZabbixClient } from './zabbix-client';
|
|
|
|
export interface MorningSummaryProblem {
|
|
eventid: string;
|
|
hostid: string;
|
|
hostName: string;
|
|
clientName: string;
|
|
triggerName: string;
|
|
severity: number;
|
|
severityLabel: string;
|
|
startedAt: Date;
|
|
durationLabel: string;
|
|
acknowledged: boolean;
|
|
needsAttention: boolean;
|
|
}
|
|
|
|
export interface MorningSummaryResolved {
|
|
eventid: string;
|
|
hostName: string;
|
|
clientName: string;
|
|
triggerName: string;
|
|
startedAt: Date;
|
|
resolvedAt: Date;
|
|
durationLabel: string;
|
|
}
|
|
|
|
export interface MorningSummary {
|
|
generatedAt: Date;
|
|
windowFrom: Date;
|
|
windowTo: Date;
|
|
isWeekendWindow: boolean;
|
|
openProblems: MorningSummaryProblem[];
|
|
resolvedOvernight: MorningSummaryResolved[];
|
|
openCount: number;
|
|
resolvedCount: number;
|
|
mttrMinutes: number | null;
|
|
clientsAffected: string[];
|
|
}
|
|
|
|
export interface WebhookConfig {
|
|
id: number;
|
|
label: string;
|
|
webhook_url: string;
|
|
enabled: boolean;
|
|
last_delivered_at: string | null;
|
|
last_status: string | null;
|
|
created_at: string;
|
|
}
|
|
|
|
export interface SummaryConfig {
|
|
weekend_suppression: boolean;
|
|
monday_extended_window: boolean;
|
|
severity_filter: number;
|
|
outages_only: boolean;
|
|
}
|
|
|
|
export interface DeliveryResult {
|
|
webhookId: number;
|
|
label: string;
|
|
success: boolean;
|
|
httpStatus?: number;
|
|
error?: string;
|
|
}
|
|
|
|
const SEVERITY_LABELS: Record<number, string> = {
|
|
0: 'Not classified',
|
|
1: 'Information',
|
|
2: 'Warning',
|
|
3: 'Average',
|
|
4: 'High',
|
|
5: 'Disaster',
|
|
};
|
|
|
|
function formatDuration(ms: number): string {
|
|
const totalMinutes = Math.floor(ms / 60000);
|
|
if (totalMinutes < 1) return '<1m';
|
|
if (totalMinutes < 60) return `${totalMinutes}m`;
|
|
const hours = Math.floor(totalMinutes / 60);
|
|
const mins = totalMinutes % 60;
|
|
return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`;
|
|
}
|
|
|
|
function makeZabbix(): ZabbixClient {
|
|
if (!process.env.ZABBIX_API_URL || !process.env.ZABBIX_API_TOKEN) {
|
|
throw new Error('Zabbix not configured: ZABBIX_API_URL and ZABBIX_API_TOKEN required');
|
|
}
|
|
return new ZabbixClient({
|
|
apiUrl: process.env.ZABBIX_API_URL,
|
|
apiToken: process.env.ZABBIX_API_TOKEN,
|
|
});
|
|
}
|
|
|
|
function getWindowStart(now: Date, config: SummaryConfig): { from: Date; isWeekend: boolean } {
|
|
const day = now.getDay(); // 0=Sun,1=Mon
|
|
if (day === 1 && config.monday_extended_window) {
|
|
const friday = new Date(now);
|
|
friday.setDate(friday.getDate() - 3);
|
|
friday.setHours(18, 0, 0, 0);
|
|
return { from: friday, isWeekend: true };
|
|
}
|
|
const yesterday = new Date(now);
|
|
yesterday.setDate(yesterday.getDate() - 1);
|
|
yesterday.setHours(18, 0, 0, 0);
|
|
return { from: yesterday, isWeekend: false };
|
|
}
|
|
|
|
function buildAdaptiveCard(summary: MorningSummary): object {
|
|
const headerText = `☀️ Morning NOC Summary${summary.isWeekendWindow ? ' — Weekend Coverage' : ''}`;
|
|
|
|
const mttrText = summary.mttrMinutes != null ? formatDuration(summary.mttrMinutes * 60000) : '—';
|
|
|
|
const statsColumns = {
|
|
type: 'ColumnSet',
|
|
columns: [
|
|
{
|
|
type: 'Column', width: 'stretch',
|
|
items: [{
|
|
type: 'TextBlock',
|
|
text: `${summary.openCount} Open`,
|
|
weight: 'Bolder', color: summary.openCount > 0 ? 'Attention' : 'Default',
|
|
wrap: true,
|
|
}],
|
|
},
|
|
{
|
|
type: 'Column', width: 'stretch',
|
|
items: [{
|
|
type: 'TextBlock',
|
|
text: `${summary.resolvedCount} Resolved`,
|
|
weight: 'Bolder', color: summary.resolvedCount > 0 ? 'Good' : 'Default',
|
|
wrap: true,
|
|
}],
|
|
},
|
|
{
|
|
type: 'Column', width: 'stretch',
|
|
items: [{
|
|
type: 'TextBlock',
|
|
text: `${mttrText} MTTR`,
|
|
weight: 'Bolder', wrap: true,
|
|
}],
|
|
},
|
|
],
|
|
};
|
|
|
|
const bodyItems: object[] = [
|
|
{ type: 'TextBlock', text: headerText, weight: 'Bolder', size: 'Large', wrap: true },
|
|
statsColumns,
|
|
{ type: 'TextBlock', text: ' ', spacing: 'None' },
|
|
];
|
|
|
|
// Open issues section
|
|
if (summary.openProblems.length > 0) {
|
|
const facts = summary.openProblems.map(p => ({
|
|
title: `${p.clientName} / ${p.hostName}`,
|
|
value: `${p.triggerName} — ${p.durationLabel}`,
|
|
}));
|
|
|
|
bodyItems.push({
|
|
type: 'Container',
|
|
style: 'attention',
|
|
items: [
|
|
{ type: 'TextBlock', text: 'Open Issues', weight: 'Bolder', color: 'Attention', wrap: true },
|
|
{ type: 'FactSet', facts },
|
|
],
|
|
});
|
|
} else {
|
|
bodyItems.push({
|
|
type: 'Container',
|
|
style: 'good',
|
|
items: [{ type: 'TextBlock', text: 'All Clear — No open issues', weight: 'Bolder', color: 'Good', wrap: true }],
|
|
});
|
|
}
|
|
|
|
// Resolved overnight section
|
|
if (summary.resolvedOvernight.length > 0) {
|
|
const shown = summary.resolvedOvernight.slice(0, 5);
|
|
const extra = summary.resolvedOvernight.length - 5;
|
|
const facts = shown.map(r => ({
|
|
title: `${r.clientName} / ${r.hostName}`,
|
|
value: `${r.triggerName} — ${r.durationLabel}`,
|
|
}));
|
|
if (extra > 0) {
|
|
facts.push({ title: '', value: `+${extra} more — all resolved` });
|
|
}
|
|
|
|
bodyItems.push({
|
|
type: 'Container',
|
|
style: 'good',
|
|
items: [
|
|
{ type: 'TextBlock', text: 'Resolved', weight: 'Bolder', color: 'Good', wrap: true },
|
|
{ type: 'FactSet', facts },
|
|
],
|
|
});
|
|
}
|
|
|
|
// Window info footer
|
|
const windowFrom = summary.windowFrom.toLocaleString('en-US', { month: 'short', day: 'numeric', hour: 'numeric', minute: '2-digit', hour12: true });
|
|
const windowTo = summary.windowTo.toLocaleString('en-US', { hour: 'numeric', minute: '2-digit', hour12: true });
|
|
bodyItems.push({
|
|
type: 'TextBlock',
|
|
text: `Window: ${windowFrom} → ${windowTo}`,
|
|
size: 'Small', color: 'Default', isSubtle: true, wrap: true, spacing: 'Small',
|
|
});
|
|
|
|
return {
|
|
$schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
|
|
type: 'AdaptiveCard',
|
|
version: '1.4',
|
|
body: bodyItems,
|
|
actions: [
|
|
{ type: 'Action.OpenUrl', title: 'Open Pulse', url: 'https://pulse.wulfconsulting.cloud' },
|
|
{ type: 'Action.OpenUrl', title: 'View Problems', url: 'https://zabbix.wulfconsulting.cloud/zabbix.php?action=problem.view' },
|
|
],
|
|
};
|
|
}
|
|
|
|
export class MorningSummaryService {
|
|
async getConfig(): Promise<SummaryConfig> {
|
|
const result = await postgresClient.query('SELECT * FROM morning_summary_config WHERE id = 1');
|
|
if (result.rows.length === 0) {
|
|
return { weekend_suppression: true, monday_extended_window: true, severity_filter: 2, outages_only: false };
|
|
}
|
|
return result.rows[0] as SummaryConfig;
|
|
}
|
|
|
|
async updateConfig(updates: Partial<SummaryConfig>): Promise<SummaryConfig> {
|
|
const fields: string[] = [];
|
|
const values: unknown[] = [];
|
|
let idx = 1;
|
|
|
|
if (updates.weekend_suppression !== undefined) { fields.push(`weekend_suppression = $${idx++}`); values.push(updates.weekend_suppression); }
|
|
if (updates.monday_extended_window !== undefined) { fields.push(`monday_extended_window = $${idx++}`); values.push(updates.monday_extended_window); }
|
|
if (updates.severity_filter !== undefined) { fields.push(`severity_filter = $${idx++}`); values.push(updates.severity_filter); }
|
|
if (updates.outages_only !== undefined) { fields.push(`outages_only = $${idx++}`); values.push(updates.outages_only); }
|
|
|
|
if (fields.length === 0) return this.getConfig();
|
|
fields.push('updated_at = NOW()');
|
|
values.push(1);
|
|
|
|
const result = await postgresClient.query(
|
|
`UPDATE morning_summary_config SET ${fields.join(', ')} WHERE id = $${idx} RETURNING *`,
|
|
values
|
|
);
|
|
return result.rows[0] as SummaryConfig;
|
|
}
|
|
|
|
async getWebhooks(): Promise<WebhookConfig[]> {
|
|
const result = await postgresClient.query('SELECT * FROM morning_summary_webhooks ORDER BY id');
|
|
return result.rows as WebhookConfig[];
|
|
}
|
|
|
|
async createWebhook(label: string, webhookUrl: string): Promise<WebhookConfig> {
|
|
const result = await postgresClient.query(
|
|
'INSERT INTO morning_summary_webhooks (label, webhook_url, enabled) VALUES ($1, $2, true) RETURNING *',
|
|
[label, webhookUrl]
|
|
);
|
|
return result.rows[0] as WebhookConfig;
|
|
}
|
|
|
|
async updateWebhook(id: number, updates: { label?: string; webhook_url?: string; enabled?: boolean }): Promise<WebhookConfig> {
|
|
const fields: string[] = [];
|
|
const values: unknown[] = [];
|
|
let idx = 1;
|
|
|
|
if (updates.label !== undefined) { fields.push(`label = $${idx++}`); values.push(updates.label); }
|
|
if (updates.webhook_url !== undefined) { fields.push(`webhook_url = $${idx++}`); values.push(updates.webhook_url); }
|
|
if (updates.enabled !== undefined) { fields.push(`enabled = $${idx++}`); values.push(updates.enabled); }
|
|
|
|
if (fields.length === 0) throw new Error('No fields to update');
|
|
values.push(id);
|
|
|
|
const result = await postgresClient.query(
|
|
`UPDATE morning_summary_webhooks SET ${fields.join(', ')} WHERE id = $${idx} RETURNING *`,
|
|
values
|
|
);
|
|
return result.rows[0] as WebhookConfig;
|
|
}
|
|
|
|
async deleteWebhook(id: number): Promise<void> {
|
|
await postgresClient.query('DELETE FROM morning_summary_webhooks WHERE id = $1', [id]);
|
|
}
|
|
|
|
async getLatestSummaryRow(): Promise<{
|
|
id: number;
|
|
generated_at: string;
|
|
window_from: string;
|
|
window_to: string;
|
|
open_count: number;
|
|
resolved_count: number;
|
|
mttr_minutes: number | null;
|
|
clients_affected: string[];
|
|
is_weekend_window: boolean;
|
|
card_payload: object | null;
|
|
delivery_status: object;
|
|
} | null> {
|
|
const result = await postgresClient.query('SELECT * FROM morning_summaries ORDER BY generated_at DESC LIMIT 1');
|
|
return result.rows.length > 0 ? result.rows[0] : null;
|
|
}
|
|
|
|
async getSummaryHistory(limit = 10): Promise<Array<{
|
|
id: number;
|
|
generated_at: string;
|
|
open_count: number;
|
|
resolved_count: number;
|
|
mttr_minutes: number | null;
|
|
is_weekend_window: boolean;
|
|
delivery_status: object;
|
|
}>> {
|
|
const result = await postgresClient.query(
|
|
'SELECT id, generated_at, open_count, resolved_count, mttr_minutes, is_weekend_window, delivery_status FROM morning_summaries ORDER BY generated_at DESC LIMIT $1',
|
|
[limit]
|
|
);
|
|
return result.rows;
|
|
}
|
|
|
|
/**
|
|
* Pull data from Zabbix and build the MorningSummary + Adaptive Card.
|
|
*/
|
|
async aggregate(overrideWindowFrom?: Date): Promise<{ summary: MorningSummary; card: object }> {
|
|
const now = new Date();
|
|
const config = await this.getConfig();
|
|
const { from: windowFrom, isWeekend } = overrideWindowFrom
|
|
? { from: overrideWindowFrom, isWeekend: false }
|
|
: getWindowStart(now, config);
|
|
|
|
const zabbix = makeZabbix();
|
|
|
|
// Fetch open problems and resolved events in parallel
|
|
const [rawProblems, rawResolved] = await Promise.all([
|
|
zabbix.getOpenProblems(config.severity_filter),
|
|
zabbix.getResolvedEvents(windowFrom, now),
|
|
]);
|
|
|
|
// Build the set of unique triggerids from both result sets
|
|
const allTriggerIds = [
|
|
...new Set([
|
|
...rawProblems.map(p => p.objectid),
|
|
...rawResolved.map(e => e.objectid),
|
|
]),
|
|
];
|
|
|
|
// Resolve triggerid → { hostid, hostName } for ENABLED hosts only.
|
|
// Triggerids tied solely to disabled hosts will be absent from this map.
|
|
const triggerHostMap = await zabbix.getTriggerEnabledHosts(allTriggerIds);
|
|
|
|
// Fetch groups for every enabled hostid so we can derive clientName
|
|
const enabledHostIds = [...new Set([...triggerHostMap.values()].map(h => h.hostid))];
|
|
const groupMap = await zabbix.getHostGroupMap(enabledHostIds);
|
|
|
|
function resolveClient(triggerid: string): { hostid: string; hostName: string; clientName: string } {
|
|
const host = triggerHostMap.get(triggerid);
|
|
if (!host) return { hostid: '', hostName: 'Unknown', clientName: 'Unknown' };
|
|
const groups = groupMap.get(host.hostid) ?? [];
|
|
const clientGroup = groups.find((g: string) => g.startsWith('Clients/'));
|
|
const clientName = clientGroup ? clientGroup.replace('Clients/', '').trim() : host.hostName;
|
|
return { hostid: host.hostid, hostName: host.hostName, clientName };
|
|
}
|
|
|
|
// Only include problems whose trigger maps to an enabled host; optionally filter to outages only
|
|
const openProblems: MorningSummaryProblem[] = rawProblems
|
|
.filter(p => triggerHostMap.has(p.objectid))
|
|
.filter(p => !config.outages_only || p.name.toLowerCase().includes('unavailable'))
|
|
.map(p => {
|
|
const { hostid, hostName, clientName } = resolveClient(p.objectid);
|
|
const severity = parseInt(p.severity, 10);
|
|
const startedAt = new Date(parseInt(p.clock, 10) * 1000);
|
|
const durationMs = now.getTime() - startedAt.getTime();
|
|
const ackArray = Array.isArray(p.acknowledges) ? p.acknowledges : [];
|
|
const acknowledged = ackArray.length > 0;
|
|
const needsAttention = !acknowledged && durationMs > 4 * 60 * 60 * 1000;
|
|
|
|
return {
|
|
eventid: p.eventid,
|
|
hostid,
|
|
hostName,
|
|
clientName,
|
|
triggerName: p.name,
|
|
severity,
|
|
severityLabel: SEVERITY_LABELS[severity] ?? 'Unknown',
|
|
startedAt,
|
|
durationLabel: formatDuration(durationMs),
|
|
acknowledged,
|
|
needsAttention,
|
|
};
|
|
});
|
|
|
|
// Sort: needs attention first, then by severity desc, then by duration desc
|
|
openProblems.sort((a, b) => {
|
|
if (a.needsAttention !== b.needsAttention) return a.needsAttention ? -1 : 1;
|
|
if (b.severity !== a.severity) return b.severity - a.severity;
|
|
return b.startedAt.getTime() - a.startedAt.getTime();
|
|
});
|
|
|
|
// Only include resolved events whose trigger maps to an enabled host; optionally filter to outages only
|
|
const resolvedOvernight: MorningSummaryResolved[] = rawResolved
|
|
.filter(ev => triggerHostMap.has(ev.objectid))
|
|
.filter(ev => !config.outages_only || ev.name.toLowerCase().includes('unavailable'))
|
|
.map(ev => {
|
|
const { hostName, clientName } = resolveClient(ev.objectid);
|
|
const startedAt = new Date(parseInt(ev.clock, 10) * 1000);
|
|
const resolvedAt = ev.r_clock ? new Date(parseInt(ev.r_clock, 10) * 1000) : now;
|
|
const durationMs = resolvedAt.getTime() - startedAt.getTime();
|
|
|
|
return {
|
|
eventid: ev.eventid,
|
|
hostName,
|
|
clientName,
|
|
triggerName: ev.name,
|
|
startedAt,
|
|
resolvedAt,
|
|
durationLabel: formatDuration(durationMs),
|
|
};
|
|
});
|
|
|
|
// Stats
|
|
const mttrMinutes = resolvedOvernight.length > 0
|
|
? Math.round(resolvedOvernight.reduce((sum, r) => {
|
|
return sum + (r.resolvedAt.getTime() - r.startedAt.getTime()) / 60000;
|
|
}, 0) / resolvedOvernight.length)
|
|
: null;
|
|
|
|
const clientsAffected = [...new Set(openProblems.map(p => p.clientName))].sort();
|
|
|
|
const summary: MorningSummary = {
|
|
generatedAt: now,
|
|
windowFrom,
|
|
windowTo: now,
|
|
isWeekendWindow: isWeekend,
|
|
openProblems,
|
|
resolvedOvernight,
|
|
openCount: openProblems.length,
|
|
resolvedCount: resolvedOvernight.length,
|
|
mttrMinutes,
|
|
clientsAffected,
|
|
};
|
|
|
|
const card = buildAdaptiveCard(summary);
|
|
|
|
// Persist
|
|
await postgresClient.query(
|
|
`INSERT INTO morning_summaries
|
|
(generated_at, window_from, window_to, open_count, resolved_count, mttr_minutes,
|
|
clients_affected, is_weekend_window, card_payload, delivery_status)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
|
|
[
|
|
now, windowFrom, now,
|
|
summary.openCount, summary.resolvedCount, summary.mttrMinutes,
|
|
summary.clientsAffected, summary.isWeekendWindow,
|
|
JSON.stringify(card), JSON.stringify({}),
|
|
]
|
|
);
|
|
|
|
return { summary, card };
|
|
}
|
|
|
|
/**
|
|
* Post an Adaptive Card to one or more webhook URLs.
|
|
* Returns per-webhook results and updates DB delivery_status.
|
|
*/
|
|
async deliver(card: object, webhookIds?: number[]): Promise<DeliveryResult[]> {
|
|
const all = await this.getWebhooks();
|
|
const targets = webhookIds
|
|
? all.filter(w => webhookIds.includes(w.id))
|
|
: all.filter(w => w.enabled);
|
|
|
|
const envelope = {
|
|
type: 'message',
|
|
attachments: [{
|
|
contentType: 'application/vnd.microsoft.card.adaptive',
|
|
contentUrl: null,
|
|
content: card,
|
|
}],
|
|
};
|
|
|
|
const results: DeliveryResult[] = await Promise.all(
|
|
targets.map(async (webhook): Promise<DeliveryResult> => {
|
|
try {
|
|
const res = await fetch(webhook.webhook_url, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(envelope),
|
|
});
|
|
|
|
const success = res.ok;
|
|
const httpStatus = res.status;
|
|
|
|
await postgresClient.query(
|
|
`UPDATE morning_summary_webhooks
|
|
SET last_delivered_at = NOW(), last_status = $1 WHERE id = $2`,
|
|
[success ? 'success' : 'failed', webhook.id]
|
|
);
|
|
|
|
return { webhookId: webhook.id, label: webhook.label, success, httpStatus };
|
|
} catch (err) {
|
|
const error = err instanceof Error ? err.message : String(err);
|
|
await postgresClient.query(
|
|
`UPDATE morning_summary_webhooks SET last_delivered_at = NOW(), last_status = 'failed' WHERE id = $1`,
|
|
[webhook.id]
|
|
);
|
|
return { webhookId: webhook.id, label: webhook.label, success: false, error };
|
|
}
|
|
})
|
|
);
|
|
|
|
// Update delivery_status on the latest summary row
|
|
const statusMap: Record<number, object> = {};
|
|
for (const r of results) {
|
|
statusMap[r.webhookId] = { success: r.success, httpStatus: r.httpStatus, error: r.error };
|
|
}
|
|
await postgresClient.query(
|
|
`UPDATE morning_summaries SET delivery_status = $1
|
|
WHERE id = (SELECT id FROM morning_summaries ORDER BY generated_at DESC LIMIT 1)`,
|
|
[JSON.stringify(statusMap)]
|
|
);
|
|
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* Full run: aggregate + deliver. Used by scheduler and /send endpoint.
|
|
*/
|
|
async run(webhookIds?: number[]): Promise<{ summary: MorningSummary; results: DeliveryResult[] }> {
|
|
const { summary, card } = await this.aggregate();
|
|
const results = await this.deliver(card, webhookIds);
|
|
console.log(`[MORNING-SUMMARY] Open: ${summary.openCount}, Resolved: ${summary.resolvedCount}, Webhooks: ${results.length}`);
|
|
return { summary, results };
|
|
}
|
|
|
|
/**
|
|
* Test send: aggregate current data and send to a single webhook only.
|
|
*/
|
|
async testSend(webhookId: number): Promise<DeliveryResult> {
|
|
const { card } = await this.aggregate();
|
|
const results = await this.deliver(card, [webhookId]);
|
|
return results[0] ?? { webhookId, label: '', success: false, error: 'Webhook not found' };
|
|
}
|
|
}
|
|
|
|
let _instance: MorningSummaryService | null = null;
|
|
export function getMorningSummaryService(): MorningSummaryService {
|
|
if (!_instance) _instance = new MorningSummaryService();
|
|
return _instance;
|
|
}
|