feat: Mimecast email integration — message logs, threat events, 120d retention, admin UI
This commit is contained in:
parent
7792e91587
commit
25bb70cfa6
11 changed files with 112285 additions and 1 deletions
435
app/admin/sync/mimecast/page.tsx
Normal file
435
app/admin/sync/mimecast/page.tsx
Normal file
|
|
@ -0,0 +1,435 @@
|
|||
'use client';
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import {
|
||||
ArrowLeft, Activity, History, Calendar, Mail, RefreshCw, Loader2,
|
||||
CheckCircle2, XCircle, AlertTriangle, Shield, Inbox, Send,
|
||||
Clock, ChevronDown, ChevronRight,
|
||||
} from 'lucide-react';
|
||||
import SyncScheduler from '@/components/admin/SyncScheduler';
|
||||
|
||||
function fmtDate(d: string | null | undefined) {
|
||||
if (!d) return 'Never';
|
||||
return new Date(d).toLocaleString(undefined, {
|
||||
month: 'short', day: 'numeric', year: 'numeric', hour: '2-digit', minute: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
function fmtNum(n: number | null | undefined) {
|
||||
if (n == null) return '—';
|
||||
return n.toLocaleString();
|
||||
}
|
||||
|
||||
function StatCard({ label, value, sub, icon: Icon, cls }: {
|
||||
label: string; value: string | number; sub?: string; icon?: React.ElementType; cls?: string;
|
||||
}) {
|
||||
return (
|
||||
<div className={`rounded-lg border p-4 flex flex-col gap-1 ${cls ?? ''}`}>
|
||||
<div className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
{Icon && <Icon className="w-3.5 h-3.5" />}{label}
|
||||
</div>
|
||||
<div className="text-2xl font-bold tabular-nums">{value}</div>
|
||||
{sub && <div className="text-xs text-muted-foreground">{sub}</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: string }) {
|
||||
const cls =
|
||||
status === 'delivered' ? 'bg-green-500/15 text-green-700' :
|
||||
status === 'rejected' ? 'bg-red-500/15 text-red-600' :
|
||||
status === 'held' ? 'bg-yellow-500/15 text-yellow-700' :
|
||||
status === 'bounced' ? 'bg-orange-500/15 text-orange-700' :
|
||||
status === 'spam' ? 'bg-purple-500/15 text-purple-700' :
|
||||
'bg-muted text-muted-foreground';
|
||||
return (
|
||||
<span className={`inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium ${cls}`}>
|
||||
{status || '—'}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function ThreatBadge({ level }: { level: string }) {
|
||||
const cls =
|
||||
level === 'high' ? 'bg-red-500/15 text-red-600' :
|
||||
level === 'medium' ? 'bg-orange-500/15 text-orange-700' :
|
||||
level === 'low' ? 'bg-yellow-500/15 text-yellow-700' :
|
||||
'bg-muted text-muted-foreground';
|
||||
return (
|
||||
<span className={`inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium ${cls}`}>
|
||||
{level || 'info'}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Status Tab ────────────────────────────────────────────────────────────────
|
||||
function StatusTab({ data, onSync, syncing }: { data: any; onSync: (t: string) => void; syncing: boolean }) {
|
||||
if (!data) return (
|
||||
<div className="flex items-center justify-center py-12">
|
||||
<Loader2 className="w-5 h-5 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
);
|
||||
|
||||
const stats = data.stats ?? {};
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Connection banner */}
|
||||
<div className="flex items-center justify-between rounded-lg border p-4 bg-muted/30">
|
||||
<div className="space-y-0.5">
|
||||
<div className="flex items-center gap-2">
|
||||
{data.connected
|
||||
? <CheckCircle2 className="w-4 h-4 text-green-500" />
|
||||
: <XCircle className="w-4 h-4 text-red-500" />}
|
||||
<p className="text-sm font-medium">
|
||||
{data.connected ? `Connected — ${data.accountName ?? 'Mimecast'}` : 'Not connected'}
|
||||
</p>
|
||||
{data.packageName && (
|
||||
<span className="text-xs text-muted-foreground">({data.packageName})</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Last sync: {fmtDate(stats.lastSync)} · Oldest message: {fmtDate(stats.oldestMessage)}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<Button size="sm" variant="outline" onClick={() => onSync('incremental')} disabled={syncing || !data.connected}>
|
||||
{syncing ? <Loader2 className="w-4 h-4 animate-spin mr-1" /> : <RefreshCw className="w-4 h-4 mr-1" />}
|
||||
Incremental
|
||||
</Button>
|
||||
<Button size="sm" onClick={() => onSync('full')} disabled={syncing || !data.connected}>
|
||||
{syncing ? <Loader2 className="w-4 h-4 animate-spin mr-1" /> : <RefreshCw className="w-4 h-4 mr-1" />}
|
||||
Full Sync (120d)
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{data.error && (
|
||||
<div className="rounded-lg border border-red-300 bg-red-50 dark:bg-red-950/20 p-3 text-sm text-red-600">
|
||||
{data.error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Stats grid */}
|
||||
<div className="grid grid-cols-2 md:grid-cols-5 gap-3">
|
||||
<StatCard label="Total Messages" value={fmtNum(stats.messages)} icon={Mail} />
|
||||
<StatCard label="Inbound" value={fmtNum(stats.inbound)} icon={Inbox} cls="border-blue-500/20 bg-blue-500/5" />
|
||||
<StatCard label="Outbound" value={fmtNum(stats.outbound)} icon={Send} cls="border-green-500/20 bg-green-500/5" />
|
||||
<StatCard label="Threat Events" value={fmtNum(stats.threats)} icon={Shield}
|
||||
cls={(stats.threats ?? 0) > 0 ? 'border-red-500/20 bg-red-500/5' : ''} />
|
||||
<StatCard label="Bodies Stored" value={fmtNum(stats.bodies)} icon={Activity} />
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border p-4 space-y-2">
|
||||
<p className="text-xs font-semibold uppercase tracking-wider text-muted-foreground">Data Retention</p>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
120-day rolling window. Messages older than 120 days are automatically purged on each sync.
|
||||
Message bodies are fetched for delivered inbound messages (up to 500 per sync run).
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Messages Tab ──────────────────────────────────────────────────────────────
|
||||
function MessagesTab() {
|
||||
const [rows, setRows] = useState<any[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [search, setSearch] = useState('');
|
||||
const [direction, setDirection] = useState('');
|
||||
const [status, setStatus] = useState('');
|
||||
const [days, setDays] = useState('7');
|
||||
|
||||
const load = () => {
|
||||
setLoading(true);
|
||||
const params = new URLSearchParams({ days, limit: '200' });
|
||||
if (search) params.set('search', search);
|
||||
if (direction) params.set('direction', direction);
|
||||
if (status) params.set('status', status);
|
||||
|
||||
fetch(`/api/mimecast/messages?${params}`)
|
||||
.then(r => r.json())
|
||||
.then(d => setRows(d.messages ?? []))
|
||||
.catch(() => setRows([]))
|
||||
.finally(() => setLoading(false));
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, [days, direction, status]);
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{/* Filters */}
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search sender, recipient, subject…"
|
||||
value={search}
|
||||
onChange={e => setSearch(e.target.value)}
|
||||
onKeyDown={e => e.key === 'Enter' && load()}
|
||||
className="flex-1 min-w-48 border rounded-md px-3 py-1.5 text-sm bg-background"
|
||||
/>
|
||||
<select value={direction} onChange={e => setDirection(e.target.value)}
|
||||
className="border rounded-md px-3 py-1.5 text-sm bg-background">
|
||||
<option value="">All directions</option>
|
||||
<option value="inbound">Inbound</option>
|
||||
<option value="outbound">Outbound</option>
|
||||
</select>
|
||||
<select value={status} onChange={e => setStatus(e.target.value)}
|
||||
className="border rounded-md px-3 py-1.5 text-sm bg-background">
|
||||
<option value="">All statuses</option>
|
||||
<option value="delivered">Delivered</option>
|
||||
<option value="rejected">Rejected</option>
|
||||
<option value="held">Held</option>
|
||||
<option value="bounced">Bounced</option>
|
||||
<option value="spam">Spam</option>
|
||||
</select>
|
||||
<select value={days} onChange={e => setDays(e.target.value)}
|
||||
className="border rounded-md px-3 py-1.5 text-sm bg-background">
|
||||
<option value="1">Last 24h</option>
|
||||
<option value="7">Last 7 days</option>
|
||||
<option value="30">Last 30 days</option>
|
||||
<option value="90">Last 90 days</option>
|
||||
<option value="120">Last 120 days</option>
|
||||
</select>
|
||||
<Button size="sm" variant="outline" onClick={load}>
|
||||
<RefreshCw className="w-4 h-4 mr-1" />Search
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<div className="flex items-center justify-center py-12">
|
||||
<Loader2 className="w-5 h-5 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
) : !rows.length ? (
|
||||
<div className="text-center py-12 text-sm text-muted-foreground">
|
||||
No messages found — run a sync first or adjust filters
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-lg border overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-muted/50 border-b">
|
||||
<tr>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">From</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">To</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Subject</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Direction</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Status</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Sent</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r: any) => (
|
||||
<tr key={r.id} className="border-b last:border-0 hover:bg-muted/30">
|
||||
<td className="px-4 py-2 text-xs truncate max-w-[180px]" title={r.sender_address}>{r.sender_address ?? '—'}</td>
|
||||
<td className="px-4 py-2 text-xs truncate max-w-[180px]" title={r.recipient_address}>{r.recipient_address ?? '—'}</td>
|
||||
<td className="px-4 py-2 text-xs truncate max-w-[200px]" title={r.subject}>{r.subject ?? '—'}</td>
|
||||
<td className="px-4 py-2 text-xs capitalize text-muted-foreground">{r.direction ?? '—'}</td>
|
||||
<td className="px-4 py-2"><StatusBadge status={r.status} /></td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground whitespace-nowrap">{fmtDate(r.sent_datetime)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Threats Tab ───────────────────────────────────────────────────────────────
|
||||
function ThreatsTab() {
|
||||
const [rows, setRows] = useState<any[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
fetch('/api/mimecast/threats?limit=200')
|
||||
.then(r => r.json())
|
||||
.then(d => setRows(d.threats ?? []))
|
||||
.catch(() => setRows([]))
|
||||
.finally(() => setLoading(false));
|
||||
}, []);
|
||||
|
||||
if (loading) return <div className="flex items-center justify-center py-12"><Loader2 className="w-5 h-5 animate-spin text-muted-foreground" /></div>;
|
||||
if (!rows.length) return <div className="text-center py-12 text-sm text-muted-foreground">No threat events — run a sync first</div>;
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-muted/50 border-b">
|
||||
<tr>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Type</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Level</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Actor</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Verdict</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">URL / File</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">When</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r: any) => (
|
||||
<tr key={r.id} className="border-b last:border-0 hover:bg-muted/30">
|
||||
<td className="px-4 py-2 text-xs capitalize">{r.event_type ?? '—'}</td>
|
||||
<td className="px-4 py-2"><ThreatBadge level={r.threat_level} /></td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground">{r.actor_email ?? '—'}</td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground capitalize">{r.verdict ?? '—'}</td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground truncate max-w-[200px]" title={r.url ?? r.file_name ?? ''}>
|
||||
{r.url ?? r.file_name ?? '—'}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground whitespace-nowrap">{fmtDate(r.event_datetime)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── History Tab ───────────────────────────────────────────────────────────────
|
||||
function HistoryTab() {
|
||||
const [rows, setRows] = useState<any[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
fetch('/api/sync/history?entityType=mimecast&limit=30')
|
||||
.then(r => r.json())
|
||||
.then(d => setRows(d.history ?? []))
|
||||
.catch(() => setRows([]))
|
||||
.finally(() => setLoading(false));
|
||||
}, []);
|
||||
|
||||
if (loading) return <div className="flex items-center justify-center py-12"><Loader2 className="w-5 h-5 animate-spin text-muted-foreground" /></div>;
|
||||
if (!rows.length) return <div className="text-center py-12 text-sm text-muted-foreground">No sync history yet</div>;
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border overflow-hidden">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-muted/50 border-b">
|
||||
<tr>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Type</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Status</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Messages</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Threats</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Started</th>
|
||||
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Duration</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r: any, i: number) => {
|
||||
const dur = r.completed_at && r.started_at
|
||||
? new Date(r.completed_at).getTime() - new Date(r.started_at).getTime()
|
||||
: null;
|
||||
const durStr = dur == null ? '—' : dur < 60000 ? `${Math.round(dur / 1000)}s` : `${Math.floor(dur / 60000)}m ${Math.round((dur % 60000) / 1000)}s`;
|
||||
const statusCls = r.status === 'completed' ? 'bg-green-500/15 text-green-700' : r.status === 'failed' ? 'bg-red-500/15 text-red-600' : 'bg-muted text-muted-foreground';
|
||||
const meta = typeof r.metadata === 'string' ? JSON.parse(r.metadata || '{}') : (r.metadata ?? {});
|
||||
|
||||
return (
|
||||
<tr key={i} className="border-b last:border-0 hover:bg-muted/30">
|
||||
<td className="px-4 py-2 capitalize text-xs">{r.sync_type ?? '—'}</td>
|
||||
<td className="px-4 py-2">
|
||||
<span className={`inline-flex rounded-full px-2 py-0.5 text-xs font-medium ${statusCls}`}>{r.status}</span>
|
||||
</td>
|
||||
<td className="px-4 py-2 tabular-nums text-xs">{fmtNum(meta.messagesUpserted ?? r.records_added)}</td>
|
||||
<td className="px-4 py-2 tabular-nums text-xs">{fmtNum(meta.threatsUpserted)}</td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground">{fmtDate(r.started_at)}</td>
|
||||
<td className="px-4 py-2 text-xs text-muted-foreground">{durStr}</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Page ──────────────────────────────────────────────────────────────────────
|
||||
export default function MimecastSyncPage() {
|
||||
const [statusData, setStatusData] = useState<any>(null);
|
||||
const [syncing, setSyncing] = useState(false);
|
||||
const [lastResult, setLastResult] = useState<any>(null);
|
||||
|
||||
const fetchStatus = () => {
|
||||
fetch('/api/mimecast/status')
|
||||
.then(r => r.json())
|
||||
.then(d => setStatusData(d))
|
||||
.catch(() => setStatusData({ configured: false, connected: false }));
|
||||
};
|
||||
|
||||
useEffect(() => { fetchStatus(); }, []);
|
||||
|
||||
const handleSync = async (syncType: string) => {
|
||||
setSyncing(true);
|
||||
setLastResult(null);
|
||||
try {
|
||||
const res = await fetch('/api/sync/mimecast', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ syncType }),
|
||||
});
|
||||
const data = await res.json();
|
||||
setLastResult(data);
|
||||
fetchStatus();
|
||||
} catch (err: any) {
|
||||
setLastResult({ error: err.message });
|
||||
} finally {
|
||||
setSyncing(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto py-4 md:py-8 px-4 space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center gap-4">
|
||||
<Link href="/admin/sync">
|
||||
<Button variant="outline" size="sm" className="gap-2">
|
||||
<ArrowLeft className="w-4 h-4" />Integrations
|
||||
</Button>
|
||||
</Link>
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="p-2 rounded-lg border border-blue-500/30 bg-blue-500/5">
|
||||
<Mail className="w-5 h-5 text-blue-500" />
|
||||
</div>
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold">Email Security — Mimecast</h1>
|
||||
<p className="text-sm text-muted-foreground">Message logs, threat events, 120-day retention</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Last sync result banner */}
|
||||
{lastResult && (
|
||||
<div className={`rounded-lg border p-3 text-sm ${lastResult.error ? 'border-red-300 bg-red-50 dark:bg-red-950/20 text-red-600' : 'border-green-300 bg-green-50 dark:bg-green-950/20 text-green-700'}`}>
|
||||
{lastResult.error
|
||||
? `Sync failed: ${lastResult.error}`
|
||||
: `Sync complete — ${fmtNum(lastResult.messagesUpserted)} messages, ${fmtNum(lastResult.threatsUpserted)} threats, ${fmtNum(lastResult.bodiesFetched)} bodies fetched in ${Math.round((lastResult.durationMs ?? 0) / 1000)}s`
|
||||
}
|
||||
{lastResult.errors?.length > 0 && (
|
||||
<div className="mt-1 text-xs opacity-80">{lastResult.errors.slice(0, 3).join(' · ')}</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<Tabs defaultValue="status" className="w-full">
|
||||
<TabsList className="grid w-full max-w-2xl grid-cols-5">
|
||||
<TabsTrigger value="status" className="gap-1.5"><Activity className="h-4 w-4" />Status</TabsTrigger>
|
||||
<TabsTrigger value="messages" className="gap-1.5"><Mail className="h-4 w-4" />Messages</TabsTrigger>
|
||||
<TabsTrigger value="threats" className="gap-1.5"><Shield className="h-4 w-4" />Threats</TabsTrigger>
|
||||
<TabsTrigger value="history" className="gap-1.5"><Clock className="h-4 w-4" />History</TabsTrigger>
|
||||
<TabsTrigger value="schedules" className="gap-1.5"><Calendar className="h-4 w-4" />Schedules</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="status" className="mt-6">
|
||||
<StatusTab data={statusData} onSync={handleSync} syncing={syncing} />
|
||||
</TabsContent>
|
||||
<TabsContent value="messages" className="mt-6"><MessagesTab /></TabsContent>
|
||||
<TabsContent value="threats" className="mt-6"><ThreatsTab /></TabsContent>
|
||||
<TabsContent value="history" className="mt-6"><HistoryTab /></TabsContent>
|
||||
<TabsContent value="schedules" className="mt-6"><SyncScheduler /></TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ const INTEGRATIONS: IntegrationCard[] = [
|
|||
{ id: 'auvik', category: 'NMS', product: 'Auvik', description: 'Network devices, topology, tenant mappings', href: '/admin/sync/auvik', logo: '/logos/auvik.ico', color: 'purple' },
|
||||
{ id: 'addigy', category: 'Apple RMM', product: 'Addigy', description: 'macOS/iOS device management, policies, compliance', href: '/admin/sync/addigy', logo: '/logos/addigy.ico', color: 'gray' },
|
||||
{ id: 'sentinelone', category: 'EDR/AV', product: 'SentinelOne', description: 'Endpoint agents, threat detections, site coverage, AV health', href: '/admin/sync/sentinelone', logo: '/logos/sentinelone.ico', color: 'purple' },
|
||||
{ id: 'mimecast', category: 'Email Security', product: 'Mimecast', description: 'Message tracking logs, threat events, SIEM data, 120-day retention', href: '/admin/sync/mimecast', logo: '/logos/mimecast.ico', color: 'blue' },
|
||||
];
|
||||
|
||||
const COLOR_MAP: Record<string, { bg: string; border: string }> = {
|
||||
|
|
@ -54,13 +55,16 @@ export default function SyncOverviewPage() {
|
|||
const [itglueSyncData, setItglueSyncData] = useState<any>(null);
|
||||
const [s1SyncData, setS1SyncData] = useState<any>(null);
|
||||
|
||||
const [mimecastData, setMimecastData] = useState<any>(null);
|
||||
|
||||
const fetchAll = async () => {
|
||||
try {
|
||||
const [intRes, atRes, itgRes, s1Res] = await Promise.all([
|
||||
const [intRes, atRes, itgRes, s1Res, mcRes] = await Promise.all([
|
||||
fetch('/api/integrations/status'),
|
||||
fetch('/api/sync/last-sync'),
|
||||
fetch('/api/itglue/sync'),
|
||||
fetch('/api/sentinelone/sync'),
|
||||
fetch('/api/mimecast/status'),
|
||||
]);
|
||||
if (intRes.ok) setStatus(await intRes.json());
|
||||
if (atRes.ok) {
|
||||
|
|
@ -69,6 +73,7 @@ export default function SyncOverviewPage() {
|
|||
}
|
||||
if (itgRes.ok) setItglueSyncData(await itgRes.json());
|
||||
if (s1Res.ok) setS1SyncData(await s1Res.json());
|
||||
if (mcRes.ok) setMimecastData(await mcRes.json());
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
} finally {
|
||||
|
|
@ -147,6 +152,16 @@ export default function SyncOverviewPage() {
|
|||
}
|
||||
if (id === 'auvik') return { lastSync: status.auvik?.lastSync, configured: status.auvik?.configured };
|
||||
if (id === 'addigy') return { lastSync: status.addigy?.lastSync, configured: status.addigy?.configured };
|
||||
if (id === 'mimecast') {
|
||||
if (!mimecastData) return null;
|
||||
const s = mimecastData.stats ?? {};
|
||||
return {
|
||||
lastSync: s.lastSync ?? null,
|
||||
connected: mimecastData.connected ?? false,
|
||||
messages: s.messages ?? 0,
|
||||
threats: s.threats ?? 0,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
|
|
@ -176,6 +191,11 @@ export default function SyncOverviewPage() {
|
|||
if (summary.infected > 0) return <AlertTriangle className="w-4 h-4 text-red-500" />;
|
||||
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
||||
}
|
||||
if (id === 'mimecast') {
|
||||
if (!summary.connected) return <Clock className="w-4 h-4 text-muted-foreground" />;
|
||||
if ((summary as any).threats > 0) return <AlertTriangle className="w-4 h-4 text-yellow-500" />;
|
||||
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
||||
}
|
||||
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
||||
};
|
||||
|
||||
|
|
@ -322,6 +342,24 @@ export default function SyncOverviewPage() {
|
|||
)}
|
||||
</>
|
||||
)}
|
||||
{intg.id === 'mimecast' && summary && (
|
||||
<>
|
||||
<div className="flex justify-between">
|
||||
<span>Last sync</span>
|
||||
<span className="font-medium text-foreground">{fmtDate((summary as any).lastSync)}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span>Messages</span>
|
||||
<span className="font-medium text-foreground">{((summary as any).messages ?? 0).toLocaleString()}</span>
|
||||
</div>
|
||||
{(summary as any).threats > 0 && (
|
||||
<div className="flex justify-between text-yellow-700">
|
||||
<span>Threat events</span>
|
||||
<span className="font-medium">{(summary as any).threats}</span>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{(intg.id === 'auvik' || intg.id === 'addigy') && (
|
||||
<div className="flex justify-between">
|
||||
<span>Status</span>
|
||||
|
|
|
|||
55
app/api/mimecast/messages/route.ts
Normal file
55
app/api/mimecast/messages/route.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { postgresClient as pg } from '@/lib/services/postgres-client';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const days = parseInt(searchParams.get('days') ?? '7');
|
||||
const limit = Math.min(parseInt(searchParams.get('limit') ?? '200'), 1000);
|
||||
const search = searchParams.get('search') ?? '';
|
||||
const direction = searchParams.get('direction') ?? '';
|
||||
const status = searchParams.get('status') ?? '';
|
||||
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - days);
|
||||
|
||||
const conditions: string[] = ['sent_datetime >= $1'];
|
||||
const params: any[] = [cutoff.toISOString()];
|
||||
let paramIdx = 2;
|
||||
|
||||
if (direction) {
|
||||
conditions.push(`direction = $${paramIdx++}`);
|
||||
params.push(direction);
|
||||
}
|
||||
if (status) {
|
||||
conditions.push(`status = $${paramIdx++}`);
|
||||
params.push(status);
|
||||
}
|
||||
if (search) {
|
||||
conditions.push(`(
|
||||
sender_address ILIKE $${paramIdx} OR
|
||||
recipient_address ILIKE $${paramIdx} OR
|
||||
subject ILIKE $${paramIdx}
|
||||
)`);
|
||||
params.push(`%${search}%`);
|
||||
paramIdx++;
|
||||
}
|
||||
|
||||
const where = conditions.join(' AND ');
|
||||
params.push(limit);
|
||||
|
||||
const result = await pg.query(`
|
||||
SELECT id, sender_address, sender_domain, recipient_address, subject,
|
||||
direction, status, action, spam_score, size_bytes, attachment_count,
|
||||
sent_datetime, received_datetime, route, reject_reason, held_reason, source_ip
|
||||
FROM mimecast_messages
|
||||
WHERE ${where}
|
||||
ORDER BY sent_datetime DESC
|
||||
LIMIT $${paramIdx}
|
||||
`, params);
|
||||
|
||||
return NextResponse.json({ messages: result.rows });
|
||||
} catch (err: any) {
|
||||
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
29
app/api/mimecast/status/route.ts
Normal file
29
app/api/mimecast/status/route.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import { NextResponse } from 'next/server';
|
||||
import { getMimecastClient } from '@/lib/services/mimecast-client';
|
||||
import { getMimecastStats } from '@/lib/services/mimecast-sync-service';
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const client = getMimecastClient();
|
||||
const [connection, stats] = await Promise.all([
|
||||
client.testConnection(),
|
||||
getMimecastStats().catch(() => null),
|
||||
]);
|
||||
|
||||
return NextResponse.json({
|
||||
configured: true,
|
||||
connected: connection.ok,
|
||||
accountName: connection.accountName,
|
||||
packageName: connection.packageName,
|
||||
error: connection.error,
|
||||
stats,
|
||||
});
|
||||
} catch (err: any) {
|
||||
return NextResponse.json({
|
||||
configured: false,
|
||||
connected: false,
|
||||
error: err.message,
|
||||
stats: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
40
app/api/mimecast/threats/route.ts
Normal file
40
app/api/mimecast/threats/route.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { postgresClient as pg } from '@/lib/services/postgres-client';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const limit = Math.min(parseInt(searchParams.get('limit') ?? '200'), 1000);
|
||||
const level = searchParams.get('level') ?? '';
|
||||
const type = searchParams.get('type') ?? '';
|
||||
|
||||
const conditions: string[] = [];
|
||||
const params: any[] = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (level) {
|
||||
conditions.push(`threat_level = $${paramIdx++}`);
|
||||
params.push(level);
|
||||
}
|
||||
if (type) {
|
||||
conditions.push(`event_type = $${paramIdx++}`);
|
||||
params.push(type);
|
||||
}
|
||||
|
||||
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
params.push(limit);
|
||||
|
||||
const result = await pg.query(`
|
||||
SELECT id, message_id, event_type, threat_level, url, file_name,
|
||||
verdict, actor_email, event_datetime
|
||||
FROM mimecast_threat_events
|
||||
${where}
|
||||
ORDER BY event_datetime DESC NULLS LAST
|
||||
LIMIT $${paramIdx}
|
||||
`, params);
|
||||
|
||||
return NextResponse.json({ threats: result.rows });
|
||||
} catch (err: any) {
|
||||
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
40
app/api/sync/mimecast/route.ts
Normal file
40
app/api/sync/mimecast/route.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { runMimecastFullSync, runMimecastIncrementalSync, getMimecastStats } from '@/lib/services/mimecast-sync-service';
|
||||
|
||||
let _syncing = false;
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
if (_syncing) {
|
||||
return NextResponse.json({ error: 'Sync already in progress' }, { status: 409 });
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await req.json().catch(() => ({}));
|
||||
const syncType = body.syncType ?? 'incremental';
|
||||
|
||||
_syncing = true;
|
||||
|
||||
const result = syncType === 'full'
|
||||
? await runMimecastFullSync()
|
||||
: await runMimecastIncrementalSync();
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
syncType,
|
||||
...result,
|
||||
});
|
||||
} catch (err: any) {
|
||||
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||
} finally {
|
||||
_syncing = false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const stats = await getMimecastStats();
|
||||
return NextResponse.json({ isSyncing: _syncing, ...stats });
|
||||
} catch (err: any) {
|
||||
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
110827
dev/mimecast-api-v2-collection.json
Normal file
110827
dev/mimecast-api-v2-collection.json
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -103,6 +103,11 @@ services:
|
|||
# ipinfo.io API token (optional)
|
||||
IPINFO_TOKEN: ${IPINFO_TOKEN:-}
|
||||
|
||||
# Mimecast API Configuration
|
||||
MIMECAST_CLIENT_ID: ${MIMECAST_CLIENT_ID}
|
||||
MIMECAST_CLIENT_SECRET: ${MIMECAST_CLIENT_SECRET}
|
||||
MIMECAST_BASE_URL: ${MIMECAST_BASE_URL:-https://api.services.mimecast.com}
|
||||
|
||||
# IT Glue Configuration
|
||||
ITGLUE_API_KEY: ${ITGLUE_API_KEY}
|
||||
|
||||
|
|
|
|||
412
lib/services/mimecast-client.ts
Normal file
412
lib/services/mimecast-client.ts
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
/**
|
||||
* Mimecast API 2.0 Client
|
||||
* Auth: OAuth2 Client Credentials — POST /oauth/token
|
||||
* Base: https://api.services.mimecast.com
|
||||
*
|
||||
* Endpoints sourced from official Postman collection (mimecast-api-v2-collection.json):
|
||||
* - Message logs: POST /api/message-finder/search (trackedEmails with pagination)
|
||||
* - SIEM batch: GET /siem/v1/batch/events/cg (returns pre-signed URLs → download JSON)
|
||||
* - Threat events: GET /threats/v1/events
|
||||
* - Message info: POST /api/message-finder/get-message-info
|
||||
* - Account: POST /api/account/get-account
|
||||
*/
|
||||
|
||||
export interface MimecastConfig {
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
interface TokenResponse {
|
||||
access_token: string;
|
||||
token_type: string;
|
||||
expires_in: number;
|
||||
}
|
||||
|
||||
interface TokenCache {
|
||||
token: string;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
export interface MimecastMessage {
|
||||
id: string;
|
||||
senderAddress: string;
|
||||
recipientAddress: string;
|
||||
subject: string;
|
||||
direction: string;
|
||||
status: string;
|
||||
action?: string;
|
||||
spamScore?: number;
|
||||
sizeBytes?: number;
|
||||
attachmentCount?: boolean | number;
|
||||
sentDateTime?: string;
|
||||
receivedDateTime?: string;
|
||||
route?: string;
|
||||
rejectReason?: string;
|
||||
heldReason?: string;
|
||||
sourceIp?: string;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface MimecastThreatEvent {
|
||||
id: string;
|
||||
messageId?: string;
|
||||
eventType: string;
|
||||
threatLevel?: string;
|
||||
url?: string;
|
||||
fileName?: string;
|
||||
verdict?: string;
|
||||
actorEmail?: string;
|
||||
eventDateTime?: string;
|
||||
analysis?: string[];
|
||||
source?: string[];
|
||||
direction?: string[];
|
||||
status?: string[];
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface MimecastMessageInfo {
|
||||
messageId: string;
|
||||
bodyText?: string;
|
||||
bodyHtml?: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface PaginatedResult<T> {
|
||||
items: T[];
|
||||
nextCursor: string | null;
|
||||
totalCount?: number;
|
||||
}
|
||||
|
||||
export class MimecastClient {
|
||||
private readonly clientId: string;
|
||||
private readonly clientSecret: string;
|
||||
private readonly baseUrl: string;
|
||||
private tokenCache: TokenCache | null = null;
|
||||
|
||||
constructor(config: MimecastConfig) {
|
||||
this.clientId = config.clientId;
|
||||
this.clientSecret = config.clientSecret;
|
||||
this.baseUrl = (config.baseUrl ?? 'https://api.services.mimecast.com').replace(/\/$/, '');
|
||||
}
|
||||
|
||||
// ── Auth ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private async getToken(): Promise<string> {
|
||||
if (this.tokenCache && Date.now() < this.tokenCache.expiresAt - 60_000) {
|
||||
return this.tokenCache.token;
|
||||
}
|
||||
|
||||
const body = new URLSearchParams({
|
||||
grant_type: 'client_credentials',
|
||||
client_id: this.clientId,
|
||||
client_secret: this.clientSecret,
|
||||
});
|
||||
|
||||
const res = await fetch(`${this.baseUrl}/oauth/token`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: body.toString(),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Mimecast auth failed (${res.status}): ${text}`);
|
||||
}
|
||||
|
||||
const data: TokenResponse = await res.json();
|
||||
this.tokenCache = {
|
||||
token: data.access_token,
|
||||
expiresAt: Date.now() + data.expires_in * 1000,
|
||||
};
|
||||
return data.access_token;
|
||||
}
|
||||
|
||||
private async request<T>(
|
||||
method: 'GET' | 'POST',
|
||||
path: string,
|
||||
body?: Record<string, any>,
|
||||
params?: Record<string, string>
|
||||
): Promise<T> {
|
||||
const token = await this.getToken();
|
||||
|
||||
let url = `${this.baseUrl}${path}`;
|
||||
if (params && Object.keys(params).length > 0) {
|
||||
url = `${url}?${new URLSearchParams(params).toString()}`;
|
||||
}
|
||||
|
||||
const res = await fetch(url, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Mimecast ${method} ${path} failed (${res.status}): ${text}`);
|
||||
}
|
||||
|
||||
return res.json() as T;
|
||||
}
|
||||
|
||||
// ── Message Tracking ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Search tracked emails via POST /api/message-finder/search
|
||||
* Response: { data: [{ trackedEmails: [...], pageToken: string }] }
|
||||
*/
|
||||
async getMessageLogs(options: {
|
||||
from: string; // ISO datetime string
|
||||
to: string;
|
||||
cursor?: string;
|
||||
pageSize?: number;
|
||||
}): Promise<PaginatedResult<MimecastMessage>> {
|
||||
const reqData: Record<string, any> = {
|
||||
from: options.from,
|
||||
to: options.to,
|
||||
pageSize: options.pageSize ?? 500,
|
||||
};
|
||||
if (options.cursor) reqData.pageToken = options.cursor;
|
||||
|
||||
const data = await this.request<any>('POST', '/api/message-finder/search', { data: [reqData] });
|
||||
|
||||
const block = data.data?.[0] ?? {};
|
||||
const tracked: any[] = block.trackedEmails ?? [];
|
||||
|
||||
return {
|
||||
items: tracked.map((m: any) => this.normalizeTrackedEmail(m)),
|
||||
nextCursor: block.pageToken ?? null,
|
||||
totalCount: tracked.length,
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeTrackedEmail(m: any): MimecastMessage {
|
||||
const sender = m.fromEnv?.emailAddress ?? m.fromHdr?.emailAddress ?? m.sender ?? '';
|
||||
const recipients: string[] = (m.to ?? []).map((t: any) =>
|
||||
typeof t === 'string' ? t : t.emailAddress ?? ''
|
||||
);
|
||||
|
||||
return {
|
||||
id: m.id,
|
||||
senderAddress: sender,
|
||||
recipientAddress: recipients.join(', '),
|
||||
subject: m.subject ?? '',
|
||||
direction: m.route?.toLowerCase().includes('inbound') ? 'inbound' : 'outbound',
|
||||
status: m.status ?? '',
|
||||
action: m.detectionLevel ?? undefined,
|
||||
spamScore: m.spamScore ?? undefined,
|
||||
sizeBytes: undefined,
|
||||
attachmentCount: m.attachments ?? 0,
|
||||
sentDateTime: m.sent ?? undefined,
|
||||
receivedDateTime: m.received ?? undefined,
|
||||
route: m.route ?? undefined,
|
||||
rejectReason: m.info ?? undefined,
|
||||
sourceIp: m.senderIP ?? undefined,
|
||||
_raw: m,
|
||||
};
|
||||
}
|
||||
|
||||
// ── SIEM Batch Events ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* GET /siem/v1/batch/events/cg — returns pre-signed download URLs for batch NDJSON files
|
||||
* Each URL points to a compressed file; we download and parse each one.
|
||||
* Response: { value: [{url, expiry, size}], "@nextPage": string }
|
||||
*/
|
||||
async getSiemBatchEventUrls(options: {
|
||||
from: string; // date only: YYYY-MM-DD
|
||||
to: string;
|
||||
type?: string;
|
||||
cursor?: string;
|
||||
pageSize?: number;
|
||||
}): Promise<{ urls: Array<{ url: string; expiry: string; size: number }>; nextCursor: string | null }> {
|
||||
const params: Record<string, string> = {
|
||||
type: options.type ?? 'mailflow',
|
||||
dateRangeStartsAt: options.from,
|
||||
dateRangeEndsAt: options.to,
|
||||
pageSize: String(options.pageSize ?? 500),
|
||||
};
|
||||
if (options.cursor) params.pageToken = options.cursor;
|
||||
|
||||
const data = await this.request<any>('GET', '/siem/v1/batch/events/cg', undefined, params);
|
||||
|
||||
return {
|
||||
urls: data.value ?? [],
|
||||
nextCursor: data['@nextPage'] ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Download and parse a SIEM batch event file (NDJSON, possibly gzipped)
|
||||
*/
|
||||
async downloadSiemBatchFile(url: string): Promise<MimecastMessage[]> {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) return [];
|
||||
|
||||
const text = await res.text();
|
||||
const lines = text.split('\n').filter(l => l.trim());
|
||||
const messages: MimecastMessage[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
messages.push(this.normalizeSiemEvent(event));
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
private normalizeSiemEvent(e: any): MimecastMessage {
|
||||
return {
|
||||
id: e.messageId ?? e.id ?? e.Messageid ?? '',
|
||||
senderAddress: e.senderAddress ?? e.Sender ?? e.sender ?? '',
|
||||
recipientAddress: e.recipientAddress ?? e.Recipient ?? e.recipient ?? '',
|
||||
subject: e.subject ?? e.Subject ?? '',
|
||||
direction: (e.Dir ?? e.direction ?? '').toLowerCase() === 'inbound' ? 'inbound' : 'outbound',
|
||||
status: e.Act ?? e.action ?? e.status ?? '',
|
||||
action: e.RejType ?? e.rejectionType ?? undefined,
|
||||
spamScore: e.SpamScore ?? e.spamScore ?? undefined,
|
||||
sizeBytes: e.MsgSize ?? e.messageSize ?? e.size ?? undefined,
|
||||
attachmentCount: e.AttCnt ?? e.attachmentCount ?? 0,
|
||||
sentDateTime: e.Datetime ?? e.datetime ?? e.timestamp ?? undefined,
|
||||
receivedDateTime: e.Datetime ?? undefined,
|
||||
route: e.Route ?? e.route ?? undefined,
|
||||
rejectReason: e.RejCode ?? e.rejectionCode ?? undefined,
|
||||
heldReason: e.HeldGroup ?? undefined,
|
||||
sourceIp: e.IP ?? e.senderIp ?? undefined,
|
||||
_raw: e,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Threat Events ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* GET /threats/v1/events
|
||||
* Response: { value: [{timestamp, analysis, source, details, status, direction, subject, sender, ...}], "@nextPage": string }
|
||||
*/
|
||||
async getThreatEvents(options: {
|
||||
cursor?: string;
|
||||
pageSize?: number;
|
||||
} = {}): Promise<PaginatedResult<MimecastThreatEvent>> {
|
||||
const params: Record<string, string> = {
|
||||
pageSize: String(options.pageSize ?? 500),
|
||||
};
|
||||
if (options.cursor) params.pageToken = options.cursor;
|
||||
|
||||
try {
|
||||
const data = await this.request<any>('GET', '/threats/v1/events', undefined, params);
|
||||
const items = (data.value ?? []).map((e: any) => this.normalizeThreatEvent(e));
|
||||
return {
|
||||
items,
|
||||
nextCursor: data['@nextPage'] ?? null,
|
||||
};
|
||||
} catch {
|
||||
return { items: [], nextCursor: null };
|
||||
}
|
||||
}
|
||||
|
||||
private normalizeThreatEvent(e: any): MimecastThreatEvent {
|
||||
const analysis: string[] = Array.isArray(e.analysis) ? e.analysis : [e.analysis].filter(Boolean);
|
||||
const eventType = analysis[0] ?? 'unknown';
|
||||
|
||||
const statusArr: string[] = Array.isArray(e.status) ? e.status : [e.status].filter(Boolean);
|
||||
const verdict = statusArr[0] ?? undefined;
|
||||
|
||||
const threatLevel = analysis.includes('malware') ? 'high'
|
||||
: analysis.includes('phishing') ? 'high'
|
||||
: analysis.includes('spam') ? 'medium'
|
||||
: 'info';
|
||||
|
||||
return {
|
||||
id: e.id ?? `threat_${e.sender ?? ''}_${e.timestamp ?? Date.now()}`,
|
||||
messageId: e.messageId ?? undefined,
|
||||
eventType,
|
||||
threatLevel,
|
||||
url: e.url ?? undefined,
|
||||
fileName: undefined,
|
||||
verdict,
|
||||
actorEmail: e.sender ?? undefined,
|
||||
eventDateTime: e.timestamp ?? undefined,
|
||||
analysis,
|
||||
source: Array.isArray(e.source) ? e.source : [e.source].filter(Boolean),
|
||||
direction: Array.isArray(e.direction) ? e.direction : [e.direction].filter(Boolean),
|
||||
status: statusArr,
|
||||
_raw: e,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Message Info (body/headers) ────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* POST /api/message-finder/get-message-info
|
||||
* Returns delivered message details including parts
|
||||
*/
|
||||
async getMessageInfo(messageId: string): Promise<MimecastMessageInfo | null> {
|
||||
try {
|
||||
const data = await this.request<any>('POST', '/api/message-finder/get-message-info', {
|
||||
data: [{ id: messageId }],
|
||||
});
|
||||
|
||||
const delivered = data.data?.[0]?.deliveredMessage;
|
||||
if (!delivered) return null;
|
||||
|
||||
const entries = Object.values(delivered) as any[];
|
||||
if (!entries.length) return null;
|
||||
|
||||
const info = entries[0]?.messageInfo ?? entries[0];
|
||||
|
||||
return {
|
||||
messageId,
|
||||
bodyText: info?.textBody ?? info?.body ?? undefined,
|
||||
bodyHtml: info?.htmlBody ?? undefined,
|
||||
headers: info?.headers ?? undefined,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Account ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* POST /api/account/get-account
|
||||
*/
|
||||
async testConnection(): Promise<{ ok: boolean; accountName?: string; packageName?: string; error?: string }> {
|
||||
try {
|
||||
const data = await this.request<any>('POST', '/api/account/get-account', { data: [{}] });
|
||||
const account = data.data?.[0] ?? {};
|
||||
return {
|
||||
ok: true,
|
||||
accountName: account.accountName ?? account.accountCode ?? 'Connected',
|
||||
packageName: account.packageName ?? undefined,
|
||||
};
|
||||
} catch (err: any) {
|
||||
return { ok: false, error: err.message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _client: MimecastClient | null = null;
|
||||
|
||||
export function getMimecastClient(): MimecastClient {
|
||||
if (!_client) {
|
||||
const clientId = process.env.MIMECAST_CLIENT_ID;
|
||||
const clientSecret = process.env.MIMECAST_CLIENT_SECRET;
|
||||
if (!clientId || !clientSecret) {
|
||||
throw new Error('MIMECAST_CLIENT_ID and MIMECAST_CLIENT_SECRET must be set');
|
||||
}
|
||||
_client = new MimecastClient({
|
||||
clientId,
|
||||
clientSecret,
|
||||
baseUrl: process.env.MIMECAST_BASE_URL ?? 'https://api.services.mimecast.com',
|
||||
});
|
||||
}
|
||||
return _client;
|
||||
}
|
||||
331
lib/services/mimecast-sync-service.ts
Normal file
331
lib/services/mimecast-sync-service.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
/**
|
||||
* Mimecast Sync Service
|
||||
* Full sync (120 days back), incremental (since last sync), body fetch, 120-day purge
|
||||
*/
|
||||
|
||||
import { getMimecastClient, MimecastMessage, MimecastThreatEvent } from './mimecast-client';
|
||||
import { postgresClient as pg } from './postgres-client';
|
||||
|
||||
export interface MimecastSyncResult {
|
||||
messagesUpserted: number;
|
||||
threatsUpserted: number;
|
||||
bodiesFetched: number;
|
||||
purgedMessages: number;
|
||||
errors: string[];
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
const RETENTION_DAYS = 120;
|
||||
const BODY_FETCH_LIMIT = 500;
|
||||
|
||||
// ── Upsert helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
async function upsertMessages(messages: MimecastMessage[]): Promise<number> {
|
||||
if (!messages.length) return 0;
|
||||
let count = 0;
|
||||
|
||||
for (const m of messages) {
|
||||
if (!m.id) continue;
|
||||
const senderDomain = m.senderAddress?.includes('@')
|
||||
? m.senderAddress.split('@')[1]?.toLowerCase()
|
||||
: null;
|
||||
|
||||
await pg.query(`
|
||||
INSERT INTO mimecast_messages (
|
||||
id, sender_address, sender_domain, recipient_address, subject,
|
||||
direction, status, action, spam_score, size_bytes, attachment_count,
|
||||
sent_datetime, received_datetime, delivery_datetime,
|
||||
route, reject_reason, held_reason, source_ip, raw, synced_at
|
||||
) VALUES (
|
||||
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,NOW()
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
action = EXCLUDED.action,
|
||||
delivery_datetime= EXCLUDED.delivery_datetime,
|
||||
reject_reason = EXCLUDED.reject_reason,
|
||||
held_reason = EXCLUDED.held_reason,
|
||||
raw = EXCLUDED.raw,
|
||||
synced_at = NOW()
|
||||
`, [
|
||||
m.id,
|
||||
m.senderAddress || null,
|
||||
senderDomain,
|
||||
m.recipientAddress || null,
|
||||
m.subject || null,
|
||||
m.direction || null,
|
||||
m.status || null,
|
||||
m.action || null,
|
||||
m.spamScore ?? null,
|
||||
m.sizeBytes ?? null,
|
||||
typeof m.attachmentCount === 'number' ? m.attachmentCount : null,
|
||||
m.sentDateTime || null,
|
||||
m.receivedDateTime || null,
|
||||
null, // delivery_datetime not in message-finder search
|
||||
m.route || null,
|
||||
m.rejectReason || null,
|
||||
m.heldReason || null,
|
||||
m.sourceIp || null,
|
||||
m._raw ? JSON.stringify(m._raw) : null,
|
||||
]);
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
async function upsertThreats(events: MimecastThreatEvent[]): Promise<number> {
|
||||
if (!events.length) return 0;
|
||||
let count = 0;
|
||||
|
||||
for (const e of events) {
|
||||
if (!e.id) continue;
|
||||
await pg.query(`
|
||||
INSERT INTO mimecast_threat_events (
|
||||
id, message_id, event_type, threat_level, url, file_name,
|
||||
verdict, actor_email, event_datetime, details, synced_at
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,NOW())
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
threat_level = EXCLUDED.threat_level,
|
||||
verdict = EXCLUDED.verdict,
|
||||
details = EXCLUDED.details,
|
||||
synced_at = NOW()
|
||||
`, [
|
||||
e.id,
|
||||
e.messageId || null,
|
||||
e.eventType || null,
|
||||
e.threatLevel || null,
|
||||
e.url || null,
|
||||
e.fileName || null,
|
||||
e.verdict || null,
|
||||
e.actorEmail || null,
|
||||
e.eventDateTime || null,
|
||||
e._raw ? JSON.stringify(e._raw) : null,
|
||||
]);
|
||||
count++;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
async function fetchAndStoreBodies(errors: string[]): Promise<number> {
|
||||
const client = getMimecastClient();
|
||||
|
||||
// Find delivered inbound messages missing body, up to limit
|
||||
const rows = await pg.query(`
|
||||
SELECT m.id FROM mimecast_messages m
|
||||
LEFT JOIN mimecast_message_bodies b ON b.message_id = m.id
|
||||
WHERE b.message_id IS NULL
|
||||
AND m.status NOT IN ('rejected','spam','bounced')
|
||||
AND m.direction = 'inbound'
|
||||
ORDER BY m.sent_datetime DESC
|
||||
LIMIT $1
|
||||
`, [BODY_FETCH_LIMIT]);
|
||||
|
||||
let fetched = 0;
|
||||
for (const row of rows.rows) {
|
||||
try {
|
||||
const info = await client.getMessageInfo(row.id);
|
||||
if (!info) continue;
|
||||
|
||||
await pg.query(`
|
||||
INSERT INTO mimecast_message_bodies (message_id, body_text, body_html, headers, fetched_at)
|
||||
VALUES ($1, $2, $3, $4, NOW())
|
||||
ON CONFLICT (message_id) DO UPDATE SET
|
||||
body_text = EXCLUDED.body_text,
|
||||
body_html = EXCLUDED.body_html,
|
||||
headers = EXCLUDED.headers,
|
||||
fetched_at = NOW()
|
||||
`, [
|
||||
info.messageId,
|
||||
info.bodyText || null,
|
||||
info.bodyHtml || null,
|
||||
info.headers ? JSON.stringify(info.headers) : null,
|
||||
]);
|
||||
fetched++;
|
||||
} catch (err: any) {
|
||||
errors.push(`Body fetch ${row.id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return fetched;
|
||||
}
|
||||
|
||||
async function purgeOldRecords(): Promise<number> {
|
||||
const cutoff = new Date();
|
||||
cutoff.setDate(cutoff.getDate() - RETENTION_DAYS);
|
||||
|
||||
const result = await pg.query(
|
||||
`DELETE FROM mimecast_messages WHERE sent_datetime < $1`,
|
||||
[cutoff.toISOString()]
|
||||
);
|
||||
return result.rowCount ?? 0;
|
||||
}
|
||||
|
||||
async function getSyncState(syncType: string): Promise<{ lastSyncedAt: Date | null; cursor: string | null }> {
|
||||
const result = await pg.query(
|
||||
`SELECT last_synced_at, cursor FROM mimecast_sync_state WHERE sync_type = $1`,
|
||||
[syncType]
|
||||
);
|
||||
const row = result.rows[0];
|
||||
return {
|
||||
lastSyncedAt: row?.last_synced_at ? new Date(row.last_synced_at) : null,
|
||||
cursor: row?.cursor ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async function saveSyncState(syncType: string, lastSyncedAt: Date, cursor: string | null): Promise<void> {
|
||||
await pg.query(`
|
||||
INSERT INTO mimecast_sync_state (sync_type, last_synced_at, cursor, updated_at)
|
||||
VALUES ($1, $2, $3, NOW())
|
||||
ON CONFLICT (sync_type) DO UPDATE SET
|
||||
last_synced_at = EXCLUDED.last_synced_at,
|
||||
cursor = EXCLUDED.cursor,
|
||||
updated_at = NOW()
|
||||
`, [syncType, lastSyncedAt.toISOString(), cursor]);
|
||||
}
|
||||
|
||||
// ── Main sync functions ───────────────────────────────────────────────────────
|
||||
|
||||
export async function syncMimecastMessages(
|
||||
fromDate: Date,
|
||||
toDate: Date,
|
||||
errors: string[]
|
||||
): Promise<number> {
|
||||
const client = getMimecastClient();
|
||||
let total = 0;
|
||||
let cursor: string | null = null;
|
||||
|
||||
const from = fromDate.toISOString();
|
||||
const to = toDate.toISOString();
|
||||
|
||||
do {
|
||||
try {
|
||||
const result = await client.getMessageLogs({ from, to, cursor: cursor ?? undefined, pageSize: 500 });
|
||||
if (result.items.length > 0) {
|
||||
total += await upsertMessages(result.items);
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
} catch (err: any) {
|
||||
errors.push(`Message sync page: ${err.message}`);
|
||||
break;
|
||||
}
|
||||
} while (cursor);
|
||||
|
||||
await saveSyncState('messages', toDate, null);
|
||||
return total;
|
||||
}
|
||||
|
||||
export async function syncMimecastThreats(errors: string[]): Promise<number> {
|
||||
const client = getMimecastClient();
|
||||
let total = 0;
|
||||
let cursor: string | null = null;
|
||||
|
||||
do {
|
||||
try {
|
||||
const result = await client.getThreatEvents({ cursor: cursor ?? undefined, pageSize: 500 });
|
||||
if (result.items.length > 0) {
|
||||
total += await upsertThreats(result.items);
|
||||
}
|
||||
cursor = result.nextCursor;
|
||||
} catch (err: any) {
|
||||
errors.push(`Threat sync: ${err.message}`);
|
||||
break;
|
||||
}
|
||||
} while (cursor);
|
||||
|
||||
await saveSyncState('threats', new Date(), null);
|
||||
return total;
|
||||
}
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function runMimecastFullSync(): Promise<MimecastSyncResult> {
|
||||
const start = Date.now();
|
||||
const errors: string[] = [];
|
||||
|
||||
const toDate = new Date();
|
||||
const fromDate = new Date();
|
||||
fromDate.setDate(fromDate.getDate() - RETENTION_DAYS);
|
||||
|
||||
const messagesUpserted = await syncMimecastMessages(fromDate, toDate, errors);
|
||||
const threatsUpserted = await syncMimecastThreats(errors);
|
||||
const bodiesFetched = await fetchAndStoreBodies(errors);
|
||||
const purgedMessages = await purgeOldRecords();
|
||||
|
||||
return {
|
||||
messagesUpserted,
|
||||
threatsUpserted,
|
||||
bodiesFetched,
|
||||
purgedMessages,
|
||||
errors,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runMimecastIncrementalSync(): Promise<MimecastSyncResult> {
|
||||
const start = Date.now();
|
||||
const errors: string[] = [];
|
||||
|
||||
const { lastSyncedAt } = await getSyncState('messages');
|
||||
|
||||
const toDate = new Date();
|
||||
const fromDate = lastSyncedAt ?? (() => {
|
||||
const d = new Date();
|
||||
d.setDate(d.getDate() - 1);
|
||||
return d;
|
||||
})();
|
||||
|
||||
const messagesUpserted = await syncMimecastMessages(fromDate, toDate, errors);
|
||||
const threatsUpserted = await syncMimecastThreats(errors);
|
||||
const bodiesFetched = await fetchAndStoreBodies(errors);
|
||||
const purgedMessages = await purgeOldRecords();
|
||||
|
||||
return {
|
||||
messagesUpserted,
|
||||
threatsUpserted,
|
||||
bodiesFetched,
|
||||
purgedMessages,
|
||||
errors,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getMimecastStats(): Promise<{
|
||||
messages: number;
|
||||
inbound: number;
|
||||
outbound: number;
|
||||
threats: number;
|
||||
bodies: number;
|
||||
lastSync: string | null;
|
||||
oldestMessage: string | null;
|
||||
}> {
|
||||
const [counts, syncState, oldest] = await Promise.all([
|
||||
pg.query(`
|
||||
SELECT
|
||||
COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE direction = 'inbound') AS inbound,
|
||||
COUNT(*) FILTER (WHERE direction = 'outbound') AS outbound
|
||||
FROM mimecast_messages
|
||||
`),
|
||||
pg.query(`SELECT last_synced_at FROM mimecast_sync_state WHERE sync_type = 'messages'`),
|
||||
pg.query(`SELECT MIN(sent_datetime) AS oldest FROM mimecast_messages`),
|
||||
]);
|
||||
|
||||
const [threatCount, bodyCount] = await Promise.all([
|
||||
pg.query(`SELECT COUNT(*) AS total FROM mimecast_threat_events`),
|
||||
pg.query(`SELECT COUNT(*) AS total FROM mimecast_message_bodies`),
|
||||
]);
|
||||
|
||||
const c = counts.rows[0] ?? {};
|
||||
|
||||
return {
|
||||
messages: Number(c.total ?? 0),
|
||||
inbound: Number(c.inbound ?? 0),
|
||||
outbound: Number(c.outbound ?? 0),
|
||||
threats: Number(threatCount.rows[0]?.total ?? 0),
|
||||
bodies: Number(bodyCount.rows[0]?.total ?? 0),
|
||||
lastSync: syncState.rows[0]?.last_synced_at ?? null,
|
||||
oldestMessage: oldest.rows[0]?.oldest ?? null,
|
||||
};
|
||||
}
|
||||
72
migrations/055_create_mimecast_tables.sql
Normal file
72
migrations/055_create_mimecast_tables.sql
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
-- ============================================================================
|
||||
-- Mimecast Email Integration Tables
|
||||
-- 120-day rolling retention for message logs, threat events, and message bodies
|
||||
-- ============================================================================
|
||||
|
||||
-- Core message tracking log
|
||||
CREATE TABLE IF NOT EXISTS mimecast_messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
sender_address TEXT,
|
||||
sender_domain TEXT,
|
||||
recipient_address TEXT,
|
||||
subject TEXT,
|
||||
direction TEXT, -- 'inbound' | 'outbound'
|
||||
status TEXT, -- 'delivered' | 'rejected' | 'bounced' | 'held' | 'spam'
|
||||
action TEXT,
|
||||
spam_score NUMERIC,
|
||||
size_bytes BIGINT,
|
||||
attachment_count INT DEFAULT 0,
|
||||
sent_datetime TIMESTAMPTZ,
|
||||
received_datetime TIMESTAMPTZ,
|
||||
delivery_datetime TIMESTAMPTZ,
|
||||
route TEXT,
|
||||
reject_reason TEXT,
|
||||
held_reason TEXT,
|
||||
source_ip TEXT,
|
||||
raw JSONB, -- Full API response for forward-compat
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Separate table for message bodies (large, for LLM use only)
|
||||
CREATE TABLE IF NOT EXISTS mimecast_message_bodies (
|
||||
message_id TEXT PRIMARY KEY REFERENCES mimecast_messages(id) ON DELETE CASCADE,
|
||||
body_text TEXT,
|
||||
body_html TEXT,
|
||||
headers JSONB,
|
||||
fetched_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- SIEM/threat events
|
||||
CREATE TABLE IF NOT EXISTS mimecast_threat_events (
|
||||
id TEXT PRIMARY KEY,
|
||||
message_id TEXT REFERENCES mimecast_messages(id) ON DELETE SET NULL,
|
||||
event_type TEXT, -- 'url_click' | 'attachment_sandbox' | 'impersonation' | 'spam' | 'virus'
|
||||
threat_level TEXT, -- 'high' | 'medium' | 'low' | 'info'
|
||||
url TEXT,
|
||||
file_name TEXT,
|
||||
verdict TEXT,
|
||||
actor_email TEXT,
|
||||
event_datetime TIMESTAMPTZ,
|
||||
details JSONB, -- Full raw event payload
|
||||
synced_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Cursor-based pagination state for resumable syncs
|
||||
CREATE TABLE IF NOT EXISTS mimecast_sync_state (
|
||||
sync_type TEXT PRIMARY KEY, -- 'messages' | 'threats' | 'held'
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
cursor TEXT,
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_sent_datetime ON mimecast_messages (sent_datetime DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_sender_domain ON mimecast_messages (sender_domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_status ON mimecast_messages (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_direction ON mimecast_messages (direction);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_recipient ON mimecast_messages (recipient_address);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_datetime ON mimecast_threat_events (event_datetime DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_type ON mimecast_threat_events (event_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_level ON mimecast_threat_events (threat_level);
|
||||
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_details ON mimecast_threat_events USING GIN (details);
|
||||
Loading…
Add table
Add a link
Reference in a new issue