feat: Mimecast email integration — message logs, threat events, 120d retention, admin UI
This commit is contained in:
parent
7792e91587
commit
25bb70cfa6
11 changed files with 112285 additions and 1 deletions
435
app/admin/sync/mimecast/page.tsx
Normal file
435
app/admin/sync/mimecast/page.tsx
Normal file
|
|
@ -0,0 +1,435 @@
|
||||||
|
'use client';
|
||||||
|
|
||||||
|
import { useState, useEffect } from 'react';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||||
|
import {
|
||||||
|
ArrowLeft, Activity, History, Calendar, Mail, RefreshCw, Loader2,
|
||||||
|
CheckCircle2, XCircle, AlertTriangle, Shield, Inbox, Send,
|
||||||
|
Clock, ChevronDown, ChevronRight,
|
||||||
|
} from 'lucide-react';
|
||||||
|
import SyncScheduler from '@/components/admin/SyncScheduler';
|
||||||
|
|
||||||
|
function fmtDate(d: string | null | undefined) {
|
||||||
|
if (!d) return 'Never';
|
||||||
|
return new Date(d).toLocaleString(undefined, {
|
||||||
|
month: 'short', day: 'numeric', year: 'numeric', hour: '2-digit', minute: '2-digit',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtNum(n: number | null | undefined) {
|
||||||
|
if (n == null) return '—';
|
||||||
|
return n.toLocaleString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function StatCard({ label, value, sub, icon: Icon, cls }: {
|
||||||
|
label: string; value: string | number; sub?: string; icon?: React.ElementType; cls?: string;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div className={`rounded-lg border p-4 flex flex-col gap-1 ${cls ?? ''}`}>
|
||||||
|
<div className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||||
|
{Icon && <Icon className="w-3.5 h-3.5" />}{label}
|
||||||
|
</div>
|
||||||
|
<div className="text-2xl font-bold tabular-nums">{value}</div>
|
||||||
|
{sub && <div className="text-xs text-muted-foreground">{sub}</div>}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function StatusBadge({ status }: { status: string }) {
|
||||||
|
const cls =
|
||||||
|
status === 'delivered' ? 'bg-green-500/15 text-green-700' :
|
||||||
|
status === 'rejected' ? 'bg-red-500/15 text-red-600' :
|
||||||
|
status === 'held' ? 'bg-yellow-500/15 text-yellow-700' :
|
||||||
|
status === 'bounced' ? 'bg-orange-500/15 text-orange-700' :
|
||||||
|
status === 'spam' ? 'bg-purple-500/15 text-purple-700' :
|
||||||
|
'bg-muted text-muted-foreground';
|
||||||
|
return (
|
||||||
|
<span className={`inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium ${cls}`}>
|
||||||
|
{status || '—'}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ThreatBadge({ level }: { level: string }) {
|
||||||
|
const cls =
|
||||||
|
level === 'high' ? 'bg-red-500/15 text-red-600' :
|
||||||
|
level === 'medium' ? 'bg-orange-500/15 text-orange-700' :
|
||||||
|
level === 'low' ? 'bg-yellow-500/15 text-yellow-700' :
|
||||||
|
'bg-muted text-muted-foreground';
|
||||||
|
return (
|
||||||
|
<span className={`inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium ${cls}`}>
|
||||||
|
{level || 'info'}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Status Tab ────────────────────────────────────────────────────────────────
|
||||||
|
function StatusTab({ data, onSync, syncing }: { data: any; onSync: (t: string) => void; syncing: boolean }) {
|
||||||
|
if (!data) return (
|
||||||
|
<div className="flex items-center justify-center py-12">
|
||||||
|
<Loader2 className="w-5 h-5 animate-spin text-muted-foreground" />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
|
||||||
|
const stats = data.stats ?? {};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-6">
|
||||||
|
{/* Connection banner */}
|
||||||
|
<div className="flex items-center justify-between rounded-lg border p-4 bg-muted/30">
|
||||||
|
<div className="space-y-0.5">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{data.connected
|
||||||
|
? <CheckCircle2 className="w-4 h-4 text-green-500" />
|
||||||
|
: <XCircle className="w-4 h-4 text-red-500" />}
|
||||||
|
<p className="text-sm font-medium">
|
||||||
|
{data.connected ? `Connected — ${data.accountName ?? 'Mimecast'}` : 'Not connected'}
|
||||||
|
</p>
|
||||||
|
{data.packageName && (
|
||||||
|
<span className="text-xs text-muted-foreground">({data.packageName})</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
Last sync: {fmtDate(stats.lastSync)} · Oldest message: {fmtDate(stats.oldestMessage)}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button size="sm" variant="outline" onClick={() => onSync('incremental')} disabled={syncing || !data.connected}>
|
||||||
|
{syncing ? <Loader2 className="w-4 h-4 animate-spin mr-1" /> : <RefreshCw className="w-4 h-4 mr-1" />}
|
||||||
|
Incremental
|
||||||
|
</Button>
|
||||||
|
<Button size="sm" onClick={() => onSync('full')} disabled={syncing || !data.connected}>
|
||||||
|
{syncing ? <Loader2 className="w-4 h-4 animate-spin mr-1" /> : <RefreshCw className="w-4 h-4 mr-1" />}
|
||||||
|
Full Sync (120d)
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{data.error && (
|
||||||
|
<div className="rounded-lg border border-red-300 bg-red-50 dark:bg-red-950/20 p-3 text-sm text-red-600">
|
||||||
|
{data.error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Stats grid */}
|
||||||
|
<div className="grid grid-cols-2 md:grid-cols-5 gap-3">
|
||||||
|
<StatCard label="Total Messages" value={fmtNum(stats.messages)} icon={Mail} />
|
||||||
|
<StatCard label="Inbound" value={fmtNum(stats.inbound)} icon={Inbox} cls="border-blue-500/20 bg-blue-500/5" />
|
||||||
|
<StatCard label="Outbound" value={fmtNum(stats.outbound)} icon={Send} cls="border-green-500/20 bg-green-500/5" />
|
||||||
|
<StatCard label="Threat Events" value={fmtNum(stats.threats)} icon={Shield}
|
||||||
|
cls={(stats.threats ?? 0) > 0 ? 'border-red-500/20 bg-red-500/5' : ''} />
|
||||||
|
<StatCard label="Bodies Stored" value={fmtNum(stats.bodies)} icon={Activity} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="rounded-lg border p-4 space-y-2">
|
||||||
|
<p className="text-xs font-semibold uppercase tracking-wider text-muted-foreground">Data Retention</p>
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
120-day rolling window. Messages older than 120 days are automatically purged on each sync.
|
||||||
|
Message bodies are fetched for delivered inbound messages (up to 500 per sync run).
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Messages Tab ──────────────────────────────────────────────────────────────
|
||||||
|
function MessagesTab() {
|
||||||
|
const [rows, setRows] = useState<any[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [search, setSearch] = useState('');
|
||||||
|
const [direction, setDirection] = useState('');
|
||||||
|
const [status, setStatus] = useState('');
|
||||||
|
const [days, setDays] = useState('7');
|
||||||
|
|
||||||
|
const load = () => {
|
||||||
|
setLoading(true);
|
||||||
|
const params = new URLSearchParams({ days, limit: '200' });
|
||||||
|
if (search) params.set('search', search);
|
||||||
|
if (direction) params.set('direction', direction);
|
||||||
|
if (status) params.set('status', status);
|
||||||
|
|
||||||
|
fetch(`/api/mimecast/messages?${params}`)
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => setRows(d.messages ?? []))
|
||||||
|
.catch(() => setRows([]))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => { load(); }, [days, direction, status]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
{/* Filters */}
|
||||||
|
<div className="flex flex-wrap gap-2">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
placeholder="Search sender, recipient, subject…"
|
||||||
|
value={search}
|
||||||
|
onChange={e => setSearch(e.target.value)}
|
||||||
|
onKeyDown={e => e.key === 'Enter' && load()}
|
||||||
|
className="flex-1 min-w-48 border rounded-md px-3 py-1.5 text-sm bg-background"
|
||||||
|
/>
|
||||||
|
<select value={direction} onChange={e => setDirection(e.target.value)}
|
||||||
|
className="border rounded-md px-3 py-1.5 text-sm bg-background">
|
||||||
|
<option value="">All directions</option>
|
||||||
|
<option value="inbound">Inbound</option>
|
||||||
|
<option value="outbound">Outbound</option>
|
||||||
|
</select>
|
||||||
|
<select value={status} onChange={e => setStatus(e.target.value)}
|
||||||
|
className="border rounded-md px-3 py-1.5 text-sm bg-background">
|
||||||
|
<option value="">All statuses</option>
|
||||||
|
<option value="delivered">Delivered</option>
|
||||||
|
<option value="rejected">Rejected</option>
|
||||||
|
<option value="held">Held</option>
|
||||||
|
<option value="bounced">Bounced</option>
|
||||||
|
<option value="spam">Spam</option>
|
||||||
|
</select>
|
||||||
|
<select value={days} onChange={e => setDays(e.target.value)}
|
||||||
|
className="border rounded-md px-3 py-1.5 text-sm bg-background">
|
||||||
|
<option value="1">Last 24h</option>
|
||||||
|
<option value="7">Last 7 days</option>
|
||||||
|
<option value="30">Last 30 days</option>
|
||||||
|
<option value="90">Last 90 days</option>
|
||||||
|
<option value="120">Last 120 days</option>
|
||||||
|
</select>
|
||||||
|
<Button size="sm" variant="outline" onClick={load}>
|
||||||
|
<RefreshCw className="w-4 h-4 mr-1" />Search
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{loading ? (
|
||||||
|
<div className="flex items-center justify-center py-12">
|
||||||
|
<Loader2 className="w-5 h-5 animate-spin text-muted-foreground" />
|
||||||
|
</div>
|
||||||
|
) : !rows.length ? (
|
||||||
|
<div className="text-center py-12 text-sm text-muted-foreground">
|
||||||
|
No messages found — run a sync first or adjust filters
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="rounded-lg border overflow-hidden">
|
||||||
|
<table className="w-full text-sm">
|
||||||
|
<thead className="bg-muted/50 border-b">
|
||||||
|
<tr>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">From</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">To</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Subject</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Direction</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Status</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Sent</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{rows.map((r: any) => (
|
||||||
|
<tr key={r.id} className="border-b last:border-0 hover:bg-muted/30">
|
||||||
|
<td className="px-4 py-2 text-xs truncate max-w-[180px]" title={r.sender_address}>{r.sender_address ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2 text-xs truncate max-w-[180px]" title={r.recipient_address}>{r.recipient_address ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2 text-xs truncate max-w-[200px]" title={r.subject}>{r.subject ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2 text-xs capitalize text-muted-foreground">{r.direction ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2"><StatusBadge status={r.status} /></td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground whitespace-nowrap">{fmtDate(r.sent_datetime)}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Threats Tab ───────────────────────────────────────────────────────────────
|
||||||
|
function ThreatsTab() {
|
||||||
|
const [rows, setRows] = useState<any[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setLoading(true);
|
||||||
|
fetch('/api/mimecast/threats?limit=200')
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => setRows(d.threats ?? []))
|
||||||
|
.catch(() => setRows([]))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (loading) return <div className="flex items-center justify-center py-12"><Loader2 className="w-5 h-5 animate-spin text-muted-foreground" /></div>;
|
||||||
|
if (!rows.length) return <div className="text-center py-12 text-sm text-muted-foreground">No threat events — run a sync first</div>;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-lg border overflow-hidden">
|
||||||
|
<table className="w-full text-sm">
|
||||||
|
<thead className="bg-muted/50 border-b">
|
||||||
|
<tr>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Type</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Level</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Actor</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Verdict</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">URL / File</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">When</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{rows.map((r: any) => (
|
||||||
|
<tr key={r.id} className="border-b last:border-0 hover:bg-muted/30">
|
||||||
|
<td className="px-4 py-2 text-xs capitalize">{r.event_type ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2"><ThreatBadge level={r.threat_level} /></td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground">{r.actor_email ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground capitalize">{r.verdict ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground truncate max-w-[200px]" title={r.url ?? r.file_name ?? ''}>
|
||||||
|
{r.url ?? r.file_name ?? '—'}
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground whitespace-nowrap">{fmtDate(r.event_datetime)}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── History Tab ───────────────────────────────────────────────────────────────
|
||||||
|
function HistoryTab() {
|
||||||
|
const [rows, setRows] = useState<any[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setLoading(true);
|
||||||
|
fetch('/api/sync/history?entityType=mimecast&limit=30')
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => setRows(d.history ?? []))
|
||||||
|
.catch(() => setRows([]))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (loading) return <div className="flex items-center justify-center py-12"><Loader2 className="w-5 h-5 animate-spin text-muted-foreground" /></div>;
|
||||||
|
if (!rows.length) return <div className="text-center py-12 text-sm text-muted-foreground">No sync history yet</div>;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="rounded-lg border overflow-hidden">
|
||||||
|
<table className="w-full text-sm">
|
||||||
|
<thead className="bg-muted/50 border-b">
|
||||||
|
<tr>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Type</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Status</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Messages</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Threats</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Started</th>
|
||||||
|
<th className="text-left px-4 py-2 font-medium text-muted-foreground">Duration</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{rows.map((r: any, i: number) => {
|
||||||
|
const dur = r.completed_at && r.started_at
|
||||||
|
? new Date(r.completed_at).getTime() - new Date(r.started_at).getTime()
|
||||||
|
: null;
|
||||||
|
const durStr = dur == null ? '—' : dur < 60000 ? `${Math.round(dur / 1000)}s` : `${Math.floor(dur / 60000)}m ${Math.round((dur % 60000) / 1000)}s`;
|
||||||
|
const statusCls = r.status === 'completed' ? 'bg-green-500/15 text-green-700' : r.status === 'failed' ? 'bg-red-500/15 text-red-600' : 'bg-muted text-muted-foreground';
|
||||||
|
const meta = typeof r.metadata === 'string' ? JSON.parse(r.metadata || '{}') : (r.metadata ?? {});
|
||||||
|
|
||||||
|
return (
|
||||||
|
<tr key={i} className="border-b last:border-0 hover:bg-muted/30">
|
||||||
|
<td className="px-4 py-2 capitalize text-xs">{r.sync_type ?? '—'}</td>
|
||||||
|
<td className="px-4 py-2">
|
||||||
|
<span className={`inline-flex rounded-full px-2 py-0.5 text-xs font-medium ${statusCls}`}>{r.status}</span>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-2 tabular-nums text-xs">{fmtNum(meta.messagesUpserted ?? r.records_added)}</td>
|
||||||
|
<td className="px-4 py-2 tabular-nums text-xs">{fmtNum(meta.threatsUpserted)}</td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground">{fmtDate(r.started_at)}</td>
|
||||||
|
<td className="px-4 py-2 text-xs text-muted-foreground">{durStr}</td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Page ──────────────────────────────────────────────────────────────────────
|
||||||
|
export default function MimecastSyncPage() {
|
||||||
|
const [statusData, setStatusData] = useState<any>(null);
|
||||||
|
const [syncing, setSyncing] = useState(false);
|
||||||
|
const [lastResult, setLastResult] = useState<any>(null);
|
||||||
|
|
||||||
|
const fetchStatus = () => {
|
||||||
|
fetch('/api/mimecast/status')
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => setStatusData(d))
|
||||||
|
.catch(() => setStatusData({ configured: false, connected: false }));
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => { fetchStatus(); }, []);
|
||||||
|
|
||||||
|
const handleSync = async (syncType: string) => {
|
||||||
|
setSyncing(true);
|
||||||
|
setLastResult(null);
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/sync/mimecast', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ syncType }),
|
||||||
|
});
|
||||||
|
const data = await res.json();
|
||||||
|
setLastResult(data);
|
||||||
|
fetchStatus();
|
||||||
|
} catch (err: any) {
|
||||||
|
setLastResult({ error: err.message });
|
||||||
|
} finally {
|
||||||
|
setSyncing(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="container mx-auto py-4 md:py-8 px-4 space-y-6">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex items-center gap-4">
|
||||||
|
<Link href="/admin/sync">
|
||||||
|
<Button variant="outline" size="sm" className="gap-2">
|
||||||
|
<ArrowLeft className="w-4 h-4" />Integrations
|
||||||
|
</Button>
|
||||||
|
</Link>
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<div className="p-2 rounded-lg border border-blue-500/30 bg-blue-500/5">
|
||||||
|
<Mail className="w-5 h-5 text-blue-500" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<h1 className="text-2xl font-bold">Email Security — Mimecast</h1>
|
||||||
|
<p className="text-sm text-muted-foreground">Message logs, threat events, 120-day retention</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Last sync result banner */}
|
||||||
|
{lastResult && (
|
||||||
|
<div className={`rounded-lg border p-3 text-sm ${lastResult.error ? 'border-red-300 bg-red-50 dark:bg-red-950/20 text-red-600' : 'border-green-300 bg-green-50 dark:bg-green-950/20 text-green-700'}`}>
|
||||||
|
{lastResult.error
|
||||||
|
? `Sync failed: ${lastResult.error}`
|
||||||
|
: `Sync complete — ${fmtNum(lastResult.messagesUpserted)} messages, ${fmtNum(lastResult.threatsUpserted)} threats, ${fmtNum(lastResult.bodiesFetched)} bodies fetched in ${Math.round((lastResult.durationMs ?? 0) / 1000)}s`
|
||||||
|
}
|
||||||
|
{lastResult.errors?.length > 0 && (
|
||||||
|
<div className="mt-1 text-xs opacity-80">{lastResult.errors.slice(0, 3).join(' · ')}</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<Tabs defaultValue="status" className="w-full">
|
||||||
|
<TabsList className="grid w-full max-w-2xl grid-cols-5">
|
||||||
|
<TabsTrigger value="status" className="gap-1.5"><Activity className="h-4 w-4" />Status</TabsTrigger>
|
||||||
|
<TabsTrigger value="messages" className="gap-1.5"><Mail className="h-4 w-4" />Messages</TabsTrigger>
|
||||||
|
<TabsTrigger value="threats" className="gap-1.5"><Shield className="h-4 w-4" />Threats</TabsTrigger>
|
||||||
|
<TabsTrigger value="history" className="gap-1.5"><Clock className="h-4 w-4" />History</TabsTrigger>
|
||||||
|
<TabsTrigger value="schedules" className="gap-1.5"><Calendar className="h-4 w-4" />Schedules</TabsTrigger>
|
||||||
|
</TabsList>
|
||||||
|
|
||||||
|
<TabsContent value="status" className="mt-6">
|
||||||
|
<StatusTab data={statusData} onSync={handleSync} syncing={syncing} />
|
||||||
|
</TabsContent>
|
||||||
|
<TabsContent value="messages" className="mt-6"><MessagesTab /></TabsContent>
|
||||||
|
<TabsContent value="threats" className="mt-6"><ThreatsTab /></TabsContent>
|
||||||
|
<TabsContent value="history" className="mt-6"><HistoryTab /></TabsContent>
|
||||||
|
<TabsContent value="schedules" className="mt-6"><SyncScheduler /></TabsContent>
|
||||||
|
</Tabs>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -23,6 +23,7 @@ const INTEGRATIONS: IntegrationCard[] = [
|
||||||
{ id: 'auvik', category: 'NMS', product: 'Auvik', description: 'Network devices, topology, tenant mappings', href: '/admin/sync/auvik', logo: '/logos/auvik.ico', color: 'purple' },
|
{ id: 'auvik', category: 'NMS', product: 'Auvik', description: 'Network devices, topology, tenant mappings', href: '/admin/sync/auvik', logo: '/logos/auvik.ico', color: 'purple' },
|
||||||
{ id: 'addigy', category: 'Apple RMM', product: 'Addigy', description: 'macOS/iOS device management, policies, compliance', href: '/admin/sync/addigy', logo: '/logos/addigy.ico', color: 'gray' },
|
{ id: 'addigy', category: 'Apple RMM', product: 'Addigy', description: 'macOS/iOS device management, policies, compliance', href: '/admin/sync/addigy', logo: '/logos/addigy.ico', color: 'gray' },
|
||||||
{ id: 'sentinelone', category: 'EDR/AV', product: 'SentinelOne', description: 'Endpoint agents, threat detections, site coverage, AV health', href: '/admin/sync/sentinelone', logo: '/logos/sentinelone.ico', color: 'purple' },
|
{ id: 'sentinelone', category: 'EDR/AV', product: 'SentinelOne', description: 'Endpoint agents, threat detections, site coverage, AV health', href: '/admin/sync/sentinelone', logo: '/logos/sentinelone.ico', color: 'purple' },
|
||||||
|
{ id: 'mimecast', category: 'Email Security', product: 'Mimecast', description: 'Message tracking logs, threat events, SIEM data, 120-day retention', href: '/admin/sync/mimecast', logo: '/logos/mimecast.ico', color: 'blue' },
|
||||||
];
|
];
|
||||||
|
|
||||||
const COLOR_MAP: Record<string, { bg: string; border: string }> = {
|
const COLOR_MAP: Record<string, { bg: string; border: string }> = {
|
||||||
|
|
@ -54,13 +55,16 @@ export default function SyncOverviewPage() {
|
||||||
const [itglueSyncData, setItglueSyncData] = useState<any>(null);
|
const [itglueSyncData, setItglueSyncData] = useState<any>(null);
|
||||||
const [s1SyncData, setS1SyncData] = useState<any>(null);
|
const [s1SyncData, setS1SyncData] = useState<any>(null);
|
||||||
|
|
||||||
|
const [mimecastData, setMimecastData] = useState<any>(null);
|
||||||
|
|
||||||
const fetchAll = async () => {
|
const fetchAll = async () => {
|
||||||
try {
|
try {
|
||||||
const [intRes, atRes, itgRes, s1Res] = await Promise.all([
|
const [intRes, atRes, itgRes, s1Res, mcRes] = await Promise.all([
|
||||||
fetch('/api/integrations/status'),
|
fetch('/api/integrations/status'),
|
||||||
fetch('/api/sync/last-sync'),
|
fetch('/api/sync/last-sync'),
|
||||||
fetch('/api/itglue/sync'),
|
fetch('/api/itglue/sync'),
|
||||||
fetch('/api/sentinelone/sync'),
|
fetch('/api/sentinelone/sync'),
|
||||||
|
fetch('/api/mimecast/status'),
|
||||||
]);
|
]);
|
||||||
if (intRes.ok) setStatus(await intRes.json());
|
if (intRes.ok) setStatus(await intRes.json());
|
||||||
if (atRes.ok) {
|
if (atRes.ok) {
|
||||||
|
|
@ -69,6 +73,7 @@ export default function SyncOverviewPage() {
|
||||||
}
|
}
|
||||||
if (itgRes.ok) setItglueSyncData(await itgRes.json());
|
if (itgRes.ok) setItglueSyncData(await itgRes.json());
|
||||||
if (s1Res.ok) setS1SyncData(await s1Res.json());
|
if (s1Res.ok) setS1SyncData(await s1Res.json());
|
||||||
|
if (mcRes.ok) setMimecastData(await mcRes.json());
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(e);
|
console.error(e);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -147,6 +152,16 @@ export default function SyncOverviewPage() {
|
||||||
}
|
}
|
||||||
if (id === 'auvik') return { lastSync: status.auvik?.lastSync, configured: status.auvik?.configured };
|
if (id === 'auvik') return { lastSync: status.auvik?.lastSync, configured: status.auvik?.configured };
|
||||||
if (id === 'addigy') return { lastSync: status.addigy?.lastSync, configured: status.addigy?.configured };
|
if (id === 'addigy') return { lastSync: status.addigy?.lastSync, configured: status.addigy?.configured };
|
||||||
|
if (id === 'mimecast') {
|
||||||
|
if (!mimecastData) return null;
|
||||||
|
const s = mimecastData.stats ?? {};
|
||||||
|
return {
|
||||||
|
lastSync: s.lastSync ?? null,
|
||||||
|
connected: mimecastData.connected ?? false,
|
||||||
|
messages: s.messages ?? 0,
|
||||||
|
threats: s.threats ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -176,6 +191,11 @@ export default function SyncOverviewPage() {
|
||||||
if (summary.infected > 0) return <AlertTriangle className="w-4 h-4 text-red-500" />;
|
if (summary.infected > 0) return <AlertTriangle className="w-4 h-4 text-red-500" />;
|
||||||
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
||||||
}
|
}
|
||||||
|
if (id === 'mimecast') {
|
||||||
|
if (!summary.connected) return <Clock className="w-4 h-4 text-muted-foreground" />;
|
||||||
|
if ((summary as any).threats > 0) return <AlertTriangle className="w-4 h-4 text-yellow-500" />;
|
||||||
|
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
||||||
|
}
|
||||||
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
return <CheckCircle2 className="w-4 h-4 text-green-500" />;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -322,6 +342,24 @@ export default function SyncOverviewPage() {
|
||||||
)}
|
)}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
{intg.id === 'mimecast' && summary && (
|
||||||
|
<>
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span>Last sync</span>
|
||||||
|
<span className="font-medium text-foreground">{fmtDate((summary as any).lastSync)}</span>
|
||||||
|
</div>
|
||||||
|
<div className="flex justify-between">
|
||||||
|
<span>Messages</span>
|
||||||
|
<span className="font-medium text-foreground">{((summary as any).messages ?? 0).toLocaleString()}</span>
|
||||||
|
</div>
|
||||||
|
{(summary as any).threats > 0 && (
|
||||||
|
<div className="flex justify-between text-yellow-700">
|
||||||
|
<span>Threat events</span>
|
||||||
|
<span className="font-medium">{(summary as any).threats}</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
{(intg.id === 'auvik' || intg.id === 'addigy') && (
|
{(intg.id === 'auvik' || intg.id === 'addigy') && (
|
||||||
<div className="flex justify-between">
|
<div className="flex justify-between">
|
||||||
<span>Status</span>
|
<span>Status</span>
|
||||||
|
|
|
||||||
55
app/api/mimecast/messages/route.ts
Normal file
55
app/api/mimecast/messages/route.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { postgresClient as pg } from '@/lib/services/postgres-client';
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest) {
|
||||||
|
try {
|
||||||
|
const { searchParams } = new URL(req.url);
|
||||||
|
const days = parseInt(searchParams.get('days') ?? '7');
|
||||||
|
const limit = Math.min(parseInt(searchParams.get('limit') ?? '200'), 1000);
|
||||||
|
const search = searchParams.get('search') ?? '';
|
||||||
|
const direction = searchParams.get('direction') ?? '';
|
||||||
|
const status = searchParams.get('status') ?? '';
|
||||||
|
|
||||||
|
const cutoff = new Date();
|
||||||
|
cutoff.setDate(cutoff.getDate() - days);
|
||||||
|
|
||||||
|
const conditions: string[] = ['sent_datetime >= $1'];
|
||||||
|
const params: any[] = [cutoff.toISOString()];
|
||||||
|
let paramIdx = 2;
|
||||||
|
|
||||||
|
if (direction) {
|
||||||
|
conditions.push(`direction = $${paramIdx++}`);
|
||||||
|
params.push(direction);
|
||||||
|
}
|
||||||
|
if (status) {
|
||||||
|
conditions.push(`status = $${paramIdx++}`);
|
||||||
|
params.push(status);
|
||||||
|
}
|
||||||
|
if (search) {
|
||||||
|
conditions.push(`(
|
||||||
|
sender_address ILIKE $${paramIdx} OR
|
||||||
|
recipient_address ILIKE $${paramIdx} OR
|
||||||
|
subject ILIKE $${paramIdx}
|
||||||
|
)`);
|
||||||
|
params.push(`%${search}%`);
|
||||||
|
paramIdx++;
|
||||||
|
}
|
||||||
|
|
||||||
|
const where = conditions.join(' AND ');
|
||||||
|
params.push(limit);
|
||||||
|
|
||||||
|
const result = await pg.query(`
|
||||||
|
SELECT id, sender_address, sender_domain, recipient_address, subject,
|
||||||
|
direction, status, action, spam_score, size_bytes, attachment_count,
|
||||||
|
sent_datetime, received_datetime, route, reject_reason, held_reason, source_ip
|
||||||
|
FROM mimecast_messages
|
||||||
|
WHERE ${where}
|
||||||
|
ORDER BY sent_datetime DESC
|
||||||
|
LIMIT $${paramIdx}
|
||||||
|
`, params);
|
||||||
|
|
||||||
|
return NextResponse.json({ messages: result.rows });
|
||||||
|
} catch (err: any) {
|
||||||
|
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
29
app/api/mimecast/status/route.ts
Normal file
29
app/api/mimecast/status/route.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
import { NextResponse } from 'next/server';
|
||||||
|
import { getMimecastClient } from '@/lib/services/mimecast-client';
|
||||||
|
import { getMimecastStats } from '@/lib/services/mimecast-sync-service';
|
||||||
|
|
||||||
|
export async function GET() {
|
||||||
|
try {
|
||||||
|
const client = getMimecastClient();
|
||||||
|
const [connection, stats] = await Promise.all([
|
||||||
|
client.testConnection(),
|
||||||
|
getMimecastStats().catch(() => null),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
configured: true,
|
||||||
|
connected: connection.ok,
|
||||||
|
accountName: connection.accountName,
|
||||||
|
packageName: connection.packageName,
|
||||||
|
error: connection.error,
|
||||||
|
stats,
|
||||||
|
});
|
||||||
|
} catch (err: any) {
|
||||||
|
return NextResponse.json({
|
||||||
|
configured: false,
|
||||||
|
connected: false,
|
||||||
|
error: err.message,
|
||||||
|
stats: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
40
app/api/mimecast/threats/route.ts
Normal file
40
app/api/mimecast/threats/route.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { postgresClient as pg } from '@/lib/services/postgres-client';
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest) {
|
||||||
|
try {
|
||||||
|
const { searchParams } = new URL(req.url);
|
||||||
|
const limit = Math.min(parseInt(searchParams.get('limit') ?? '200'), 1000);
|
||||||
|
const level = searchParams.get('level') ?? '';
|
||||||
|
const type = searchParams.get('type') ?? '';
|
||||||
|
|
||||||
|
const conditions: string[] = [];
|
||||||
|
const params: any[] = [];
|
||||||
|
let paramIdx = 1;
|
||||||
|
|
||||||
|
if (level) {
|
||||||
|
conditions.push(`threat_level = $${paramIdx++}`);
|
||||||
|
params.push(level);
|
||||||
|
}
|
||||||
|
if (type) {
|
||||||
|
conditions.push(`event_type = $${paramIdx++}`);
|
||||||
|
params.push(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||||
|
params.push(limit);
|
||||||
|
|
||||||
|
const result = await pg.query(`
|
||||||
|
SELECT id, message_id, event_type, threat_level, url, file_name,
|
||||||
|
verdict, actor_email, event_datetime
|
||||||
|
FROM mimecast_threat_events
|
||||||
|
${where}
|
||||||
|
ORDER BY event_datetime DESC NULLS LAST
|
||||||
|
LIMIT $${paramIdx}
|
||||||
|
`, params);
|
||||||
|
|
||||||
|
return NextResponse.json({ threats: result.rows });
|
||||||
|
} catch (err: any) {
|
||||||
|
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
40
app/api/sync/mimecast/route.ts
Normal file
40
app/api/sync/mimecast/route.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { runMimecastFullSync, runMimecastIncrementalSync, getMimecastStats } from '@/lib/services/mimecast-sync-service';
|
||||||
|
|
||||||
|
let _syncing = false;
|
||||||
|
|
||||||
|
export async function POST(req: NextRequest) {
|
||||||
|
if (_syncing) {
|
||||||
|
return NextResponse.json({ error: 'Sync already in progress' }, { status: 409 });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const body = await req.json().catch(() => ({}));
|
||||||
|
const syncType = body.syncType ?? 'incremental';
|
||||||
|
|
||||||
|
_syncing = true;
|
||||||
|
|
||||||
|
const result = syncType === 'full'
|
||||||
|
? await runMimecastFullSync()
|
||||||
|
: await runMimecastIncrementalSync();
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
syncType,
|
||||||
|
...result,
|
||||||
|
});
|
||||||
|
} catch (err: any) {
|
||||||
|
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||||
|
} finally {
|
||||||
|
_syncing = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET() {
|
||||||
|
try {
|
||||||
|
const stats = await getMimecastStats();
|
||||||
|
return NextResponse.json({ isSyncing: _syncing, ...stats });
|
||||||
|
} catch (err: any) {
|
||||||
|
return NextResponse.json({ error: err.message }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
110827
dev/mimecast-api-v2-collection.json
Normal file
110827
dev/mimecast-api-v2-collection.json
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -103,6 +103,11 @@ services:
|
||||||
# ipinfo.io API token (optional)
|
# ipinfo.io API token (optional)
|
||||||
IPINFO_TOKEN: ${IPINFO_TOKEN:-}
|
IPINFO_TOKEN: ${IPINFO_TOKEN:-}
|
||||||
|
|
||||||
|
# Mimecast API Configuration
|
||||||
|
MIMECAST_CLIENT_ID: ${MIMECAST_CLIENT_ID}
|
||||||
|
MIMECAST_CLIENT_SECRET: ${MIMECAST_CLIENT_SECRET}
|
||||||
|
MIMECAST_BASE_URL: ${MIMECAST_BASE_URL:-https://api.services.mimecast.com}
|
||||||
|
|
||||||
# IT Glue Configuration
|
# IT Glue Configuration
|
||||||
ITGLUE_API_KEY: ${ITGLUE_API_KEY}
|
ITGLUE_API_KEY: ${ITGLUE_API_KEY}
|
||||||
|
|
||||||
|
|
|
||||||
412
lib/services/mimecast-client.ts
Normal file
412
lib/services/mimecast-client.ts
Normal file
|
|
@ -0,0 +1,412 @@
|
||||||
|
/**
|
||||||
|
* Mimecast API 2.0 Client
|
||||||
|
* Auth: OAuth2 Client Credentials — POST /oauth/token
|
||||||
|
* Base: https://api.services.mimecast.com
|
||||||
|
*
|
||||||
|
* Endpoints sourced from official Postman collection (mimecast-api-v2-collection.json):
|
||||||
|
* - Message logs: POST /api/message-finder/search (trackedEmails with pagination)
|
||||||
|
* - SIEM batch: GET /siem/v1/batch/events/cg (returns pre-signed URLs → download JSON)
|
||||||
|
* - Threat events: GET /threats/v1/events
|
||||||
|
* - Message info: POST /api/message-finder/get-message-info
|
||||||
|
* - Account: POST /api/account/get-account
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface MimecastConfig {
|
||||||
|
clientId: string;
|
||||||
|
clientSecret: string;
|
||||||
|
baseUrl?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TokenResponse {
|
||||||
|
access_token: string;
|
||||||
|
token_type: string;
|
||||||
|
expires_in: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TokenCache {
|
||||||
|
token: string;
|
||||||
|
expiresAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MimecastMessage {
|
||||||
|
id: string;
|
||||||
|
senderAddress: string;
|
||||||
|
recipientAddress: string;
|
||||||
|
subject: string;
|
||||||
|
direction: string;
|
||||||
|
status: string;
|
||||||
|
action?: string;
|
||||||
|
spamScore?: number;
|
||||||
|
sizeBytes?: number;
|
||||||
|
attachmentCount?: boolean | number;
|
||||||
|
sentDateTime?: string;
|
||||||
|
receivedDateTime?: string;
|
||||||
|
route?: string;
|
||||||
|
rejectReason?: string;
|
||||||
|
heldReason?: string;
|
||||||
|
sourceIp?: string;
|
||||||
|
[key: string]: any;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MimecastThreatEvent {
|
||||||
|
id: string;
|
||||||
|
messageId?: string;
|
||||||
|
eventType: string;
|
||||||
|
threatLevel?: string;
|
||||||
|
url?: string;
|
||||||
|
fileName?: string;
|
||||||
|
verdict?: string;
|
||||||
|
actorEmail?: string;
|
||||||
|
eventDateTime?: string;
|
||||||
|
analysis?: string[];
|
||||||
|
source?: string[];
|
||||||
|
direction?: string[];
|
||||||
|
status?: string[];
|
||||||
|
[key: string]: any;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MimecastMessageInfo {
|
||||||
|
messageId: string;
|
||||||
|
bodyText?: string;
|
||||||
|
bodyHtml?: string;
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PaginatedResult<T> {
|
||||||
|
items: T[];
|
||||||
|
nextCursor: string | null;
|
||||||
|
totalCount?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class MimecastClient {
|
||||||
|
private readonly clientId: string;
|
||||||
|
private readonly clientSecret: string;
|
||||||
|
private readonly baseUrl: string;
|
||||||
|
private tokenCache: TokenCache | null = null;
|
||||||
|
|
||||||
|
constructor(config: MimecastConfig) {
|
||||||
|
this.clientId = config.clientId;
|
||||||
|
this.clientSecret = config.clientSecret;
|
||||||
|
this.baseUrl = (config.baseUrl ?? 'https://api.services.mimecast.com').replace(/\/$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Auth ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
private async getToken(): Promise<string> {
|
||||||
|
if (this.tokenCache && Date.now() < this.tokenCache.expiresAt - 60_000) {
|
||||||
|
return this.tokenCache.token;
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = new URLSearchParams({
|
||||||
|
grant_type: 'client_credentials',
|
||||||
|
client_id: this.clientId,
|
||||||
|
client_secret: this.clientSecret,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await fetch(`${this.baseUrl}/oauth/token`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||||
|
body: body.toString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`Mimecast auth failed (${res.status}): ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data: TokenResponse = await res.json();
|
||||||
|
this.tokenCache = {
|
||||||
|
token: data.access_token,
|
||||||
|
expiresAt: Date.now() + data.expires_in * 1000,
|
||||||
|
};
|
||||||
|
return data.access_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async request<T>(
|
||||||
|
method: 'GET' | 'POST',
|
||||||
|
path: string,
|
||||||
|
body?: Record<string, any>,
|
||||||
|
params?: Record<string, string>
|
||||||
|
): Promise<T> {
|
||||||
|
const token = await this.getToken();
|
||||||
|
|
||||||
|
let url = `${this.baseUrl}${path}`;
|
||||||
|
if (params && Object.keys(params).length > 0) {
|
||||||
|
url = `${url}?${new URLSearchParams(params).toString()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${token}`,
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
Accept: 'application/json',
|
||||||
|
},
|
||||||
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text();
|
||||||
|
throw new Error(`Mimecast ${method} ${path} failed (${res.status}): ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.json() as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Message Tracking ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Search tracked emails via POST /api/message-finder/search
|
||||||
|
* Response: { data: [{ trackedEmails: [...], pageToken: string }] }
|
||||||
|
*/
|
||||||
|
async getMessageLogs(options: {
|
||||||
|
from: string; // ISO datetime string
|
||||||
|
to: string;
|
||||||
|
cursor?: string;
|
||||||
|
pageSize?: number;
|
||||||
|
}): Promise<PaginatedResult<MimecastMessage>> {
|
||||||
|
const reqData: Record<string, any> = {
|
||||||
|
from: options.from,
|
||||||
|
to: options.to,
|
||||||
|
pageSize: options.pageSize ?? 500,
|
||||||
|
};
|
||||||
|
if (options.cursor) reqData.pageToken = options.cursor;
|
||||||
|
|
||||||
|
const data = await this.request<any>('POST', '/api/message-finder/search', { data: [reqData] });
|
||||||
|
|
||||||
|
const block = data.data?.[0] ?? {};
|
||||||
|
const tracked: any[] = block.trackedEmails ?? [];
|
||||||
|
|
||||||
|
return {
|
||||||
|
items: tracked.map((m: any) => this.normalizeTrackedEmail(m)),
|
||||||
|
nextCursor: block.pageToken ?? null,
|
||||||
|
totalCount: tracked.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeTrackedEmail(m: any): MimecastMessage {
|
||||||
|
const sender = m.fromEnv?.emailAddress ?? m.fromHdr?.emailAddress ?? m.sender ?? '';
|
||||||
|
const recipients: string[] = (m.to ?? []).map((t: any) =>
|
||||||
|
typeof t === 'string' ? t : t.emailAddress ?? ''
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: m.id,
|
||||||
|
senderAddress: sender,
|
||||||
|
recipientAddress: recipients.join(', '),
|
||||||
|
subject: m.subject ?? '',
|
||||||
|
direction: m.route?.toLowerCase().includes('inbound') ? 'inbound' : 'outbound',
|
||||||
|
status: m.status ?? '',
|
||||||
|
action: m.detectionLevel ?? undefined,
|
||||||
|
spamScore: m.spamScore ?? undefined,
|
||||||
|
sizeBytes: undefined,
|
||||||
|
attachmentCount: m.attachments ?? 0,
|
||||||
|
sentDateTime: m.sent ?? undefined,
|
||||||
|
receivedDateTime: m.received ?? undefined,
|
||||||
|
route: m.route ?? undefined,
|
||||||
|
rejectReason: m.info ?? undefined,
|
||||||
|
sourceIp: m.senderIP ?? undefined,
|
||||||
|
_raw: m,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── SIEM Batch Events ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /siem/v1/batch/events/cg — returns pre-signed download URLs for batch NDJSON files
|
||||||
|
* Each URL points to a compressed file; we download and parse each one.
|
||||||
|
* Response: { value: [{url, expiry, size}], "@nextPage": string }
|
||||||
|
*/
|
||||||
|
async getSiemBatchEventUrls(options: {
|
||||||
|
from: string; // date only: YYYY-MM-DD
|
||||||
|
to: string;
|
||||||
|
type?: string;
|
||||||
|
cursor?: string;
|
||||||
|
pageSize?: number;
|
||||||
|
}): Promise<{ urls: Array<{ url: string; expiry: string; size: number }>; nextCursor: string | null }> {
|
||||||
|
const params: Record<string, string> = {
|
||||||
|
type: options.type ?? 'mailflow',
|
||||||
|
dateRangeStartsAt: options.from,
|
||||||
|
dateRangeEndsAt: options.to,
|
||||||
|
pageSize: String(options.pageSize ?? 500),
|
||||||
|
};
|
||||||
|
if (options.cursor) params.pageToken = options.cursor;
|
||||||
|
|
||||||
|
const data = await this.request<any>('GET', '/siem/v1/batch/events/cg', undefined, params);
|
||||||
|
|
||||||
|
return {
|
||||||
|
urls: data.value ?? [],
|
||||||
|
nextCursor: data['@nextPage'] ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download and parse a SIEM batch event file (NDJSON, possibly gzipped)
|
||||||
|
*/
|
||||||
|
async downloadSiemBatchFile(url: string): Promise<MimecastMessage[]> {
|
||||||
|
const res = await fetch(url);
|
||||||
|
if (!res.ok) return [];
|
||||||
|
|
||||||
|
const text = await res.text();
|
||||||
|
const lines = text.split('\n').filter(l => l.trim());
|
||||||
|
const messages: MimecastMessage[] = [];
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
try {
|
||||||
|
const event = JSON.parse(line);
|
||||||
|
messages.push(this.normalizeSiemEvent(event));
|
||||||
|
} catch {
|
||||||
|
// skip malformed lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return messages;
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeSiemEvent(e: any): MimecastMessage {
|
||||||
|
return {
|
||||||
|
id: e.messageId ?? e.id ?? e.Messageid ?? '',
|
||||||
|
senderAddress: e.senderAddress ?? e.Sender ?? e.sender ?? '',
|
||||||
|
recipientAddress: e.recipientAddress ?? e.Recipient ?? e.recipient ?? '',
|
||||||
|
subject: e.subject ?? e.Subject ?? '',
|
||||||
|
direction: (e.Dir ?? e.direction ?? '').toLowerCase() === 'inbound' ? 'inbound' : 'outbound',
|
||||||
|
status: e.Act ?? e.action ?? e.status ?? '',
|
||||||
|
action: e.RejType ?? e.rejectionType ?? undefined,
|
||||||
|
spamScore: e.SpamScore ?? e.spamScore ?? undefined,
|
||||||
|
sizeBytes: e.MsgSize ?? e.messageSize ?? e.size ?? undefined,
|
||||||
|
attachmentCount: e.AttCnt ?? e.attachmentCount ?? 0,
|
||||||
|
sentDateTime: e.Datetime ?? e.datetime ?? e.timestamp ?? undefined,
|
||||||
|
receivedDateTime: e.Datetime ?? undefined,
|
||||||
|
route: e.Route ?? e.route ?? undefined,
|
||||||
|
rejectReason: e.RejCode ?? e.rejectionCode ?? undefined,
|
||||||
|
heldReason: e.HeldGroup ?? undefined,
|
||||||
|
sourceIp: e.IP ?? e.senderIp ?? undefined,
|
||||||
|
_raw: e,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Threat Events ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /threats/v1/events
|
||||||
|
* Response: { value: [{timestamp, analysis, source, details, status, direction, subject, sender, ...}], "@nextPage": string }
|
||||||
|
*/
|
||||||
|
async getThreatEvents(options: {
|
||||||
|
cursor?: string;
|
||||||
|
pageSize?: number;
|
||||||
|
} = {}): Promise<PaginatedResult<MimecastThreatEvent>> {
|
||||||
|
const params: Record<string, string> = {
|
||||||
|
pageSize: String(options.pageSize ?? 500),
|
||||||
|
};
|
||||||
|
if (options.cursor) params.pageToken = options.cursor;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = await this.request<any>('GET', '/threats/v1/events', undefined, params);
|
||||||
|
const items = (data.value ?? []).map((e: any) => this.normalizeThreatEvent(e));
|
||||||
|
return {
|
||||||
|
items,
|
||||||
|
nextCursor: data['@nextPage'] ?? null,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return { items: [], nextCursor: null };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeThreatEvent(e: any): MimecastThreatEvent {
|
||||||
|
const analysis: string[] = Array.isArray(e.analysis) ? e.analysis : [e.analysis].filter(Boolean);
|
||||||
|
const eventType = analysis[0] ?? 'unknown';
|
||||||
|
|
||||||
|
const statusArr: string[] = Array.isArray(e.status) ? e.status : [e.status].filter(Boolean);
|
||||||
|
const verdict = statusArr[0] ?? undefined;
|
||||||
|
|
||||||
|
const threatLevel = analysis.includes('malware') ? 'high'
|
||||||
|
: analysis.includes('phishing') ? 'high'
|
||||||
|
: analysis.includes('spam') ? 'medium'
|
||||||
|
: 'info';
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: e.id ?? `threat_${e.sender ?? ''}_${e.timestamp ?? Date.now()}`,
|
||||||
|
messageId: e.messageId ?? undefined,
|
||||||
|
eventType,
|
||||||
|
threatLevel,
|
||||||
|
url: e.url ?? undefined,
|
||||||
|
fileName: undefined,
|
||||||
|
verdict,
|
||||||
|
actorEmail: e.sender ?? undefined,
|
||||||
|
eventDateTime: e.timestamp ?? undefined,
|
||||||
|
analysis,
|
||||||
|
source: Array.isArray(e.source) ? e.source : [e.source].filter(Boolean),
|
||||||
|
direction: Array.isArray(e.direction) ? e.direction : [e.direction].filter(Boolean),
|
||||||
|
status: statusArr,
|
||||||
|
_raw: e,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Message Info (body/headers) ────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/message-finder/get-message-info
|
||||||
|
* Returns delivered message details including parts
|
||||||
|
*/
|
||||||
|
async getMessageInfo(messageId: string): Promise<MimecastMessageInfo | null> {
|
||||||
|
try {
|
||||||
|
const data = await this.request<any>('POST', '/api/message-finder/get-message-info', {
|
||||||
|
data: [{ id: messageId }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const delivered = data.data?.[0]?.deliveredMessage;
|
||||||
|
if (!delivered) return null;
|
||||||
|
|
||||||
|
const entries = Object.values(delivered) as any[];
|
||||||
|
if (!entries.length) return null;
|
||||||
|
|
||||||
|
const info = entries[0]?.messageInfo ?? entries[0];
|
||||||
|
|
||||||
|
return {
|
||||||
|
messageId,
|
||||||
|
bodyText: info?.textBody ?? info?.body ?? undefined,
|
||||||
|
bodyHtml: info?.htmlBody ?? undefined,
|
||||||
|
headers: info?.headers ?? undefined,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Account ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/account/get-account
|
||||||
|
*/
|
||||||
|
async testConnection(): Promise<{ ok: boolean; accountName?: string; packageName?: string; error?: string }> {
|
||||||
|
try {
|
||||||
|
const data = await this.request<any>('POST', '/api/account/get-account', { data: [{}] });
|
||||||
|
const account = data.data?.[0] ?? {};
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
accountName: account.accountName ?? account.accountCode ?? 'Connected',
|
||||||
|
packageName: account.packageName ?? undefined,
|
||||||
|
};
|
||||||
|
} catch (err: any) {
|
||||||
|
return { ok: false, error: err.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let _client: MimecastClient | null = null;
|
||||||
|
|
||||||
|
export function getMimecastClient(): MimecastClient {
|
||||||
|
if (!_client) {
|
||||||
|
const clientId = process.env.MIMECAST_CLIENT_ID;
|
||||||
|
const clientSecret = process.env.MIMECAST_CLIENT_SECRET;
|
||||||
|
if (!clientId || !clientSecret) {
|
||||||
|
throw new Error('MIMECAST_CLIENT_ID and MIMECAST_CLIENT_SECRET must be set');
|
||||||
|
}
|
||||||
|
_client = new MimecastClient({
|
||||||
|
clientId,
|
||||||
|
clientSecret,
|
||||||
|
baseUrl: process.env.MIMECAST_BASE_URL ?? 'https://api.services.mimecast.com',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return _client;
|
||||||
|
}
|
||||||
331
lib/services/mimecast-sync-service.ts
Normal file
331
lib/services/mimecast-sync-service.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
||||||
|
/**
|
||||||
|
* Mimecast Sync Service
|
||||||
|
* Full sync (120 days back), incremental (since last sync), body fetch, 120-day purge
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { getMimecastClient, MimecastMessage, MimecastThreatEvent } from './mimecast-client';
|
||||||
|
import { postgresClient as pg } from './postgres-client';
|
||||||
|
|
||||||
|
export interface MimecastSyncResult {
|
||||||
|
messagesUpserted: number;
|
||||||
|
threatsUpserted: number;
|
||||||
|
bodiesFetched: number;
|
||||||
|
purgedMessages: number;
|
||||||
|
errors: string[];
|
||||||
|
durationMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RETENTION_DAYS = 120;
|
||||||
|
const BODY_FETCH_LIMIT = 500;
|
||||||
|
|
||||||
|
// ── Upsert helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function upsertMessages(messages: MimecastMessage[]): Promise<number> {
|
||||||
|
if (!messages.length) return 0;
|
||||||
|
let count = 0;
|
||||||
|
|
||||||
|
for (const m of messages) {
|
||||||
|
if (!m.id) continue;
|
||||||
|
const senderDomain = m.senderAddress?.includes('@')
|
||||||
|
? m.senderAddress.split('@')[1]?.toLowerCase()
|
||||||
|
: null;
|
||||||
|
|
||||||
|
await pg.query(`
|
||||||
|
INSERT INTO mimecast_messages (
|
||||||
|
id, sender_address, sender_domain, recipient_address, subject,
|
||||||
|
direction, status, action, spam_score, size_bytes, attachment_count,
|
||||||
|
sent_datetime, received_datetime, delivery_datetime,
|
||||||
|
route, reject_reason, held_reason, source_ip, raw, synced_at
|
||||||
|
) VALUES (
|
||||||
|
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,NOW()
|
||||||
|
)
|
||||||
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
|
status = EXCLUDED.status,
|
||||||
|
action = EXCLUDED.action,
|
||||||
|
delivery_datetime= EXCLUDED.delivery_datetime,
|
||||||
|
reject_reason = EXCLUDED.reject_reason,
|
||||||
|
held_reason = EXCLUDED.held_reason,
|
||||||
|
raw = EXCLUDED.raw,
|
||||||
|
synced_at = NOW()
|
||||||
|
`, [
|
||||||
|
m.id,
|
||||||
|
m.senderAddress || null,
|
||||||
|
senderDomain,
|
||||||
|
m.recipientAddress || null,
|
||||||
|
m.subject || null,
|
||||||
|
m.direction || null,
|
||||||
|
m.status || null,
|
||||||
|
m.action || null,
|
||||||
|
m.spamScore ?? null,
|
||||||
|
m.sizeBytes ?? null,
|
||||||
|
typeof m.attachmentCount === 'number' ? m.attachmentCount : null,
|
||||||
|
m.sentDateTime || null,
|
||||||
|
m.receivedDateTime || null,
|
||||||
|
null, // delivery_datetime not in message-finder search
|
||||||
|
m.route || null,
|
||||||
|
m.rejectReason || null,
|
||||||
|
m.heldReason || null,
|
||||||
|
m.sourceIp || null,
|
||||||
|
m._raw ? JSON.stringify(m._raw) : null,
|
||||||
|
]);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function upsertThreats(events: MimecastThreatEvent[]): Promise<number> {
|
||||||
|
if (!events.length) return 0;
|
||||||
|
let count = 0;
|
||||||
|
|
||||||
|
for (const e of events) {
|
||||||
|
if (!e.id) continue;
|
||||||
|
await pg.query(`
|
||||||
|
INSERT INTO mimecast_threat_events (
|
||||||
|
id, message_id, event_type, threat_level, url, file_name,
|
||||||
|
verdict, actor_email, event_datetime, details, synced_at
|
||||||
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,NOW())
|
||||||
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
|
threat_level = EXCLUDED.threat_level,
|
||||||
|
verdict = EXCLUDED.verdict,
|
||||||
|
details = EXCLUDED.details,
|
||||||
|
synced_at = NOW()
|
||||||
|
`, [
|
||||||
|
e.id,
|
||||||
|
e.messageId || null,
|
||||||
|
e.eventType || null,
|
||||||
|
e.threatLevel || null,
|
||||||
|
e.url || null,
|
||||||
|
e.fileName || null,
|
||||||
|
e.verdict || null,
|
||||||
|
e.actorEmail || null,
|
||||||
|
e.eventDateTime || null,
|
||||||
|
e._raw ? JSON.stringify(e._raw) : null,
|
||||||
|
]);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchAndStoreBodies(errors: string[]): Promise<number> {
|
||||||
|
const client = getMimecastClient();
|
||||||
|
|
||||||
|
// Find delivered inbound messages missing body, up to limit
|
||||||
|
const rows = await pg.query(`
|
||||||
|
SELECT m.id FROM mimecast_messages m
|
||||||
|
LEFT JOIN mimecast_message_bodies b ON b.message_id = m.id
|
||||||
|
WHERE b.message_id IS NULL
|
||||||
|
AND m.status NOT IN ('rejected','spam','bounced')
|
||||||
|
AND m.direction = 'inbound'
|
||||||
|
ORDER BY m.sent_datetime DESC
|
||||||
|
LIMIT $1
|
||||||
|
`, [BODY_FETCH_LIMIT]);
|
||||||
|
|
||||||
|
let fetched = 0;
|
||||||
|
for (const row of rows.rows) {
|
||||||
|
try {
|
||||||
|
const info = await client.getMessageInfo(row.id);
|
||||||
|
if (!info) continue;
|
||||||
|
|
||||||
|
await pg.query(`
|
||||||
|
INSERT INTO mimecast_message_bodies (message_id, body_text, body_html, headers, fetched_at)
|
||||||
|
VALUES ($1, $2, $3, $4, NOW())
|
||||||
|
ON CONFLICT (message_id) DO UPDATE SET
|
||||||
|
body_text = EXCLUDED.body_text,
|
||||||
|
body_html = EXCLUDED.body_html,
|
||||||
|
headers = EXCLUDED.headers,
|
||||||
|
fetched_at = NOW()
|
||||||
|
`, [
|
||||||
|
info.messageId,
|
||||||
|
info.bodyText || null,
|
||||||
|
info.bodyHtml || null,
|
||||||
|
info.headers ? JSON.stringify(info.headers) : null,
|
||||||
|
]);
|
||||||
|
fetched++;
|
||||||
|
} catch (err: any) {
|
||||||
|
errors.push(`Body fetch ${row.id}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fetched;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function purgeOldRecords(): Promise<number> {
|
||||||
|
const cutoff = new Date();
|
||||||
|
cutoff.setDate(cutoff.getDate() - RETENTION_DAYS);
|
||||||
|
|
||||||
|
const result = await pg.query(
|
||||||
|
`DELETE FROM mimecast_messages WHERE sent_datetime < $1`,
|
||||||
|
[cutoff.toISOString()]
|
||||||
|
);
|
||||||
|
return result.rowCount ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getSyncState(syncType: string): Promise<{ lastSyncedAt: Date | null; cursor: string | null }> {
|
||||||
|
const result = await pg.query(
|
||||||
|
`SELECT last_synced_at, cursor FROM mimecast_sync_state WHERE sync_type = $1`,
|
||||||
|
[syncType]
|
||||||
|
);
|
||||||
|
const row = result.rows[0];
|
||||||
|
return {
|
||||||
|
lastSyncedAt: row?.last_synced_at ? new Date(row.last_synced_at) : null,
|
||||||
|
cursor: row?.cursor ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveSyncState(syncType: string, lastSyncedAt: Date, cursor: string | null): Promise<void> {
|
||||||
|
await pg.query(`
|
||||||
|
INSERT INTO mimecast_sync_state (sync_type, last_synced_at, cursor, updated_at)
|
||||||
|
VALUES ($1, $2, $3, NOW())
|
||||||
|
ON CONFLICT (sync_type) DO UPDATE SET
|
||||||
|
last_synced_at = EXCLUDED.last_synced_at,
|
||||||
|
cursor = EXCLUDED.cursor,
|
||||||
|
updated_at = NOW()
|
||||||
|
`, [syncType, lastSyncedAt.toISOString(), cursor]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Main sync functions ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function syncMimecastMessages(
|
||||||
|
fromDate: Date,
|
||||||
|
toDate: Date,
|
||||||
|
errors: string[]
|
||||||
|
): Promise<number> {
|
||||||
|
const client = getMimecastClient();
|
||||||
|
let total = 0;
|
||||||
|
let cursor: string | null = null;
|
||||||
|
|
||||||
|
const from = fromDate.toISOString();
|
||||||
|
const to = toDate.toISOString();
|
||||||
|
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
const result = await client.getMessageLogs({ from, to, cursor: cursor ?? undefined, pageSize: 500 });
|
||||||
|
if (result.items.length > 0) {
|
||||||
|
total += await upsertMessages(result.items);
|
||||||
|
}
|
||||||
|
cursor = result.nextCursor;
|
||||||
|
} catch (err: any) {
|
||||||
|
errors.push(`Message sync page: ${err.message}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} while (cursor);
|
||||||
|
|
||||||
|
await saveSyncState('messages', toDate, null);
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function syncMimecastThreats(errors: string[]): Promise<number> {
|
||||||
|
const client = getMimecastClient();
|
||||||
|
let total = 0;
|
||||||
|
let cursor: string | null = null;
|
||||||
|
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
const result = await client.getThreatEvents({ cursor: cursor ?? undefined, pageSize: 500 });
|
||||||
|
if (result.items.length > 0) {
|
||||||
|
total += await upsertThreats(result.items);
|
||||||
|
}
|
||||||
|
cursor = result.nextCursor;
|
||||||
|
} catch (err: any) {
|
||||||
|
errors.push(`Threat sync: ${err.message}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} while (cursor);
|
||||||
|
|
||||||
|
await saveSyncState('threats', new Date(), null);
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Public API ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function runMimecastFullSync(): Promise<MimecastSyncResult> {
|
||||||
|
const start = Date.now();
|
||||||
|
const errors: string[] = [];
|
||||||
|
|
||||||
|
const toDate = new Date();
|
||||||
|
const fromDate = new Date();
|
||||||
|
fromDate.setDate(fromDate.getDate() - RETENTION_DAYS);
|
||||||
|
|
||||||
|
const messagesUpserted = await syncMimecastMessages(fromDate, toDate, errors);
|
||||||
|
const threatsUpserted = await syncMimecastThreats(errors);
|
||||||
|
const bodiesFetched = await fetchAndStoreBodies(errors);
|
||||||
|
const purgedMessages = await purgeOldRecords();
|
||||||
|
|
||||||
|
return {
|
||||||
|
messagesUpserted,
|
||||||
|
threatsUpserted,
|
||||||
|
bodiesFetched,
|
||||||
|
purgedMessages,
|
||||||
|
errors,
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runMimecastIncrementalSync(): Promise<MimecastSyncResult> {
|
||||||
|
const start = Date.now();
|
||||||
|
const errors: string[] = [];
|
||||||
|
|
||||||
|
const { lastSyncedAt } = await getSyncState('messages');
|
||||||
|
|
||||||
|
const toDate = new Date();
|
||||||
|
const fromDate = lastSyncedAt ?? (() => {
|
||||||
|
const d = new Date();
|
||||||
|
d.setDate(d.getDate() - 1);
|
||||||
|
return d;
|
||||||
|
})();
|
||||||
|
|
||||||
|
const messagesUpserted = await syncMimecastMessages(fromDate, toDate, errors);
|
||||||
|
const threatsUpserted = await syncMimecastThreats(errors);
|
||||||
|
const bodiesFetched = await fetchAndStoreBodies(errors);
|
||||||
|
const purgedMessages = await purgeOldRecords();
|
||||||
|
|
||||||
|
return {
|
||||||
|
messagesUpserted,
|
||||||
|
threatsUpserted,
|
||||||
|
bodiesFetched,
|
||||||
|
purgedMessages,
|
||||||
|
errors,
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getMimecastStats(): Promise<{
|
||||||
|
messages: number;
|
||||||
|
inbound: number;
|
||||||
|
outbound: number;
|
||||||
|
threats: number;
|
||||||
|
bodies: number;
|
||||||
|
lastSync: string | null;
|
||||||
|
oldestMessage: string | null;
|
||||||
|
}> {
|
||||||
|
const [counts, syncState, oldest] = await Promise.all([
|
||||||
|
pg.query(`
|
||||||
|
SELECT
|
||||||
|
COUNT(*) AS total,
|
||||||
|
COUNT(*) FILTER (WHERE direction = 'inbound') AS inbound,
|
||||||
|
COUNT(*) FILTER (WHERE direction = 'outbound') AS outbound
|
||||||
|
FROM mimecast_messages
|
||||||
|
`),
|
||||||
|
pg.query(`SELECT last_synced_at FROM mimecast_sync_state WHERE sync_type = 'messages'`),
|
||||||
|
pg.query(`SELECT MIN(sent_datetime) AS oldest FROM mimecast_messages`),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const [threatCount, bodyCount] = await Promise.all([
|
||||||
|
pg.query(`SELECT COUNT(*) AS total FROM mimecast_threat_events`),
|
||||||
|
pg.query(`SELECT COUNT(*) AS total FROM mimecast_message_bodies`),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const c = counts.rows[0] ?? {};
|
||||||
|
|
||||||
|
return {
|
||||||
|
messages: Number(c.total ?? 0),
|
||||||
|
inbound: Number(c.inbound ?? 0),
|
||||||
|
outbound: Number(c.outbound ?? 0),
|
||||||
|
threats: Number(threatCount.rows[0]?.total ?? 0),
|
||||||
|
bodies: Number(bodyCount.rows[0]?.total ?? 0),
|
||||||
|
lastSync: syncState.rows[0]?.last_synced_at ?? null,
|
||||||
|
oldestMessage: oldest.rows[0]?.oldest ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
72
migrations/055_create_mimecast_tables.sql
Normal file
72
migrations/055_create_mimecast_tables.sql
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
-- ============================================================================
|
||||||
|
-- Mimecast Email Integration Tables
|
||||||
|
-- 120-day rolling retention for message logs, threat events, and message bodies
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Core message tracking log
|
||||||
|
CREATE TABLE IF NOT EXISTS mimecast_messages (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
sender_address TEXT,
|
||||||
|
sender_domain TEXT,
|
||||||
|
recipient_address TEXT,
|
||||||
|
subject TEXT,
|
||||||
|
direction TEXT, -- 'inbound' | 'outbound'
|
||||||
|
status TEXT, -- 'delivered' | 'rejected' | 'bounced' | 'held' | 'spam'
|
||||||
|
action TEXT,
|
||||||
|
spam_score NUMERIC,
|
||||||
|
size_bytes BIGINT,
|
||||||
|
attachment_count INT DEFAULT 0,
|
||||||
|
sent_datetime TIMESTAMPTZ,
|
||||||
|
received_datetime TIMESTAMPTZ,
|
||||||
|
delivery_datetime TIMESTAMPTZ,
|
||||||
|
route TEXT,
|
||||||
|
reject_reason TEXT,
|
||||||
|
held_reason TEXT,
|
||||||
|
source_ip TEXT,
|
||||||
|
raw JSONB, -- Full API response for forward-compat
|
||||||
|
synced_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Separate table for message bodies (large, for LLM use only)
|
||||||
|
CREATE TABLE IF NOT EXISTS mimecast_message_bodies (
|
||||||
|
message_id TEXT PRIMARY KEY REFERENCES mimecast_messages(id) ON DELETE CASCADE,
|
||||||
|
body_text TEXT,
|
||||||
|
body_html TEXT,
|
||||||
|
headers JSONB,
|
||||||
|
fetched_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- SIEM/threat events
|
||||||
|
CREATE TABLE IF NOT EXISTS mimecast_threat_events (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
message_id TEXT REFERENCES mimecast_messages(id) ON DELETE SET NULL,
|
||||||
|
event_type TEXT, -- 'url_click' | 'attachment_sandbox' | 'impersonation' | 'spam' | 'virus'
|
||||||
|
threat_level TEXT, -- 'high' | 'medium' | 'low' | 'info'
|
||||||
|
url TEXT,
|
||||||
|
file_name TEXT,
|
||||||
|
verdict TEXT,
|
||||||
|
actor_email TEXT,
|
||||||
|
event_datetime TIMESTAMPTZ,
|
||||||
|
details JSONB, -- Full raw event payload
|
||||||
|
synced_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Cursor-based pagination state for resumable syncs
|
||||||
|
CREATE TABLE IF NOT EXISTS mimecast_sync_state (
|
||||||
|
sync_type TEXT PRIMARY KEY, -- 'messages' | 'threats' | 'held'
|
||||||
|
last_synced_at TIMESTAMPTZ,
|
||||||
|
cursor TEXT,
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_sent_datetime ON mimecast_messages (sent_datetime DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_sender_domain ON mimecast_messages (sender_domain);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_status ON mimecast_messages (status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_direction ON mimecast_messages (direction);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_messages_recipient ON mimecast_messages (recipient_address);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_datetime ON mimecast_threat_events (event_datetime DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_type ON mimecast_threat_events (event_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_level ON mimecast_threat_events (threat_level);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mimecast_threat_events_details ON mimecast_threat_events USING GIN (details);
|
||||||
Loading…
Add table
Add a link
Reference in a new issue