lorentz/wulf-pulse
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fix: Mimecast 7-day chunk windows, 30-day retention limit, two-pass from+to domain filter

Browse source
This commit is contained in:
lorentz 2026-03-17 16:53:46 -04:00
parent 22bea85648
commit b6c2681a69
1 changed files with 25 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

40
lib/services/mimecast-sync-service.ts
View file

@ -15,8 +15,9 @@ export interface MimecastSyncResult {
durationMs: number;
}
const RETENTION_DAYS = 120;
const RETENTION_DAYS = 30; // API max lookback is ~30 days
const BODY_FETCH_LIMIT = 500;
const CHUNK_DAYS = 7; // Pages the 30-day window in 7-day chunks to avoid 5000-result cap
// ── Upsert helpers ────────────────────────────────────────────────────────────
@ -222,23 +223,32 @@ export async function syncMimecastMessages(
errors: string[]
): Promise<number> {
const client = getMimecastClient();
let total = 0;
// Two passes: outbound (from Wulf) + inbound (to Wulf), deduplicated by ID
const [outbound, inbound] = await Promise.all([
fetchAllPages(client, fromDate, toDate, { fromFilter: 'wulfconsulting.com' }, errors),
fetchAllPages(client, fromDate, toDate, { toFilter: 'wulfconsulting.com' }, errors),
]);
const seen = new Set<string>();
const all: MimecastMessage[] = [];
for (const m of [...outbound, ...inbound]) {
if (m.id && !seen.has(m.id)) {
seen.add(m.id);
all.push(m);
}
// Break range into CHUNK_DAYS windows to avoid the 5000-result cap per request
const chunks: Array<{ chunkFrom: Date; chunkTo: Date }> = [];
let cursor = new Date(fromDate);
while (cursor < toDate) {
const chunkEnd = new Date(cursor);
chunkEnd.setDate(chunkEnd.getDate() + CHUNK_DAYS);
chunks.push({ chunkFrom: new Date(cursor), chunkTo: chunkEnd > toDate ? toDate : chunkEnd });
cursor = new Date(chunkEnd);
}
for (const { chunkFrom, chunkTo } of chunks) {
const [outbound, inbound] = await Promise.all([
fetchAllPages(client, chunkFrom, chunkTo, { fromFilter: 'wulfconsulting.com' }, errors),
fetchAllPages(client, chunkFrom, chunkTo, { toFilter: 'wulfconsulting.com' }, errors),
]);
const seen = new Set<string>();
const all: MimecastMessage[] = [];
for (const m of [...outbound, ...inbound]) {
if (m.id && !seen.has(m.id)) { seen.add(m.id); all.push(m); }
}
total += await upsertMessages(all);
}
const total = await upsertMessages(all);
await saveSyncState('messages', toDate, null);
return total;
}