From b6c2681a691ab76ee51e2519275095ad66dc22f5 Mon Sep 17 00:00:00 2001
From: lorentz <lorentz@wulfconsulting.com>
Date: Tue, 17 Mar 2026 16:53:46 -0400
Subject: [PATCH] fix: Mimecast 7-day chunk windows, 30-day retention limit,
 two-pass from+to domain filter

---
 lib/services/mimecast-sync-service.ts | 40 +++++++++++++++++----------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/lib/services/mimecast-sync-service.ts b/lib/services/mimecast-sync-service.ts
index 6b94333..02dd2fe 100644
--- a/lib/services/mimecast-sync-service.ts
+++ b/lib/services/mimecast-sync-service.ts
@@ -15,8 +15,9 @@ export interface MimecastSyncResult {
   durationMs: number;
 }
 
-const RETENTION_DAYS = 120;
+const RETENTION_DAYS = 30;   // API max lookback is ~30 days
 const BODY_FETCH_LIMIT = 500;
+const CHUNK_DAYS = 7;         // Pages the 30-day window in 7-day chunks to avoid 5000-result cap
 
 // ── Upsert helpers ────────────────────────────────────────────────────────────
 
@@ -222,23 +223,32 @@ export async function syncMimecastMessages(
   errors: string[]
 ): Promise<number> {
   const client = getMimecastClient();
+  let total = 0;
 
-  // Two passes: outbound (from Wulf) + inbound (to Wulf), deduplicated by ID
-  const [outbound, inbound] = await Promise.all([
-    fetchAllPages(client, fromDate, toDate, { fromFilter: 'wulfconsulting.com' }, errors),
-    fetchAllPages(client, fromDate, toDate, { toFilter: 'wulfconsulting.com' }, errors),
-  ]);
-
-  const seen = new Set<string>();
-  const all: MimecastMessage[] = [];
-  for (const m of [...outbound, ...inbound]) {
-    if (m.id && !seen.has(m.id)) {
-      seen.add(m.id);
-      all.push(m);
-    }
+  // Break range into CHUNK_DAYS windows to avoid the 5000-result cap per request
+  const chunks: Array<{ chunkFrom: Date; chunkTo: Date }> = [];
+  let cursor = new Date(fromDate);
+  while (cursor < toDate) {
+    const chunkEnd = new Date(cursor);
+    chunkEnd.setDate(chunkEnd.getDate() + CHUNK_DAYS);
+    chunks.push({ chunkFrom: new Date(cursor), chunkTo: chunkEnd > toDate ? toDate : chunkEnd });
+    cursor = new Date(chunkEnd);
+  }
+
+  for (const { chunkFrom, chunkTo } of chunks) {
+    const [outbound, inbound] = await Promise.all([
+      fetchAllPages(client, chunkFrom, chunkTo, { fromFilter: 'wulfconsulting.com' }, errors),
+      fetchAllPages(client, chunkFrom, chunkTo, { toFilter: 'wulfconsulting.com' }, errors),
+    ]);
+
+    const seen = new Set<string>();
+    const all: MimecastMessage[] = [];
+    for (const m of [...outbound, ...inbound]) {
+      if (m.id && !seen.has(m.id)) { seen.add(m.id); all.push(m); }
+    }
+    total += await upsertMessages(all);
   }
 
-  const total = await upsertMessages(all);
   await saveSyncState('messages', toDate, null);
   return total;
 }