From 758b7e7f15a6b5d1fa801b1a23ab928884841537 Mon Sep 17 00:00:00 2001 From: lorentz Date: Tue, 2 Jun 2026 20:25:14 -0400 Subject: [PATCH] feat(engagement): replace Graph email counts with real-time mimecast data - Broaden mimecast retention from 30 days to 18 months rolling - Re-enable mimecast-sync schedule (was disabled since March 17) - Full sync triggered: 35,559 messages loaded for last 30 days - Users list API: LATERAL join on mimecast_messages for emails_sent/received - User detail API: add emails{d7,d30,d90} field from mimecast - Engagement page: prefer mimecast email counts in detail panel sub-label Graph API has 48-72hr reporting lag; mimecast is same-day --- app/api/engagement/user/[userId]/route.ts | 21 +++++++++++++++++++++ app/api/engagement/users/route.ts | 12 ++++++++++-- app/engagement/page.tsx | 3 ++- lib/services/mimecast-sync-service.ts | 9 +++++---- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/app/api/engagement/user/[userId]/route.ts b/app/api/engagement/user/[userId]/route.ts index e424169..2b86848 100644 --- a/app/api/engagement/user/[userId]/route.ts +++ b/app/api/engagement/user/[userId]/route.ts @@ -207,6 +207,22 @@ export async function GET( [user.email, periodDays] ).catch(() => null); + // Mimecast real-time email counts + const mimecastEmailsResult = await postgresClient.query( + `SELECT + COUNT(*) FILTER (WHERE LOWER(sender_address) = LOWER($1) AND direction IN ('outbound','internal') AND sent_datetime >= NOW() - INTERVAL '7 days') AS sent_d7, + COUNT(*) FILTER (WHERE LOWER(sender_address) = LOWER($1) AND direction IN ('outbound','internal') AND sent_datetime >= NOW() - INTERVAL '30 days') AS sent_d30, + COUNT(*) FILTER (WHERE LOWER(sender_address) = LOWER($1) AND direction IN ('outbound','internal') AND sent_datetime >= NOW() - INTERVAL '90 days') AS sent_d90, + COUNT(*) FILTER (WHERE LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted') AND sent_datetime >= NOW() - INTERVAL '7 days') AS received_d7, + COUNT(*) FILTER (WHERE LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted') AND sent_datetime >= NOW() - INTERVAL '30 days') AS received_d30, + COUNT(*) FILTER (WHERE LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted') AND sent_datetime >= NOW() - INTERVAL '90 days') AS received_d90 + FROM mimecast_messages + WHERE LOWER(sender_address) = LOWER($1) + OR (LOWER(recipient_address) = LOWER($1) AND direction IN ('inbound','internal') AND status IN ('archived','accepted'))`, + [user.email] + ).catch(() => null); + const me = mimecastEmailsResult?.rows[0]; + // After-hours meetings (5:30 PM – 7:00 AM America/New_York) const afterHoursMeetingsResult = await postgresClient.query( `SELECT COUNT(*) as count @@ -496,6 +512,11 @@ export async function GET( messagesPct: totalMessages > 0 ? Math.round((afterHoursMessages / totalMessages) * 100) : 0, meetingsPct: totalMeetings > 0 ? Math.round((afterHoursMeetings / totalMeetings) * 100) : 0, }, + emails: me ? { + d7: { sent: parseInt(me.sent_d7 ?? 0), received: parseInt(me.received_d7 ?? 0) }, + d30: { sent: parseInt(me.sent_d30 ?? 0), received: parseInt(me.received_d30 ?? 0) }, + d90: { sent: parseInt(me.sent_d90 ?? 0), received: parseInt(me.received_d90 ?? 0) }, + } : null, snapshots: snapshotsResult.rows, hours: hours ? { diff --git a/app/api/engagement/users/route.ts b/app/api/engagement/users/route.ts index f86ffb2..1f6df95 100644 --- a/app/api/engagement/users/route.ts +++ b/app/api/engagement/users/route.ts @@ -82,8 +82,8 @@ export async function GET(request: NextRequest) { COALESCE(es.teams_calls, 0) as teams_calls, COALESCE(es.teams_meetings_attended, 0) as teams_meetings_attended, COALESCE(es.teams_meetings_organized, 0) as teams_meetings_organized, - COALESCE(es.emails_sent, 0) as emails_sent, - COALESCE(es.emails_received, 0) as emails_received, + COALESCE(mc.emails_sent, 0) as emails_sent, + COALESCE(mc.emails_received, 0) as emails_received, COALESCE(es.emails_read, 0) as emails_read, COALESCE(es.audio_duration_seconds, 0) as audio_duration_seconds, COALESCE(es.meeting_duration_seconds, 0) as meeting_duration_seconds, @@ -139,6 +139,14 @@ export async function GET(request: NextRequest) { WHERE start_time >= NOW() - INTERVAL '${interval}' GROUP BY host_email ) zm ON LOWER(r.email) = LOWER(zm.host_email) + LEFT JOIN LATERAL ( + SELECT + COUNT(*) FILTER (WHERE LOWER(mm.sender_address) = LOWER(gu.email) AND mm.direction IN ('outbound', 'internal')) AS emails_sent, + COUNT(*) FILTER (WHERE LOWER(mm.recipient_address) = LOWER(gu.email) AND mm.direction IN ('inbound', 'internal') AND mm.status IN ('archived', 'accepted')) AS emails_received + FROM mimecast_messages mm + WHERE (LOWER(mm.sender_address) = LOWER(gu.email) OR LOWER(mm.recipient_address) = LOWER(gu.email)) + AND mm.sent_datetime >= NOW() - INTERVAL '${interval}' + ) mc ON true WHERE gu.account_enabled = true AND LOWER(gu.email) LIKE '%@wulfconsulting.%' AND LOWER(gu.email) NOT LIKE '%#ext#%' diff --git a/app/engagement/page.tsx b/app/engagement/page.tsx index 86af00a..091565a 100644 --- a/app/engagement/page.tsx +++ b/app/engagement/page.tsx @@ -146,6 +146,7 @@ interface UserDetail { end_date_time: string | null; }>; }>; + emails: Record<'d7' | 'd30' | 'd90', { sent: number; received: number }> | null; meetingCounts: { total: number; withClients: number }; zoom: { calls: Record<'d7' | 'd30' | 'd90', { total: number; client: number; outbound: number; inbound: number; durationSeconds: number }>; @@ -989,7 +990,7 @@ export default function EngagementPage() { }, ...(snap ? [{ label: 'Messages', - sub: `${snap.emails_sent} emails`, + sub: userDetail.emails?.[pKey] ? `${userDetail.emails[pKey].sent} emails sent` : `${snap.emails_sent} emails`, value: snap.teams_chat_messages + snap.teams_private_messages, peerMax: userDetail.peerMax?.messages || (period === 'D7' ? 200 : period === 'D30' ? 800 : 2400), prevValue: null, diff --git a/lib/services/mimecast-sync-service.ts b/lib/services/mimecast-sync-service.ts index 02dd2fe..ecc1043 100644 --- a/lib/services/mimecast-sync-service.ts +++ b/lib/services/mimecast-sync-service.ts @@ -1,6 +1,6 @@ /** * Mimecast Sync Service - * Full sync (120 days back), incremental (since last sync), body fetch, 120-day purge + * Full sync (18 months back), incremental (since last sync), body fetch, 18-month rolling purge */ import { getMimecastClient, MimecastMessage, MimecastThreatEvent } from './mimecast-client'; @@ -15,9 +15,10 @@ export interface MimecastSyncResult { durationMs: number; } -const RETENTION_DAYS = 30; // API max lookback is ~30 days +const RETENTION_DAYS = 548; // 18-month rolling retention in DB +const FULL_SYNC_DAYS = 548; // How far back a full sync reaches const BODY_FETCH_LIMIT = 500; -const CHUNK_DAYS = 7; // Pages the 30-day window in 7-day chunks to avoid 5000-result cap +const CHUNK_DAYS = 7; // Pages the window in 7-day chunks to avoid 5000-result cap per request // ── Upsert helpers ──────────────────────────────────────────────────────────── @@ -283,7 +284,7 @@ export async function runMimecastFullSync(): Promise { const toDate = new Date(); const fromDate = new Date(); - fromDate.setDate(fromDate.getDate() - RETENTION_DAYS); + fromDate.setDate(fromDate.getDate() - FULL_SYNC_DAYS); const messagesUpserted = await syncMimecastMessages(fromDate, toDate, errors); const threatsUpserted = await syncMimecastThreats(errors);