From ca2c9c701d65db83d50a315fb930f70876a0c712 Mon Sep 17 00:00:00 2001 From: lorentz Date: Wed, 8 Apr 2026 11:19:37 +0000 Subject: [PATCH] feat: SHAPE historical import admin UI - New DB tables: shape_import_runs, shape_import_logs (raw SQL applied) - src/lib/shape-import/run-import.ts: importable core logic extracted from CLI script - POST /api/admin/shape-import: starts run async, returns runId immediately - GET /api/admin/shape-import: lists last 20 runs - GET /api/admin/shape-import/[id]: polls log lines and run status - /admin/shape-import page: drive ID config, dry run / execute (with confirmation), live log panel (2s poll), stats summary, run history with log replay - Admin index: added SHAPE Historical Import card --- ondeck/prisma/schema.prisma | 31 + ondeck/src/app/(dashboard)/admin/page.tsx | 10 +- .../(dashboard)/admin/shape-import/page.tsx | 46 ++ .../app/api/admin/shape-import/[id]/route.ts | 33 + .../src/app/api/admin/shape-import/route.ts | 71 ++ .../components/admin/shape-import-panel.tsx | 409 ++++++++++ ondeck/src/lib/shape-import/run-import.ts | 770 ++++++++++++++++++ 7 files changed, 1369 insertions(+), 1 deletion(-) create mode 100644 ondeck/src/app/(dashboard)/admin/shape-import/page.tsx create mode 100644 ondeck/src/app/api/admin/shape-import/[id]/route.ts create mode 100644 ondeck/src/app/api/admin/shape-import/route.ts create mode 100644 ondeck/src/components/admin/shape-import-panel.tsx create mode 100644 ondeck/src/lib/shape-import/run-import.ts diff --git a/ondeck/prisma/schema.prisma b/ondeck/prisma/schema.prisma index 1092ecc..e1aafdc 100644 --- a/ondeck/prisma/schema.prisma +++ b/ondeck/prisma/schema.prisma @@ -40,6 +40,7 @@ model User { createdPolicyGroups PolicyGroup[] advocateClients Client[] @relation("ClientAdvocate") clientMemberships ClientMember[] + shapeImportRuns ShapeImportRun[] @@map("users") } @@ -446,6 +447,36 @@ model Notification { @@map("notifications") } +model ShapeImportRun { + id String @id @default(cuid()) + startedAt DateTime @default(now()) @map("started_at") + completedAt DateTime? @map("completed_at") + status String // pending | running | completed | failed + dryRun Boolean @default(true) @map("dry_run") + driveId String @map("drive_id") + triggeredBy String? @map("triggered_by") + stats Json? + + user User? @relation(fields: [triggeredBy], references: [id]) + logLines ShapeImportLog[] + + @@index([startedAt]) + @@map("shape_import_runs") +} + +model ShapeImportLog { + id String @id @default(cuid()) + runId String @map("run_id") + seq Int + line String @db.Text + createdAt DateTime @default(now()) @map("created_at") + + run ShapeImportRun @relation(fields: [runId], references: [id], onDelete: Cascade) + + @@index([runId, seq]) + @@map("shape_import_logs") +} + model NotificationPreference { id String @id @default(cuid()) userId String @map("user_id") diff --git a/ondeck/src/app/(dashboard)/admin/page.tsx b/ondeck/src/app/(dashboard)/admin/page.tsx index 875f111..f7085b0 100644 --- a/ondeck/src/app/(dashboard)/admin/page.tsx +++ b/ondeck/src/app/(dashboard)/admin/page.tsx @@ -4,7 +4,7 @@ import { redirect } from 'next/navigation' import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' import { Button } from '@/components/ui/button' import Link from 'next/link' -import { Shapes, FileText, Users, Database, Settings, ClipboardList } from 'lucide-react' +import { Shapes, FileText, Users, Database, Settings, ClipboardList, History } from 'lucide-react' export default async function AdminPage() { const session = await getServerSession(authOptions) @@ -67,6 +67,14 @@ export default async function AdminPage() { color: 'text-red-600', bgColor: 'bg-red-100', }, + { + title: 'SHAPE Historical Import', + description: 'Import historical task completion data from SharePoint SHAPE Excel files', + icon: History, + href: '/admin/shape-import', + color: 'text-teal-600', + bgColor: 'bg-teal-100', + }, ] return ( diff --git a/ondeck/src/app/(dashboard)/admin/shape-import/page.tsx b/ondeck/src/app/(dashboard)/admin/shape-import/page.tsx new file mode 100644 index 0000000..23c52e5 --- /dev/null +++ b/ondeck/src/app/(dashboard)/admin/shape-import/page.tsx @@ -0,0 +1,46 @@ +import { getServerSession } from 'next-auth' +import { authOptions } from '@/lib/auth' +import { redirect } from 'next/navigation' +import { prisma } from '@/lib/db' +import { ShapeImportPanel } from '@/components/admin/shape-import-panel' +import { ArrowLeft } from 'lucide-react' +import Link from 'next/link' + +export default async function ShapeImportPage() { + const session = await getServerSession(authOptions) + if (!session?.user) redirect('/auth/signin') + const roles = (session.user as any)?.roles || [] + if (!roles.includes('Admin')) redirect('/dashboard') + + const runs = await prisma.shapeImportRun.findMany({ + orderBy: { startedAt: 'desc' }, + take: 20, + include: { user: { select: { displayName: true, email: true } } }, + }) + + const serialized = runs.map((r) => ({ + id: r.id, + startedAt: r.startedAt.toISOString(), + completedAt: r.completedAt?.toISOString() ?? null, + status: r.status, + dryRun: r.dryRun, + driveId: r.driveId, + stats: r.stats as Record | null, + user: r.user ?? null, + })) + + return ( +
+
+ + Back to Admin + +

SHAPE Historical Import

+

+ Import historical task completion data from SharePoint SHAPE Excel files into Horizon. +

+
+ +
+ ) +} diff --git a/ondeck/src/app/api/admin/shape-import/[id]/route.ts b/ondeck/src/app/api/admin/shape-import/[id]/route.ts new file mode 100644 index 0000000..a67d78b --- /dev/null +++ b/ondeck/src/app/api/admin/shape-import/[id]/route.ts @@ -0,0 +1,33 @@ +import { NextRequest, NextResponse } from 'next/server' +import { getServerSession } from 'next-auth' +import { authOptions } from '@/lib/auth' +import { prisma } from '@/lib/db' + +export async function GET( + req: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const session = await getServerSession(authOptions) + if (!session?.user) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + const roles = (session.user as any)?.roles || [] + if (!roles.includes('Admin')) return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) + + const { id } = await params + const afterParam = req.nextUrl.searchParams.get('after') + const after = afterParam !== null ? parseInt(afterParam) : -1 + + const run = await prisma.shapeImportRun.findUnique({ + where: { id }, + select: { id: true, status: true, dryRun: true, startedAt: true, completedAt: true, stats: true }, + }) + if (!run) return NextResponse.json({ error: 'Not found' }, { status: 404 }) + + const logs = await prisma.shapeImportLog.findMany({ + where: { runId: id, seq: { gt: after } }, + orderBy: { seq: 'asc' }, + take: 500, + select: { seq: true, line: true }, + }) + + return NextResponse.json({ run, logs }) +} diff --git a/ondeck/src/app/api/admin/shape-import/route.ts b/ondeck/src/app/api/admin/shape-import/route.ts new file mode 100644 index 0000000..e3f630d --- /dev/null +++ b/ondeck/src/app/api/admin/shape-import/route.ts @@ -0,0 +1,71 @@ +import { NextRequest, NextResponse } from 'next/server' +import { getServerSession } from 'next-auth' +import { authOptions } from '@/lib/auth' +import { prisma } from '@/lib/db' +import { runShapeImport } from '@/lib/shape-import/run-import' +import { randomBytes } from 'crypto' + +const DEFAULT_DRIVE_ID = 'b!OYuzIexQkkOvfEPyMJPzzZHfzTrOCOdPhTWgTlzKs6M0ZWVrAc6LR4LjWl4QFEzm' + +function cuid() { + return 'c' + randomBytes(11).toString('base64url').slice(0, 24) +} + +export async function GET() { + const session = await getServerSession(authOptions) + if (!session?.user) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + const roles = (session.user as any)?.roles || [] + if (!roles.includes('Admin')) return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) + + const runs = await prisma.shapeImportRun.findMany({ + orderBy: { startedAt: 'desc' }, + take: 20, + include: { user: { select: { displayName: true, email: true } } }, + }) + + return NextResponse.json({ runs }) +} + +export async function POST(req: NextRequest) { + const session = await getServerSession(authOptions) + if (!session?.user) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + const roles = (session.user as any)?.roles || [] + if (!roles.includes('Admin')) return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) + + const body = await req.json().catch(() => ({})) + const dryRun: boolean = body.dryRun !== false + const driveId: string = body.driveId || DEFAULT_DRIVE_ID + const userId = (session.user as any).id as string + + const run = await prisma.shapeImportRun.create({ + data: { id: cuid(), status: 'running', dryRun, driveId, triggeredBy: userId }, + }) + + let seq = 0 + + const onLog = async (line: string) => { + await prisma.shapeImportLog.create({ + data: { id: cuid(), runId: run.id, seq: seq++, line }, + }) + } + + // Run async — don't await so the response returns immediately + ;(async () => { + try { + const stats = await runShapeImport({ dryRun, driveId, onLog }) + await prisma.shapeImportRun.update({ + where: { id: run.id }, + data: { status: 'completed', completedAt: new Date(), stats: stats as any }, + }) + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + await onLog(`FATAL ERROR: ${msg}`) + await prisma.shapeImportRun.update({ + where: { id: run.id }, + data: { status: 'failed', completedAt: new Date() }, + }) + } + })() + + return NextResponse.json({ runId: run.id }) +} diff --git a/ondeck/src/components/admin/shape-import-panel.tsx b/ondeck/src/components/admin/shape-import-panel.tsx new file mode 100644 index 0000000..eb5d53e --- /dev/null +++ b/ondeck/src/components/admin/shape-import-panel.tsx @@ -0,0 +1,409 @@ +'use client' + +import { useState, useEffect, useRef, useCallback } from 'react' +import { toast } from 'sonner' +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' +import { Button } from '@/components/ui/button' +import { Badge } from '@/components/ui/badge' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogFooter, +} from '@/components/ui/dialog' +import { + PlayCircle, + Loader2, + CheckCircle2, + XCircle, + Clock, + FileText, + ChevronDown, + ChevronUp, + AlertTriangle, +} from 'lucide-react' + +const DEFAULT_DRIVE_ID = 'b!OYuzIexQkkOvfEPyMJPzzZHfzTrOCOdPhTWgTlzKs6M0ZWVrAc6LR4LjWl4QFEzm' + +interface Run { + id: string + startedAt: string + completedAt: string | null + status: string + dryRun: boolean + driveId: string + stats: Record | null + user?: { displayName: string | null; email: string } | null +} + +interface LogLine { + seq: number + line: string +} + +function StatusBadge({ status }: { status: string }) { + if (status === 'completed') return Completed + if (status === 'failed') return Failed + if (status === 'running') return Running + return Pending +} + +function StatLine({ label, value }: { label: string; value: number | string }) { + return ( +
+ {label} + {value} +
+ ) +} + +export function ShapeImportPanel({ initialRuns }: { initialRuns: Run[] }) { + const [runs, setRuns] = useState(initialRuns) + const [driveId, setDriveId] = useState(DEFAULT_DRIVE_ID) + const [activeRunId, setActiveRunId] = useState(null) + const [activeRunStatus, setActiveRunStatus] = useState('idle') + const [logLines, setLogLines] = useState([]) + const [lastSeq, setLastSeq] = useState(-1) + const [viewingRunId, setViewingRunId] = useState(null) + const [confirmOpen, setConfirmOpen] = useState(false) + const [starting, setStarting] = useState(false) + const logRef = useRef(null) + const pollRef = useRef | null>(null) + + const scrollToBottom = useCallback(() => { + if (logRef.current) logRef.current.scrollTop = logRef.current.scrollHeight + }, []) + + useEffect(() => { + scrollToBottom() + }, [logLines, scrollToBottom]) + + const stopPolling = useCallback(() => { + if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null } + }, []) + + const pollRun = useCallback(async (runId: string, currentSeq: number) => { + try { + const res = await fetch(`/api/admin/shape-import/${runId}?after=${currentSeq}`) + if (!res.ok) return + const data = await res.json() + + if (data.logs && data.logs.length > 0) { + setLogLines((prev) => { + const existing = new Set(prev.map((l) => l.seq)) + const newLines = (data.logs as LogLine[]).filter((l) => !existing.has(l.seq)) + return [...prev, ...newLines] + }) + const maxSeq = Math.max(...(data.logs as LogLine[]).map((l) => l.seq)) + setLastSeq(maxSeq) + currentSeq = maxSeq + } + + if (data.run?.status === 'completed' || data.run?.status === 'failed') { + setActiveRunStatus(data.run.status) + stopPolling() + setRuns((prev) => + prev.map((r) => (r.id === runId ? { ...r, status: data.run.status, completedAt: data.run.completedAt, stats: data.run.stats } : r)) + ) + } + } catch { + // transient — keep polling + } + }, [stopPolling]) + + const startPolling = useCallback((runId: string) => { + stopPolling() + let seq = -1 + pollRef.current = setInterval(async () => { + try { + const res = await fetch(`/api/admin/shape-import/${runId}?after=${seq}`) + if (!res.ok) return + const data = await res.json() + if (data.logs && data.logs.length > 0) { + setLogLines((prev) => { + const existing = new Set(prev.map((l) => l.seq)) + const newLines = (data.logs as LogLine[]).filter((l) => !existing.has(l.seq)) + return [...prev, ...newLines] + }) + seq = Math.max(...(data.logs as LogLine[]).map((l) => l.seq)) + setLastSeq(seq) + } + if (data.run?.status === 'completed' || data.run?.status === 'failed') { + setActiveRunStatus(data.run.status) + stopPolling() + setRuns((prev) => + prev.map((r) => (r.id === runId ? { ...r, status: data.run.status, completedAt: data.run.completedAt, stats: data.run.stats } : r)) + ) + if (data.run.status === 'completed') toast.success('Import completed') + else toast.error('Import failed — check the log for details') + } + } catch { /* keep polling */ } + }, 2000) + }, [stopPolling]) + + useEffect(() => () => stopPolling(), [stopPolling]) + + const startRun = async (dryRun: boolean) => { + setStarting(true) + setLogLines([]) + setLastSeq(-1) + try { + const res = await fetch('/api/admin/shape-import', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ dryRun, driveId }), + }) + const data = await res.json() + if (!res.ok) throw new Error(data.error || 'Failed to start') + const newRun: Run = { id: data.runId, startedAt: new Date().toISOString(), completedAt: null, status: 'running', dryRun, driveId, stats: null } + setRuns((prev) => [newRun, ...prev]) + setActiveRunId(data.runId) + setViewingRunId(data.runId) + setActiveRunStatus('running') + startPolling(data.runId) + toast.info(dryRun ? 'Dry run started — results will appear in the log' : 'Import started') + } catch (err: any) { + toast.error(err.message || 'Failed to start import') + } finally { + setStarting(false) + } + } + + const loadRunLog = async (runId: string) => { + setViewingRunId(runId) + setLogLines([]) + setLastSeq(-1) + try { + const res = await fetch(`/api/admin/shape-import/${runId}?after=-1`) + if (!res.ok) return + const data = await res.json() + setLogLines(data.logs || []) + if (data.logs?.length) setLastSeq(data.logs[data.logs.length - 1].seq) + } catch { /* ignore */ } + } + + const isRunning = activeRunStatus === 'running' && activeRunId !== null + + return ( +
+ {/* Configuration */} + + + + + SHAPE Historical Import + + + +

+ Import historical task completion data from SHAPE Excel files on SharePoint into Horizon. + Always run a dry run first — it reads SharePoint and shows what would be + created/updated without writing anything to the database. +

+ +
+ + setDriveId(e.target.value)} + disabled={isRunning} + className="font-mono text-xs" + /> +

+ Path: Claims/SHAPE Accounts/ within this drive. Default is the current production drive. +

+
+ +
+ + + +
+ + {isRunning && ( +
+ + Import running — log updating live below… +
+ )} + + + + {/* Log viewer */} + {viewingRunId && ( + + + + + + Import Log + {isRunning && viewingRunId === activeRunId && ( + Live + )} + + {(() => { + const run = runs.find((r) => r.id === viewingRunId) + return run ? : null + })()} + + + + {/* Stats summary (completed runs) */} + {(() => { + const run = runs.find((r) => r.id === viewingRunId) + if (!run?.stats || run.status === 'running') return null + const s = run.stats + return ( +
+
+

Clients

+ + +
+
+

Tasks

+ + + +
+
+

Other

+ + + +
+
+

Issues

+ + +
+
+ ) + })()} + + {/* Log */} + 
+              {logLines.length === 0
+                ? '(waiting for output...)'
+                : logLines.map((l) => l.line).join('\n')}
+
+ + + )} + + {/* Run history */} + + + + + Run History + + + + {runs.length === 0 ? ( +

No runs yet

+ ) : ( +
+ {runs.map((run) => ( + + ))} +
+ )} + + + + {/* Execute confirmation dialog */} + + + + + + Execute SHAPE Import + + +
+

This will write data to the database:

+
    +
  • Create or update tasks for matched SHAPE clients
    • +
  • Mark historical tasks as Completed or N/A
    • +
  • Assign claims advocates to clients (if not already set)
    • +
  • Delete duplicate tasks
    • +
+

+ Run a dry run first and review the log if you haven't already. + This action cannot be automatically undone. +

+
+ + + + + + +
+ ) +} diff --git a/ondeck/src/lib/shape-import/run-import.ts b/ondeck/src/lib/shape-import/run-import.ts new file mode 100644 index 0000000..3b8be13 --- /dev/null +++ b/ondeck/src/lib/shape-import/run-import.ts @@ -0,0 +1,770 @@ +/** + * SHAPE Historical Import — core logic, importable by both CLI and API. + * + * Call runShapeImport({ dryRun, driveId, onLog }) to execute. + */ + +import * as zlib from 'zlib' +import { prisma } from '@/lib/db' + +// ─── Types ──────────────────────────────────────────────────────────────────── + +export interface ImportStats { + filesProcessed: number + clientsMatched: number + clientsUnmatched: number + tasksUpdated: number + tasksAssigned: number + tasksCreated: number + duplicatesDeleted: number + adHocCreated: number + advocatesAssigned: number + errors: string[] + unmatchedClients: Array<{ folder: string; file: string; excelName: string }> + fuzzyMatches: Array<{ + folder: string + file: string + excelName: string + dbName: string + method: string + }> +} + +interface ParsedRow { + taskName: string + daysAfterRenewal: number + dateCompleted: string + notes: string +} + +interface ParsedBlock { + clientName: string + effectiveDate: Date + isShape2: boolean + rows: ParsedRow[] + additionalServices: string[] +} + +interface DbTemplate { + id: string + name: string + daysOffset: number + designationId: string | null +} + +interface DbClient { + id: string + name: string + claimsAdvocateId: string | null +} + +interface DbState { + clientsByNormalizedName: Map + allClients: DbClient[] + shapeTemplates: DbTemplate[] + shape2Templates: DbTemplate[] + shapeDesignationId: string + shape2DesignationId: string +} + +interface DriveItem { + id: string + name: string + file?: { mimeType: string } + folder?: { childCount: number } +} + +// ─── Team member map ────────────────────────────────────────────────────────── + +export const TEAM_MEMBER_MAP: Record = { + CHRIS: { userId: 'cmkm36yc90021y7vb9783nitl', displayName: 'Christine Gove' }, + DAWN: { userId: 'cmkm36xfr000ny7vbcajejjux', displayName: 'Dawn Boland' }, + Jeanne: { userId: 'cmkm370f80065y7vbrugc9jub', displayName: 'Jeanne Strong' }, + LUKE: { userId: 'cmkm36xd4000jy7vbhanp66s9', displayName: 'Luke Billman' }, + MIMI: { userId: 'cmkm36zr0004wy7vbztyynj35', displayName: 'Mimi Rawlings' }, +} + +const TEMPLATE_NAME_ALIASES: Record = { + 'request 125 day loss runs': 'request 120 day loss runs', + 'request 89 day loss runs': 'request 90 day loss runs', + 'request 89 day loss runs (if being marketed)': 'request 90 day loss runs (if being marketed)', + 'claim review (six month)': 'claim review', + 'claims review': 'claim review', +} + +// ─── Graph API ──────────────────────────────────────────────────────────────── + +let _cachedToken: { token: string; expiresAt: number } | null = null + +async function getGraphToken(): Promise { + if (_cachedToken && Date.now() < _cachedToken.expiresAt - 30_000) return _cachedToken.token + const tenantId = process.env.AZURE_AD_TENANT_ID! + const clientId = process.env.AZURE_AD_CLIENT_ID! + const clientSecret = process.env.AZURE_AD_CLIENT_SECRET! + const body = new URLSearchParams({ + grant_type: 'client_credentials', + client_id: clientId, + client_secret: clientSecret, + scope: 'https://graph.microsoft.com/.default', + }) + const res = await fetch(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: body.toString(), + }) + if (!res.ok) throw new Error(`Token fetch failed: ${res.status}`) + const data = (await res.json()) as { access_token: string; expires_in: number } + _cachedToken = { token: data.access_token, expiresAt: Date.now() + data.expires_in * 1000 } + return _cachedToken.token +} + +async function graphGet(url: string): Promise { + const token = await getGraphToken() + const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` } }) + if (!res.ok) { + const text = await res.text() + throw new Error(`Graph API error ${res.status}: ${text.slice(0, 200)}`) + } + return res.json() +} + +async function downloadFile(driveId: string, itemId: string): Promise { + const token = await getGraphToken() + const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content` + const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` }, redirect: 'follow' }) + if (!res.ok) throw new Error(`Download failed: ${res.status}`) + const ab = await res.arrayBuffer() + return Buffer.from(ab) +} + +async function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)) +} + +async function listChildren(driveId: string, itemId: string): Promise { + const data = (await graphGet( + `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/children?$top=200` + )) as { value: DriveItem[] } + return data.value ?? [] +} + +async function discoverFiles( + driveId: string, + log: (line: string) => Promise +): Promise> { + await log('Discovering files from SharePoint...') + const files: Array<{ folder: string; itemId: string; fileName: string }> = [] + + const rootData = (await graphGet( + `https://graph.microsoft.com/v1.0/drives/${driveId}/root:/Claims%2FSHAPE%20Accounts:/children` + )) as { value: DriveItem[] } + const rootChildren = rootData.value ?? [] + + for (const item of rootChildren) { + if (!item.folder) continue + const memberInfo = TEAM_MEMBER_MAP[item.name] + if (!memberInfo) continue + await log(` Scanning folder: ${item.name}`) + const children = await listChildren(driveId, item.id) + await sleep(150) + for (const child of children) { + if (!child.file) continue + if (!child.name.toLowerCase().endsWith('.xlsx')) continue + if (child.name.toLowerCase().startsWith('archive')) continue + if ( + child.name.toLowerCase().includes('template') || + child.name.toLowerCase().includes('master workbook') || + child.name.toLowerCase().startsWith('all shape') + ) continue + files.push({ folder: item.name, itemId: child.id, fileName: child.name }) + } + } + + await log(` Found ${files.length} Excel files`) + return files +} + +// ─── ZIP / XLSX Parser ──────────────────────────────────────────────────────── + +function parseZip(buffer: Buffer): Map { + const entries = new Map() + let eocdOffset = -1 + const searchStart = Math.max(0, buffer.length - 65558) + for (let i = buffer.length - 22; i >= searchStart; i--) { + if (buffer.readUInt32LE(i) === 0x06054b50) { eocdOffset = i; break } + } + if (eocdOffset < 0) throw new Error('Not a valid ZIP file (EOCD not found)') + const cdOffset = buffer.readUInt32LE(eocdOffset + 16) + const cdEntries = buffer.readUInt16LE(eocdOffset + 10) + let pos = cdOffset + for (let i = 0; i < cdEntries; i++) { + if (buffer.readUInt32LE(pos) !== 0x02014b50) throw new Error(`Invalid central directory entry at offset ${pos}`) + const method = buffer.readUInt16LE(pos + 10) + const compSize = buffer.readUInt32LE(pos + 20) + const fileNameLen = buffer.readUInt16LE(pos + 28) + const extraLen = buffer.readUInt16LE(pos + 30) + const commentLen = buffer.readUInt16LE(pos + 32) + const localOffset = buffer.readUInt32LE(pos + 42) + const fileName = buffer.slice(pos + 46, pos + 46 + fileNameLen).toString('utf-8') + if (buffer.readUInt32LE(localOffset) !== 0x04034b50) throw new Error(`Invalid local header for ${fileName}`) + const localFileNameLen = buffer.readUInt16LE(localOffset + 26) + const localExtraLen = buffer.readUInt16LE(localOffset + 28) + const dataStart = localOffset + 30 + localFileNameLen + localExtraLen + const compData = buffer.slice(dataStart, dataStart + compSize) + let data: Buffer + if (method === 0) data = compData + else if (method === 8) data = zlib.inflateRawSync(compData) + else data = Buffer.alloc(0) + entries.set(fileName.toLowerCase(), data) + pos += 46 + fileNameLen + extraLen + commentLen + } + return entries +} + +function decodeXmlEntities(s: string): string { + return s + .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') + .replace(/"/g, '"').replace(/'/g, "'") + .replace(/&#x([0-9A-Fa-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16))) + .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10))) +} + +function parseSharedStrings(xmlBuf: Buffer | undefined): string[] { + if (!xmlBuf || xmlBuf.length === 0) return [] + const xml = xmlBuf.toString('utf-8') + const strings: string[] = [] + const siRegex = /([\s\S]*?)<\/si>/g + let match: RegExpExecArray | null + while ((match = siRegex.exec(xml)) !== null) { + const texts: string[] = [] + const tRegex = /]*)?>([^<]*)<\/t>/g + let tm: RegExpExecArray | null + while ((tm = tRegex.exec(match[1])) !== null) texts.push(decodeXmlEntities(tm[1])) + strings.push(texts.join('')) + } + return strings +} + +function colLetterToNum(letters: string): number { + let n = 0 + for (let i = 0; i < letters.length; i++) n = n * 26 + (letters.charCodeAt(i) - 64) + return n +} + +function parseSheetToGrid(sheetBuf: Buffer, sharedStrings: string[]): { get: (row: number, col: number) => string; maxRow: number } { + const cells = new Map() + let maxRow = 0 + const xml = sheetBuf.toString('utf-8').replace(/]*\/>/g, '') + const cRegex = /]*)>([\s\S]*?)<\/c>/g + let m: RegExpExecArray | null + while ((m = cRegex.exec(xml)) !== null) { + const colStr = m[1]; const rowNum = parseInt(m[2]); const attrs = m[3]; const inner = m[4] + if (rowNum > maxRow) maxRow = rowNum + const typeMatch = attrs.match(/\bt="([^"]+)"/); const type = typeMatch ? typeMatch[1] : 'n' + let value = '' + const vMatch = inner.match(/([^<]*)<\/v>/) + const isMatch = inner.match(/]*)?>([^<]*)<\/t><\/is>/) + if (isMatch) value = decodeXmlEntities(isMatch[1]) + else if (vMatch) { + if (type === 's') value = sharedStrings[parseInt(vMatch[1])] ?? '' + else if (type === 'str') value = decodeXmlEntities(vMatch[1]) + else if (type === 'b') value = vMatch[1] === '1' ? 'TRUE' : 'FALSE' + else value = vMatch[1] + } + cells.set(`${rowNum}:${colLetterToNum(colStr)}`, value) + } + return { get(row, col) { return cells.get(`${row}:${col}`) ?? '' }, maxRow } +} + +function findTargetSheet(zipEntries: Map): Buffer | null { + const wbBuf = zipEntries.get('xl/workbook.xml') + if (!wbBuf) return zipEntries.get('xl/worksheets/sheet1.xml') ?? null + const wb = wbBuf.toString('utf-8') + const relsBuf = zipEntries.get('xl/_rels/workbook.xml.rels') + const ridToPath = new Map() + if (relsBuf) { + const rels = relsBuf.toString('utf-8') + const relRegex = /]+Target="([^"]+)"/g + let rm: RegExpExecArray | null + while ((rm = relRegex.exec(rels)) !== null) { + let target = rm[2] + if (target.startsWith('/')) target = target.slice(1) + else if (!target.startsWith('xl/')) target = 'xl/' + target + ridToPath.set(rm[1], target) + } + } + const sheetEntries: Array<{ name: string; rid: string }> = [] + const sheetTagRegex = /]*\/?>)/g + let sm: RegExpExecArray | null + while ((sm = sheetTagRegex.exec(wb)) !== null) { + const attrs = sm[1] + const nameMatch = attrs.match(/\bname="([^"]+)"/) + const ridMatch = attrs.match(/\br:id="([^"]+)"/) + if (nameMatch && ridMatch) sheetEntries.push({ name: nameMatch[1], rid: ridMatch[1] }) + } + const resolveSheet = (rid: string): Buffer | null => { + const path = ridToPath.get(rid); if (!path) return null + return zipEntries.get(path.toLowerCase()) ?? null + } + for (const p of ['shape2', 'shape', 'client']) { + const match = sheetEntries.find((s) => s.name.toLowerCase() === p) + if (match) { const buf = resolveSheet(match.rid); if (buf) return buf } + } + if (sheetEntries.length > 0) { const buf = resolveSheet(sheetEntries[0].rid); if (buf) return buf } + return zipEntries.get('xl/worksheets/sheet1.xml') ?? null +} + +function excelSerialToDate(serial: number): Date { + return new Date((serial - 25569) * 86400 * 1000) +} + +function parseExcelBlocks( + grid: { get: (r: number, c: number) => string; maxRow: number }, + fileName: string +): ParsedBlock[] { + const blocks: ParsedBlock[] = [] + const isShape2 = /(SHAPE2|SHAPE 2)/i.test(fileName) + let row = 1 + while (row <= grid.maxRow) { + let clientNameCol = -1 + for (let c = 1; c <= 5; c++) { + if (grid.get(row, c).trim().toLowerCase().includes('client name')) { clientNameCol = c; break } + } + if (clientNameCol < 0) { row++; continue } + let clientName = '' + for (let dc = 1; dc <= 3; dc++) { + const candidate = grid.get(row, clientNameCol + dc).trim() + if (candidate && !candidate.toLowerCase().includes('client name')) { clientName = candidate; break } + } + if (!clientName) { + clientName = fileName + .replace(/^SHAPE\S*\s*[-–]\s*/i, '').replace(/\s*[-–]\s*SHAPE.*$/i, '') + .replace(/\s*SHAPE\s*Services?\s*Checklist.*$/i, '').replace(/\s*SHAPE\b.*/i, '') + .replace(/\.xlsx$/i, '').trim() + } + if (!clientName) { row++; continue } + row++ + let effectiveDateSerial = 0; let foundDateRow = row + const EXCEL_DATE_MIN = 40000; const EXCEL_DATE_MAX = 60000 + for (let r = row; r <= Math.min(row + 4, grid.maxRow); r++) { + let foundDateInRow = false + for (let c = 1; c <= 8; c++) { + const cellVal = grid.get(r, c).trim() + if (cellVal.toLowerCase().includes('effective date')) { + for (let dc = 1; dc <= 4; dc++) { + const val = grid.get(r, c + dc).trim(); const serial = parseFloat(val) + if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) { + effectiveDateSerial = serial; foundDateInRow = true; break + } + } + if (foundDateInRow) break + } + if (!foundDateInRow && c > 1) { + const serial = parseFloat(cellVal) + if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) { + effectiveDateSerial = serial; foundDateInRow = true; break + } + } + } + if (foundDateInRow) { foundDateRow = r; break } + } + const effectiveDate = effectiveDateSerial > 0 ? excelSerialToDate(effectiveDateSerial) : new Date() + row = foundDateRow + 1 + for (let r = row; r <= Math.min(row + 3, grid.maxRow); r++) { + let foundPolicies = false + for (let c = 1; c <= 5; c++) { + if (grid.get(r, c).trim().toLowerCase().startsWith('policies')) { foundPolicies = true; break } + } + if (foundPolicies) { row = r + 1; break } + let hasTask = false + for (let c = 1; c <= 8; c++) { + if (grid.get(r, c).trim().toLowerCase() === 'task') { hasTask = true; break } + } + if (hasTask) break + } + let headerRow = -1; let taskCol = 0 + for (let r = row; r <= Math.min(row + 5, grid.maxRow); r++) { + for (let c = 1; c <= 8; c++) { + if (grid.get(r, c).trim().toLowerCase() === 'task') { headerRow = r; taskCol = c; break } + } + if (headerRow >= 0) break + } + if (headerRow < 0) { row++; continue } + let daysAfterCol = 0; let dateCompletedCol = 0; let notesCol = 0 + for (let c = taskCol + 1; c <= taskCol + 8; c++) { + const hdr = grid.get(headerRow, c).trim().toLowerCase() + if (hdr.includes('days after')) daysAfterCol = c + else if (hdr.includes('date completed') || hdr === 'completed' || hdr === 'date complete') dateCompletedCol = c + else if (hdr.includes('notes')) notesCol = c + } + row = headerRow + 1 + const taskRows: ParsedRow[] = []; const additionalServices: string[] = []; let inAdditional = false + while (row <= grid.maxRow) { + let nextBlock = false + for (let c = 1; c <= 5; c++) { + if (grid.get(row, c).trim().toLowerCase().includes('client name')) { nextBlock = true; break } + } + if (nextBlock) break + const taskName = grid.get(row, taskCol).trim() + if (!taskName) { row++; continue } + if (taskName.toLowerCase().includes('additional services')) { inAdditional = true; row++; continue } + const daysAfterRaw = daysAfterCol > 0 ? grid.get(row, daysAfterCol).trim() : '' + const isDaysNumeric = daysAfterRaw !== '' && !isNaN(parseFloat(daysAfterRaw)) + if (!isDaysNumeric) { if (taskName.length > 3) additionalServices.push(taskName); row++; continue } + if (inAdditional) { additionalServices.push(taskName); row++; continue } + const daysAfterRenewal = parseFloat(daysAfterRaw) + const dateCompletedRaw = dateCompletedCol > 0 ? grid.get(row, dateCompletedCol).trim() : '' + const notes = notesCol > 0 ? grid.get(row, notesCol).trim() : '' + taskRows.push({ taskName, daysAfterRenewal, dateCompleted: dateCompletedRaw, notes }) + row++ + } + blocks.push({ clientName, effectiveDate, isShape2, rows: taskRows, additionalServices }) + } + return blocks +} + +function parseExcelFile(buf: Buffer, fileName: string): ParsedBlock[] { + const zipEntries = parseZip(buf) + const sharedStrings = parseSharedStrings(zipEntries.get('xl/sharedstrings.xml')) + const sheetBuf = findTargetSheet(zipEntries) + if (!sheetBuf) return [] + return parseExcelBlocks(parseSheetToGrid(sheetBuf, sharedStrings), fileName) +} + +// ─── Client name matching ───────────────────────────────────────────────────── + +function normalizeClientName(name: string): string { + return name.toLowerCase().replace(/^the\s+/, '').replace(/&/g, 'and') + .replace(/\b(incorporated|inc|llc|l\.l\.c|corp|corporation|co|ltd|limited|lp|l\.p|company|enterprises|enterprise|services|group|associates|solutions|management|consulting)\b\.?/gi, '') + .replace(/[.,;:'"()\-_#@!?]/g, '').replace(/\s+/g, ' ').trim() +} + +function tokenize(s: string): Set { + return new Set(s.split(/\s+/).filter((t) => t.length > 1)) +} + +function jaccardSimilarity(a: Set, b: Set): number { + if (a.size === 0 && b.size === 0) return 1 + let intersection = 0 + for (const t of a) if (b.has(t)) intersection++ + const union = a.size + b.size - intersection + return union === 0 ? 0 : intersection / union +} + +function matchClient(excelName: string, dbState: DbState): { client: DbClient; method: string } | null { + const normalized = normalizeClientName(excelName) + if (!normalized) return null + const exactCandidates = dbState.clientsByNormalizedName.get(normalized) + if (exactCandidates && exactCandidates.length > 0) return { client: exactCandidates[0], method: 'exact' } + for (const [key, clients] of dbState.clientsByNormalizedName) { + if (key.includes(normalized) || normalized.includes(key)) { + if (clients.length > 0) return { client: clients[0], method: 'contains' } + } + } + const aTokens = tokenize(normalized); let bestScore = 0; let bestClient: DbClient | null = null + for (const [key, clients] of dbState.clientsByNormalizedName) { + const score = jaccardSimilarity(aTokens, tokenize(key)) + if (score > bestScore && score >= 0.65) { bestScore = score; bestClient = clients[0] } + } + if (bestClient) return { client: bestClient, method: `jaccard(${bestScore.toFixed(2)})` } + return null +} + +// ─── Template matching ──────────────────────────────────────────────────────── + +function normalizeTemplateName(name: string): string { + const lowered = name.toLowerCase().trim() + return TEMPLATE_NAME_ALIASES[lowered] ?? lowered +} + +function findTemplate(row: ParsedRow, templates: DbTemplate[]): DbTemplate | null { + const normalized = normalizeTemplateName(row.taskName) + for (const t of templates) if (normalizeTemplateName(t.name) === normalized) return t + for (const t of templates) { + const tn = normalizeTemplateName(t.name) + if (tn.includes(normalized) || normalized.includes(tn)) return t + } + if (row.daysAfterRenewal > 0) { + const approxOffset = -(365 - row.daysAfterRenewal) + let closest: DbTemplate | null = null; let closestDiff = Infinity + for (const t of templates) { + const diff = Math.abs(t.daysOffset - approxOffset) + if (diff < closestDiff && diff <= 15) { closestDiff = diff; closest = t } + } + if (closest) return closest + } + return null +} + +// ─── DB state ──────────────────────────────────────────────────────────────── + +async function loadDbState(log: (l: string) => Promise): Promise { + await log('Loading database state...') + const designations = await prisma.designation.findMany({ + where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } }, + }) + const shapeDes = designations.find((d) => d.name === 'Shape') + const shape2Des = designations.find((d) => d.name === 'Shape 2' || d.name === 'Shape2') + if (!shapeDes || !shape2Des) throw new Error(`Could not find Shape/Shape2 designations. Found: ${designations.map((d) => d.name).join(', ')}`) + + const allTemplates = await prisma.taskTemplate.findMany({ + where: { designationId: { in: [shapeDes.id, shape2Des.id] }, isActive: true }, + }) + const shapeTemplates = allTemplates.filter((t) => t.designationId === shapeDes.id) + const shape2Templates = allTemplates.filter((t) => t.designationId === shape2Des.id) + await log(` Loaded ${shapeTemplates.length} Shape templates, ${shape2Templates.length} Shape2 templates`) + + const clients = await prisma.client.findMany({ + where: { OR: [{ designationId: { in: [shapeDes.id, shape2Des.id] } }, { designation2Id: { in: [shapeDes.id, shape2Des.id] } }] }, + select: { id: true, name: true, claimsAdvocateId: true }, + }) + await log(` Loaded ${clients.length} SHAPE clients`) + + const clientsByNormalizedName = new Map() + for (const c of clients) { + const key = normalizeClientName(c.name) + if (!clientsByNormalizedName.has(key)) clientsByNormalizedName.set(key, []) + clientsByNormalizedName.get(key)!.push({ id: c.id, name: c.name, claimsAdvocateId: c.claimsAdvocateId }) + } + + return { + clientsByNormalizedName, + allClients: clients.map((c) => ({ id: c.id, name: c.name, claimsAdvocateId: c.claimsAdvocateId })), + shapeTemplates: shapeTemplates.map((t) => ({ id: t.id, name: t.name, daysOffset: t.daysOffset, designationId: t.designationId })), + shape2Templates: shape2Templates.map((t) => ({ id: t.id, name: t.name, daysOffset: t.daysOffset, designationId: t.designationId })), + shapeDesignationId: shapeDes.id, + shape2DesignationId: shape2Des.id, + } +} + +// ─── Template fixes ─────────────────────────────────────────────────────────── + +async function fixTemplates(shapeDesignationId: string, shape2DesignationId: string, dryRun: boolean, log: (l: string) => Promise): Promise { + await log('\nFixing template data...') + const fix1 = await prisma.taskTemplate.updateMany({ where: { name: 'Request 125 day loss runs', daysOffset: -125 }, data: { name: 'Request 120 day loss runs', daysOffset: -120 } }) + if (fix1.count > 0) await log(` Fixed ${fix1.count} template(s): "Request 125..." → "Request 120..."`) + const fix2 = await prisma.taskTemplate.updateMany({ where: { daysOffset: -89 }, data: { daysOffset: -90 } }) + await prisma.taskTemplate.updateMany({ where: { name: 'Request 89 day loss runs' }, data: { name: 'Request 90 day loss runs' } }) + await prisma.taskTemplate.updateMany({ where: { name: 'Request 89 day loss runs (if being marketed)' }, data: { name: 'Request 90 day loss runs (if being marketed)' } }) + if (fix2.count > 0) await log(` Fixed ${fix2.count} template(s): offset -89 → -90`) + const existing185 = await prisma.taskTemplate.findFirst({ where: { name: 'Claim Review', daysOffset: -185, designationId: shapeDesignationId } }) + if (!existing185) { + if (!dryRun) { + await prisma.taskTemplate.create({ data: { name: 'Claim Review', department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: -185, defaultPriority: 'MEDIUM', isActive: true, displayOrder: 4, designationId: shapeDesignationId } }) + await log(' Created missing "Claim Review" template at -185 days (Shape)') + } else { + await log(' [DRY RUN] Would create missing "Claim Review" template at -185 days (Shape)') + } + } else { + await log(' "Claim Review" at -185 already exists — skipping') + } +} + +// ─── Task processing ────────────────────────────────────────────────────────── + +interface DateCompletedResult { status: 'COMPLETED' | 'NA' | 'NOT_STARTED'; completedAt: Date | null; naReason: string | null } + +function interpretDateCompleted(raw: string): DateCompletedResult { + const normalized = raw.trim().toLowerCase() + if (normalized === '' || normalized === '??' || normalized === '?') return { status: 'NOT_STARTED', completedAt: null, naReason: null } + if (normalized === 'n/a' || normalized === 'na') return { status: 'NA', completedAt: null, naReason: 'Historical — marked N/A in SHAPE tracker' } + const serial = parseFloat(raw) + if (!isNaN(serial) && serial > 40000 && serial < 60000) return { status: 'COMPLETED', completedAt: excelSerialToDate(serial), naReason: null } + return { status: 'NOT_STARTED', completedAt: null, naReason: null } +} + +async function findMatchingTasks(clientId: string, templateId: string, dueDate: Date): Promise<{ match: string | null; duplicateIds: string[] }> { + const windowMs = 5 * 86400 * 1000 + const candidates = await prisma.task.findMany({ + where: { clientId, templateId, dueDate: { gte: new Date(dueDate.getTime() - windowMs), lte: new Date(dueDate.getTime() + windowMs) } }, + orderBy: { createdAt: 'asc' }, + select: { id: true }, + }) + if (candidates.length === 0) return { match: null, duplicateIds: [] } + return { match: candidates[0].id, duplicateIds: candidates.slice(1).map((c) => c.id) } +} + +async function ensureTaskAssignment(taskId: string, userId: string, stats: ImportStats, dryRun: boolean): Promise { + if (dryRun) { stats.tasksAssigned++; return } + try { + await prisma.taskAssignment.upsert({ where: { taskId_userId: { taskId, userId } }, create: { taskId, userId }, update: {} }) + stats.tasksAssigned++ + } catch { /* already exists */ } +} + +async function processTaskRow(clientId: string, row: ParsedRow, template: DbTemplate, effectiveDate: Date, advocateUserId: string, stats: ImportStats, dryRun: boolean): Promise { + const dueDate = new Date(effectiveDate) + dueDate.setDate(dueDate.getDate() + Math.round(row.daysAfterRenewal)) + const { status, completedAt, naReason } = interpretDateCompleted(row.dateCompleted) + const { match: taskId, duplicateIds } = await findMatchingTasks(clientId, template.id, dueDate) + + if (duplicateIds.length > 0) { + if (!dryRun) await prisma.task.deleteMany({ where: { id: { in: duplicateIds } } }) + stats.duplicatesDeleted += duplicateIds.length + } + + if (taskId) { + await ensureTaskAssignment(taskId, advocateUserId, stats, dryRun) + if (status === 'COMPLETED' || status === 'NA') { + const existing = await prisma.task.findUnique({ where: { id: taskId }, select: { status: true, notes: true } }) + if (existing && existing.status === 'NOT_STARTED') { + if (!dryRun) { + await prisma.task.update({ where: { id: taskId }, data: { status, completedAt: completedAt ?? undefined, completedBy: status === 'COMPLETED' ? advocateUserId : undefined, naReason: naReason ?? undefined, notes: row.notes && !existing.notes ? row.notes : existing.notes ?? undefined } }) + } + stats.tasksUpdated++ + } + } + } else { + if (status === 'COMPLETED' || status === 'NA') { + if (!dryRun) { + const newTask = await prisma.task.create({ data: { title: template.name, department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: template.daysOffset, dueDate, status, priority: 'MEDIUM', clientId, templateId: template.id, completedAt: completedAt ?? undefined, completedBy: status === 'COMPLETED' ? advocateUserId : undefined, naReason: naReason ?? undefined, notes: row.notes || undefined, isAdHoc: false, createdBy: advocateUserId } }) + await ensureTaskAssignment(newTask.id, advocateUserId, stats, dryRun) + } else { stats.tasksAssigned++ } + stats.tasksCreated++ + } else { + if (!dryRun) { + const newTask = await prisma.task.create({ data: { title: template.name, department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: template.daysOffset, dueDate, status: 'NOT_STARTED', priority: 'MEDIUM', clientId, templateId: template.id, notes: row.notes || undefined, isAdHoc: false, createdBy: advocateUserId } }) + await ensureTaskAssignment(newTask.id, advocateUserId, stats, dryRun) + } else { stats.tasksAssigned++ } + stats.tasksCreated++ + } + } +} + +async function createAdHocTask(clientId: string, text: string, effectiveDate: Date, advocateUserId: string, stats: ImportStats, dryRun: boolean): Promise { + const title = text.slice(0, 500) + const existing = await prisma.task.findFirst({ where: { clientId, title, isAdHoc: true } }) + if (existing) return + if (!dryRun) { + const task = await prisma.task.create({ data: { title, department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: 0, dueDate: effectiveDate, status: 'COMPLETED', priority: 'MEDIUM', clientId, isAdHoc: true, completedAt: effectiveDate, completedBy: advocateUserId, notes: text, createdBy: advocateUserId } }) + await ensureTaskAssignment(task.id, advocateUserId, stats, dryRun) + } else { stats.tasksAssigned++ } + stats.adHocCreated++ +} + +async function assignAdvocate(clientId: string, advocateUserId: string, currentAdvocateId: string | null, stats: ImportStats, dryRun: boolean): Promise { + if (currentAdvocateId) return + if (!dryRun) await prisma.client.update({ where: { id: clientId }, data: { claimsAdvocateId: advocateUserId } }) + stats.advocatesAssigned++ +} + +async function processFile(folder: string, itemId: string, fileName: string, driveId: string, dbState: DbState, stats: ImportStats, dryRun: boolean, log: (l: string) => Promise): Promise { + const advocateInfo = TEAM_MEMBER_MAP[folder]! + let buf: Buffer + try { buf = await downloadFile(driveId, itemId); await sleep(150) } + catch (err) { stats.errors.push(`${folder}/${fileName}: Download failed — ${(err as Error).message}`); return } + + let blocks: ParsedBlock[] + try { blocks = parseExcelFile(buf, fileName) } + catch (err) { stats.errors.push(`${folder}/${fileName}: Parse failed — ${(err as Error).message}`); return } + + if (blocks.length === 0) { stats.errors.push(`${folder}/${fileName}: No data blocks found`); return } + stats.filesProcessed++ + + for (const block of blocks) { + const matchResult = matchClient(block.clientName, dbState) + if (!matchResult) { + stats.clientsUnmatched++ + stats.unmatchedClients.push({ folder, file: fileName, excelName: block.clientName }) + continue + } + const { client, method } = matchResult + stats.clientsMatched++ + if (method !== 'exact') stats.fuzzyMatches.push({ folder, file: fileName, excelName: block.clientName, dbName: client.name, method }) + await assignAdvocate(client.id, advocateInfo.userId, client.claimsAdvocateId, stats, dryRun) + client.claimsAdvocateId = client.claimsAdvocateId ?? advocateInfo.userId + const templates = block.isShape2 ? dbState.shape2Templates : dbState.shapeTemplates + for (const row of block.rows) { + const template = findTemplate(row, templates) + if (!template) { + const { status } = interpretDateCompleted(row.dateCompleted) + if (status === 'COMPLETED' || status === 'NA') { + const adHocTitle = `[Unmatched Task] ${row.taskName}${row.notes ? ': ' + row.notes : ''}` + await createAdHocTask(client.id, adHocTitle, block.effectiveDate, advocateInfo.userId, stats, dryRun) + } + continue + } + try { await processTaskRow(client.id, row, template, block.effectiveDate, advocateInfo.userId, stats, dryRun) } + catch (err) { stats.errors.push(`${folder}/${fileName} / ${block.clientName} / "${row.taskName}": ${(err as Error).message}`) } + } + for (const svc of block.additionalServices) { + try { await createAdHocTask(client.id, svc, block.effectiveDate, advocateInfo.userId, stats, dryRun) } + catch (err) { stats.errors.push(`${folder}/${fileName} / ${block.clientName} / adHoc "${svc.slice(0, 40)}": ${(err as Error).message}`) } + } + } +} + +// ─── Public entry point ─────────────────────────────────────────────────────── + +export async function runShapeImport(opts: { + dryRun: boolean + driveId: string + onLog: (line: string) => Promise +}): Promise { + const { dryRun, driveId, onLog } = opts + + const stats: ImportStats = { + filesProcessed: 0, clientsMatched: 0, clientsUnmatched: 0, + tasksUpdated: 0, tasksAssigned: 0, tasksCreated: 0, + duplicatesDeleted: 0, adHocCreated: 0, advocatesAssigned: 0, + errors: [], unmatchedClients: [], fuzzyMatches: [], + } + + await onLog(`SHAPE Historical Import — ${new Date().toISOString()}`) + await onLog(`Mode: ${dryRun ? 'DRY RUN (no changes written)' : 'EXECUTE'}`) + + const designations = await prisma.designation.findMany({ where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } } }) + const shapeDes = designations.find((d) => d.name === 'Shape') + const shape2Des = designations.find((d) => d.name === 'Shape 2' || d.name === 'Shape2') + if (!shapeDes || !shape2Des) throw new Error('Shape designations not found') + + await fixTemplates(shapeDes.id, shape2Des.id, dryRun, onLog) + const dbState = await loadDbState(onLog) + const files = await discoverFiles(driveId, onLog) + + await onLog(`\nProcessing ${files.length} files...`) + for (let i = 0; i < files.length; i++) { + const { folder, itemId, fileName } = files[i] + const pct = Math.round(((i + 1) / files.length) * 100) + await onLog(`[${pct}%] ${i + 1}/${files.length} — ${folder}/${fileName}`) + await processFile(folder, itemId, fileName, driveId, dbState, stats, dryRun, onLog) + } + + await onLog('') + await onLog('═'.repeat(60)) + await onLog('SHAPE Historical Import Report') + await onLog('═'.repeat(60)) + await onLog(`Mode: ${dryRun ? 'DRY RUN (no changes written)' : 'EXECUTE (changes written to DB)'}`) + await onLog('') + await onLog(`Files processed: ${stats.filesProcessed}`) + await onLog(`Clients matched: ${stats.clientsMatched}`) + await onLog(` - exact: ${stats.clientsMatched - stats.fuzzyMatches.length}`) + await onLog(` - fuzzy: ${stats.fuzzyMatches.length}`) + await onLog(`Clients unmatched: ${stats.clientsUnmatched}`) + await onLog(`Tasks updated: ${stats.tasksUpdated}`) + await onLog(`Tasks assigned: ${stats.tasksAssigned}`) + await onLog(`Tasks created: ${stats.tasksCreated}`) + await onLog(`Duplicates deleted: ${stats.duplicatesDeleted}`) + await onLog(`Ad-hoc created: ${stats.adHocCreated}`) + await onLog(`Advocates assigned: ${stats.advocatesAssigned}`) + await onLog(`Errors: ${stats.errors.length}`) + + if (stats.fuzzyMatches.length > 0) { + await onLog('') + await onLog('── Fuzzy Client Matches ──────────────────────────────────────') + for (const m of stats.fuzzyMatches) await onLog(` [${m.folder}] "${m.excelName}" → "${m.dbName}" (${m.method})`) + } + if (stats.unmatchedClients.length > 0) { + await onLog('') + await onLog('── Unmatched Clients (manual action required) ─────────────────') + for (const u of stats.unmatchedClients) await onLog(` [${u.folder}] ${u.file} → "${u.excelName}"`) + } + if (stats.errors.length > 0) { + await onLog('') + await onLog('── Errors ──────────────────────────────────────────────────────') + for (const e of stats.errors) await onLog(` ERROR: ${e}`) + } + await onLog('═'.repeat(60)) + + return stats +}