From 851487a8ddf06dd01ec8a5eac398b1b3416aa5b1 Mon Sep 17 00:00:00 2001 From: lorentz Date: Wed, 8 Apr 2026 10:05:10 +0000 Subject: [PATCH] fix: advocate dropdown on client list shows claims dept only --- .claude/settings.local.json | 13 +- ondeck/scripts/import-shape-historical.ts | 1541 +++++++++++++++++ ondeck/src/components/clients/client-list.tsx | 11 +- 3 files changed, 1560 insertions(+), 5 deletions(-) create mode 100644 ondeck/scripts/import-shape-historical.ts diff --git a/.claude/settings.local.json b/.claude/settings.local.json index f80e902..af600e2 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -13,7 +13,18 @@ "Bash(git push:*)", "Bash(curl -s -X POST https://login.microsoftonline.com/480673c9-b57f-4366-b0ea-e317cbc6c160/oauth2/v2.0/token -H 'Content-Type: application/x-www-form-urlencoded' -d 'client_id=f6c778cf-c850-47d0-85e7-aff1d1c3288b&client_secret=0Wi8Q~6hvmszI1W4WeK3iI~NeSixFggyhcC_Qavk&scope=https://graph.microsoft.com/.default&grant_type=client_credentials')", "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(d.get\\('id',''\\), d.get\\('name',''\\), d.get\\('error',''\\)\\)\")", - "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(json.dumps\\(d, indent=2\\)\\)\")" + "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); print\\(json.dumps\\(d, indent=2\\)\\)\")", + "Bash(psql:*)", + "Bash(npx prisma:*)", + "Bash(node -e ':*)", + "Bash(node:*)", + "Bash(grep -E \"^buffer$|^bl$|^readable\")", + "Bash(ls /opt/projects/OnDeck/ondeck/.env*)", + "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); deps={**d.get\\('dependencies',{}\\), **d.get\\('devDependencies',{}\\)}; [print\\(k,v\\) for k,v in deps.items\\(\\) if any\\(x in k.lower\\(\\) for x in ['xlsx','excel','sheet','xml','zip']\\)]\")", + "Bash(python3 -c \"import sys,json; d=json.load\\(sys.stdin\\); deps={**d.get\\('dependencies',{}\\), **d.get\\('devDependencies',{}\\)}; [print\\(k\\) for k in sorted\\(deps.keys\\(\\)\\)]\")", + "Bash(npx tsx:*)", + "Bash(tee /tmp/shape-import-console.txt)", + "Bash(python3:*)" ] } } diff --git a/ondeck/scripts/import-shape-historical.ts b/ondeck/scripts/import-shape-historical.ts new file mode 100644 index 0000000..c461272 --- /dev/null +++ b/ondeck/scripts/import-shape-historical.ts @@ -0,0 +1,1541 @@ +/** + * SHAPE Historical Data Import Script + * + * Imports historical task completion data from SharePoint Excel files into OnDeck. + * Each team member has a folder in Claims/SHAPE Accounts/{MEMBER_FOLDER}/{Client}.xlsx + * + * Usage: + * npx tsx scripts/import-shape-historical.ts # dry run + * npx tsx scripts/import-shape-historical.ts --execute # write to DB + * + * Output is written to /tmp/shape-import-report.txt (override with --output=/path) + */ + +import { PrismaClient } from '@prisma/client' +import { PrismaPg } from '@prisma/adapter-pg' +import { Pool } from 'pg' +import * as zlib from 'zlib' +import * as fs from 'fs' +import * as dotenv from 'dotenv' + +dotenv.config() + +// ─── Configuration ──────────────────────────────────────────────────────────── + +const DRY_RUN = !process.argv.includes('--execute') +const OUTPUT_FILE = + process.argv.find((a) => a.startsWith('--output='))?.split('=')[1] ?? + '/tmp/shape-import-report.txt' + +const DRIVE_ID = + 'b!OYuzIexQkkOvfEPyMJPzzZHfzTrOCOdPhTWgTlzKs6M0ZWVrAc6LR4LjWl4QFEzm' + +const TEAM_MEMBER_MAP: Record = + { + CHRIS: { + userId: 'cmkm36yc90021y7vb9783nitl', + displayName: 'Christine Gove', + }, + DAWN: { + userId: 'cmkm36xfr000ny7vbcajejjux', + displayName: 'Dawn Boland', + }, + Jeanne: { + userId: 'cmkm370f80065y7vbrugc9jub', + displayName: 'Jeanne Strong', + }, + LUKE: { + userId: 'cmkm36xd4000jy7vbhanp66s9', + displayName: 'Luke Billman', + }, + MIMI: { + userId: 'cmkm36zr0004wy7vbztyynj35', + displayName: 'Mimi Rawlings', + }, + } + +// Template name aliases: old/variant names → canonical name (after fix) +const TEMPLATE_NAME_ALIASES: Record = { + 'request 125 day loss runs': 'request 120 day loss runs', + 'request 89 day loss runs': 'request 90 day loss runs', + 'request 89 day loss runs (if being marketed)': + 'request 90 day loss runs (if being marketed)', + 'claim review (six month)': 'claim review', + 'claims review': 'claim review', +} + +// ─── DB Setup ───────────────────────────────────────────────────────────────── + +const pool = new Pool({ connectionString: process.env.DATABASE_URL }) +const adapter = new PrismaPg(pool) +const prisma = new PrismaClient({ adapter }) + +// ─── Output / Logging ───────────────────────────────────────────────────────── + +const outputLines: string[] = [] + +function log(line = '') { + console.log(line) + outputLines.push(line) +} + +function writeOutput() { + fs.writeFileSync(OUTPUT_FILE, outputLines.join('\n') + '\n', 'utf-8') + console.log(`\nReport written to ${OUTPUT_FILE}`) +} + +// ─── Types ──────────────────────────────────────────────────────────────────── + +interface ParsedRow { + taskName: string + daysAfterRenewal: number + dateCompleted: string // raw: numeric string | 'n/a' | '??' | '' + notes: string +} + +interface ParsedBlock { + clientName: string + effectiveDate: Date + isShape2: boolean + rows: ParsedRow[] + additionalServices: string[] +} + +interface DbTemplate { + id: string + name: string + daysOffset: number + designationId: string | null +} + +interface DbClient { + id: string + name: string + claimsAdvocateId: string | null +} + +interface DbState { + clientsByNormalizedName: Map + allClients: DbClient[] + shapeTemplates: DbTemplate[] + shape2Templates: DbTemplate[] + shapeDesignationId: string + shape2DesignationId: string +} + +interface ImportStats { + filesProcessed: number + clientsMatched: number + clientsUnmatched: number + tasksUpdated: number + tasksAssigned: number + tasksCreated: number + duplicatesDeleted: number + adHocCreated: number + advocatesAssigned: number + errors: string[] + unmatchedClients: Array<{ folder: string; file: string; excelName: string }> + fuzzyMatches: Array<{ + folder: string + file: string + excelName: string + dbName: string + method: string + }> +} + +// ─── Graph API ──────────────────────────────────────────────────────────────── + +let _cachedToken: { token: string; expiresAt: number } | null = null + +async function getGraphToken(): Promise { + if (_cachedToken && Date.now() < _cachedToken.expiresAt - 30_000) { + return _cachedToken.token + } + const tenantId = process.env.AZURE_AD_TENANT_ID! + const clientId = process.env.AZURE_AD_CLIENT_ID! + const clientSecret = process.env.AZURE_AD_CLIENT_SECRET! + const body = new URLSearchParams({ + grant_type: 'client_credentials', + client_id: clientId, + client_secret: clientSecret, + scope: 'https://graph.microsoft.com/.default', + }) + const res = await fetch( + `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, + { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: body.toString(), + } + ) + if (!res.ok) throw new Error(`Token fetch failed: ${res.status}`) + const data = (await res.json()) as { + access_token: string + expires_in: number + } + _cachedToken = { + token: data.access_token, + expiresAt: Date.now() + data.expires_in * 1000, + } + return _cachedToken.token +} + +async function graphGet(url: string): Promise { + const token = await getGraphToken() + const res = await fetch(url, { + headers: { Authorization: `Bearer ${token}` }, + }) + if (!res.ok) { + const text = await res.text() + throw new Error(`Graph API error ${res.status}: ${text.slice(0, 200)}`) + } + return res.json() +} + +async function downloadFile(itemId: string): Promise { + const token = await getGraphToken() + const url = `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/items/${itemId}/content` + const res = await fetch(url, { + headers: { Authorization: `Bearer ${token}` }, + redirect: 'follow', + }) + if (!res.ok) throw new Error(`Download failed: ${res.status}`) + const ab = await res.arrayBuffer() + return Buffer.from(ab) +} + +async function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)) +} + +interface DriveItem { + id: string + name: string + file?: { mimeType: string } + folder?: { childCount: number } +} + +async function listChildren(itemId: string): Promise { + const data = (await graphGet( + `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/items/${itemId}/children?$top=200` + )) as { value: DriveItem[] } + return data.value ?? [] +} + +async function discoverFiles(): Promise< + Array<{ folder: string; itemId: string; fileName: string }> +> { + log('Discovering files from SharePoint...') + const files: Array<{ folder: string; itemId: string; fileName: string }> = [] + + const rootData = (await graphGet( + `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/root:/Claims%2FSHAPE%20Accounts:/children` + )) as { value: DriveItem[] } + const rootChildren = rootData.value ?? [] + + for (const item of rootChildren) { + if (!item.folder) continue + const memberInfo = TEAM_MEMBER_MAP[item.name] + if (!memberInfo) continue + + log(` Scanning folder: ${item.name}`) + const children = await listChildren(item.id) + await sleep(150) + + for (const child of children) { + if (!child.file) continue + if (!child.name.toLowerCase().endsWith('.xlsx')) continue + if (child.name.toLowerCase().startsWith('archive')) continue + // Skip template/master files + if ( + child.name.toLowerCase().includes('template') || + child.name.toLowerCase().includes('master workbook') || + child.name.toLowerCase().startsWith('all shape') + ) + continue + + files.push({ folder: item.name, itemId: child.id, fileName: child.name }) + } + } + + log(` Found ${files.length} Excel files`) + return files +} + +// ─── ZIP Parser ─────────────────────────────────────────────────────────────── + +function parseZip(buffer: Buffer): Map { + const entries = new Map() + + // Find End of Central Directory (EOCD) record — signature 0x06054b50 + let eocdOffset = -1 + const searchStart = Math.max(0, buffer.length - 65558) + for (let i = buffer.length - 22; i >= searchStart; i--) { + if (buffer.readUInt32LE(i) === 0x06054b50) { + eocdOffset = i + break + } + } + if (eocdOffset < 0) throw new Error('Not a valid ZIP file (EOCD not found)') + + const cdOffset = buffer.readUInt32LE(eocdOffset + 16) + const cdEntries = buffer.readUInt16LE(eocdOffset + 10) + + let pos = cdOffset + for (let i = 0; i < cdEntries; i++) { + if (buffer.readUInt32LE(pos) !== 0x02014b50) { + throw new Error(`Invalid central directory entry at offset ${pos}`) + } + const method = buffer.readUInt16LE(pos + 10) + const compSize = buffer.readUInt32LE(pos + 20) + const fileNameLen = buffer.readUInt16LE(pos + 28) + const extraLen = buffer.readUInt16LE(pos + 30) + const commentLen = buffer.readUInt16LE(pos + 32) + const localOffset = buffer.readUInt32LE(pos + 42) + const fileName = buffer.slice(pos + 46, pos + 46 + fileNameLen).toString('utf-8') + + // Read local file header + if (buffer.readUInt32LE(localOffset) !== 0x04034b50) { + throw new Error(`Invalid local header for ${fileName}`) + } + const localFileNameLen = buffer.readUInt16LE(localOffset + 26) + const localExtraLen = buffer.readUInt16LE(localOffset + 28) + const dataStart = localOffset + 30 + localFileNameLen + localExtraLen + + const compData = buffer.slice(dataStart, dataStart + compSize) + let data: Buffer + if (method === 0) { + data = compData // stored + } else if (method === 8) { + data = zlib.inflateRawSync(compData) // deflate + } else { + data = Buffer.alloc(0) // unsupported, skip + } + + // Store with lowercase key for case-insensitive lookup + entries.set(fileName.toLowerCase(), data) + + pos += 46 + fileNameLen + extraLen + commentLen + } + + return entries +} + +// ─── XLSX Parsing ───────────────────────────────────────────────────────────── + +function decodeXmlEntities(s: string): string { + return s + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#x([0-9A-Fa-f]+);/gi, (_, h) => + String.fromCharCode(parseInt(h, 16)) + ) + .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10))) +} + +function parseSharedStrings(xmlBuf: Buffer | undefined): string[] { + if (!xmlBuf || xmlBuf.length === 0) return [] + const xml = xmlBuf.toString('utf-8') + const strings: string[] = [] + const siRegex = /([\s\S]*?)<\/si>/g + let match: RegExpExecArray | null + while ((match = siRegex.exec(xml)) !== null) { + const siContent = match[1] + const texts: string[] = [] + const tRegex = /]*)?>([^<]*)<\/t>/g + let tm: RegExpExecArray | null + while ((tm = tRegex.exec(siContent)) !== null) { + texts.push(decodeXmlEntities(tm[1])) + } + strings.push(texts.join('')) + } + return strings +} + +function colLetterToNum(letters: string): number { + let n = 0 + for (let i = 0; i < letters.length; i++) { + n = n * 26 + (letters.charCodeAt(i) - 64) + } + return n +} + +function numToColLetter(n: number): string { + let s = '' + while (n > 0) { + s = String.fromCharCode(((n - 1) % 26) + 65) + s + n = Math.floor((n - 1) / 26) + } + return s +} + +function parseSheetToGrid( + sheetBuf: Buffer, + sharedStrings: string[] +): { get: (row: number, col: number) => string; maxRow: number } { + const cells = new Map() + let maxRow = 0 + // Strip self-closing tags first — they represent empty cells and must not + // be confused with the opening tag of a non-empty cell that follows. + const xml = sheetBuf.toString('utf-8').replace(/]*\/>/g, '') + + // Match each ... block + const cRegex = /]*)>([\s\S]*?)<\/c>/g + let m: RegExpExecArray | null + while ((m = cRegex.exec(xml)) !== null) { + const colStr = m[1] + const rowNum = parseInt(m[2]) + const attrs = m[3] + const inner = m[4] + + if (rowNum > maxRow) maxRow = rowNum + + const typeMatch = attrs.match(/\bt="([^"]+)"/) + const type = typeMatch ? typeMatch[1] : 'n' + + let value = '' + const vMatch = inner.match(/([^<]*)<\/v>/) + const isMatch = inner.match(/]*)?>([^<]*)<\/t><\/is>/) + + if (isMatch) { + value = decodeXmlEntities(isMatch[1]) + } else if (vMatch) { + if (type === 's') { + value = sharedStrings[parseInt(vMatch[1])] ?? '' + } else if (type === 'str') { + value = decodeXmlEntities(vMatch[1]) + } else if (type === 'b') { + value = vMatch[1] === '1' ? 'TRUE' : 'FALSE' + } else { + value = vMatch[1] // number/date + } + } + + const col = colLetterToNum(colStr) + cells.set(`${rowNum}:${col}`, value) + } + + return { + get(row: number, col: number): string { + return cells.get(`${row}:${col}`) ?? '' + }, + maxRow, + } +} + +function findTargetSheet( + zipEntries: Map +): Buffer | null { + // Parse workbook.xml to find sheet r:ids, then map via rels + const wbBuf = zipEntries.get('xl/workbook.xml') + if (!wbBuf) return zipEntries.get('xl/worksheets/sheet1.xml') ?? null + + const wb = wbBuf.toString('utf-8') + const relsBuf = zipEntries.get('xl/_rels/workbook.xml.rels') + + // Build r:id → file path map from rels + // Paths in xl/_rels/workbook.xml.rels are relative to xl/ + // e.g. Target="worksheets/sheet1.xml" → actual path is "xl/worksheets/sheet1.xml" + const ridToPath = new Map() + if (relsBuf) { + const rels = relsBuf.toString('utf-8') + const relRegex = /]+Target="([^"]+)"/g + let rm: RegExpExecArray | null + while ((rm = relRegex.exec(rels)) !== null) { + let target = rm[2] + if (target.startsWith('/')) { + target = target.slice(1) // absolute paths: strip leading / + } else if (!target.startsWith('xl/')) { + target = 'xl/' + target // relative to xl/: prepend xl/ + } + ridToPath.set(rm[1], target) + } + } + + // Find sheet entries — attributes may appear in any order, parse flexibly + const sheetEntries: Array<{ name: string; rid: string }> = [] + const sheetTagRegex = /]*\/?>)/g + let sm: RegExpExecArray | null + while ((sm = sheetTagRegex.exec(wb)) !== null) { + const attrs = sm[1] + const nameMatch = attrs.match(/\bname="([^"]+)"/) + const ridMatch = attrs.match(/\br:id="([^"]+)"/) + if (nameMatch && ridMatch) { + sheetEntries.push({ name: nameMatch[1], rid: ridMatch[1] }) + } + } + + const resolveSheet = (rid: string): Buffer | null => { + const path = ridToPath.get(rid) + if (!path) return null + return zipEntries.get(path.toLowerCase()) ?? null + } + + // Prefer SHAPE2 > SHAPE > CLIENT, then fall back to first sheet + const priority = ['shape2', 'shape', 'client'] + for (const p of priority) { + const match = sheetEntries.find((s) => s.name.toLowerCase() === p) + if (match) { + const buf = resolveSheet(match.rid) + if (buf) return buf + } + } + + if (sheetEntries.length > 0) { + const buf = resolveSheet(sheetEntries[0].rid) + if (buf) return buf + } + + return zipEntries.get('xl/worksheets/sheet1.xml') ?? null +} + +function excelSerialToDate(serial: number): Date { + // Excel epoch: Jan 0, 1900 (effectively Dec 30, 1899) + // But Excel has a leap year bug treating 1900 as leap year, so offset is 25569 + return new Date((serial - 25569) * 86400 * 1000) +} + +// ─── Block Parser ───────────────────────────────────────────────────────────── + +function parseExcelBlocks( + grid: { get: (r: number, c: number) => string; maxRow: number }, + fileName: string +): ParsedBlock[] { + const blocks: ParsedBlock[] = [] + const isShape2 = /(SHAPE2|SHAPE 2)/i.test(fileName) + + let row = 1 + while (row <= grid.maxRow) { + // Scan up to 10 columns for "Client Name:" + let clientNameCol = -1 + for (let c = 1; c <= 5; c++) { + if (grid.get(row, c).trim().toLowerCase().includes('client name')) { + clientNameCol = c + break + } + } + + if (clientNameCol < 0) { + row++ + continue + } + + // Client name: first try the cell to the right, then up to 3 columns right + let clientName = '' + for (let dc = 1; dc <= 3; dc++) { + const candidate = grid.get(row, clientNameCol + dc).trim() + if (candidate && !candidate.toLowerCase().includes('client name')) { + clientName = candidate + break + } + } + // Fallback: extract from filename (strip known suffixes) + if (!clientName) { + clientName = fileName + .replace(/^SHAPE\S*\s*[-–]\s*/i, '') // strip leading "SHAPE2 - " prefix + .replace(/\s*[-–]\s*SHAPE.*$/i, '') + .replace(/\s*SHAPE\s*Services?\s*Checklist.*$/i, '') + .replace(/\s*SHAPE\b.*/i, '') + .replace(/\.xlsx$/i, '') + .trim() + } + if (!clientName) { + row++ + continue + } + + row++ + + // Find effective date (within next 5 rows) + // Strategy 1: look for "Effective Date:" label, take numeric value in adjacent cell + // Strategy 2 (fallback): look for any Excel-range serial (40000–60000) in next rows + let effectiveDateSerial = 0 + let foundDateRow = row + const EXCEL_DATE_MIN = 40000 + const EXCEL_DATE_MAX = 60000 + + for (let r = row; r <= Math.min(row + 4, grid.maxRow); r++) { + let foundDateInRow = false + for (let c = 1; c <= 8; c++) { + const cellVal = grid.get(r, c).trim() + if (cellVal.toLowerCase().includes('effective date')) { + // Date value is in next column(s) + for (let dc = 1; dc <= 4; dc++) { + const val = grid.get(r, c + dc).trim() + const serial = parseFloat(val) + if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) { + effectiveDateSerial = serial + foundDateInRow = true + break + } + } + if (foundDateInRow) break + } + // Fallback: any valid date serial in this row (not the first col which may be a number label) + if (!foundDateInRow && c > 1) { + const serial = parseFloat(cellVal) + if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) { + effectiveDateSerial = serial + foundDateInRow = true + break + } + } + } + if (foundDateInRow) { + foundDateRow = r + break + } + } + + // If no effective date found, use current date as fallback (for files like Minniefield + // where date is stored as display-only formatting, not cell value) + const effectiveDate = effectiveDateSerial > 0 + ? excelSerialToDate(effectiveDateSerial) + : new Date() + row = foundDateRow + 1 + + // Skip "Policies:" row if present + for (let r = row; r <= Math.min(row + 3, grid.maxRow); r++) { + let foundPolicies = false + for (let c = 1; c <= 5; c++) { + if (grid.get(r, c).trim().toLowerCase().startsWith('policies')) { + foundPolicies = true + break + } + } + if (foundPolicies) { + row = r + 1 + break + } + // Check if this is the header row — stop skipping + let hasTask = false + for (let c = 1; c <= 8; c++) { + if (grid.get(r, c).trim().toLowerCase() === 'task') { + hasTask = true + break + } + } + if (hasTask) break + } + + // Find header row containing "Task" + let headerRow = -1 + let taskCol = 0 + for (let r = row; r <= Math.min(row + 5, grid.maxRow); r++) { + for (let c = 1; c <= 8; c++) { + if (grid.get(r, c).trim().toLowerCase() === 'task') { + headerRow = r + taskCol = c + break + } + } + if (headerRow >= 0) break + } + + if (headerRow < 0) { + row++ + continue + } + + // Map header columns (scan right of taskCol) + let daysAfterCol = 0 + let dateCompletedCol = 0 + let notesCol = 0 + + for (let c = taskCol + 1; c <= taskCol + 8; c++) { + const hdr = grid.get(headerRow, c).trim().toLowerCase() + if (hdr.includes('days after')) daysAfterCol = c + else if ( + hdr.includes('date completed') || + hdr === 'completed' || + hdr === 'date complete' + ) + dateCompletedCol = c + else if (hdr.includes('notes')) notesCol = c + } + + row = headerRow + 1 + + // Parse data rows + const taskRows: ParsedRow[] = [] + const additionalServices: string[] = [] + let inAdditional = false + + while (row <= grid.maxRow) { + // Check if next block starts + let nextBlock = false + for (let c = 1; c <= 5; c++) { + if (grid.get(row, c).trim().toLowerCase().includes('client name')) { + nextBlock = true + break + } + } + if (nextBlock) break + + const taskName = grid.get(row, taskCol).trim() + + if (!taskName) { + row++ + continue + } + + // Detect "Additional Services Provided" section header + if (taskName.toLowerCase().includes('additional services')) { + inAdditional = true + row++ + continue + } + + const daysAfterRaw = + daysAfterCol > 0 ? grid.get(row, daysAfterCol).trim() : '' + const isDaysNumeric = daysAfterRaw !== '' && !isNaN(parseFloat(daysAfterRaw)) + + // If no numeric days value, treat as additional service / free text + if (!isDaysNumeric) { + if (taskName.length > 3) { + additionalServices.push(taskName) + } + row++ + continue + } + + if (inAdditional) { + additionalServices.push(taskName) + row++ + continue + } + + const daysAfterRenewal = parseFloat(daysAfterRaw) + const dateCompletedRaw = + dateCompletedCol > 0 ? grid.get(row, dateCompletedCol).trim() : '' + const notes = notesCol > 0 ? grid.get(row, notesCol).trim() : '' + + taskRows.push({ + taskName, + daysAfterRenewal, + dateCompleted: dateCompletedRaw, + notes, + }) + + row++ + } + + blocks.push({ + clientName, + effectiveDate, + isShape2, + rows: taskRows, + additionalServices, + }) + } + + return blocks +} + +function parseExcelFile(buf: Buffer, fileName: string): ParsedBlock[] { + const zipEntries = parseZip(buf) + const sharedStringsBuf = zipEntries.get('xl/sharedstrings.xml') + const sharedStrings = parseSharedStrings(sharedStringsBuf) + const sheetBuf = findTargetSheet(zipEntries) + if (!sheetBuf) return [] + const grid = parseSheetToGrid(sheetBuf, sharedStrings) + return parseExcelBlocks(grid, fileName) +} + +// ─── Client Name Normalization / Matching ───────────────────────────────────── + +function normalizeClientName(name: string): string { + return name + .toLowerCase() + .replace(/^the\s+/, '') + .replace(/&/g, 'and') + .replace( + /\b(incorporated|inc|llc|l\.l\.c|corp|corporation|co|ltd|limited|lp|l\.p|company|enterprises|enterprise|services|group|associates|solutions|management|consulting)\b\.?/gi, + '' + ) + .replace(/[.,;:'"()\-_#@!?]/g, '') + .replace(/\s+/g, ' ') + .trim() +} + +function tokenize(s: string): Set { + return new Set( + s + .split(/\s+/) + .filter((t) => t.length > 1) + ) +} + +function jaccardSimilarity(a: Set, b: Set): number { + if (a.size === 0 && b.size === 0) return 1 + let intersection = 0 + for (const t of a) { + if (b.has(t)) intersection++ + } + const union = a.size + b.size - intersection + return union === 0 ? 0 : intersection / union +} + +function matchClient( + excelName: string, + dbState: DbState +): { client: DbClient; method: string } | null { + const normalized = normalizeClientName(excelName) + if (!normalized) return null + + // 1. Exact normalized match + const exactCandidates = dbState.clientsByNormalizedName.get(normalized) + if (exactCandidates && exactCandidates.length > 0) { + return { client: exactCandidates[0], method: 'exact' } + } + + // 2. Contains match + for (const [key, clients] of dbState.clientsByNormalizedName) { + if (key.includes(normalized) || normalized.includes(key)) { + if (clients.length > 0) { + return { client: clients[0], method: 'contains' } + } + } + } + + // 3. Token Jaccard similarity ≥ 0.65 + const aTokens = tokenize(normalized) + let bestScore = 0 + let bestClient: DbClient | null = null + + for (const [key, clients] of dbState.clientsByNormalizedName) { + const bTokens = tokenize(key) + const score = jaccardSimilarity(aTokens, bTokens) + if (score > bestScore && score >= 0.65) { + bestScore = score + bestClient = clients[0] + } + } + + if (bestClient) { + return { client: bestClient, method: `jaccard(${bestScore.toFixed(2)})` } + } + + return null +} + +// ─── Template Matching ──────────────────────────────────────────────────────── + +function normalizeTemplateName(name: string): string { + const lowered = name.toLowerCase().trim() + return TEMPLATE_NAME_ALIASES[lowered] ?? lowered +} + +function findTemplate( + row: ParsedRow, + templates: DbTemplate[] +): DbTemplate | null { + const normalized = normalizeTemplateName(row.taskName) + + // 1. Exact normalized name match + for (const t of templates) { + if (normalizeTemplateName(t.name) === normalized) return t + } + + // 2. Contains match + for (const t of templates) { + const tn = normalizeTemplateName(t.name) + if (tn.includes(normalized) || normalized.includes(tn)) return t + } + + // 3. Days-based fallback (approximate offset from daysAfterRenewal) + // daysOffset ≈ -(365 - daysAfterRenewal) but let's use a 15-day window + if (row.daysAfterRenewal > 0) { + const approxOffset = -(365 - row.daysAfterRenewal) + let closest: DbTemplate | null = null + let closestDiff = Infinity + for (const t of templates) { + const diff = Math.abs(t.daysOffset - approxOffset) + if (diff < closestDiff && diff <= 15) { + closestDiff = diff + closest = t + } + } + if (closest) return closest + } + + return null +} + +// ─── DB State Loading ───────────────────────────────────────────────────────── + +async function loadDbState(): Promise { + log('Loading database state...') + + // Load designations + const designations = await prisma.designation.findMany({ + where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } }, + }) + const shapeDes = designations.find((d) => d.name === 'Shape') + const shape2Des = designations.find( + (d) => d.name === 'Shape 2' || d.name === 'Shape2' + ) + + if (!shapeDes || !shape2Des) { + throw new Error( + `Could not find Shape/Shape2 designations. Found: ${designations.map((d) => d.name).join(', ')}` + ) + } + + // Load templates + const allTemplates = await prisma.taskTemplate.findMany({ + where: { + designationId: { in: [shapeDes.id, shape2Des.id] }, + isActive: true, + }, + }) + + const shapeTemplates = allTemplates.filter( + (t) => t.designationId === shapeDes.id + ) + const shape2Templates = allTemplates.filter( + (t) => t.designationId === shape2Des.id + ) + + log( + ` Loaded ${shapeTemplates.length} Shape templates, ${shape2Templates.length} Shape2 templates` + ) + + // Load SHAPE clients (those with shape designation on either designation slot) + const clients = await prisma.client.findMany({ + where: { + OR: [ + { designationId: { in: [shapeDes.id, shape2Des.id] } }, + { designation2Id: { in: [shapeDes.id, shape2Des.id] } }, + ], + }, + select: { id: true, name: true, claimsAdvocateId: true }, + }) + + log(` Loaded ${clients.length} SHAPE clients`) + + // Build lookup map + const clientsByNormalizedName = new Map() + for (const c of clients) { + const key = normalizeClientName(c.name) + if (!clientsByNormalizedName.has(key)) { + clientsByNormalizedName.set(key, []) + } + clientsByNormalizedName.get(key)!.push({ + id: c.id, + name: c.name, + claimsAdvocateId: c.claimsAdvocateId, + }) + } + + return { + clientsByNormalizedName, + allClients: clients.map((c) => ({ + id: c.id, + name: c.name, + claimsAdvocateId: c.claimsAdvocateId, + })), + shapeTemplates: shapeTemplates.map((t) => ({ + id: t.id, + name: t.name, + daysOffset: t.daysOffset, + designationId: t.designationId, + })), + shape2Templates: shape2Templates.map((t) => ({ + id: t.id, + name: t.name, + daysOffset: t.daysOffset, + designationId: t.designationId, + })), + shapeDesignationId: shapeDes.id, + shape2DesignationId: shape2Des.id, + } +} + +// ─── Template Fixes ─────────────────────────────────────────────────────────── + +async function fixTemplates( + shapeDesignationId: string, + shape2DesignationId: string +): Promise { + log('\nFixing template data...') + + // Fix 1: "Request 125 day loss runs" → "Request 120 day loss runs" at -120 + const fix1 = await prisma.taskTemplate.updateMany({ + where: { name: 'Request 125 day loss runs', daysOffset: -125 }, + data: { name: 'Request 120 day loss runs', daysOffset: -120 }, + }) + if (fix1.count > 0) + log(` Fixed ${fix1.count} template(s): "Request 125..." → "Request 120..."`) + + // Fix 2: "Request 89 day loss runs" → "Request 90 day loss runs" at -90 + const fix2 = await prisma.taskTemplate.updateMany({ + where: { daysOffset: -89 }, + data: { daysOffset: -90 }, + }) + const fix2b = await prisma.taskTemplate.updateMany({ + where: { + name: { in: ['Request 89 day loss runs', 'Request 89 day loss runs (if being marketed)'] }, + }, + data: { name: undefined }, // name update handled separately since two variants exist + }) + // Update names specifically + await prisma.taskTemplate.updateMany({ + where: { name: 'Request 89 day loss runs' }, + data: { name: 'Request 90 day loss runs' }, + }) + await prisma.taskTemplate.updateMany({ + where: { name: 'Request 89 day loss runs (if being marketed)' }, + data: { name: 'Request 90 day loss runs (if being marketed)' }, + }) + if (fix2.count > 0) + log(` Fixed ${fix2.count} template(s): offset -89 → -90, updated names`) + + // Fix 3: Add missing "Claim Review" at -185 for Shape designation + const existing185 = await prisma.taskTemplate.findFirst({ + where: { + name: 'Claim Review', + daysOffset: -185, + designationId: shapeDesignationId, + }, + }) + if (!existing185) { + if (!DRY_RUN) { + await prisma.taskTemplate.create({ + data: { + name: 'Claim Review', + department: 'CLAIMS', + timing: 'PRE_RENEWAL', + daysOffset: -185, + defaultPriority: 'MEDIUM', + isActive: true, + displayOrder: 4, // after 90-day review (order 3), before mod factor (order 5) + designationId: shapeDesignationId, + }, + }) + log(' Created missing "Claim Review" template at -185 days (Shape)') + } else { + log( + ' [DRY RUN] Would create missing "Claim Review" template at -185 days (Shape)' + ) + } + } else { + log(' "Claim Review" at -185 already exists — skipping') + } +} + +// ─── Task Processing ────────────────────────────────────────────────────────── + +async function findMatchingTasks( + clientId: string, + templateId: string, + dueDate: Date +): Promise<{ match: string | null; duplicateIds: string[] }> { + const windowMs = 5 * 86400 * 1000 // ±5 days + const candidates = await prisma.task.findMany({ + where: { + clientId, + templateId, + dueDate: { + gte: new Date(dueDate.getTime() - windowMs), + lte: new Date(dueDate.getTime() + windowMs), + }, + }, + orderBy: { createdAt: 'asc' }, + select: { id: true, status: true, notes: true }, + }) + + if (candidates.length === 0) return { match: null, duplicateIds: [] } + return { + match: candidates[0].id, + duplicateIds: candidates.slice(1).map((c) => c.id), + } +} + +interface DateCompletedResult { + status: 'COMPLETED' | 'NA' | 'NOT_STARTED' + completedAt: Date | null + naReason: string | null +} + +function interpretDateCompleted( + raw: string, + advocateUserId: string +): DateCompletedResult { + const normalized = raw.trim().toLowerCase() + + if (normalized === '' || normalized === '??' || normalized === '?') { + return { status: 'NOT_STARTED', completedAt: null, naReason: null } + } + + if (normalized === 'n/a' || normalized === 'na') { + return { + status: 'NA', + completedAt: null, + naReason: 'Historical — marked N/A in SHAPE tracker', + } + } + + // Try to parse as Excel serial date + const serial = parseFloat(raw) + if (!isNaN(serial) && serial > 40000 && serial < 60000) { + return { + status: 'COMPLETED', + completedAt: excelSerialToDate(serial), + naReason: null, + } + } + + // Unknown value — treat as not started + return { status: 'NOT_STARTED', completedAt: null, naReason: null } +} + +async function processTaskRow( + clientId: string, + row: ParsedRow, + template: DbTemplate, + effectiveDate: Date, + advocateUserId: string, + stats: ImportStats +): Promise { + // Calculate expected due date: effectiveDate + daysAfterRenewal + const dueDate = new Date(effectiveDate) + dueDate.setDate(dueDate.getDate() + Math.round(row.daysAfterRenewal)) + + const { status, completedAt, naReason } = interpretDateCompleted( + row.dateCompleted, + advocateUserId + ) + + const { match: taskId, duplicateIds } = await findMatchingTasks( + clientId, + template.id, + dueDate + ) + + // Delete duplicates + if (duplicateIds.length > 0) { + if (!DRY_RUN) { + await prisma.task.deleteMany({ where: { id: { in: duplicateIds } } }) + } + stats.duplicatesDeleted += duplicateIds.length + } + + if (taskId) { + // Assign advocate to existing task (regardless of completion status) + await ensureTaskAssignment(taskId, advocateUserId, stats) + + // Update task if it has completion data + if (status === 'COMPLETED' || status === 'NA') { + const existing = await prisma.task.findUnique({ + where: { id: taskId }, + select: { status: true, notes: true }, + }) + + if (existing && existing.status === 'NOT_STARTED') { + if (!DRY_RUN) { + await prisma.task.update({ + where: { id: taskId }, + data: { + status, + completedAt: completedAt ?? undefined, + completedBy: status === 'COMPLETED' ? advocateUserId : undefined, + naReason: naReason ?? undefined, + notes: + row.notes && !existing.notes + ? row.notes + : existing.notes ?? undefined, + }, + }) + } + stats.tasksUpdated++ + } + } + } else { + // Task doesn't exist — create it if it has completion data + if (status === 'COMPLETED' || status === 'NA') { + if (!DRY_RUN) { + const newTask = await prisma.task.create({ + data: { + title: template.name, + department: 'CLAIMS', + timing: 'PRE_RENEWAL', + daysOffset: template.daysOffset, + dueDate, + status, + priority: 'MEDIUM', + clientId, + templateId: template.id, + completedAt: completedAt ?? undefined, + completedBy: status === 'COMPLETED' ? advocateUserId : undefined, + naReason: naReason ?? undefined, + notes: row.notes || undefined, + isAdHoc: false, + createdBy: advocateUserId, + }, + }) + await ensureTaskAssignment(newTask.id, advocateUserId, stats) + } else { + stats.tasksAssigned++ // count the would-be assignment + } + stats.tasksCreated++ + } else { + // NOT_STARTED and no existing task — this shouldn't happen for SHAPE clients + // but if it does, create the task and assign it + if (!DRY_RUN) { + const newTask = await prisma.task.create({ + data: { + title: template.name, + department: 'CLAIMS', + timing: 'PRE_RENEWAL', + daysOffset: template.daysOffset, + dueDate, + status: 'NOT_STARTED', + priority: 'MEDIUM', + clientId, + templateId: template.id, + notes: row.notes || undefined, + isAdHoc: false, + createdBy: advocateUserId, + }, + }) + await ensureTaskAssignment(newTask.id, advocateUserId, stats) + } else { + stats.tasksAssigned++ + } + stats.tasksCreated++ + } + } +} + +async function ensureTaskAssignment( + taskId: string, + userId: string, + stats: ImportStats +): Promise { + if (DRY_RUN) { + stats.tasksAssigned++ + return + } + try { + await prisma.taskAssignment.upsert({ + where: { taskId_userId: { taskId, userId } }, + create: { taskId, userId }, + update: {}, + }) + stats.tasksAssigned++ + } catch { + // Unique constraint violation means it already exists — fine + } +} + +async function createAdHocTask( + clientId: string, + text: string, + effectiveDate: Date, + advocateUserId: string, + stats: ImportStats +): Promise { + const title = text.slice(0, 500) + + // Idempotency check + const existing = await prisma.task.findFirst({ + where: { clientId, title, isAdHoc: true }, + }) + if (existing) return + + if (!DRY_RUN) { + const task = await prisma.task.create({ + data: { + title, + department: 'CLAIMS', + timing: 'PRE_RENEWAL', + daysOffset: 0, + dueDate: effectiveDate, + status: 'COMPLETED', + priority: 'MEDIUM', + clientId, + isAdHoc: true, + completedAt: effectiveDate, + completedBy: advocateUserId, + notes: text, + createdBy: advocateUserId, + }, + }) + await ensureTaskAssignment(task.id, advocateUserId, stats) + } else { + stats.tasksAssigned++ + } + stats.adHocCreated++ +} + +async function assignAdvocate( + clientId: string, + advocateUserId: string, + currentAdvocateId: string | null, + stats: ImportStats +): Promise { + if (currentAdvocateId) return // Never overwrite existing assignment + if (!DRY_RUN) { + await prisma.client.update({ + where: { id: clientId }, + data: { claimsAdvocateId: advocateUserId }, + }) + } + stats.advocatesAssigned++ +} + +// ─── File Processing ────────────────────────────────────────────────────────── + +async function processFile( + folder: string, + itemId: string, + fileName: string, + dbState: DbState, + stats: ImportStats +): Promise { + const advocateInfo = TEAM_MEMBER_MAP[folder]! + + let buf: Buffer + try { + buf = await downloadFile(itemId) + await sleep(150) + } catch (err) { + const msg = `${folder}/${fileName}: Download failed — ${(err as Error).message}` + stats.errors.push(msg) + return + } + + let blocks: ParsedBlock[] + try { + blocks = parseExcelFile(buf, fileName) + } catch (err) { + const msg = `${folder}/${fileName}: Parse failed — ${(err as Error).message}` + stats.errors.push(msg) + return + } + + if (blocks.length === 0) { + stats.errors.push(`${folder}/${fileName}: No data blocks found`) + return + } + + stats.filesProcessed++ + + for (const block of blocks) { + // Match client + const matchResult = matchClient(block.clientName, dbState) + if (!matchResult) { + stats.clientsUnmatched++ + stats.unmatchedClients.push({ + folder, + file: fileName, + excelName: block.clientName, + }) + continue + } + + const { client, method } = matchResult + stats.clientsMatched++ + + if (method !== 'exact') { + stats.fuzzyMatches.push({ + folder, + file: fileName, + excelName: block.clientName, + dbName: client.name, + method, + }) + } + + // Assign advocate at client level + await assignAdvocate( + client.id, + advocateInfo.userId, + client.claimsAdvocateId, + stats + ) + + // Update our cached advocate status so subsequent blocks don't re-assign + client.claimsAdvocateId = client.claimsAdvocateId ?? advocateInfo.userId + + // Choose templates based on SHAPE vs SHAPE2 + const templates = block.isShape2 + ? dbState.shape2Templates + : dbState.shapeTemplates + + // Process each task row + for (const row of block.rows) { + const template = findTemplate(row, templates) + if (!template) { + // Unmatched template — add as additional service if it has completion data + const { status } = interpretDateCompleted(row.dateCompleted, advocateInfo.userId) + if (status === 'COMPLETED' || status === 'NA') { + const adHocTitle = `[Unmatched Task] ${row.taskName}${row.notes ? ': ' + row.notes : ''}` + await createAdHocTask( + client.id, + adHocTitle, + block.effectiveDate, + advocateInfo.userId, + stats + ) + } + continue + } + + try { + await processTaskRow( + client.id, + row, + template, + block.effectiveDate, + advocateInfo.userId, + stats + ) + } catch (err) { + stats.errors.push( + `${folder}/${fileName} / ${block.clientName} / "${row.taskName}": ${(err as Error).message}` + ) + } + } + + // Create ad-hoc tasks for additional services + for (const svc of block.additionalServices) { + try { + await createAdHocTask( + client.id, + svc, + block.effectiveDate, + advocateInfo.userId, + stats + ) + } catch (err) { + stats.errors.push( + `${folder}/${fileName} / ${block.clientName} / adHoc "${svc.slice(0, 40)}": ${(err as Error).message}` + ) + } + } + } +} + +// ─── Report ─────────────────────────────────────────────────────────────────── + +function printReport(stats: ImportStats): void { + log('') + log('═'.repeat(60)) + log('SHAPE Historical Import Report') + log('═'.repeat(60)) + log(`Mode: ${DRY_RUN ? 'DRY RUN (no changes written)' : 'EXECUTE (changes written to DB)'}`) + log('') + log(`Files processed: ${stats.filesProcessed}`) + log(`Clients matched: ${stats.clientsMatched}`) + log(` - exact: ${stats.clientsMatched - stats.fuzzyMatches.length}`) + log(` - fuzzy: ${stats.fuzzyMatches.length}`) + log(`Clients unmatched: ${stats.clientsUnmatched}`) + log(`Tasks updated: ${stats.tasksUpdated} (completed/NA status set)`) + log(`Tasks assigned: ${stats.tasksAssigned} (advocate linked)`) + log(`Tasks created: ${stats.tasksCreated} (missing from DB)`) + log(`Duplicates deleted: ${stats.duplicatesDeleted}`) + log(`Ad-hoc created: ${stats.adHocCreated}`) + log(`Advocates assigned: ${stats.advocatesAssigned} (client-level)`) + log(`Errors: ${stats.errors.length}`) + + if (stats.fuzzyMatches.length > 0) { + log('') + log('── Fuzzy Client Matches (review for correctness) ──────────────') + for (const m of stats.fuzzyMatches) { + log(` [${m.folder}] "${m.excelName}" → "${m.dbName}" (${m.method})`) + } + } + + if (stats.unmatchedClients.length > 0) { + log('') + log('── Unmatched Clients (manual action required) ──────────────────') + for (const u of stats.unmatchedClients) { + log(` [${u.folder}] ${u.file} → "${u.excelName}"`) + } + } + + if (stats.errors.length > 0) { + log('') + log('── Errors ──────────────────────────────────────────────────────') + for (const e of stats.errors) { + log(` ERROR: ${e}`) + } + } + + log('') + log('═'.repeat(60)) +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + log(`SHAPE Historical Import — ${new Date().toISOString()}`) + log( + `Mode: ${DRY_RUN ? 'DRY RUN (pass --execute to write)' : 'EXECUTE'}` + ) + + const stats: ImportStats = { + filesProcessed: 0, + clientsMatched: 0, + clientsUnmatched: 0, + tasksUpdated: 0, + tasksAssigned: 0, + tasksCreated: 0, + duplicatesDeleted: 0, + adHocCreated: 0, + advocatesAssigned: 0, + errors: [], + unmatchedClients: [], + fuzzyMatches: [], + } + + // Step 1: Fix template data + const designations = await prisma.designation.findMany({ + where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } }, + }) + const shapeDes = designations.find((d) => d.name === 'Shape') + const shape2Des = designations.find( + (d) => d.name === 'Shape 2' || d.name === 'Shape2' + ) + if (!shapeDes || !shape2Des) throw new Error('Shape designations not found') + + await fixTemplates(shapeDes.id, shape2Des.id) + + // Step 2: Load DB state + const dbState = await loadDbState() + + // Step 3: Discover SharePoint files + const files = await discoverFiles() + + // Step 4: Process each file + log(`\nProcessing ${files.length} files...`) + for (let i = 0; i < files.length; i++) { + const { folder, itemId, fileName } = files[i] + const pct = Math.round(((i + 1) / files.length) * 100) + process.stdout.write( + `\r [${pct}%] ${i + 1}/${files.length} — ${folder}/${fileName.slice(0, 40)}` + ) + await processFile(folder, itemId, fileName, dbState, stats) + } + process.stdout.write('\n') + + // Step 5: Print report + printReport(stats) + writeOutput() +} + +main() + .catch((e) => { + console.error('\nFatal error:', e) + process.exit(1) + }) + .finally(async () => { + await prisma.$disconnect() + await pool.end() + }) diff --git a/ondeck/src/components/clients/client-list.tsx b/ondeck/src/components/clients/client-list.tsx index ef704c9..6e73c4c 100644 --- a/ondeck/src/components/clients/client-list.tsx +++ b/ondeck/src/components/clients/client-list.tsx @@ -21,6 +21,7 @@ interface SimpleUser { id: string displayName: string | null email: string + department?: string | null } interface ClientWithRelations extends Client { @@ -82,7 +83,7 @@ export function ClientList({ initialClients = [], designations = [] }: ClientLis } useEffect(() => { - fetch('/api/users?isActive=true&limit=200') + fetch('/api/users?isActive=true&limit=200&fields=department') .then((r) => r.json()) .then((d) => setUsers(d.users || [])) .catch(() => {}) @@ -142,9 +143,11 @@ export function ClientList({ initialClients = [], designations = [] }: ClientLis All Advocates - {users.map((u) => ( - {u.displayName || u.email} - ))} + {users + .filter((u) => u.department?.toLowerCase().includes('claims')) + .map((u) => ( + {u.displayName || u.email} + ))}