lorentz
ca2c9c701d
- New DB tables: shape_import_runs, shape_import_logs (raw SQL applied) - src/lib/shape-import/run-import.ts: importable core logic extracted from CLI script - POST /api/admin/shape-import: starts run async, returns runId immediately - GET /api/admin/shape-import: lists last 20 runs - GET /api/admin/shape-import/[id]: polls log lines and run status - /admin/shape-import page: drive ID config, dry run / execute (with confirmation), live log panel (2s poll), stats summary, run history with log replay - Admin index: added SHAPE Historical Import card
770 lines
36 KiB
TypeScript
770 lines
36 KiB
TypeScript
/**
|
||
* SHAPE Historical Import — core logic, importable by both CLI and API.
|
||
*
|
||
* Call runShapeImport({ dryRun, driveId, onLog }) to execute.
|
||
*/
|
||
|
||
import * as zlib from 'zlib'
|
||
import { prisma } from '@/lib/db'
|
||
|
||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||
|
||
export interface ImportStats {
|
||
filesProcessed: number
|
||
clientsMatched: number
|
||
clientsUnmatched: number
|
||
tasksUpdated: number
|
||
tasksAssigned: number
|
||
tasksCreated: number
|
||
duplicatesDeleted: number
|
||
adHocCreated: number
|
||
advocatesAssigned: number
|
||
errors: string[]
|
||
unmatchedClients: Array<{ folder: string; file: string; excelName: string }>
|
||
fuzzyMatches: Array<{
|
||
folder: string
|
||
file: string
|
||
excelName: string
|
||
dbName: string
|
||
method: string
|
||
}>
|
||
}
|
||
|
||
interface ParsedRow {
|
||
taskName: string
|
||
daysAfterRenewal: number
|
||
dateCompleted: string
|
||
notes: string
|
||
}
|
||
|
||
interface ParsedBlock {
|
||
clientName: string
|
||
effectiveDate: Date
|
||
isShape2: boolean
|
||
rows: ParsedRow[]
|
||
additionalServices: string[]
|
||
}
|
||
|
||
interface DbTemplate {
|
||
id: string
|
||
name: string
|
||
daysOffset: number
|
||
designationId: string | null
|
||
}
|
||
|
||
interface DbClient {
|
||
id: string
|
||
name: string
|
||
claimsAdvocateId: string | null
|
||
}
|
||
|
||
interface DbState {
|
||
clientsByNormalizedName: Map<string, DbClient[]>
|
||
allClients: DbClient[]
|
||
shapeTemplates: DbTemplate[]
|
||
shape2Templates: DbTemplate[]
|
||
shapeDesignationId: string
|
||
shape2DesignationId: string
|
||
}
|
||
|
||
interface DriveItem {
|
||
id: string
|
||
name: string
|
||
file?: { mimeType: string }
|
||
folder?: { childCount: number }
|
||
}
|
||
|
||
// ─── Team member map ──────────────────────────────────────────────────────────
|
||
|
||
export const TEAM_MEMBER_MAP: Record<string, { userId: string; displayName: string }> = {
|
||
CHRIS: { userId: 'cmkm36yc90021y7vb9783nitl', displayName: 'Christine Gove' },
|
||
DAWN: { userId: 'cmkm36xfr000ny7vbcajejjux', displayName: 'Dawn Boland' },
|
||
Jeanne: { userId: 'cmkm370f80065y7vbrugc9jub', displayName: 'Jeanne Strong' },
|
||
LUKE: { userId: 'cmkm36xd4000jy7vbhanp66s9', displayName: 'Luke Billman' },
|
||
MIMI: { userId: 'cmkm36zr0004wy7vbztyynj35', displayName: 'Mimi Rawlings' },
|
||
}
|
||
|
||
const TEMPLATE_NAME_ALIASES: Record<string, string> = {
|
||
'request 125 day loss runs': 'request 120 day loss runs',
|
||
'request 89 day loss runs': 'request 90 day loss runs',
|
||
'request 89 day loss runs (if being marketed)': 'request 90 day loss runs (if being marketed)',
|
||
'claim review (six month)': 'claim review',
|
||
'claims review': 'claim review',
|
||
}
|
||
|
||
// ─── Graph API ────────────────────────────────────────────────────────────────
|
||
|
||
let _cachedToken: { token: string; expiresAt: number } | null = null
|
||
|
||
async function getGraphToken(): Promise<string> {
|
||
if (_cachedToken && Date.now() < _cachedToken.expiresAt - 30_000) return _cachedToken.token
|
||
const tenantId = process.env.AZURE_AD_TENANT_ID!
|
||
const clientId = process.env.AZURE_AD_CLIENT_ID!
|
||
const clientSecret = process.env.AZURE_AD_CLIENT_SECRET!
|
||
const body = new URLSearchParams({
|
||
grant_type: 'client_credentials',
|
||
client_id: clientId,
|
||
client_secret: clientSecret,
|
||
scope: 'https://graph.microsoft.com/.default',
|
||
})
|
||
const res = await fetch(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: body.toString(),
|
||
})
|
||
if (!res.ok) throw new Error(`Token fetch failed: ${res.status}`)
|
||
const data = (await res.json()) as { access_token: string; expires_in: number }
|
||
_cachedToken = { token: data.access_token, expiresAt: Date.now() + data.expires_in * 1000 }
|
||
return _cachedToken.token
|
||
}
|
||
|
||
async function graphGet(url: string): Promise<unknown> {
|
||
const token = await getGraphToken()
|
||
const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` } })
|
||
if (!res.ok) {
|
||
const text = await res.text()
|
||
throw new Error(`Graph API error ${res.status}: ${text.slice(0, 200)}`)
|
||
}
|
||
return res.json()
|
||
}
|
||
|
||
async function downloadFile(driveId: string, itemId: string): Promise<Buffer> {
|
||
const token = await getGraphToken()
|
||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`
|
||
const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` }, redirect: 'follow' })
|
||
if (!res.ok) throw new Error(`Download failed: ${res.status}`)
|
||
const ab = await res.arrayBuffer()
|
||
return Buffer.from(ab)
|
||
}
|
||
|
||
async function sleep(ms: number) {
|
||
return new Promise((r) => setTimeout(r, ms))
|
||
}
|
||
|
||
async function listChildren(driveId: string, itemId: string): Promise<DriveItem[]> {
|
||
const data = (await graphGet(
|
||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/children?$top=200`
|
||
)) as { value: DriveItem[] }
|
||
return data.value ?? []
|
||
}
|
||
|
||
async function discoverFiles(
|
||
driveId: string,
|
||
log: (line: string) => Promise<void>
|
||
): Promise<Array<{ folder: string; itemId: string; fileName: string }>> {
|
||
await log('Discovering files from SharePoint...')
|
||
const files: Array<{ folder: string; itemId: string; fileName: string }> = []
|
||
|
||
const rootData = (await graphGet(
|
||
`https://graph.microsoft.com/v1.0/drives/${driveId}/root:/Claims%2FSHAPE%20Accounts:/children`
|
||
)) as { value: DriveItem[] }
|
||
const rootChildren = rootData.value ?? []
|
||
|
||
for (const item of rootChildren) {
|
||
if (!item.folder) continue
|
||
const memberInfo = TEAM_MEMBER_MAP[item.name]
|
||
if (!memberInfo) continue
|
||
await log(` Scanning folder: ${item.name}`)
|
||
const children = await listChildren(driveId, item.id)
|
||
await sleep(150)
|
||
for (const child of children) {
|
||
if (!child.file) continue
|
||
if (!child.name.toLowerCase().endsWith('.xlsx')) continue
|
||
if (child.name.toLowerCase().startsWith('archive')) continue
|
||
if (
|
||
child.name.toLowerCase().includes('template') ||
|
||
child.name.toLowerCase().includes('master workbook') ||
|
||
child.name.toLowerCase().startsWith('all shape')
|
||
) continue
|
||
files.push({ folder: item.name, itemId: child.id, fileName: child.name })
|
||
}
|
||
}
|
||
|
||
await log(` Found ${files.length} Excel files`)
|
||
return files
|
||
}
|
||
|
||
// ─── ZIP / XLSX Parser ────────────────────────────────────────────────────────
|
||
|
||
function parseZip(buffer: Buffer): Map<string, Buffer> {
|
||
const entries = new Map<string, Buffer>()
|
||
let eocdOffset = -1
|
||
const searchStart = Math.max(0, buffer.length - 65558)
|
||
for (let i = buffer.length - 22; i >= searchStart; i--) {
|
||
if (buffer.readUInt32LE(i) === 0x06054b50) { eocdOffset = i; break }
|
||
}
|
||
if (eocdOffset < 0) throw new Error('Not a valid ZIP file (EOCD not found)')
|
||
const cdOffset = buffer.readUInt32LE(eocdOffset + 16)
|
||
const cdEntries = buffer.readUInt16LE(eocdOffset + 10)
|
||
let pos = cdOffset
|
||
for (let i = 0; i < cdEntries; i++) {
|
||
if (buffer.readUInt32LE(pos) !== 0x02014b50) throw new Error(`Invalid central directory entry at offset ${pos}`)
|
||
const method = buffer.readUInt16LE(pos + 10)
|
||
const compSize = buffer.readUInt32LE(pos + 20)
|
||
const fileNameLen = buffer.readUInt16LE(pos + 28)
|
||
const extraLen = buffer.readUInt16LE(pos + 30)
|
||
const commentLen = buffer.readUInt16LE(pos + 32)
|
||
const localOffset = buffer.readUInt32LE(pos + 42)
|
||
const fileName = buffer.slice(pos + 46, pos + 46 + fileNameLen).toString('utf-8')
|
||
if (buffer.readUInt32LE(localOffset) !== 0x04034b50) throw new Error(`Invalid local header for ${fileName}`)
|
||
const localFileNameLen = buffer.readUInt16LE(localOffset + 26)
|
||
const localExtraLen = buffer.readUInt16LE(localOffset + 28)
|
||
const dataStart = localOffset + 30 + localFileNameLen + localExtraLen
|
||
const compData = buffer.slice(dataStart, dataStart + compSize)
|
||
let data: Buffer
|
||
if (method === 0) data = compData
|
||
else if (method === 8) data = zlib.inflateRawSync(compData)
|
||
else data = Buffer.alloc(0)
|
||
entries.set(fileName.toLowerCase(), data)
|
||
pos += 46 + fileNameLen + extraLen + commentLen
|
||
}
|
||
return entries
|
||
}
|
||
|
||
function decodeXmlEntities(s: string): string {
|
||
return s
|
||
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||
.replace(/"/g, '"').replace(/'/g, "'")
|
||
.replace(/&#x([0-9A-Fa-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
|
||
.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
|
||
}
|
||
|
||
function parseSharedStrings(xmlBuf: Buffer | undefined): string[] {
|
||
if (!xmlBuf || xmlBuf.length === 0) return []
|
||
const xml = xmlBuf.toString('utf-8')
|
||
const strings: string[] = []
|
||
const siRegex = /<si>([\s\S]*?)<\/si>/g
|
||
let match: RegExpExecArray | null
|
||
while ((match = siRegex.exec(xml)) !== null) {
|
||
const texts: string[] = []
|
||
const tRegex = /<t(?:\s[^>]*)?>([^<]*)<\/t>/g
|
||
let tm: RegExpExecArray | null
|
||
while ((tm = tRegex.exec(match[1])) !== null) texts.push(decodeXmlEntities(tm[1]))
|
||
strings.push(texts.join(''))
|
||
}
|
||
return strings
|
||
}
|
||
|
||
function colLetterToNum(letters: string): number {
|
||
let n = 0
|
||
for (let i = 0; i < letters.length; i++) n = n * 26 + (letters.charCodeAt(i) - 64)
|
||
return n
|
||
}
|
||
|
||
function parseSheetToGrid(sheetBuf: Buffer, sharedStrings: string[]): { get: (row: number, col: number) => string; maxRow: number } {
|
||
const cells = new Map<string, string>()
|
||
let maxRow = 0
|
||
const xml = sheetBuf.toString('utf-8').replace(/<c\b[^>]*\/>/g, '')
|
||
const cRegex = /<c\s+r="([A-Z]+)(\d+)"([^>]*)>([\s\S]*?)<\/c>/g
|
||
let m: RegExpExecArray | null
|
||
while ((m = cRegex.exec(xml)) !== null) {
|
||
const colStr = m[1]; const rowNum = parseInt(m[2]); const attrs = m[3]; const inner = m[4]
|
||
if (rowNum > maxRow) maxRow = rowNum
|
||
const typeMatch = attrs.match(/\bt="([^"]+)"/); const type = typeMatch ? typeMatch[1] : 'n'
|
||
let value = ''
|
||
const vMatch = inner.match(/<v>([^<]*)<\/v>/)
|
||
const isMatch = inner.match(/<is><t(?:[^>]*)?>([^<]*)<\/t><\/is>/)
|
||
if (isMatch) value = decodeXmlEntities(isMatch[1])
|
||
else if (vMatch) {
|
||
if (type === 's') value = sharedStrings[parseInt(vMatch[1])] ?? ''
|
||
else if (type === 'str') value = decodeXmlEntities(vMatch[1])
|
||
else if (type === 'b') value = vMatch[1] === '1' ? 'TRUE' : 'FALSE'
|
||
else value = vMatch[1]
|
||
}
|
||
cells.set(`${rowNum}:${colLetterToNum(colStr)}`, value)
|
||
}
|
||
return { get(row, col) { return cells.get(`${row}:${col}`) ?? '' }, maxRow }
|
||
}
|
||
|
||
function findTargetSheet(zipEntries: Map<string, Buffer>): Buffer | null {
|
||
const wbBuf = zipEntries.get('xl/workbook.xml')
|
||
if (!wbBuf) return zipEntries.get('xl/worksheets/sheet1.xml') ?? null
|
||
const wb = wbBuf.toString('utf-8')
|
||
const relsBuf = zipEntries.get('xl/_rels/workbook.xml.rels')
|
||
const ridToPath = new Map<string, string>()
|
||
if (relsBuf) {
|
||
const rels = relsBuf.toString('utf-8')
|
||
const relRegex = /<Relationship\s+Id="([^"]+)"[^>]+Target="([^"]+)"/g
|
||
let rm: RegExpExecArray | null
|
||
while ((rm = relRegex.exec(rels)) !== null) {
|
||
let target = rm[2]
|
||
if (target.startsWith('/')) target = target.slice(1)
|
||
else if (!target.startsWith('xl/')) target = 'xl/' + target
|
||
ridToPath.set(rm[1], target)
|
||
}
|
||
}
|
||
const sheetEntries: Array<{ name: string; rid: string }> = []
|
||
const sheetTagRegex = /<sheet\b([^>]*\/?>)/g
|
||
let sm: RegExpExecArray | null
|
||
while ((sm = sheetTagRegex.exec(wb)) !== null) {
|
||
const attrs = sm[1]
|
||
const nameMatch = attrs.match(/\bname="([^"]+)"/)
|
||
const ridMatch = attrs.match(/\br:id="([^"]+)"/)
|
||
if (nameMatch && ridMatch) sheetEntries.push({ name: nameMatch[1], rid: ridMatch[1] })
|
||
}
|
||
const resolveSheet = (rid: string): Buffer | null => {
|
||
const path = ridToPath.get(rid); if (!path) return null
|
||
return zipEntries.get(path.toLowerCase()) ?? null
|
||
}
|
||
for (const p of ['shape2', 'shape', 'client']) {
|
||
const match = sheetEntries.find((s) => s.name.toLowerCase() === p)
|
||
if (match) { const buf = resolveSheet(match.rid); if (buf) return buf }
|
||
}
|
||
if (sheetEntries.length > 0) { const buf = resolveSheet(sheetEntries[0].rid); if (buf) return buf }
|
||
return zipEntries.get('xl/worksheets/sheet1.xml') ?? null
|
||
}
|
||
|
||
function excelSerialToDate(serial: number): Date {
|
||
return new Date((serial - 25569) * 86400 * 1000)
|
||
}
|
||
|
||
function parseExcelBlocks(
|
||
grid: { get: (r: number, c: number) => string; maxRow: number },
|
||
fileName: string
|
||
): ParsedBlock[] {
|
||
const blocks: ParsedBlock[] = []
|
||
const isShape2 = /(SHAPE2|SHAPE 2)/i.test(fileName)
|
||
let row = 1
|
||
while (row <= grid.maxRow) {
|
||
let clientNameCol = -1
|
||
for (let c = 1; c <= 5; c++) {
|
||
if (grid.get(row, c).trim().toLowerCase().includes('client name')) { clientNameCol = c; break }
|
||
}
|
||
if (clientNameCol < 0) { row++; continue }
|
||
let clientName = ''
|
||
for (let dc = 1; dc <= 3; dc++) {
|
||
const candidate = grid.get(row, clientNameCol + dc).trim()
|
||
if (candidate && !candidate.toLowerCase().includes('client name')) { clientName = candidate; break }
|
||
}
|
||
if (!clientName) {
|
||
clientName = fileName
|
||
.replace(/^SHAPE\S*\s*[-–]\s*/i, '').replace(/\s*[-–]\s*SHAPE.*$/i, '')
|
||
.replace(/\s*SHAPE\s*Services?\s*Checklist.*$/i, '').replace(/\s*SHAPE\b.*/i, '')
|
||
.replace(/\.xlsx$/i, '').trim()
|
||
}
|
||
if (!clientName) { row++; continue }
|
||
row++
|
||
let effectiveDateSerial = 0; let foundDateRow = row
|
||
const EXCEL_DATE_MIN = 40000; const EXCEL_DATE_MAX = 60000
|
||
for (let r = row; r <= Math.min(row + 4, grid.maxRow); r++) {
|
||
let foundDateInRow = false
|
||
for (let c = 1; c <= 8; c++) {
|
||
const cellVal = grid.get(r, c).trim()
|
||
if (cellVal.toLowerCase().includes('effective date')) {
|
||
for (let dc = 1; dc <= 4; dc++) {
|
||
const val = grid.get(r, c + dc).trim(); const serial = parseFloat(val)
|
||
if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) {
|
||
effectiveDateSerial = serial; foundDateInRow = true; break
|
||
}
|
||
}
|
||
if (foundDateInRow) break
|
||
}
|
||
if (!foundDateInRow && c > 1) {
|
||
const serial = parseFloat(cellVal)
|
||
if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) {
|
||
effectiveDateSerial = serial; foundDateInRow = true; break
|
||
}
|
||
}
|
||
}
|
||
if (foundDateInRow) { foundDateRow = r; break }
|
||
}
|
||
const effectiveDate = effectiveDateSerial > 0 ? excelSerialToDate(effectiveDateSerial) : new Date()
|
||
row = foundDateRow + 1
|
||
for (let r = row; r <= Math.min(row + 3, grid.maxRow); r++) {
|
||
let foundPolicies = false
|
||
for (let c = 1; c <= 5; c++) {
|
||
if (grid.get(r, c).trim().toLowerCase().startsWith('policies')) { foundPolicies = true; break }
|
||
}
|
||
if (foundPolicies) { row = r + 1; break }
|
||
let hasTask = false
|
||
for (let c = 1; c <= 8; c++) {
|
||
if (grid.get(r, c).trim().toLowerCase() === 'task') { hasTask = true; break }
|
||
}
|
||
if (hasTask) break
|
||
}
|
||
let headerRow = -1; let taskCol = 0
|
||
for (let r = row; r <= Math.min(row + 5, grid.maxRow); r++) {
|
||
for (let c = 1; c <= 8; c++) {
|
||
if (grid.get(r, c).trim().toLowerCase() === 'task') { headerRow = r; taskCol = c; break }
|
||
}
|
||
if (headerRow >= 0) break
|
||
}
|
||
if (headerRow < 0) { row++; continue }
|
||
let daysAfterCol = 0; let dateCompletedCol = 0; let notesCol = 0
|
||
for (let c = taskCol + 1; c <= taskCol + 8; c++) {
|
||
const hdr = grid.get(headerRow, c).trim().toLowerCase()
|
||
if (hdr.includes('days after')) daysAfterCol = c
|
||
else if (hdr.includes('date completed') || hdr === 'completed' || hdr === 'date complete') dateCompletedCol = c
|
||
else if (hdr.includes('notes')) notesCol = c
|
||
}
|
||
row = headerRow + 1
|
||
const taskRows: ParsedRow[] = []; const additionalServices: string[] = []; let inAdditional = false
|
||
while (row <= grid.maxRow) {
|
||
let nextBlock = false
|
||
for (let c = 1; c <= 5; c++) {
|
||
if (grid.get(row, c).trim().toLowerCase().includes('client name')) { nextBlock = true; break }
|
||
}
|
||
if (nextBlock) break
|
||
const taskName = grid.get(row, taskCol).trim()
|
||
if (!taskName) { row++; continue }
|
||
if (taskName.toLowerCase().includes('additional services')) { inAdditional = true; row++; continue }
|
||
const daysAfterRaw = daysAfterCol > 0 ? grid.get(row, daysAfterCol).trim() : ''
|
||
const isDaysNumeric = daysAfterRaw !== '' && !isNaN(parseFloat(daysAfterRaw))
|
||
if (!isDaysNumeric) { if (taskName.length > 3) additionalServices.push(taskName); row++; continue }
|
||
if (inAdditional) { additionalServices.push(taskName); row++; continue }
|
||
const daysAfterRenewal = parseFloat(daysAfterRaw)
|
||
const dateCompletedRaw = dateCompletedCol > 0 ? grid.get(row, dateCompletedCol).trim() : ''
|
||
const notes = notesCol > 0 ? grid.get(row, notesCol).trim() : ''
|
||
taskRows.push({ taskName, daysAfterRenewal, dateCompleted: dateCompletedRaw, notes })
|
||
row++
|
||
}
|
||
blocks.push({ clientName, effectiveDate, isShape2, rows: taskRows, additionalServices })
|
||
}
|
||
return blocks
|
||
}
|
||
|
||
function parseExcelFile(buf: Buffer, fileName: string): ParsedBlock[] {
|
||
const zipEntries = parseZip(buf)
|
||
const sharedStrings = parseSharedStrings(zipEntries.get('xl/sharedstrings.xml'))
|
||
const sheetBuf = findTargetSheet(zipEntries)
|
||
if (!sheetBuf) return []
|
||
return parseExcelBlocks(parseSheetToGrid(sheetBuf, sharedStrings), fileName)
|
||
}
|
||
|
||
// ─── Client name matching ─────────────────────────────────────────────────────
|
||
|
||
function normalizeClientName(name: string): string {
|
||
return name.toLowerCase().replace(/^the\s+/, '').replace(/&/g, 'and')
|
||
.replace(/\b(incorporated|inc|llc|l\.l\.c|corp|corporation|co|ltd|limited|lp|l\.p|company|enterprises|enterprise|services|group|associates|solutions|management|consulting)\b\.?/gi, '')
|
||
.replace(/[.,;:'"()\-_#@!?]/g, '').replace(/\s+/g, ' ').trim()
|
||
}
|
||
|
||
function tokenize(s: string): Set<string> {
|
||
return new Set(s.split(/\s+/).filter((t) => t.length > 1))
|
||
}
|
||
|
||
function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
|
||
if (a.size === 0 && b.size === 0) return 1
|
||
let intersection = 0
|
||
for (const t of a) if (b.has(t)) intersection++
|
||
const union = a.size + b.size - intersection
|
||
return union === 0 ? 0 : intersection / union
|
||
}
|
||
|
||
function matchClient(excelName: string, dbState: DbState): { client: DbClient; method: string } | null {
|
||
const normalized = normalizeClientName(excelName)
|
||
if (!normalized) return null
|
||
const exactCandidates = dbState.clientsByNormalizedName.get(normalized)
|
||
if (exactCandidates && exactCandidates.length > 0) return { client: exactCandidates[0], method: 'exact' }
|
||
for (const [key, clients] of dbState.clientsByNormalizedName) {
|
||
if (key.includes(normalized) || normalized.includes(key)) {
|
||
if (clients.length > 0) return { client: clients[0], method: 'contains' }
|
||
}
|
||
}
|
||
const aTokens = tokenize(normalized); let bestScore = 0; let bestClient: DbClient | null = null
|
||
for (const [key, clients] of dbState.clientsByNormalizedName) {
|
||
const score = jaccardSimilarity(aTokens, tokenize(key))
|
||
if (score > bestScore && score >= 0.65) { bestScore = score; bestClient = clients[0] }
|
||
}
|
||
if (bestClient) return { client: bestClient, method: `jaccard(${bestScore.toFixed(2)})` }
|
||
return null
|
||
}
|
||
|
||
// ─── Template matching ────────────────────────────────────────────────────────
|
||
|
||
function normalizeTemplateName(name: string): string {
|
||
const lowered = name.toLowerCase().trim()
|
||
return TEMPLATE_NAME_ALIASES[lowered] ?? lowered
|
||
}
|
||
|
||
function findTemplate(row: ParsedRow, templates: DbTemplate[]): DbTemplate | null {
|
||
const normalized = normalizeTemplateName(row.taskName)
|
||
for (const t of templates) if (normalizeTemplateName(t.name) === normalized) return t
|
||
for (const t of templates) {
|
||
const tn = normalizeTemplateName(t.name)
|
||
if (tn.includes(normalized) || normalized.includes(tn)) return t
|
||
}
|
||
if (row.daysAfterRenewal > 0) {
|
||
const approxOffset = -(365 - row.daysAfterRenewal)
|
||
let closest: DbTemplate | null = null; let closestDiff = Infinity
|
||
for (const t of templates) {
|
||
const diff = Math.abs(t.daysOffset - approxOffset)
|
||
if (diff < closestDiff && diff <= 15) { closestDiff = diff; closest = t }
|
||
}
|
||
if (closest) return closest
|
||
}
|
||
return null
|
||
}
|
||
|
||
// ─── DB state ────────────────────────────────────────────────────────────────
|
||
|
||
async function loadDbState(log: (l: string) => Promise<void>): Promise<DbState> {
|
||
await log('Loading database state...')
|
||
const designations = await prisma.designation.findMany({
|
||
where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } },
|
||
})
|
||
const shapeDes = designations.find((d) => d.name === 'Shape')
|
||
const shape2Des = designations.find((d) => d.name === 'Shape 2' || d.name === 'Shape2')
|
||
if (!shapeDes || !shape2Des) throw new Error(`Could not find Shape/Shape2 designations. Found: ${designations.map((d) => d.name).join(', ')}`)
|
||
|
||
const allTemplates = await prisma.taskTemplate.findMany({
|
||
where: { designationId: { in: [shapeDes.id, shape2Des.id] }, isActive: true },
|
||
})
|
||
const shapeTemplates = allTemplates.filter((t) => t.designationId === shapeDes.id)
|
||
const shape2Templates = allTemplates.filter((t) => t.designationId === shape2Des.id)
|
||
await log(` Loaded ${shapeTemplates.length} Shape templates, ${shape2Templates.length} Shape2 templates`)
|
||
|
||
const clients = await prisma.client.findMany({
|
||
where: { OR: [{ designationId: { in: [shapeDes.id, shape2Des.id] } }, { designation2Id: { in: [shapeDes.id, shape2Des.id] } }] },
|
||
select: { id: true, name: true, claimsAdvocateId: true },
|
||
})
|
||
await log(` Loaded ${clients.length} SHAPE clients`)
|
||
|
||
const clientsByNormalizedName = new Map<string, DbClient[]>()
|
||
for (const c of clients) {
|
||
const key = normalizeClientName(c.name)
|
||
if (!clientsByNormalizedName.has(key)) clientsByNormalizedName.set(key, [])
|
||
clientsByNormalizedName.get(key)!.push({ id: c.id, name: c.name, claimsAdvocateId: c.claimsAdvocateId })
|
||
}
|
||
|
||
return {
|
||
clientsByNormalizedName,
|
||
allClients: clients.map((c) => ({ id: c.id, name: c.name, claimsAdvocateId: c.claimsAdvocateId })),
|
||
shapeTemplates: shapeTemplates.map((t) => ({ id: t.id, name: t.name, daysOffset: t.daysOffset, designationId: t.designationId })),
|
||
shape2Templates: shape2Templates.map((t) => ({ id: t.id, name: t.name, daysOffset: t.daysOffset, designationId: t.designationId })),
|
||
shapeDesignationId: shapeDes.id,
|
||
shape2DesignationId: shape2Des.id,
|
||
}
|
||
}
|
||
|
||
// ─── Template fixes ───────────────────────────────────────────────────────────
|
||
|
||
async function fixTemplates(shapeDesignationId: string, shape2DesignationId: string, dryRun: boolean, log: (l: string) => Promise<void>): Promise<void> {
|
||
await log('\nFixing template data...')
|
||
const fix1 = await prisma.taskTemplate.updateMany({ where: { name: 'Request 125 day loss runs', daysOffset: -125 }, data: { name: 'Request 120 day loss runs', daysOffset: -120 } })
|
||
if (fix1.count > 0) await log(` Fixed ${fix1.count} template(s): "Request 125..." → "Request 120..."`)
|
||
const fix2 = await prisma.taskTemplate.updateMany({ where: { daysOffset: -89 }, data: { daysOffset: -90 } })
|
||
await prisma.taskTemplate.updateMany({ where: { name: 'Request 89 day loss runs' }, data: { name: 'Request 90 day loss runs' } })
|
||
await prisma.taskTemplate.updateMany({ where: { name: 'Request 89 day loss runs (if being marketed)' }, data: { name: 'Request 90 day loss runs (if being marketed)' } })
|
||
if (fix2.count > 0) await log(` Fixed ${fix2.count} template(s): offset -89 → -90`)
|
||
const existing185 = await prisma.taskTemplate.findFirst({ where: { name: 'Claim Review', daysOffset: -185, designationId: shapeDesignationId } })
|
||
if (!existing185) {
|
||
if (!dryRun) {
|
||
await prisma.taskTemplate.create({ data: { name: 'Claim Review', department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: -185, defaultPriority: 'MEDIUM', isActive: true, displayOrder: 4, designationId: shapeDesignationId } })
|
||
await log(' Created missing "Claim Review" template at -185 days (Shape)')
|
||
} else {
|
||
await log(' [DRY RUN] Would create missing "Claim Review" template at -185 days (Shape)')
|
||
}
|
||
} else {
|
||
await log(' "Claim Review" at -185 already exists — skipping')
|
||
}
|
||
}
|
||
|
||
// ─── Task processing ──────────────────────────────────────────────────────────
|
||
|
||
interface DateCompletedResult { status: 'COMPLETED' | 'NA' | 'NOT_STARTED'; completedAt: Date | null; naReason: string | null }
|
||
|
||
function interpretDateCompleted(raw: string): DateCompletedResult {
|
||
const normalized = raw.trim().toLowerCase()
|
||
if (normalized === '' || normalized === '??' || normalized === '?') return { status: 'NOT_STARTED', completedAt: null, naReason: null }
|
||
if (normalized === 'n/a' || normalized === 'na') return { status: 'NA', completedAt: null, naReason: 'Historical — marked N/A in SHAPE tracker' }
|
||
const serial = parseFloat(raw)
|
||
if (!isNaN(serial) && serial > 40000 && serial < 60000) return { status: 'COMPLETED', completedAt: excelSerialToDate(serial), naReason: null }
|
||
return { status: 'NOT_STARTED', completedAt: null, naReason: null }
|
||
}
|
||
|
||
async function findMatchingTasks(clientId: string, templateId: string, dueDate: Date): Promise<{ match: string | null; duplicateIds: string[] }> {
|
||
const windowMs = 5 * 86400 * 1000
|
||
const candidates = await prisma.task.findMany({
|
||
where: { clientId, templateId, dueDate: { gte: new Date(dueDate.getTime() - windowMs), lte: new Date(dueDate.getTime() + windowMs) } },
|
||
orderBy: { createdAt: 'asc' },
|
||
select: { id: true },
|
||
})
|
||
if (candidates.length === 0) return { match: null, duplicateIds: [] }
|
||
return { match: candidates[0].id, duplicateIds: candidates.slice(1).map((c) => c.id) }
|
||
}
|
||
|
||
async function ensureTaskAssignment(taskId: string, userId: string, stats: ImportStats, dryRun: boolean): Promise<void> {
|
||
if (dryRun) { stats.tasksAssigned++; return }
|
||
try {
|
||
await prisma.taskAssignment.upsert({ where: { taskId_userId: { taskId, userId } }, create: { taskId, userId }, update: {} })
|
||
stats.tasksAssigned++
|
||
} catch { /* already exists */ }
|
||
}
|
||
|
||
async function processTaskRow(clientId: string, row: ParsedRow, template: DbTemplate, effectiveDate: Date, advocateUserId: string, stats: ImportStats, dryRun: boolean): Promise<void> {
|
||
const dueDate = new Date(effectiveDate)
|
||
dueDate.setDate(dueDate.getDate() + Math.round(row.daysAfterRenewal))
|
||
const { status, completedAt, naReason } = interpretDateCompleted(row.dateCompleted)
|
||
const { match: taskId, duplicateIds } = await findMatchingTasks(clientId, template.id, dueDate)
|
||
|
||
if (duplicateIds.length > 0) {
|
||
if (!dryRun) await prisma.task.deleteMany({ where: { id: { in: duplicateIds } } })
|
||
stats.duplicatesDeleted += duplicateIds.length
|
||
}
|
||
|
||
if (taskId) {
|
||
await ensureTaskAssignment(taskId, advocateUserId, stats, dryRun)
|
||
if (status === 'COMPLETED' || status === 'NA') {
|
||
const existing = await prisma.task.findUnique({ where: { id: taskId }, select: { status: true, notes: true } })
|
||
if (existing && existing.status === 'NOT_STARTED') {
|
||
if (!dryRun) {
|
||
await prisma.task.update({ where: { id: taskId }, data: { status, completedAt: completedAt ?? undefined, completedBy: status === 'COMPLETED' ? advocateUserId : undefined, naReason: naReason ?? undefined, notes: row.notes && !existing.notes ? row.notes : existing.notes ?? undefined } })
|
||
}
|
||
stats.tasksUpdated++
|
||
}
|
||
}
|
||
} else {
|
||
if (status === 'COMPLETED' || status === 'NA') {
|
||
if (!dryRun) {
|
||
const newTask = await prisma.task.create({ data: { title: template.name, department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: template.daysOffset, dueDate, status, priority: 'MEDIUM', clientId, templateId: template.id, completedAt: completedAt ?? undefined, completedBy: status === 'COMPLETED' ? advocateUserId : undefined, naReason: naReason ?? undefined, notes: row.notes || undefined, isAdHoc: false, createdBy: advocateUserId } })
|
||
await ensureTaskAssignment(newTask.id, advocateUserId, stats, dryRun)
|
||
} else { stats.tasksAssigned++ }
|
||
stats.tasksCreated++
|
||
} else {
|
||
if (!dryRun) {
|
||
const newTask = await prisma.task.create({ data: { title: template.name, department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: template.daysOffset, dueDate, status: 'NOT_STARTED', priority: 'MEDIUM', clientId, templateId: template.id, notes: row.notes || undefined, isAdHoc: false, createdBy: advocateUserId } })
|
||
await ensureTaskAssignment(newTask.id, advocateUserId, stats, dryRun)
|
||
} else { stats.tasksAssigned++ }
|
||
stats.tasksCreated++
|
||
}
|
||
}
|
||
}
|
||
|
||
async function createAdHocTask(clientId: string, text: string, effectiveDate: Date, advocateUserId: string, stats: ImportStats, dryRun: boolean): Promise<void> {
|
||
const title = text.slice(0, 500)
|
||
const existing = await prisma.task.findFirst({ where: { clientId, title, isAdHoc: true } })
|
||
if (existing) return
|
||
if (!dryRun) {
|
||
const task = await prisma.task.create({ data: { title, department: 'CLAIMS', timing: 'PRE_RENEWAL', daysOffset: 0, dueDate: effectiveDate, status: 'COMPLETED', priority: 'MEDIUM', clientId, isAdHoc: true, completedAt: effectiveDate, completedBy: advocateUserId, notes: text, createdBy: advocateUserId } })
|
||
await ensureTaskAssignment(task.id, advocateUserId, stats, dryRun)
|
||
} else { stats.tasksAssigned++ }
|
||
stats.adHocCreated++
|
||
}
|
||
|
||
async function assignAdvocate(clientId: string, advocateUserId: string, currentAdvocateId: string | null, stats: ImportStats, dryRun: boolean): Promise<void> {
|
||
if (currentAdvocateId) return
|
||
if (!dryRun) await prisma.client.update({ where: { id: clientId }, data: { claimsAdvocateId: advocateUserId } })
|
||
stats.advocatesAssigned++
|
||
}
|
||
|
||
async function processFile(folder: string, itemId: string, fileName: string, driveId: string, dbState: DbState, stats: ImportStats, dryRun: boolean, log: (l: string) => Promise<void>): Promise<void> {
|
||
const advocateInfo = TEAM_MEMBER_MAP[folder]!
|
||
let buf: Buffer
|
||
try { buf = await downloadFile(driveId, itemId); await sleep(150) }
|
||
catch (err) { stats.errors.push(`${folder}/${fileName}: Download failed — ${(err as Error).message}`); return }
|
||
|
||
let blocks: ParsedBlock[]
|
||
try { blocks = parseExcelFile(buf, fileName) }
|
||
catch (err) { stats.errors.push(`${folder}/${fileName}: Parse failed — ${(err as Error).message}`); return }
|
||
|
||
if (blocks.length === 0) { stats.errors.push(`${folder}/${fileName}: No data blocks found`); return }
|
||
stats.filesProcessed++
|
||
|
||
for (const block of blocks) {
|
||
const matchResult = matchClient(block.clientName, dbState)
|
||
if (!matchResult) {
|
||
stats.clientsUnmatched++
|
||
stats.unmatchedClients.push({ folder, file: fileName, excelName: block.clientName })
|
||
continue
|
||
}
|
||
const { client, method } = matchResult
|
||
stats.clientsMatched++
|
||
if (method !== 'exact') stats.fuzzyMatches.push({ folder, file: fileName, excelName: block.clientName, dbName: client.name, method })
|
||
await assignAdvocate(client.id, advocateInfo.userId, client.claimsAdvocateId, stats, dryRun)
|
||
client.claimsAdvocateId = client.claimsAdvocateId ?? advocateInfo.userId
|
||
const templates = block.isShape2 ? dbState.shape2Templates : dbState.shapeTemplates
|
||
for (const row of block.rows) {
|
||
const template = findTemplate(row, templates)
|
||
if (!template) {
|
||
const { status } = interpretDateCompleted(row.dateCompleted)
|
||
if (status === 'COMPLETED' || status === 'NA') {
|
||
const adHocTitle = `[Unmatched Task] ${row.taskName}${row.notes ? ': ' + row.notes : ''}`
|
||
await createAdHocTask(client.id, adHocTitle, block.effectiveDate, advocateInfo.userId, stats, dryRun)
|
||
}
|
||
continue
|
||
}
|
||
try { await processTaskRow(client.id, row, template, block.effectiveDate, advocateInfo.userId, stats, dryRun) }
|
||
catch (err) { stats.errors.push(`${folder}/${fileName} / ${block.clientName} / "${row.taskName}": ${(err as Error).message}`) }
|
||
}
|
||
for (const svc of block.additionalServices) {
|
||
try { await createAdHocTask(client.id, svc, block.effectiveDate, advocateInfo.userId, stats, dryRun) }
|
||
catch (err) { stats.errors.push(`${folder}/${fileName} / ${block.clientName} / adHoc "${svc.slice(0, 40)}": ${(err as Error).message}`) }
|
||
}
|
||
}
|
||
}
|
||
|
||
// ─── Public entry point ───────────────────────────────────────────────────────
|
||
|
||
export async function runShapeImport(opts: {
|
||
dryRun: boolean
|
||
driveId: string
|
||
onLog: (line: string) => Promise<void>
|
||
}): Promise<ImportStats> {
|
||
const { dryRun, driveId, onLog } = opts
|
||
|
||
const stats: ImportStats = {
|
||
filesProcessed: 0, clientsMatched: 0, clientsUnmatched: 0,
|
||
tasksUpdated: 0, tasksAssigned: 0, tasksCreated: 0,
|
||
duplicatesDeleted: 0, adHocCreated: 0, advocatesAssigned: 0,
|
||
errors: [], unmatchedClients: [], fuzzyMatches: [],
|
||
}
|
||
|
||
await onLog(`SHAPE Historical Import — ${new Date().toISOString()}`)
|
||
await onLog(`Mode: ${dryRun ? 'DRY RUN (no changes written)' : 'EXECUTE'}`)
|
||
|
||
const designations = await prisma.designation.findMany({ where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } } })
|
||
const shapeDes = designations.find((d) => d.name === 'Shape')
|
||
const shape2Des = designations.find((d) => d.name === 'Shape 2' || d.name === 'Shape2')
|
||
if (!shapeDes || !shape2Des) throw new Error('Shape designations not found')
|
||
|
||
await fixTemplates(shapeDes.id, shape2Des.id, dryRun, onLog)
|
||
const dbState = await loadDbState(onLog)
|
||
const files = await discoverFiles(driveId, onLog)
|
||
|
||
await onLog(`\nProcessing ${files.length} files...`)
|
||
for (let i = 0; i < files.length; i++) {
|
||
const { folder, itemId, fileName } = files[i]
|
||
const pct = Math.round(((i + 1) / files.length) * 100)
|
||
await onLog(`[${pct}%] ${i + 1}/${files.length} — ${folder}/${fileName}`)
|
||
await processFile(folder, itemId, fileName, driveId, dbState, stats, dryRun, onLog)
|
||
}
|
||
|
||
await onLog('')
|
||
await onLog('═'.repeat(60))
|
||
await onLog('SHAPE Historical Import Report')
|
||
await onLog('═'.repeat(60))
|
||
await onLog(`Mode: ${dryRun ? 'DRY RUN (no changes written)' : 'EXECUTE (changes written to DB)'}`)
|
||
await onLog('')
|
||
await onLog(`Files processed: ${stats.filesProcessed}`)
|
||
await onLog(`Clients matched: ${stats.clientsMatched}`)
|
||
await onLog(` - exact: ${stats.clientsMatched - stats.fuzzyMatches.length}`)
|
||
await onLog(` - fuzzy: ${stats.fuzzyMatches.length}`)
|
||
await onLog(`Clients unmatched: ${stats.clientsUnmatched}`)
|
||
await onLog(`Tasks updated: ${stats.tasksUpdated}`)
|
||
await onLog(`Tasks assigned: ${stats.tasksAssigned}`)
|
||
await onLog(`Tasks created: ${stats.tasksCreated}`)
|
||
await onLog(`Duplicates deleted: ${stats.duplicatesDeleted}`)
|
||
await onLog(`Ad-hoc created: ${stats.adHocCreated}`)
|
||
await onLog(`Advocates assigned: ${stats.advocatesAssigned}`)
|
||
await onLog(`Errors: ${stats.errors.length}`)
|
||
|
||
if (stats.fuzzyMatches.length > 0) {
|
||
await onLog('')
|
||
await onLog('── Fuzzy Client Matches ──────────────────────────────────────')
|
||
for (const m of stats.fuzzyMatches) await onLog(` [${m.folder}] "${m.excelName}" → "${m.dbName}" (${m.method})`)
|
||
}
|
||
if (stats.unmatchedClients.length > 0) {
|
||
await onLog('')
|
||
await onLog('── Unmatched Clients (manual action required) ─────────────────')
|
||
for (const u of stats.unmatchedClients) await onLog(` [${u.folder}] ${u.file} → "${u.excelName}"`)
|
||
}
|
||
if (stats.errors.length > 0) {
|
||
await onLog('')
|
||
await onLog('── Errors ──────────────────────────────────────────────────────')
|
||
for (const e of stats.errors) await onLog(` ERROR: ${e}`)
|
||
}
|
||
await onLog('═'.repeat(60))
|
||
|
||
return stats
|
||
}
|