1541 lines
46 KiB
TypeScript
1541 lines
46 KiB
TypeScript
/**
|
||
* SHAPE Historical Data Import Script
|
||
*
|
||
* Imports historical task completion data from SharePoint Excel files into Horizon.
|
||
* Each team member has a folder in Claims/SHAPE Accounts/{MEMBER_FOLDER}/{Client}.xlsx
|
||
*
|
||
* Usage:
|
||
* npx tsx scripts/import-shape-historical.ts # dry run
|
||
* npx tsx scripts/import-shape-historical.ts --execute # write to DB
|
||
*
|
||
* Output is written to /tmp/shape-import-report.txt (override with --output=/path)
|
||
*/
|
||
|
||
import { PrismaClient } from '@prisma/client'
|
||
import { PrismaPg } from '@prisma/adapter-pg'
|
||
import { Pool } from 'pg'
|
||
import * as zlib from 'zlib'
|
||
import * as fs from 'fs'
|
||
import * as dotenv from 'dotenv'
|
||
|
||
dotenv.config()
|
||
|
||
// ─── Configuration ────────────────────────────────────────────────────────────
|
||
|
||
const DRY_RUN = !process.argv.includes('--execute')
|
||
const OUTPUT_FILE =
|
||
process.argv.find((a) => a.startsWith('--output='))?.split('=')[1] ??
|
||
'/tmp/shape-import-report.txt'
|
||
|
||
const DRIVE_ID =
|
||
'b!OYuzIexQkkOvfEPyMJPzzZHfzTrOCOdPhTWgTlzKs6M0ZWVrAc6LR4LjWl4QFEzm'
|
||
|
||
const TEAM_MEMBER_MAP: Record<string, { userId: string; displayName: string }> =
|
||
{
|
||
CHRIS: {
|
||
userId: 'cmkm36yc90021y7vb9783nitl',
|
||
displayName: 'Christine Gove',
|
||
},
|
||
DAWN: {
|
||
userId: 'cmkm36xfr000ny7vbcajejjux',
|
||
displayName: 'Dawn Boland',
|
||
},
|
||
Jeanne: {
|
||
userId: 'cmkm370f80065y7vbrugc9jub',
|
||
displayName: 'Jeanne Strong',
|
||
},
|
||
LUKE: {
|
||
userId: 'cmkm36xd4000jy7vbhanp66s9',
|
||
displayName: 'Luke Billman',
|
||
},
|
||
MIMI: {
|
||
userId: 'cmkm36zr0004wy7vbztyynj35',
|
||
displayName: 'Mimi Rawlings',
|
||
},
|
||
}
|
||
|
||
// Template name aliases: old/variant names → canonical name (after fix)
|
||
const TEMPLATE_NAME_ALIASES: Record<string, string> = {
|
||
'request 125 day loss runs': 'request 120 day loss runs',
|
||
'request 89 day loss runs': 'request 90 day loss runs',
|
||
'request 89 day loss runs (if being marketed)':
|
||
'request 90 day loss runs (if being marketed)',
|
||
'claim review (six month)': 'claim review',
|
||
'claims review': 'claim review',
|
||
}
|
||
|
||
// ─── DB Setup ─────────────────────────────────────────────────────────────────
|
||
|
||
const pool = new Pool({ connectionString: process.env.DATABASE_URL })
|
||
const adapter = new PrismaPg(pool)
|
||
const prisma = new PrismaClient({ adapter })
|
||
|
||
// ─── Output / Logging ─────────────────────────────────────────────────────────
|
||
|
||
const outputLines: string[] = []
|
||
|
||
function log(line = '') {
|
||
console.log(line)
|
||
outputLines.push(line)
|
||
}
|
||
|
||
function writeOutput() {
|
||
fs.writeFileSync(OUTPUT_FILE, outputLines.join('\n') + '\n', 'utf-8')
|
||
console.log(`\nReport written to ${OUTPUT_FILE}`)
|
||
}
|
||
|
||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||
|
||
interface ParsedRow {
|
||
taskName: string
|
||
daysAfterRenewal: number
|
||
dateCompleted: string // raw: numeric string | 'n/a' | '??' | ''
|
||
notes: string
|
||
}
|
||
|
||
interface ParsedBlock {
|
||
clientName: string
|
||
effectiveDate: Date
|
||
isShape2: boolean
|
||
rows: ParsedRow[]
|
||
additionalServices: string[]
|
||
}
|
||
|
||
interface DbTemplate {
|
||
id: string
|
||
name: string
|
||
daysOffset: number
|
||
designationId: string | null
|
||
}
|
||
|
||
interface DbClient {
|
||
id: string
|
||
name: string
|
||
claimsAdvocateId: string | null
|
||
}
|
||
|
||
interface DbState {
|
||
clientsByNormalizedName: Map<string, DbClient[]>
|
||
allClients: DbClient[]
|
||
shapeTemplates: DbTemplate[]
|
||
shape2Templates: DbTemplate[]
|
||
shapeDesignationId: string
|
||
shape2DesignationId: string
|
||
}
|
||
|
||
interface ImportStats {
|
||
filesProcessed: number
|
||
clientsMatched: number
|
||
clientsUnmatched: number
|
||
tasksUpdated: number
|
||
tasksAssigned: number
|
||
tasksCreated: number
|
||
duplicatesDeleted: number
|
||
adHocCreated: number
|
||
advocatesAssigned: number
|
||
errors: string[]
|
||
unmatchedClients: Array<{ folder: string; file: string; excelName: string }>
|
||
fuzzyMatches: Array<{
|
||
folder: string
|
||
file: string
|
||
excelName: string
|
||
dbName: string
|
||
method: string
|
||
}>
|
||
}
|
||
|
||
// ─── Graph API ────────────────────────────────────────────────────────────────
|
||
|
||
let _cachedToken: { token: string; expiresAt: number } | null = null
|
||
|
||
async function getGraphToken(): Promise<string> {
|
||
if (_cachedToken && Date.now() < _cachedToken.expiresAt - 30_000) {
|
||
return _cachedToken.token
|
||
}
|
||
const tenantId = process.env.AZURE_AD_TENANT_ID!
|
||
const clientId = process.env.AZURE_AD_CLIENT_ID!
|
||
const clientSecret = process.env.AZURE_AD_CLIENT_SECRET!
|
||
const body = new URLSearchParams({
|
||
grant_type: 'client_credentials',
|
||
client_id: clientId,
|
||
client_secret: clientSecret,
|
||
scope: 'https://graph.microsoft.com/.default',
|
||
})
|
||
const res = await fetch(
|
||
`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`,
|
||
{
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: body.toString(),
|
||
}
|
||
)
|
||
if (!res.ok) throw new Error(`Token fetch failed: ${res.status}`)
|
||
const data = (await res.json()) as {
|
||
access_token: string
|
||
expires_in: number
|
||
}
|
||
_cachedToken = {
|
||
token: data.access_token,
|
||
expiresAt: Date.now() + data.expires_in * 1000,
|
||
}
|
||
return _cachedToken.token
|
||
}
|
||
|
||
async function graphGet(url: string): Promise<unknown> {
|
||
const token = await getGraphToken()
|
||
const res = await fetch(url, {
|
||
headers: { Authorization: `Bearer ${token}` },
|
||
})
|
||
if (!res.ok) {
|
||
const text = await res.text()
|
||
throw new Error(`Graph API error ${res.status}: ${text.slice(0, 200)}`)
|
||
}
|
||
return res.json()
|
||
}
|
||
|
||
async function downloadFile(itemId: string): Promise<Buffer> {
|
||
const token = await getGraphToken()
|
||
const url = `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/items/${itemId}/content`
|
||
const res = await fetch(url, {
|
||
headers: { Authorization: `Bearer ${token}` },
|
||
redirect: 'follow',
|
||
})
|
||
if (!res.ok) throw new Error(`Download failed: ${res.status}`)
|
||
const ab = await res.arrayBuffer()
|
||
return Buffer.from(ab)
|
||
}
|
||
|
||
async function sleep(ms: number) {
|
||
return new Promise((r) => setTimeout(r, ms))
|
||
}
|
||
|
||
interface DriveItem {
|
||
id: string
|
||
name: string
|
||
file?: { mimeType: string }
|
||
folder?: { childCount: number }
|
||
}
|
||
|
||
async function listChildren(itemId: string): Promise<DriveItem[]> {
|
||
const data = (await graphGet(
|
||
`https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/items/${itemId}/children?$top=200`
|
||
)) as { value: DriveItem[] }
|
||
return data.value ?? []
|
||
}
|
||
|
||
async function discoverFiles(): Promise<
|
||
Array<{ folder: string; itemId: string; fileName: string }>
|
||
> {
|
||
log('Discovering files from SharePoint...')
|
||
const files: Array<{ folder: string; itemId: string; fileName: string }> = []
|
||
|
||
const rootData = (await graphGet(
|
||
`https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/root:/Claims%2FSHAPE%20Accounts:/children`
|
||
)) as { value: DriveItem[] }
|
||
const rootChildren = rootData.value ?? []
|
||
|
||
for (const item of rootChildren) {
|
||
if (!item.folder) continue
|
||
const memberInfo = TEAM_MEMBER_MAP[item.name]
|
||
if (!memberInfo) continue
|
||
|
||
log(` Scanning folder: ${item.name}`)
|
||
const children = await listChildren(item.id)
|
||
await sleep(150)
|
||
|
||
for (const child of children) {
|
||
if (!child.file) continue
|
||
if (!child.name.toLowerCase().endsWith('.xlsx')) continue
|
||
if (child.name.toLowerCase().startsWith('archive')) continue
|
||
// Skip template/master files
|
||
if (
|
||
child.name.toLowerCase().includes('template') ||
|
||
child.name.toLowerCase().includes('master workbook') ||
|
||
child.name.toLowerCase().startsWith('all shape')
|
||
)
|
||
continue
|
||
|
||
files.push({ folder: item.name, itemId: child.id, fileName: child.name })
|
||
}
|
||
}
|
||
|
||
log(` Found ${files.length} Excel files`)
|
||
return files
|
||
}
|
||
|
||
// ─── ZIP Parser ───────────────────────────────────────────────────────────────
|
||
|
||
function parseZip(buffer: Buffer): Map<string, Buffer> {
|
||
const entries = new Map<string, Buffer>()
|
||
|
||
// Find End of Central Directory (EOCD) record — signature 0x06054b50
|
||
let eocdOffset = -1
|
||
const searchStart = Math.max(0, buffer.length - 65558)
|
||
for (let i = buffer.length - 22; i >= searchStart; i--) {
|
||
if (buffer.readUInt32LE(i) === 0x06054b50) {
|
||
eocdOffset = i
|
||
break
|
||
}
|
||
}
|
||
if (eocdOffset < 0) throw new Error('Not a valid ZIP file (EOCD not found)')
|
||
|
||
const cdOffset = buffer.readUInt32LE(eocdOffset + 16)
|
||
const cdEntries = buffer.readUInt16LE(eocdOffset + 10)
|
||
|
||
let pos = cdOffset
|
||
for (let i = 0; i < cdEntries; i++) {
|
||
if (buffer.readUInt32LE(pos) !== 0x02014b50) {
|
||
throw new Error(`Invalid central directory entry at offset ${pos}`)
|
||
}
|
||
const method = buffer.readUInt16LE(pos + 10)
|
||
const compSize = buffer.readUInt32LE(pos + 20)
|
||
const fileNameLen = buffer.readUInt16LE(pos + 28)
|
||
const extraLen = buffer.readUInt16LE(pos + 30)
|
||
const commentLen = buffer.readUInt16LE(pos + 32)
|
||
const localOffset = buffer.readUInt32LE(pos + 42)
|
||
const fileName = buffer.slice(pos + 46, pos + 46 + fileNameLen).toString('utf-8')
|
||
|
||
// Read local file header
|
||
if (buffer.readUInt32LE(localOffset) !== 0x04034b50) {
|
||
throw new Error(`Invalid local header for ${fileName}`)
|
||
}
|
||
const localFileNameLen = buffer.readUInt16LE(localOffset + 26)
|
||
const localExtraLen = buffer.readUInt16LE(localOffset + 28)
|
||
const dataStart = localOffset + 30 + localFileNameLen + localExtraLen
|
||
|
||
const compData = buffer.slice(dataStart, dataStart + compSize)
|
||
let data: Buffer
|
||
if (method === 0) {
|
||
data = compData // stored
|
||
} else if (method === 8) {
|
||
data = zlib.inflateRawSync(compData) // deflate
|
||
} else {
|
||
data = Buffer.alloc(0) // unsupported, skip
|
||
}
|
||
|
||
// Store with lowercase key for case-insensitive lookup
|
||
entries.set(fileName.toLowerCase(), data)
|
||
|
||
pos += 46 + fileNameLen + extraLen + commentLen
|
||
}
|
||
|
||
return entries
|
||
}
|
||
|
||
// ─── XLSX Parsing ─────────────────────────────────────────────────────────────
|
||
|
||
function decodeXmlEntities(s: string): string {
|
||
return s
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/&#x([0-9A-Fa-f]+);/gi, (_, h) =>
|
||
String.fromCharCode(parseInt(h, 16))
|
||
)
|
||
.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
|
||
}
|
||
|
||
function parseSharedStrings(xmlBuf: Buffer | undefined): string[] {
|
||
if (!xmlBuf || xmlBuf.length === 0) return []
|
||
const xml = xmlBuf.toString('utf-8')
|
||
const strings: string[] = []
|
||
const siRegex = /<si>([\s\S]*?)<\/si>/g
|
||
let match: RegExpExecArray | null
|
||
while ((match = siRegex.exec(xml)) !== null) {
|
||
const siContent = match[1]
|
||
const texts: string[] = []
|
||
const tRegex = /<t(?:\s[^>]*)?>([^<]*)<\/t>/g
|
||
let tm: RegExpExecArray | null
|
||
while ((tm = tRegex.exec(siContent)) !== null) {
|
||
texts.push(decodeXmlEntities(tm[1]))
|
||
}
|
||
strings.push(texts.join(''))
|
||
}
|
||
return strings
|
||
}
|
||
|
||
function colLetterToNum(letters: string): number {
|
||
let n = 0
|
||
for (let i = 0; i < letters.length; i++) {
|
||
n = n * 26 + (letters.charCodeAt(i) - 64)
|
||
}
|
||
return n
|
||
}
|
||
|
||
function numToColLetter(n: number): string {
|
||
let s = ''
|
||
while (n > 0) {
|
||
s = String.fromCharCode(((n - 1) % 26) + 65) + s
|
||
n = Math.floor((n - 1) / 26)
|
||
}
|
||
return s
|
||
}
|
||
|
||
function parseSheetToGrid(
|
||
sheetBuf: Buffer,
|
||
sharedStrings: string[]
|
||
): { get: (row: number, col: number) => string; maxRow: number } {
|
||
const cells = new Map<string, string>()
|
||
let maxRow = 0
|
||
// Strip self-closing <c .../> tags first — they represent empty cells and must not
|
||
// be confused with the opening tag of a non-empty cell that follows.
|
||
const xml = sheetBuf.toString('utf-8').replace(/<c\b[^>]*\/>/g, '')
|
||
|
||
// Match each <c r="..." ...>...</c> block
|
||
const cRegex = /<c\s+r="([A-Z]+)(\d+)"([^>]*)>([\s\S]*?)<\/c>/g
|
||
let m: RegExpExecArray | null
|
||
while ((m = cRegex.exec(xml)) !== null) {
|
||
const colStr = m[1]
|
||
const rowNum = parseInt(m[2])
|
||
const attrs = m[3]
|
||
const inner = m[4]
|
||
|
||
if (rowNum > maxRow) maxRow = rowNum
|
||
|
||
const typeMatch = attrs.match(/\bt="([^"]+)"/)
|
||
const type = typeMatch ? typeMatch[1] : 'n'
|
||
|
||
let value = ''
|
||
const vMatch = inner.match(/<v>([^<]*)<\/v>/)
|
||
const isMatch = inner.match(/<is><t(?:[^>]*)?>([^<]*)<\/t><\/is>/)
|
||
|
||
if (isMatch) {
|
||
value = decodeXmlEntities(isMatch[1])
|
||
} else if (vMatch) {
|
||
if (type === 's') {
|
||
value = sharedStrings[parseInt(vMatch[1])] ?? ''
|
||
} else if (type === 'str') {
|
||
value = decodeXmlEntities(vMatch[1])
|
||
} else if (type === 'b') {
|
||
value = vMatch[1] === '1' ? 'TRUE' : 'FALSE'
|
||
} else {
|
||
value = vMatch[1] // number/date
|
||
}
|
||
}
|
||
|
||
const col = colLetterToNum(colStr)
|
||
cells.set(`${rowNum}:${col}`, value)
|
||
}
|
||
|
||
return {
|
||
get(row: number, col: number): string {
|
||
return cells.get(`${row}:${col}`) ?? ''
|
||
},
|
||
maxRow,
|
||
}
|
||
}
|
||
|
||
function findTargetSheet(
|
||
zipEntries: Map<string, Buffer>
|
||
): Buffer | null {
|
||
// Parse workbook.xml to find sheet r:ids, then map via rels
|
||
const wbBuf = zipEntries.get('xl/workbook.xml')
|
||
if (!wbBuf) return zipEntries.get('xl/worksheets/sheet1.xml') ?? null
|
||
|
||
const wb = wbBuf.toString('utf-8')
|
||
const relsBuf = zipEntries.get('xl/_rels/workbook.xml.rels')
|
||
|
||
// Build r:id → file path map from rels
|
||
// Paths in xl/_rels/workbook.xml.rels are relative to xl/
|
||
// e.g. Target="worksheets/sheet1.xml" → actual path is "xl/worksheets/sheet1.xml"
|
||
const ridToPath = new Map<string, string>()
|
||
if (relsBuf) {
|
||
const rels = relsBuf.toString('utf-8')
|
||
const relRegex = /<Relationship\s+Id="([^"]+)"[^>]+Target="([^"]+)"/g
|
||
let rm: RegExpExecArray | null
|
||
while ((rm = relRegex.exec(rels)) !== null) {
|
||
let target = rm[2]
|
||
if (target.startsWith('/')) {
|
||
target = target.slice(1) // absolute paths: strip leading /
|
||
} else if (!target.startsWith('xl/')) {
|
||
target = 'xl/' + target // relative to xl/: prepend xl/
|
||
}
|
||
ridToPath.set(rm[1], target)
|
||
}
|
||
}
|
||
|
||
// Find sheet entries — attributes may appear in any order, parse flexibly
|
||
const sheetEntries: Array<{ name: string; rid: string }> = []
|
||
const sheetTagRegex = /<sheet\b([^>]*\/?>)/g
|
||
let sm: RegExpExecArray | null
|
||
while ((sm = sheetTagRegex.exec(wb)) !== null) {
|
||
const attrs = sm[1]
|
||
const nameMatch = attrs.match(/\bname="([^"]+)"/)
|
||
const ridMatch = attrs.match(/\br:id="([^"]+)"/)
|
||
if (nameMatch && ridMatch) {
|
||
sheetEntries.push({ name: nameMatch[1], rid: ridMatch[1] })
|
||
}
|
||
}
|
||
|
||
const resolveSheet = (rid: string): Buffer | null => {
|
||
const path = ridToPath.get(rid)
|
||
if (!path) return null
|
||
return zipEntries.get(path.toLowerCase()) ?? null
|
||
}
|
||
|
||
// Prefer SHAPE2 > SHAPE > CLIENT, then fall back to first sheet
|
||
const priority = ['shape2', 'shape', 'client']
|
||
for (const p of priority) {
|
||
const match = sheetEntries.find((s) => s.name.toLowerCase() === p)
|
||
if (match) {
|
||
const buf = resolveSheet(match.rid)
|
||
if (buf) return buf
|
||
}
|
||
}
|
||
|
||
if (sheetEntries.length > 0) {
|
||
const buf = resolveSheet(sheetEntries[0].rid)
|
||
if (buf) return buf
|
||
}
|
||
|
||
return zipEntries.get('xl/worksheets/sheet1.xml') ?? null
|
||
}
|
||
|
||
function excelSerialToDate(serial: number): Date {
|
||
// Excel epoch: Jan 0, 1900 (effectively Dec 30, 1899)
|
||
// But Excel has a leap year bug treating 1900 as leap year, so offset is 25569
|
||
return new Date((serial - 25569) * 86400 * 1000)
|
||
}
|
||
|
||
// ─── Block Parser ─────────────────────────────────────────────────────────────
|
||
|
||
function parseExcelBlocks(
|
||
grid: { get: (r: number, c: number) => string; maxRow: number },
|
||
fileName: string
|
||
): ParsedBlock[] {
|
||
const blocks: ParsedBlock[] = []
|
||
const isShape2 = /(SHAPE2|SHAPE 2)/i.test(fileName)
|
||
|
||
let row = 1
|
||
while (row <= grid.maxRow) {
|
||
// Scan up to 10 columns for "Client Name:"
|
||
let clientNameCol = -1
|
||
for (let c = 1; c <= 5; c++) {
|
||
if (grid.get(row, c).trim().toLowerCase().includes('client name')) {
|
||
clientNameCol = c
|
||
break
|
||
}
|
||
}
|
||
|
||
if (clientNameCol < 0) {
|
||
row++
|
||
continue
|
||
}
|
||
|
||
// Client name: first try the cell to the right, then up to 3 columns right
|
||
let clientName = ''
|
||
for (let dc = 1; dc <= 3; dc++) {
|
||
const candidate = grid.get(row, clientNameCol + dc).trim()
|
||
if (candidate && !candidate.toLowerCase().includes('client name')) {
|
||
clientName = candidate
|
||
break
|
||
}
|
||
}
|
||
// Fallback: extract from filename (strip known suffixes)
|
||
if (!clientName) {
|
||
clientName = fileName
|
||
.replace(/^SHAPE\S*\s*[-–]\s*/i, '') // strip leading "SHAPE2 - " prefix
|
||
.replace(/\s*[-–]\s*SHAPE.*$/i, '')
|
||
.replace(/\s*SHAPE\s*Services?\s*Checklist.*$/i, '')
|
||
.replace(/\s*SHAPE\b.*/i, '')
|
||
.replace(/\.xlsx$/i, '')
|
||
.trim()
|
||
}
|
||
if (!clientName) {
|
||
row++
|
||
continue
|
||
}
|
||
|
||
row++
|
||
|
||
// Find effective date (within next 5 rows)
|
||
// Strategy 1: look for "Effective Date:" label, take numeric value in adjacent cell
|
||
// Strategy 2 (fallback): look for any Excel-range serial (40000–60000) in next rows
|
||
let effectiveDateSerial = 0
|
||
let foundDateRow = row
|
||
const EXCEL_DATE_MIN = 40000
|
||
const EXCEL_DATE_MAX = 60000
|
||
|
||
for (let r = row; r <= Math.min(row + 4, grid.maxRow); r++) {
|
||
let foundDateInRow = false
|
||
for (let c = 1; c <= 8; c++) {
|
||
const cellVal = grid.get(r, c).trim()
|
||
if (cellVal.toLowerCase().includes('effective date')) {
|
||
// Date value is in next column(s)
|
||
for (let dc = 1; dc <= 4; dc++) {
|
||
const val = grid.get(r, c + dc).trim()
|
||
const serial = parseFloat(val)
|
||
if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) {
|
||
effectiveDateSerial = serial
|
||
foundDateInRow = true
|
||
break
|
||
}
|
||
}
|
||
if (foundDateInRow) break
|
||
}
|
||
// Fallback: any valid date serial in this row (not the first col which may be a number label)
|
||
if (!foundDateInRow && c > 1) {
|
||
const serial = parseFloat(cellVal)
|
||
if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) {
|
||
effectiveDateSerial = serial
|
||
foundDateInRow = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if (foundDateInRow) {
|
||
foundDateRow = r
|
||
break
|
||
}
|
||
}
|
||
|
||
// If no effective date found, use current date as fallback (for files like Minniefield
|
||
// where date is stored as display-only formatting, not cell value)
|
||
const effectiveDate = effectiveDateSerial > 0
|
||
? excelSerialToDate(effectiveDateSerial)
|
||
: new Date()
|
||
row = foundDateRow + 1
|
||
|
||
// Skip "Policies:" row if present
|
||
for (let r = row; r <= Math.min(row + 3, grid.maxRow); r++) {
|
||
let foundPolicies = false
|
||
for (let c = 1; c <= 5; c++) {
|
||
if (grid.get(r, c).trim().toLowerCase().startsWith('policies')) {
|
||
foundPolicies = true
|
||
break
|
||
}
|
||
}
|
||
if (foundPolicies) {
|
||
row = r + 1
|
||
break
|
||
}
|
||
// Check if this is the header row — stop skipping
|
||
let hasTask = false
|
||
for (let c = 1; c <= 8; c++) {
|
||
if (grid.get(r, c).trim().toLowerCase() === 'task') {
|
||
hasTask = true
|
||
break
|
||
}
|
||
}
|
||
if (hasTask) break
|
||
}
|
||
|
||
// Find header row containing "Task"
|
||
let headerRow = -1
|
||
let taskCol = 0
|
||
for (let r = row; r <= Math.min(row + 5, grid.maxRow); r++) {
|
||
for (let c = 1; c <= 8; c++) {
|
||
if (grid.get(r, c).trim().toLowerCase() === 'task') {
|
||
headerRow = r
|
||
taskCol = c
|
||
break
|
||
}
|
||
}
|
||
if (headerRow >= 0) break
|
||
}
|
||
|
||
if (headerRow < 0) {
|
||
row++
|
||
continue
|
||
}
|
||
|
||
// Map header columns (scan right of taskCol)
|
||
let daysAfterCol = 0
|
||
let dateCompletedCol = 0
|
||
let notesCol = 0
|
||
|
||
for (let c = taskCol + 1; c <= taskCol + 8; c++) {
|
||
const hdr = grid.get(headerRow, c).trim().toLowerCase()
|
||
if (hdr.includes('days after')) daysAfterCol = c
|
||
else if (
|
||
hdr.includes('date completed') ||
|
||
hdr === 'completed' ||
|
||
hdr === 'date complete'
|
||
)
|
||
dateCompletedCol = c
|
||
else if (hdr.includes('notes')) notesCol = c
|
||
}
|
||
|
||
row = headerRow + 1
|
||
|
||
// Parse data rows
|
||
const taskRows: ParsedRow[] = []
|
||
const additionalServices: string[] = []
|
||
let inAdditional = false
|
||
|
||
while (row <= grid.maxRow) {
|
||
// Check if next block starts
|
||
let nextBlock = false
|
||
for (let c = 1; c <= 5; c++) {
|
||
if (grid.get(row, c).trim().toLowerCase().includes('client name')) {
|
||
nextBlock = true
|
||
break
|
||
}
|
||
}
|
||
if (nextBlock) break
|
||
|
||
const taskName = grid.get(row, taskCol).trim()
|
||
|
||
if (!taskName) {
|
||
row++
|
||
continue
|
||
}
|
||
|
||
// Detect "Additional Services Provided" section header
|
||
if (taskName.toLowerCase().includes('additional services')) {
|
||
inAdditional = true
|
||
row++
|
||
continue
|
||
}
|
||
|
||
const daysAfterRaw =
|
||
daysAfterCol > 0 ? grid.get(row, daysAfterCol).trim() : ''
|
||
const isDaysNumeric = daysAfterRaw !== '' && !isNaN(parseFloat(daysAfterRaw))
|
||
|
||
// If no numeric days value, treat as additional service / free text
|
||
if (!isDaysNumeric) {
|
||
if (taskName.length > 3) {
|
||
additionalServices.push(taskName)
|
||
}
|
||
row++
|
||
continue
|
||
}
|
||
|
||
if (inAdditional) {
|
||
additionalServices.push(taskName)
|
||
row++
|
||
continue
|
||
}
|
||
|
||
const daysAfterRenewal = parseFloat(daysAfterRaw)
|
||
const dateCompletedRaw =
|
||
dateCompletedCol > 0 ? grid.get(row, dateCompletedCol).trim() : ''
|
||
const notes = notesCol > 0 ? grid.get(row, notesCol).trim() : ''
|
||
|
||
taskRows.push({
|
||
taskName,
|
||
daysAfterRenewal,
|
||
dateCompleted: dateCompletedRaw,
|
||
notes,
|
||
})
|
||
|
||
row++
|
||
}
|
||
|
||
blocks.push({
|
||
clientName,
|
||
effectiveDate,
|
||
isShape2,
|
||
rows: taskRows,
|
||
additionalServices,
|
||
})
|
||
}
|
||
|
||
return blocks
|
||
}
|
||
|
||
function parseExcelFile(buf: Buffer, fileName: string): ParsedBlock[] {
|
||
const zipEntries = parseZip(buf)
|
||
const sharedStringsBuf = zipEntries.get('xl/sharedstrings.xml')
|
||
const sharedStrings = parseSharedStrings(sharedStringsBuf)
|
||
const sheetBuf = findTargetSheet(zipEntries)
|
||
if (!sheetBuf) return []
|
||
const grid = parseSheetToGrid(sheetBuf, sharedStrings)
|
||
return parseExcelBlocks(grid, fileName)
|
||
}
|
||
|
||
// ─── Client Name Normalization / Matching ─────────────────────────────────────
|
||
|
||
function normalizeClientName(name: string): string {
|
||
return name
|
||
.toLowerCase()
|
||
.replace(/^the\s+/, '')
|
||
.replace(/&/g, 'and')
|
||
.replace(
|
||
/\b(incorporated|inc|llc|l\.l\.c|corp|corporation|co|ltd|limited|lp|l\.p|company|enterprises|enterprise|services|group|associates|solutions|management|consulting)\b\.?/gi,
|
||
''
|
||
)
|
||
.replace(/[.,;:'"()\-_#@!?]/g, '')
|
||
.replace(/\s+/g, ' ')
|
||
.trim()
|
||
}
|
||
|
||
function tokenize(s: string): Set<string> {
|
||
return new Set(
|
||
s
|
||
.split(/\s+/)
|
||
.filter((t) => t.length > 1)
|
||
)
|
||
}
|
||
|
||
function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
|
||
if (a.size === 0 && b.size === 0) return 1
|
||
let intersection = 0
|
||
for (const t of a) {
|
||
if (b.has(t)) intersection++
|
||
}
|
||
const union = a.size + b.size - intersection
|
||
return union === 0 ? 0 : intersection / union
|
||
}
|
||
|
||
function matchClient(
|
||
excelName: string,
|
||
dbState: DbState
|
||
): { client: DbClient; method: string } | null {
|
||
const normalized = normalizeClientName(excelName)
|
||
if (!normalized) return null
|
||
|
||
// 1. Exact normalized match
|
||
const exactCandidates = dbState.clientsByNormalizedName.get(normalized)
|
||
if (exactCandidates && exactCandidates.length > 0) {
|
||
return { client: exactCandidates[0], method: 'exact' }
|
||
}
|
||
|
||
// 2. Contains match
|
||
for (const [key, clients] of dbState.clientsByNormalizedName) {
|
||
if (key.includes(normalized) || normalized.includes(key)) {
|
||
if (clients.length > 0) {
|
||
return { client: clients[0], method: 'contains' }
|
||
}
|
||
}
|
||
}
|
||
|
||
// 3. Token Jaccard similarity ≥ 0.65
|
||
const aTokens = tokenize(normalized)
|
||
let bestScore = 0
|
||
let bestClient: DbClient | null = null
|
||
|
||
for (const [key, clients] of dbState.clientsByNormalizedName) {
|
||
const bTokens = tokenize(key)
|
||
const score = jaccardSimilarity(aTokens, bTokens)
|
||
if (score > bestScore && score >= 0.65) {
|
||
bestScore = score
|
||
bestClient = clients[0]
|
||
}
|
||
}
|
||
|
||
if (bestClient) {
|
||
return { client: bestClient, method: `jaccard(${bestScore.toFixed(2)})` }
|
||
}
|
||
|
||
return null
|
||
}
|
||
|
||
// ─── Template Matching ────────────────────────────────────────────────────────
|
||
|
||
function normalizeTemplateName(name: string): string {
|
||
const lowered = name.toLowerCase().trim()
|
||
return TEMPLATE_NAME_ALIASES[lowered] ?? lowered
|
||
}
|
||
|
||
function findTemplate(
|
||
row: ParsedRow,
|
||
templates: DbTemplate[]
|
||
): DbTemplate | null {
|
||
const normalized = normalizeTemplateName(row.taskName)
|
||
|
||
// 1. Exact normalized name match
|
||
for (const t of templates) {
|
||
if (normalizeTemplateName(t.name) === normalized) return t
|
||
}
|
||
|
||
// 2. Contains match
|
||
for (const t of templates) {
|
||
const tn = normalizeTemplateName(t.name)
|
||
if (tn.includes(normalized) || normalized.includes(tn)) return t
|
||
}
|
||
|
||
// 3. Days-based fallback (approximate offset from daysAfterRenewal)
|
||
// daysOffset ≈ -(365 - daysAfterRenewal) but let's use a 15-day window
|
||
if (row.daysAfterRenewal > 0) {
|
||
const approxOffset = -(365 - row.daysAfterRenewal)
|
||
let closest: DbTemplate | null = null
|
||
let closestDiff = Infinity
|
||
for (const t of templates) {
|
||
const diff = Math.abs(t.daysOffset - approxOffset)
|
||
if (diff < closestDiff && diff <= 15) {
|
||
closestDiff = diff
|
||
closest = t
|
||
}
|
||
}
|
||
if (closest) return closest
|
||
}
|
||
|
||
return null
|
||
}
|
||
|
||
// ─── DB State Loading ─────────────────────────────────────────────────────────
|
||
|
||
async function loadDbState(): Promise<DbState> {
|
||
log('Loading database state...')
|
||
|
||
// Load designations
|
||
const designations = await prisma.designation.findMany({
|
||
where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } },
|
||
})
|
||
const shapeDes = designations.find((d) => d.name === 'Shape')
|
||
const shape2Des = designations.find(
|
||
(d) => d.name === 'Shape 2' || d.name === 'Shape2'
|
||
)
|
||
|
||
if (!shapeDes || !shape2Des) {
|
||
throw new Error(
|
||
`Could not find Shape/Shape2 designations. Found: ${designations.map((d) => d.name).join(', ')}`
|
||
)
|
||
}
|
||
|
||
// Load templates
|
||
const allTemplates = await prisma.taskTemplate.findMany({
|
||
where: {
|
||
designationId: { in: [shapeDes.id, shape2Des.id] },
|
||
isActive: true,
|
||
},
|
||
})
|
||
|
||
const shapeTemplates = allTemplates.filter(
|
||
(t) => t.designationId === shapeDes.id
|
||
)
|
||
const shape2Templates = allTemplates.filter(
|
||
(t) => t.designationId === shape2Des.id
|
||
)
|
||
|
||
log(
|
||
` Loaded ${shapeTemplates.length} Shape templates, ${shape2Templates.length} Shape2 templates`
|
||
)
|
||
|
||
// Load SHAPE clients (those with shape designation on either designation slot)
|
||
const clients = await prisma.client.findMany({
|
||
where: {
|
||
OR: [
|
||
{ designationId: { in: [shapeDes.id, shape2Des.id] } },
|
||
{ designation2Id: { in: [shapeDes.id, shape2Des.id] } },
|
||
],
|
||
},
|
||
select: { id: true, name: true, claimsAdvocateId: true },
|
||
})
|
||
|
||
log(` Loaded ${clients.length} SHAPE clients`)
|
||
|
||
// Build lookup map
|
||
const clientsByNormalizedName = new Map<string, DbClient[]>()
|
||
for (const c of clients) {
|
||
const key = normalizeClientName(c.name)
|
||
if (!clientsByNormalizedName.has(key)) {
|
||
clientsByNormalizedName.set(key, [])
|
||
}
|
||
clientsByNormalizedName.get(key)!.push({
|
||
id: c.id,
|
||
name: c.name,
|
||
claimsAdvocateId: c.claimsAdvocateId,
|
||
})
|
||
}
|
||
|
||
return {
|
||
clientsByNormalizedName,
|
||
allClients: clients.map((c) => ({
|
||
id: c.id,
|
||
name: c.name,
|
||
claimsAdvocateId: c.claimsAdvocateId,
|
||
})),
|
||
shapeTemplates: shapeTemplates.map((t) => ({
|
||
id: t.id,
|
||
name: t.name,
|
||
daysOffset: t.daysOffset,
|
||
designationId: t.designationId,
|
||
})),
|
||
shape2Templates: shape2Templates.map((t) => ({
|
||
id: t.id,
|
||
name: t.name,
|
||
daysOffset: t.daysOffset,
|
||
designationId: t.designationId,
|
||
})),
|
||
shapeDesignationId: shapeDes.id,
|
||
shape2DesignationId: shape2Des.id,
|
||
}
|
||
}
|
||
|
||
// ─── Template Fixes ───────────────────────────────────────────────────────────
|
||
|
||
async function fixTemplates(
|
||
shapeDesignationId: string,
|
||
shape2DesignationId: string
|
||
): Promise<void> {
|
||
log('\nFixing template data...')
|
||
|
||
// Fix 1: "Request 125 day loss runs" → "Request 120 day loss runs" at -120
|
||
const fix1 = await prisma.taskTemplate.updateMany({
|
||
where: { name: 'Request 125 day loss runs', daysOffset: -125 },
|
||
data: { name: 'Request 120 day loss runs', daysOffset: -120 },
|
||
})
|
||
if (fix1.count > 0)
|
||
log(` Fixed ${fix1.count} template(s): "Request 125..." → "Request 120..."`)
|
||
|
||
// Fix 2: "Request 89 day loss runs" → "Request 90 day loss runs" at -90
|
||
const fix2 = await prisma.taskTemplate.updateMany({
|
||
where: { daysOffset: -89 },
|
||
data: { daysOffset: -90 },
|
||
})
|
||
const fix2b = await prisma.taskTemplate.updateMany({
|
||
where: {
|
||
name: { in: ['Request 89 day loss runs', 'Request 89 day loss runs (if being marketed)'] },
|
||
},
|
||
data: { name: undefined }, // name update handled separately since two variants exist
|
||
})
|
||
// Update names specifically
|
||
await prisma.taskTemplate.updateMany({
|
||
where: { name: 'Request 89 day loss runs' },
|
||
data: { name: 'Request 90 day loss runs' },
|
||
})
|
||
await prisma.taskTemplate.updateMany({
|
||
where: { name: 'Request 89 day loss runs (if being marketed)' },
|
||
data: { name: 'Request 90 day loss runs (if being marketed)' },
|
||
})
|
||
if (fix2.count > 0)
|
||
log(` Fixed ${fix2.count} template(s): offset -89 → -90, updated names`)
|
||
|
||
// Fix 3: Add missing "Claim Review" at -185 for Shape designation
|
||
const existing185 = await prisma.taskTemplate.findFirst({
|
||
where: {
|
||
name: 'Claim Review',
|
||
daysOffset: -185,
|
||
designationId: shapeDesignationId,
|
||
},
|
||
})
|
||
if (!existing185) {
|
||
if (!DRY_RUN) {
|
||
await prisma.taskTemplate.create({
|
||
data: {
|
||
name: 'Claim Review',
|
||
department: 'CLAIMS',
|
||
timing: 'PRE_RENEWAL',
|
||
daysOffset: -185,
|
||
defaultPriority: 'MEDIUM',
|
||
isActive: true,
|
||
displayOrder: 4, // after 90-day review (order 3), before mod factor (order 5)
|
||
designationId: shapeDesignationId,
|
||
},
|
||
})
|
||
log(' Created missing "Claim Review" template at -185 days (Shape)')
|
||
} else {
|
||
log(
|
||
' [DRY RUN] Would create missing "Claim Review" template at -185 days (Shape)'
|
||
)
|
||
}
|
||
} else {
|
||
log(' "Claim Review" at -185 already exists — skipping')
|
||
}
|
||
}
|
||
|
||
// ─── Task Processing ──────────────────────────────────────────────────────────
|
||
|
||
async function findMatchingTasks(
|
||
clientId: string,
|
||
templateId: string,
|
||
dueDate: Date
|
||
): Promise<{ match: string | null; duplicateIds: string[] }> {
|
||
const windowMs = 5 * 86400 * 1000 // ±5 days
|
||
const candidates = await prisma.task.findMany({
|
||
where: {
|
||
clientId,
|
||
templateId,
|
||
dueDate: {
|
||
gte: new Date(dueDate.getTime() - windowMs),
|
||
lte: new Date(dueDate.getTime() + windowMs),
|
||
},
|
||
},
|
||
orderBy: { createdAt: 'asc' },
|
||
select: { id: true, status: true, notes: true },
|
||
})
|
||
|
||
if (candidates.length === 0) return { match: null, duplicateIds: [] }
|
||
return {
|
||
match: candidates[0].id,
|
||
duplicateIds: candidates.slice(1).map((c) => c.id),
|
||
}
|
||
}
|
||
|
||
interface DateCompletedResult {
|
||
status: 'COMPLETED' | 'NA' | 'NOT_STARTED'
|
||
completedAt: Date | null
|
||
naReason: string | null
|
||
}
|
||
|
||
function interpretDateCompleted(
|
||
raw: string,
|
||
advocateUserId: string
|
||
): DateCompletedResult {
|
||
const normalized = raw.trim().toLowerCase()
|
||
|
||
if (normalized === '' || normalized === '??' || normalized === '?') {
|
||
return { status: 'NOT_STARTED', completedAt: null, naReason: null }
|
||
}
|
||
|
||
if (normalized === 'n/a' || normalized === 'na') {
|
||
return {
|
||
status: 'NA',
|
||
completedAt: null,
|
||
naReason: 'Historical — marked N/A in SHAPE tracker',
|
||
}
|
||
}
|
||
|
||
// Try to parse as Excel serial date
|
||
const serial = parseFloat(raw)
|
||
if (!isNaN(serial) && serial > 40000 && serial < 60000) {
|
||
return {
|
||
status: 'COMPLETED',
|
||
completedAt: excelSerialToDate(serial),
|
||
naReason: null,
|
||
}
|
||
}
|
||
|
||
// Unknown value — treat as not started
|
||
return { status: 'NOT_STARTED', completedAt: null, naReason: null }
|
||
}
|
||
|
||
async function processTaskRow(
|
||
clientId: string,
|
||
row: ParsedRow,
|
||
template: DbTemplate,
|
||
effectiveDate: Date,
|
||
advocateUserId: string,
|
||
stats: ImportStats
|
||
): Promise<void> {
|
||
// Calculate expected due date: effectiveDate + daysAfterRenewal
|
||
const dueDate = new Date(effectiveDate)
|
||
dueDate.setDate(dueDate.getDate() + Math.round(row.daysAfterRenewal))
|
||
|
||
const { status, completedAt, naReason } = interpretDateCompleted(
|
||
row.dateCompleted,
|
||
advocateUserId
|
||
)
|
||
|
||
const { match: taskId, duplicateIds } = await findMatchingTasks(
|
||
clientId,
|
||
template.id,
|
||
dueDate
|
||
)
|
||
|
||
// Delete duplicates
|
||
if (duplicateIds.length > 0) {
|
||
if (!DRY_RUN) {
|
||
await prisma.task.deleteMany({ where: { id: { in: duplicateIds } } })
|
||
}
|
||
stats.duplicatesDeleted += duplicateIds.length
|
||
}
|
||
|
||
if (taskId) {
|
||
// Assign advocate to existing task (regardless of completion status)
|
||
await ensureTaskAssignment(taskId, advocateUserId, stats)
|
||
|
||
// Update task if it has completion data
|
||
if (status === 'COMPLETED' || status === 'NA') {
|
||
const existing = await prisma.task.findUnique({
|
||
where: { id: taskId },
|
||
select: { status: true, notes: true },
|
||
})
|
||
|
||
if (existing && existing.status === 'NOT_STARTED') {
|
||
if (!DRY_RUN) {
|
||
await prisma.task.update({
|
||
where: { id: taskId },
|
||
data: {
|
||
status,
|
||
completedAt: completedAt ?? undefined,
|
||
completedBy: status === 'COMPLETED' ? advocateUserId : undefined,
|
||
naReason: naReason ?? undefined,
|
||
notes:
|
||
row.notes && !existing.notes
|
||
? row.notes
|
||
: existing.notes ?? undefined,
|
||
},
|
||
})
|
||
}
|
||
stats.tasksUpdated++
|
||
}
|
||
}
|
||
} else {
|
||
// Task doesn't exist — create it if it has completion data
|
||
if (status === 'COMPLETED' || status === 'NA') {
|
||
if (!DRY_RUN) {
|
||
const newTask = await prisma.task.create({
|
||
data: {
|
||
title: template.name,
|
||
department: 'CLAIMS',
|
||
timing: 'PRE_RENEWAL',
|
||
daysOffset: template.daysOffset,
|
||
dueDate,
|
||
status,
|
||
priority: 'MEDIUM',
|
||
clientId,
|
||
templateId: template.id,
|
||
completedAt: completedAt ?? undefined,
|
||
completedBy: status === 'COMPLETED' ? advocateUserId : undefined,
|
||
naReason: naReason ?? undefined,
|
||
notes: row.notes || undefined,
|
||
isAdHoc: false,
|
||
createdBy: advocateUserId,
|
||
},
|
||
})
|
||
await ensureTaskAssignment(newTask.id, advocateUserId, stats)
|
||
} else {
|
||
stats.tasksAssigned++ // count the would-be assignment
|
||
}
|
||
stats.tasksCreated++
|
||
} else {
|
||
// NOT_STARTED and no existing task — this shouldn't happen for SHAPE clients
|
||
// but if it does, create the task and assign it
|
||
if (!DRY_RUN) {
|
||
const newTask = await prisma.task.create({
|
||
data: {
|
||
title: template.name,
|
||
department: 'CLAIMS',
|
||
timing: 'PRE_RENEWAL',
|
||
daysOffset: template.daysOffset,
|
||
dueDate,
|
||
status: 'NOT_STARTED',
|
||
priority: 'MEDIUM',
|
||
clientId,
|
||
templateId: template.id,
|
||
notes: row.notes || undefined,
|
||
isAdHoc: false,
|
||
createdBy: advocateUserId,
|
||
},
|
||
})
|
||
await ensureTaskAssignment(newTask.id, advocateUserId, stats)
|
||
} else {
|
||
stats.tasksAssigned++
|
||
}
|
||
stats.tasksCreated++
|
||
}
|
||
}
|
||
}
|
||
|
||
async function ensureTaskAssignment(
|
||
taskId: string,
|
||
userId: string,
|
||
stats: ImportStats
|
||
): Promise<void> {
|
||
if (DRY_RUN) {
|
||
stats.tasksAssigned++
|
||
return
|
||
}
|
||
try {
|
||
await prisma.taskAssignment.upsert({
|
||
where: { taskId_userId: { taskId, userId } },
|
||
create: { taskId, userId },
|
||
update: {},
|
||
})
|
||
stats.tasksAssigned++
|
||
} catch {
|
||
// Unique constraint violation means it already exists — fine
|
||
}
|
||
}
|
||
|
||
async function createAdHocTask(
|
||
clientId: string,
|
||
text: string,
|
||
effectiveDate: Date,
|
||
advocateUserId: string,
|
||
stats: ImportStats
|
||
): Promise<void> {
|
||
const title = text.slice(0, 500)
|
||
|
||
// Idempotency check
|
||
const existing = await prisma.task.findFirst({
|
||
where: { clientId, title, isAdHoc: true },
|
||
})
|
||
if (existing) return
|
||
|
||
if (!DRY_RUN) {
|
||
const task = await prisma.task.create({
|
||
data: {
|
||
title,
|
||
department: 'CLAIMS',
|
||
timing: 'PRE_RENEWAL',
|
||
daysOffset: 0,
|
||
dueDate: effectiveDate,
|
||
status: 'COMPLETED',
|
||
priority: 'MEDIUM',
|
||
clientId,
|
||
isAdHoc: true,
|
||
completedAt: effectiveDate,
|
||
completedBy: advocateUserId,
|
||
notes: text,
|
||
createdBy: advocateUserId,
|
||
},
|
||
})
|
||
await ensureTaskAssignment(task.id, advocateUserId, stats)
|
||
} else {
|
||
stats.tasksAssigned++
|
||
}
|
||
stats.adHocCreated++
|
||
}
|
||
|
||
async function assignAdvocate(
|
||
clientId: string,
|
||
advocateUserId: string,
|
||
currentAdvocateId: string | null,
|
||
stats: ImportStats
|
||
): Promise<void> {
|
||
if (currentAdvocateId) return // Never overwrite existing assignment
|
||
if (!DRY_RUN) {
|
||
await prisma.client.update({
|
||
where: { id: clientId },
|
||
data: { claimsAdvocateId: advocateUserId },
|
||
})
|
||
}
|
||
stats.advocatesAssigned++
|
||
}
|
||
|
||
// ─── File Processing ──────────────────────────────────────────────────────────
|
||
|
||
async function processFile(
|
||
folder: string,
|
||
itemId: string,
|
||
fileName: string,
|
||
dbState: DbState,
|
||
stats: ImportStats
|
||
): Promise<void> {
|
||
const advocateInfo = TEAM_MEMBER_MAP[folder]!
|
||
|
||
let buf: Buffer
|
||
try {
|
||
buf = await downloadFile(itemId)
|
||
await sleep(150)
|
||
} catch (err) {
|
||
const msg = `${folder}/${fileName}: Download failed — ${(err as Error).message}`
|
||
stats.errors.push(msg)
|
||
return
|
||
}
|
||
|
||
let blocks: ParsedBlock[]
|
||
try {
|
||
blocks = parseExcelFile(buf, fileName)
|
||
} catch (err) {
|
||
const msg = `${folder}/${fileName}: Parse failed — ${(err as Error).message}`
|
||
stats.errors.push(msg)
|
||
return
|
||
}
|
||
|
||
if (blocks.length === 0) {
|
||
stats.errors.push(`${folder}/${fileName}: No data blocks found`)
|
||
return
|
||
}
|
||
|
||
stats.filesProcessed++
|
||
|
||
for (const block of blocks) {
|
||
// Match client
|
||
const matchResult = matchClient(block.clientName, dbState)
|
||
if (!matchResult) {
|
||
stats.clientsUnmatched++
|
||
stats.unmatchedClients.push({
|
||
folder,
|
||
file: fileName,
|
||
excelName: block.clientName,
|
||
})
|
||
continue
|
||
}
|
||
|
||
const { client, method } = matchResult
|
||
stats.clientsMatched++
|
||
|
||
if (method !== 'exact') {
|
||
stats.fuzzyMatches.push({
|
||
folder,
|
||
file: fileName,
|
||
excelName: block.clientName,
|
||
dbName: client.name,
|
||
method,
|
||
})
|
||
}
|
||
|
||
// Assign advocate at client level
|
||
await assignAdvocate(
|
||
client.id,
|
||
advocateInfo.userId,
|
||
client.claimsAdvocateId,
|
||
stats
|
||
)
|
||
|
||
// Update our cached advocate status so subsequent blocks don't re-assign
|
||
client.claimsAdvocateId = client.claimsAdvocateId ?? advocateInfo.userId
|
||
|
||
// Choose templates based on SHAPE vs SHAPE2
|
||
const templates = block.isShape2
|
||
? dbState.shape2Templates
|
||
: dbState.shapeTemplates
|
||
|
||
// Process each task row
|
||
for (const row of block.rows) {
|
||
const template = findTemplate(row, templates)
|
||
if (!template) {
|
||
// Unmatched template — add as additional service if it has completion data
|
||
const { status } = interpretDateCompleted(row.dateCompleted, advocateInfo.userId)
|
||
if (status === 'COMPLETED' || status === 'NA') {
|
||
const adHocTitle = `[Unmatched Task] ${row.taskName}${row.notes ? ': ' + row.notes : ''}`
|
||
await createAdHocTask(
|
||
client.id,
|
||
adHocTitle,
|
||
block.effectiveDate,
|
||
advocateInfo.userId,
|
||
stats
|
||
)
|
||
}
|
||
continue
|
||
}
|
||
|
||
try {
|
||
await processTaskRow(
|
||
client.id,
|
||
row,
|
||
template,
|
||
block.effectiveDate,
|
||
advocateInfo.userId,
|
||
stats
|
||
)
|
||
} catch (err) {
|
||
stats.errors.push(
|
||
`${folder}/${fileName} / ${block.clientName} / "${row.taskName}": ${(err as Error).message}`
|
||
)
|
||
}
|
||
}
|
||
|
||
// Create ad-hoc tasks for additional services
|
||
for (const svc of block.additionalServices) {
|
||
try {
|
||
await createAdHocTask(
|
||
client.id,
|
||
svc,
|
||
block.effectiveDate,
|
||
advocateInfo.userId,
|
||
stats
|
||
)
|
||
} catch (err) {
|
||
stats.errors.push(
|
||
`${folder}/${fileName} / ${block.clientName} / adHoc "${svc.slice(0, 40)}": ${(err as Error).message}`
|
||
)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ─── Report ───────────────────────────────────────────────────────────────────
|
||
|
||
function printReport(stats: ImportStats): void {
|
||
log('')
|
||
log('═'.repeat(60))
|
||
log('SHAPE Historical Import Report')
|
||
log('═'.repeat(60))
|
||
log(`Mode: ${DRY_RUN ? 'DRY RUN (no changes written)' : 'EXECUTE (changes written to DB)'}`)
|
||
log('')
|
||
log(`Files processed: ${stats.filesProcessed}`)
|
||
log(`Clients matched: ${stats.clientsMatched}`)
|
||
log(` - exact: ${stats.clientsMatched - stats.fuzzyMatches.length}`)
|
||
log(` - fuzzy: ${stats.fuzzyMatches.length}`)
|
||
log(`Clients unmatched: ${stats.clientsUnmatched}`)
|
||
log(`Tasks updated: ${stats.tasksUpdated} (completed/NA status set)`)
|
||
log(`Tasks assigned: ${stats.tasksAssigned} (advocate linked)`)
|
||
log(`Tasks created: ${stats.tasksCreated} (missing from DB)`)
|
||
log(`Duplicates deleted: ${stats.duplicatesDeleted}`)
|
||
log(`Ad-hoc created: ${stats.adHocCreated}`)
|
||
log(`Advocates assigned: ${stats.advocatesAssigned} (client-level)`)
|
||
log(`Errors: ${stats.errors.length}`)
|
||
|
||
if (stats.fuzzyMatches.length > 0) {
|
||
log('')
|
||
log('── Fuzzy Client Matches (review for correctness) ──────────────')
|
||
for (const m of stats.fuzzyMatches) {
|
||
log(` [${m.folder}] "${m.excelName}" → "${m.dbName}" (${m.method})`)
|
||
}
|
||
}
|
||
|
||
if (stats.unmatchedClients.length > 0) {
|
||
log('')
|
||
log('── Unmatched Clients (manual action required) ──────────────────')
|
||
for (const u of stats.unmatchedClients) {
|
||
log(` [${u.folder}] ${u.file} → "${u.excelName}"`)
|
||
}
|
||
}
|
||
|
||
if (stats.errors.length > 0) {
|
||
log('')
|
||
log('── Errors ──────────────────────────────────────────────────────')
|
||
for (const e of stats.errors) {
|
||
log(` ERROR: ${e}`)
|
||
}
|
||
}
|
||
|
||
log('')
|
||
log('═'.repeat(60))
|
||
}
|
||
|
||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||
|
||
async function main() {
|
||
log(`SHAPE Historical Import — ${new Date().toISOString()}`)
|
||
log(
|
||
`Mode: ${DRY_RUN ? 'DRY RUN (pass --execute to write)' : 'EXECUTE'}`
|
||
)
|
||
|
||
const stats: ImportStats = {
|
||
filesProcessed: 0,
|
||
clientsMatched: 0,
|
||
clientsUnmatched: 0,
|
||
tasksUpdated: 0,
|
||
tasksAssigned: 0,
|
||
tasksCreated: 0,
|
||
duplicatesDeleted: 0,
|
||
adHocCreated: 0,
|
||
advocatesAssigned: 0,
|
||
errors: [],
|
||
unmatchedClients: [],
|
||
fuzzyMatches: [],
|
||
}
|
||
|
||
// Step 1: Fix template data
|
||
const designations = await prisma.designation.findMany({
|
||
where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } },
|
||
})
|
||
const shapeDes = designations.find((d) => d.name === 'Shape')
|
||
const shape2Des = designations.find(
|
||
(d) => d.name === 'Shape 2' || d.name === 'Shape2'
|
||
)
|
||
if (!shapeDes || !shape2Des) throw new Error('Shape designations not found')
|
||
|
||
await fixTemplates(shapeDes.id, shape2Des.id)
|
||
|
||
// Step 2: Load DB state
|
||
const dbState = await loadDbState()
|
||
|
||
// Step 3: Discover SharePoint files
|
||
const files = await discoverFiles()
|
||
|
||
// Step 4: Process each file
|
||
log(`\nProcessing ${files.length} files...`)
|
||
for (let i = 0; i < files.length; i++) {
|
||
const { folder, itemId, fileName } = files[i]
|
||
const pct = Math.round(((i + 1) / files.length) * 100)
|
||
process.stdout.write(
|
||
`\r [${pct}%] ${i + 1}/${files.length} — ${folder}/${fileName.slice(0, 40)}`
|
||
)
|
||
await processFile(folder, itemId, fileName, dbState, stats)
|
||
}
|
||
process.stdout.write('\n')
|
||
|
||
// Step 5: Print report
|
||
printReport(stats)
|
||
writeOutput()
|
||
}
|
||
|
||
main()
|
||
.catch((e) => {
|
||
console.error('\nFatal error:', e)
|
||
process.exit(1)
|
||
})
|
||
.finally(async () => {
|
||
await prisma.$disconnect()
|
||
await pool.end()
|
||
})
|