seubert-claims/ondeck/scripts/import-shape-historical.ts

/**
 * SHAPE Historical Data Import Script
 *
 * Imports historical task completion data from SharePoint Excel files into Horizon.
 * Each team member has a folder in Claims/SHAPE Accounts/{MEMBER_FOLDER}/{Client}.xlsx
 *
 * Usage:
 *   npx tsx scripts/import-shape-historical.ts              # dry run
 *   npx tsx scripts/import-shape-historical.ts --execute    # write to DB
 *
 * Output is written to /tmp/shape-import-report.txt (override with --output=/path)
 */

import { PrismaClient } from '@prisma/client'
import { PrismaPg } from '@prisma/adapter-pg'
import { Pool } from 'pg'
import * as zlib from 'zlib'
import * as fs from 'fs'
import * as dotenv from 'dotenv'

dotenv.config()

// ─── Configuration ────────────────────────────────────────────────────────────

const DRY_RUN = !process.argv.includes('--execute')
const OUTPUT_FILE =
  process.argv.find((a) => a.startsWith('--output='))?.split('=')[1] ??
  '/tmp/shape-import-report.txt'

const DRIVE_ID =
  'b!OYuzIexQkkOvfEPyMJPzzZHfzTrOCOdPhTWgTlzKs6M0ZWVrAc6LR4LjWl4QFEzm'

const TEAM_MEMBER_MAP: Record<string, { userId: string; displayName: string }> =
  {
    CHRIS: {
      userId: 'cmkm36yc90021y7vb9783nitl',
      displayName: 'Christine Gove',
    },
    DAWN: {
      userId: 'cmkm36xfr000ny7vbcajejjux',
      displayName: 'Dawn Boland',
    },
    Jeanne: {
      userId: 'cmkm370f80065y7vbrugc9jub',
      displayName: 'Jeanne Strong',
    },
    LUKE: {
      userId: 'cmkm36xd4000jy7vbhanp66s9',
      displayName: 'Luke Billman',
    },
    MIMI: {
      userId: 'cmkm36zr0004wy7vbztyynj35',
      displayName: 'Mimi Rawlings',
    },
  }

// Template name aliases: old/variant names → canonical name (after fix)
const TEMPLATE_NAME_ALIASES: Record<string, string> = {
  'request 125 day loss runs': 'request 120 day loss runs',
  'request 89 day loss runs': 'request 90 day loss runs',
  'request 89 day loss runs (if being marketed)':
    'request 90 day loss runs (if being marketed)',
  'claim review (six month)': 'claim review',
  'claims review': 'claim review',
}

// ─── DB Setup ─────────────────────────────────────────────────────────────────

const pool = new Pool({ connectionString: process.env.DATABASE_URL })
const adapter = new PrismaPg(pool)
const prisma = new PrismaClient({ adapter })

// ─── Output / Logging ─────────────────────────────────────────────────────────

const outputLines: string[] = []

function log(line = '') {
  console.log(line)
  outputLines.push(line)
}

function writeOutput() {
  fs.writeFileSync(OUTPUT_FILE, outputLines.join('\n') + '\n', 'utf-8')
  console.log(`\nReport written to ${OUTPUT_FILE}`)
}

// ─── Types ────────────────────────────────────────────────────────────────────

interface ParsedRow {
  taskName: string
  daysAfterRenewal: number
  dateCompleted: string // raw: numeric string | 'n/a' | '??' | ''
  notes: string
}

interface ParsedBlock {
  clientName: string
  effectiveDate: Date
  isShape2: boolean
  rows: ParsedRow[]
  additionalServices: string[]
}

interface DbTemplate {
  id: string
  name: string
  daysOffset: number
  designationId: string | null
}

interface DbClient {
  id: string
  name: string
  claimsAdvocateId: string | null
}

interface DbState {
  clientsByNormalizedName: Map<string, DbClient[]>
  allClients: DbClient[]
  shapeTemplates: DbTemplate[]
  shape2Templates: DbTemplate[]
  shapeDesignationId: string
  shape2DesignationId: string
}

interface ImportStats {
  filesProcessed: number
  clientsMatched: number
  clientsUnmatched: number
  tasksUpdated: number
  tasksAssigned: number
  tasksCreated: number
  duplicatesDeleted: number
  adHocCreated: number
  advocatesAssigned: number
  errors: string[]
  unmatchedClients: Array<{ folder: string; file: string; excelName: string }>
  fuzzyMatches: Array<{
    folder: string
    file: string
    excelName: string
    dbName: string
    method: string
  }>
}

// ─── Graph API ────────────────────────────────────────────────────────────────

let _cachedToken: { token: string; expiresAt: number } | null = null

async function getGraphToken(): Promise<string> {
  if (_cachedToken && Date.now() < _cachedToken.expiresAt - 30_000) {
    return _cachedToken.token
  }
  const tenantId = process.env.AZURE_AD_TENANT_ID!
  const clientId = process.env.AZURE_AD_CLIENT_ID!
  const clientSecret = process.env.AZURE_AD_CLIENT_SECRET!
  const body = new URLSearchParams({
    grant_type: 'client_credentials',
    client_id: clientId,
    client_secret: clientSecret,
    scope: 'https://graph.microsoft.com/.default',
  })
  const res = await fetch(
    `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`,
    {
      method: 'POST',
      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
      body: body.toString(),
    }
  )
  if (!res.ok) throw new Error(`Token fetch failed: ${res.status}`)
  const data = (await res.json()) as {
    access_token: string
    expires_in: number
  }
  _cachedToken = {
    token: data.access_token,
    expiresAt: Date.now() + data.expires_in * 1000,
  }
  return _cachedToken.token
}

async function graphGet(url: string): Promise<unknown> {
  const token = await getGraphToken()
  const res = await fetch(url, {
    headers: { Authorization: `Bearer ${token}` },
  })
  if (!res.ok) {
    const text = await res.text()
    throw new Error(`Graph API error ${res.status}: ${text.slice(0, 200)}`)
  }
  return res.json()
}

async function downloadFile(itemId: string): Promise<Buffer> {
  const token = await getGraphToken()
  const url = `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/items/${itemId}/content`
  const res = await fetch(url, {
    headers: { Authorization: `Bearer ${token}` },
    redirect: 'follow',
  })
  if (!res.ok) throw new Error(`Download failed: ${res.status}`)
  const ab = await res.arrayBuffer()
  return Buffer.from(ab)
}

async function sleep(ms: number) {
  return new Promise((r) => setTimeout(r, ms))
}

interface DriveItem {
  id: string
  name: string
  file?: { mimeType: string }
  folder?: { childCount: number }
}

async function listChildren(itemId: string): Promise<DriveItem[]> {
  const data = (await graphGet(
    `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/items/${itemId}/children?$top=200`
  )) as { value: DriveItem[] }
  return data.value ?? []
}

async function discoverFiles(): Promise<
  Array<{ folder: string; itemId: string; fileName: string }>
> {
  log('Discovering files from SharePoint...')
  const files: Array<{ folder: string; itemId: string; fileName: string }> = []

  const rootData = (await graphGet(
    `https://graph.microsoft.com/v1.0/drives/${DRIVE_ID}/root:/Claims%2FSHAPE%20Accounts:/children`
  )) as { value: DriveItem[] }
  const rootChildren = rootData.value ?? []

  for (const item of rootChildren) {
    if (!item.folder) continue
    const memberInfo = TEAM_MEMBER_MAP[item.name]
    if (!memberInfo) continue

    log(`  Scanning folder: ${item.name}`)
    const children = await listChildren(item.id)
    await sleep(150)

    for (const child of children) {
      if (!child.file) continue
      if (!child.name.toLowerCase().endsWith('.xlsx')) continue
      if (child.name.toLowerCase().startsWith('archive')) continue
      // Skip template/master files
      if (
        child.name.toLowerCase().includes('template') ||
        child.name.toLowerCase().includes('master workbook') ||
        child.name.toLowerCase().startsWith('all shape')
      )
        continue

      files.push({ folder: item.name, itemId: child.id, fileName: child.name })
    }
  }

  log(`  Found ${files.length} Excel files`)
  return files
}

// ─── ZIP Parser ───────────────────────────────────────────────────────────────

function parseZip(buffer: Buffer): Map<string, Buffer> {
  const entries = new Map<string, Buffer>()

  // Find End of Central Directory (EOCD) record — signature 0x06054b50
  let eocdOffset = -1
  const searchStart = Math.max(0, buffer.length - 65558)
  for (let i = buffer.length - 22; i >= searchStart; i--) {
    if (buffer.readUInt32LE(i) === 0x06054b50) {
      eocdOffset = i
      break
    }
  }
  if (eocdOffset < 0) throw new Error('Not a valid ZIP file (EOCD not found)')

  const cdOffset = buffer.readUInt32LE(eocdOffset + 16)
  const cdEntries = buffer.readUInt16LE(eocdOffset + 10)

  let pos = cdOffset
  for (let i = 0; i < cdEntries; i++) {
    if (buffer.readUInt32LE(pos) !== 0x02014b50) {
      throw new Error(`Invalid central directory entry at offset ${pos}`)
    }
    const method = buffer.readUInt16LE(pos + 10)
    const compSize = buffer.readUInt32LE(pos + 20)
    const fileNameLen = buffer.readUInt16LE(pos + 28)
    const extraLen = buffer.readUInt16LE(pos + 30)
    const commentLen = buffer.readUInt16LE(pos + 32)
    const localOffset = buffer.readUInt32LE(pos + 42)
    const fileName = buffer.slice(pos + 46, pos + 46 + fileNameLen).toString('utf-8')

    // Read local file header
    if (buffer.readUInt32LE(localOffset) !== 0x04034b50) {
      throw new Error(`Invalid local header for ${fileName}`)
    }
    const localFileNameLen = buffer.readUInt16LE(localOffset + 26)
    const localExtraLen = buffer.readUInt16LE(localOffset + 28)
    const dataStart = localOffset + 30 + localFileNameLen + localExtraLen

    const compData = buffer.slice(dataStart, dataStart + compSize)
    let data: Buffer
    if (method === 0) {
      data = compData // stored
    } else if (method === 8) {
      data = zlib.inflateRawSync(compData) // deflate
    } else {
      data = Buffer.alloc(0) // unsupported, skip
    }

    // Store with lowercase key for case-insensitive lookup
    entries.set(fileName.toLowerCase(), data)

    pos += 46 + fileNameLen + extraLen + commentLen
  }

  return entries
}

// ─── XLSX Parsing ─────────────────────────────────────────────────────────────

function decodeXmlEntities(s: string): string {
  return s
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&apos;/g, "'")
    .replace(/&#x([0-9A-Fa-f]+);/gi, (_, h) =>
      String.fromCharCode(parseInt(h, 16))
    )
    .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
}

function parseSharedStrings(xmlBuf: Buffer | undefined): string[] {
  if (!xmlBuf || xmlBuf.length === 0) return []
  const xml = xmlBuf.toString('utf-8')
  const strings: string[] = []
  const siRegex = /<si>([\s\S]*?)<\/si>/g
  let match: RegExpExecArray | null
  while ((match = siRegex.exec(xml)) !== null) {
    const siContent = match[1]
    const texts: string[] = []
    const tRegex = /<t(?:\s[^>]*)?>([^<]*)<\/t>/g
    let tm: RegExpExecArray | null
    while ((tm = tRegex.exec(siContent)) !== null) {
      texts.push(decodeXmlEntities(tm[1]))
    }
    strings.push(texts.join(''))
  }
  return strings
}

function colLetterToNum(letters: string): number {
  let n = 0
  for (let i = 0; i < letters.length; i++) {
    n = n * 26 + (letters.charCodeAt(i) - 64)
  }
  return n
}

function numToColLetter(n: number): string {
  let s = ''
  while (n > 0) {
    s = String.fromCharCode(((n - 1) % 26) + 65) + s
    n = Math.floor((n - 1) / 26)
  }
  return s
}

function parseSheetToGrid(
  sheetBuf: Buffer,
  sharedStrings: string[]
): { get: (row: number, col: number) => string; maxRow: number } {
  const cells = new Map<string, string>()
  let maxRow = 0
  // Strip self-closing <c .../> tags first — they represent empty cells and must not
  // be confused with the opening tag of a non-empty cell that follows.
  const xml = sheetBuf.toString('utf-8').replace(/<c\b[^>]*\/>/g, '')

  // Match each <c r="..." ...>...</c> block
  const cRegex = /<c\s+r="([A-Z]+)(\d+)"([^>]*)>([\s\S]*?)<\/c>/g
  let m: RegExpExecArray | null
  while ((m = cRegex.exec(xml)) !== null) {
    const colStr = m[1]
    const rowNum = parseInt(m[2])
    const attrs = m[3]
    const inner = m[4]

    if (rowNum > maxRow) maxRow = rowNum

    const typeMatch = attrs.match(/\bt="([^"]+)"/)
    const type = typeMatch ? typeMatch[1] : 'n'

    let value = ''
    const vMatch = inner.match(/<v>([^<]*)<\/v>/)
    const isMatch = inner.match(/<is><t(?:[^>]*)?>([^<]*)<\/t><\/is>/)

    if (isMatch) {
      value = decodeXmlEntities(isMatch[1])
    } else if (vMatch) {
      if (type === 's') {
        value = sharedStrings[parseInt(vMatch[1])] ?? ''
      } else if (type === 'str') {
        value = decodeXmlEntities(vMatch[1])
      } else if (type === 'b') {
        value = vMatch[1] === '1' ? 'TRUE' : 'FALSE'
      } else {
        value = vMatch[1] // number/date
      }
    }

    const col = colLetterToNum(colStr)
    cells.set(`${rowNum}:${col}`, value)
  }

  return {
    get(row: number, col: number): string {
      return cells.get(`${row}:${col}`) ?? ''
    },
    maxRow,
  }
}

function findTargetSheet(
  zipEntries: Map<string, Buffer>
): Buffer | null {
  // Parse workbook.xml to find sheet r:ids, then map via rels
  const wbBuf = zipEntries.get('xl/workbook.xml')
  if (!wbBuf) return zipEntries.get('xl/worksheets/sheet1.xml') ?? null

  const wb = wbBuf.toString('utf-8')
  const relsBuf = zipEntries.get('xl/_rels/workbook.xml.rels')

  // Build r:id → file path map from rels
  // Paths in xl/_rels/workbook.xml.rels are relative to xl/
  // e.g. Target="worksheets/sheet1.xml" → actual path is "xl/worksheets/sheet1.xml"
  const ridToPath = new Map<string, string>()
  if (relsBuf) {
    const rels = relsBuf.toString('utf-8')
    const relRegex = /<Relationship\s+Id="([^"]+)"[^>]+Target="([^"]+)"/g
    let rm: RegExpExecArray | null
    while ((rm = relRegex.exec(rels)) !== null) {
      let target = rm[2]
      if (target.startsWith('/')) {
        target = target.slice(1) // absolute paths: strip leading /
      } else if (!target.startsWith('xl/')) {
        target = 'xl/' + target // relative to xl/: prepend xl/
      }
      ridToPath.set(rm[1], target)
    }
  }

  // Find sheet entries — attributes may appear in any order, parse flexibly
  const sheetEntries: Array<{ name: string; rid: string }> = []
  const sheetTagRegex = /<sheet\b([^>]*\/?>)/g
  let sm: RegExpExecArray | null
  while ((sm = sheetTagRegex.exec(wb)) !== null) {
    const attrs = sm[1]
    const nameMatch = attrs.match(/\bname="([^"]+)"/)
    const ridMatch = attrs.match(/\br:id="([^"]+)"/)
    if (nameMatch && ridMatch) {
      sheetEntries.push({ name: nameMatch[1], rid: ridMatch[1] })
    }
  }

  const resolveSheet = (rid: string): Buffer | null => {
    const path = ridToPath.get(rid)
    if (!path) return null
    return zipEntries.get(path.toLowerCase()) ?? null
  }

  // Prefer SHAPE2 > SHAPE > CLIENT, then fall back to first sheet
  const priority = ['shape2', 'shape', 'client']
  for (const p of priority) {
    const match = sheetEntries.find((s) => s.name.toLowerCase() === p)
    if (match) {
      const buf = resolveSheet(match.rid)
      if (buf) return buf
    }
  }

  if (sheetEntries.length > 0) {
    const buf = resolveSheet(sheetEntries[0].rid)
    if (buf) return buf
  }

  return zipEntries.get('xl/worksheets/sheet1.xml') ?? null
}

function excelSerialToDate(serial: number): Date {
  // Excel epoch: Jan 0, 1900 (effectively Dec 30, 1899)
  // But Excel has a leap year bug treating 1900 as leap year, so offset is 25569
  return new Date((serial - 25569) * 86400 * 1000)
}

// ─── Block Parser ─────────────────────────────────────────────────────────────

function parseExcelBlocks(
  grid: { get: (r: number, c: number) => string; maxRow: number },
  fileName: string
): ParsedBlock[] {
  const blocks: ParsedBlock[] = []
  const isShape2 = /(SHAPE2|SHAPE 2)/i.test(fileName)

  let row = 1
  while (row <= grid.maxRow) {
    // Scan up to 10 columns for "Client Name:"
    let clientNameCol = -1
    for (let c = 1; c <= 5; c++) {
      if (grid.get(row, c).trim().toLowerCase().includes('client name')) {
        clientNameCol = c
        break
      }
    }

    if (clientNameCol < 0) {
      row++
      continue
    }

    // Client name: first try the cell to the right, then up to 3 columns right
    let clientName = ''
    for (let dc = 1; dc <= 3; dc++) {
      const candidate = grid.get(row, clientNameCol + dc).trim()
      if (candidate && !candidate.toLowerCase().includes('client name')) {
        clientName = candidate
        break
      }
    }
    // Fallback: extract from filename (strip known suffixes)
    if (!clientName) {
      clientName = fileName
        .replace(/^SHAPE\S*\s*[-–]\s*/i, '')  // strip leading "SHAPE2 - " prefix
        .replace(/\s*[-–]\s*SHAPE.*$/i, '')
        .replace(/\s*SHAPE\s*Services?\s*Checklist.*$/i, '')
        .replace(/\s*SHAPE\b.*/i, '')
        .replace(/\.xlsx$/i, '')
        .trim()
    }
    if (!clientName) {
      row++
      continue
    }

    row++

    // Find effective date (within next 5 rows)
    // Strategy 1: look for "Effective Date:" label, take numeric value in adjacent cell
    // Strategy 2 (fallback): look for any Excel-range serial (40000–60000) in next rows
    let effectiveDateSerial = 0
    let foundDateRow = row
    const EXCEL_DATE_MIN = 40000
    const EXCEL_DATE_MAX = 60000

    for (let r = row; r <= Math.min(row + 4, grid.maxRow); r++) {
      let foundDateInRow = false
      for (let c = 1; c <= 8; c++) {
        const cellVal = grid.get(r, c).trim()
        if (cellVal.toLowerCase().includes('effective date')) {
          // Date value is in next column(s)
          for (let dc = 1; dc <= 4; dc++) {
            const val = grid.get(r, c + dc).trim()
            const serial = parseFloat(val)
            if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) {
              effectiveDateSerial = serial
              foundDateInRow = true
              break
            }
          }
          if (foundDateInRow) break
        }
        // Fallback: any valid date serial in this row (not the first col which may be a number label)
        if (!foundDateInRow && c > 1) {
          const serial = parseFloat(cellVal)
          if (!isNaN(serial) && serial > EXCEL_DATE_MIN && serial < EXCEL_DATE_MAX) {
            effectiveDateSerial = serial
            foundDateInRow = true
            break
          }
        }
      }
      if (foundDateInRow) {
        foundDateRow = r
        break
      }
    }

    // If no effective date found, use current date as fallback (for files like Minniefield
    // where date is stored as display-only formatting, not cell value)
    const effectiveDate = effectiveDateSerial > 0
      ? excelSerialToDate(effectiveDateSerial)
      : new Date()
    row = foundDateRow + 1

    // Skip "Policies:" row if present
    for (let r = row; r <= Math.min(row + 3, grid.maxRow); r++) {
      let foundPolicies = false
      for (let c = 1; c <= 5; c++) {
        if (grid.get(r, c).trim().toLowerCase().startsWith('policies')) {
          foundPolicies = true
          break
        }
      }
      if (foundPolicies) {
        row = r + 1
        break
      }
      // Check if this is the header row — stop skipping
      let hasTask = false
      for (let c = 1; c <= 8; c++) {
        if (grid.get(r, c).trim().toLowerCase() === 'task') {
          hasTask = true
          break
        }
      }
      if (hasTask) break
    }

    // Find header row containing "Task"
    let headerRow = -1
    let taskCol = 0
    for (let r = row; r <= Math.min(row + 5, grid.maxRow); r++) {
      for (let c = 1; c <= 8; c++) {
        if (grid.get(r, c).trim().toLowerCase() === 'task') {
          headerRow = r
          taskCol = c
          break
        }
      }
      if (headerRow >= 0) break
    }

    if (headerRow < 0) {
      row++
      continue
    }

    // Map header columns (scan right of taskCol)
    let daysAfterCol = 0
    let dateCompletedCol = 0
    let notesCol = 0

    for (let c = taskCol + 1; c <= taskCol + 8; c++) {
      const hdr = grid.get(headerRow, c).trim().toLowerCase()
      if (hdr.includes('days after')) daysAfterCol = c
      else if (
        hdr.includes('date completed') ||
        hdr === 'completed' ||
        hdr === 'date complete'
      )
        dateCompletedCol = c
      else if (hdr.includes('notes')) notesCol = c
    }

    row = headerRow + 1

    // Parse data rows
    const taskRows: ParsedRow[] = []
    const additionalServices: string[] = []
    let inAdditional = false

    while (row <= grid.maxRow) {
      // Check if next block starts
      let nextBlock = false
      for (let c = 1; c <= 5; c++) {
        if (grid.get(row, c).trim().toLowerCase().includes('client name')) {
          nextBlock = true
          break
        }
      }
      if (nextBlock) break

      const taskName = grid.get(row, taskCol).trim()

      if (!taskName) {
        row++
        continue
      }

      // Detect "Additional Services Provided" section header
      if (taskName.toLowerCase().includes('additional services')) {
        inAdditional = true
        row++
        continue
      }

      const daysAfterRaw =
        daysAfterCol > 0 ? grid.get(row, daysAfterCol).trim() : ''
      const isDaysNumeric = daysAfterRaw !== '' && !isNaN(parseFloat(daysAfterRaw))

      // If no numeric days value, treat as additional service / free text
      if (!isDaysNumeric) {
        if (taskName.length > 3) {
          additionalServices.push(taskName)
        }
        row++
        continue
      }

      if (inAdditional) {
        additionalServices.push(taskName)
        row++
        continue
      }

      const daysAfterRenewal = parseFloat(daysAfterRaw)
      const dateCompletedRaw =
        dateCompletedCol > 0 ? grid.get(row, dateCompletedCol).trim() : ''
      const notes = notesCol > 0 ? grid.get(row, notesCol).trim() : ''

      taskRows.push({
        taskName,
        daysAfterRenewal,
        dateCompleted: dateCompletedRaw,
        notes,
      })

      row++
    }

    blocks.push({
      clientName,
      effectiveDate,
      isShape2,
      rows: taskRows,
      additionalServices,
    })
  }

  return blocks
}

function parseExcelFile(buf: Buffer, fileName: string): ParsedBlock[] {
  const zipEntries = parseZip(buf)
  const sharedStringsBuf = zipEntries.get('xl/sharedstrings.xml')
  const sharedStrings = parseSharedStrings(sharedStringsBuf)
  const sheetBuf = findTargetSheet(zipEntries)
  if (!sheetBuf) return []
  const grid = parseSheetToGrid(sheetBuf, sharedStrings)
  return parseExcelBlocks(grid, fileName)
}

// ─── Client Name Normalization / Matching ─────────────────────────────────────

function normalizeClientName(name: string): string {
  return name
    .toLowerCase()
    .replace(/^the\s+/, '')
    .replace(/&/g, 'and')
    .replace(
      /\b(incorporated|inc|llc|l\.l\.c|corp|corporation|co|ltd|limited|lp|l\.p|company|enterprises|enterprise|services|group|associates|solutions|management|consulting)\b\.?/gi,
      ''
    )
    .replace(/[.,;:'"()\-_#@!?]/g, '')
    .replace(/\s+/g, ' ')
    .trim()
}

function tokenize(s: string): Set<string> {
  return new Set(
    s
      .split(/\s+/)
      .filter((t) => t.length > 1)
  )
}

function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
  if (a.size === 0 && b.size === 0) return 1
  let intersection = 0
  for (const t of a) {
    if (b.has(t)) intersection++
  }
  const union = a.size + b.size - intersection
  return union === 0 ? 0 : intersection / union
}

function matchClient(
  excelName: string,
  dbState: DbState
): { client: DbClient; method: string } | null {
  const normalized = normalizeClientName(excelName)
  if (!normalized) return null

  // 1. Exact normalized match
  const exactCandidates = dbState.clientsByNormalizedName.get(normalized)
  if (exactCandidates && exactCandidates.length > 0) {
    return { client: exactCandidates[0], method: 'exact' }
  }

  // 2. Contains match
  for (const [key, clients] of dbState.clientsByNormalizedName) {
    if (key.includes(normalized) || normalized.includes(key)) {
      if (clients.length > 0) {
        return { client: clients[0], method: 'contains' }
      }
    }
  }

  // 3. Token Jaccard similarity ≥ 0.65
  const aTokens = tokenize(normalized)
  let bestScore = 0
  let bestClient: DbClient | null = null

  for (const [key, clients] of dbState.clientsByNormalizedName) {
    const bTokens = tokenize(key)
    const score = jaccardSimilarity(aTokens, bTokens)
    if (score > bestScore && score >= 0.65) {
      bestScore = score
      bestClient = clients[0]
    }
  }

  if (bestClient) {
    return { client: bestClient, method: `jaccard(${bestScore.toFixed(2)})` }
  }

  return null
}

// ─── Template Matching ────────────────────────────────────────────────────────

function normalizeTemplateName(name: string): string {
  const lowered = name.toLowerCase().trim()
  return TEMPLATE_NAME_ALIASES[lowered] ?? lowered
}

function findTemplate(
  row: ParsedRow,
  templates: DbTemplate[]
): DbTemplate | null {
  const normalized = normalizeTemplateName(row.taskName)

  // 1. Exact normalized name match
  for (const t of templates) {
    if (normalizeTemplateName(t.name) === normalized) return t
  }

  // 2. Contains match
  for (const t of templates) {
    const tn = normalizeTemplateName(t.name)
    if (tn.includes(normalized) || normalized.includes(tn)) return t
  }

  // 3. Days-based fallback (approximate offset from daysAfterRenewal)
  // daysOffset ≈ -(365 - daysAfterRenewal) but let's use a 15-day window
  if (row.daysAfterRenewal > 0) {
    const approxOffset = -(365 - row.daysAfterRenewal)
    let closest: DbTemplate | null = null
    let closestDiff = Infinity
    for (const t of templates) {
      const diff = Math.abs(t.daysOffset - approxOffset)
      if (diff < closestDiff && diff <= 15) {
        closestDiff = diff
        closest = t
      }
    }
    if (closest) return closest
  }

  return null
}

// ─── DB State Loading ─────────────────────────────────────────────────────────

async function loadDbState(): Promise<DbState> {
  log('Loading database state...')

  // Load designations
  const designations = await prisma.designation.findMany({
    where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } },
  })
  const shapeDes = designations.find((d) => d.name === 'Shape')
  const shape2Des = designations.find(
    (d) => d.name === 'Shape 2' || d.name === 'Shape2'
  )

  if (!shapeDes || !shape2Des) {
    throw new Error(
      `Could not find Shape/Shape2 designations. Found: ${designations.map((d) => d.name).join(', ')}`
    )
  }

  // Load templates
  const allTemplates = await prisma.taskTemplate.findMany({
    where: {
      designationId: { in: [shapeDes.id, shape2Des.id] },
      isActive: true,
    },
  })

  const shapeTemplates = allTemplates.filter(
    (t) => t.designationId === shapeDes.id
  )
  const shape2Templates = allTemplates.filter(
    (t) => t.designationId === shape2Des.id
  )

  log(
    `  Loaded ${shapeTemplates.length} Shape templates, ${shape2Templates.length} Shape2 templates`
  )

  // Load SHAPE clients (those with shape designation on either designation slot)
  const clients = await prisma.client.findMany({
    where: {
      OR: [
        { designationId: { in: [shapeDes.id, shape2Des.id] } },
        { designation2Id: { in: [shapeDes.id, shape2Des.id] } },
      ],
    },
    select: { id: true, name: true, claimsAdvocateId: true },
  })

  log(`  Loaded ${clients.length} SHAPE clients`)

  // Build lookup map
  const clientsByNormalizedName = new Map<string, DbClient[]>()
  for (const c of clients) {
    const key = normalizeClientName(c.name)
    if (!clientsByNormalizedName.has(key)) {
      clientsByNormalizedName.set(key, [])
    }
    clientsByNormalizedName.get(key)!.push({
      id: c.id,
      name: c.name,
      claimsAdvocateId: c.claimsAdvocateId,
    })
  }

  return {
    clientsByNormalizedName,
    allClients: clients.map((c) => ({
      id: c.id,
      name: c.name,
      claimsAdvocateId: c.claimsAdvocateId,
    })),
    shapeTemplates: shapeTemplates.map((t) => ({
      id: t.id,
      name: t.name,
      daysOffset: t.daysOffset,
      designationId: t.designationId,
    })),
    shape2Templates: shape2Templates.map((t) => ({
      id: t.id,
      name: t.name,
      daysOffset: t.daysOffset,
      designationId: t.designationId,
    })),
    shapeDesignationId: shapeDes.id,
    shape2DesignationId: shape2Des.id,
  }
}

// ─── Template Fixes ───────────────────────────────────────────────────────────

async function fixTemplates(
  shapeDesignationId: string,
  shape2DesignationId: string
): Promise<void> {
  log('\nFixing template data...')

  // Fix 1: "Request 125 day loss runs" → "Request 120 day loss runs" at -120
  const fix1 = await prisma.taskTemplate.updateMany({
    where: { name: 'Request 125 day loss runs', daysOffset: -125 },
    data: { name: 'Request 120 day loss runs', daysOffset: -120 },
  })
  if (fix1.count > 0)
    log(`  Fixed ${fix1.count} template(s): "Request 125..." → "Request 120..."`)

  // Fix 2: "Request 89 day loss runs" → "Request 90 day loss runs" at -90
  const fix2 = await prisma.taskTemplate.updateMany({
    where: { daysOffset: -89 },
    data: { daysOffset: -90 },
  })
  const fix2b = await prisma.taskTemplate.updateMany({
    where: {
      name: { in: ['Request 89 day loss runs', 'Request 89 day loss runs (if being marketed)'] },
    },
    data: { name: undefined }, // name update handled separately since two variants exist
  })
  // Update names specifically
  await prisma.taskTemplate.updateMany({
    where: { name: 'Request 89 day loss runs' },
    data: { name: 'Request 90 day loss runs' },
  })
  await prisma.taskTemplate.updateMany({
    where: { name: 'Request 89 day loss runs (if being marketed)' },
    data: { name: 'Request 90 day loss runs (if being marketed)' },
  })
  if (fix2.count > 0)
    log(`  Fixed ${fix2.count} template(s): offset -89 → -90, updated names`)

  // Fix 3: Add missing "Claim Review" at -185 for Shape designation
  const existing185 = await prisma.taskTemplate.findFirst({
    where: {
      name: 'Claim Review',
      daysOffset: -185,
      designationId: shapeDesignationId,
    },
  })
  if (!existing185) {
    if (!DRY_RUN) {
      await prisma.taskTemplate.create({
        data: {
          name: 'Claim Review',
          department: 'CLAIMS',
          timing: 'PRE_RENEWAL',
          daysOffset: -185,
          defaultPriority: 'MEDIUM',
          isActive: true,
          displayOrder: 4, // after 90-day review (order 3), before mod factor (order 5)
          designationId: shapeDesignationId,
        },
      })
      log('  Created missing "Claim Review" template at -185 days (Shape)')
    } else {
      log(
        '  [DRY RUN] Would create missing "Claim Review" template at -185 days (Shape)'
      )
    }
  } else {
    log('  "Claim Review" at -185 already exists — skipping')
  }
}

// ─── Task Processing ──────────────────────────────────────────────────────────

async function findMatchingTasks(
  clientId: string,
  templateId: string,
  dueDate: Date
): Promise<{ match: string | null; duplicateIds: string[] }> {
  const windowMs = 5 * 86400 * 1000 // ±5 days
  const candidates = await prisma.task.findMany({
    where: {
      clientId,
      templateId,
      dueDate: {
        gte: new Date(dueDate.getTime() - windowMs),
        lte: new Date(dueDate.getTime() + windowMs),
      },
    },
    orderBy: { createdAt: 'asc' },
    select: { id: true, status: true, notes: true },
  })

  if (candidates.length === 0) return { match: null, duplicateIds: [] }
  return {
    match: candidates[0].id,
    duplicateIds: candidates.slice(1).map((c) => c.id),
  }
}

interface DateCompletedResult {
  status: 'COMPLETED' | 'NA' | 'NOT_STARTED'
  completedAt: Date | null
  naReason: string | null
}

function interpretDateCompleted(
  raw: string,
  advocateUserId: string
): DateCompletedResult {
  const normalized = raw.trim().toLowerCase()

  if (normalized === '' || normalized === '??' || normalized === '?') {
    return { status: 'NOT_STARTED', completedAt: null, naReason: null }
  }

  if (normalized === 'n/a' || normalized === 'na') {
    return {
      status: 'NA',
      completedAt: null,
      naReason: 'Historical — marked N/A in SHAPE tracker',
    }
  }

  // Try to parse as Excel serial date
  const serial = parseFloat(raw)
  if (!isNaN(serial) && serial > 40000 && serial < 60000) {
    return {
      status: 'COMPLETED',
      completedAt: excelSerialToDate(serial),
      naReason: null,
    }
  }

  // Unknown value — treat as not started
  return { status: 'NOT_STARTED', completedAt: null, naReason: null }
}

async function processTaskRow(
  clientId: string,
  row: ParsedRow,
  template: DbTemplate,
  effectiveDate: Date,
  advocateUserId: string,
  stats: ImportStats
): Promise<void> {
  // Calculate expected due date: effectiveDate + daysAfterRenewal
  const dueDate = new Date(effectiveDate)
  dueDate.setDate(dueDate.getDate() + Math.round(row.daysAfterRenewal))

  const { status, completedAt, naReason } = interpretDateCompleted(
    row.dateCompleted,
    advocateUserId
  )

  const { match: taskId, duplicateIds } = await findMatchingTasks(
    clientId,
    template.id,
    dueDate
  )

  // Delete duplicates
  if (duplicateIds.length > 0) {
    if (!DRY_RUN) {
      await prisma.task.deleteMany({ where: { id: { in: duplicateIds } } })
    }
    stats.duplicatesDeleted += duplicateIds.length
  }

  if (taskId) {
    // Assign advocate to existing task (regardless of completion status)
    await ensureTaskAssignment(taskId, advocateUserId, stats)

    // Update task if it has completion data
    if (status === 'COMPLETED' || status === 'NA') {
      const existing = await prisma.task.findUnique({
        where: { id: taskId },
        select: { status: true, notes: true },
      })

      if (existing && existing.status === 'NOT_STARTED') {
        if (!DRY_RUN) {
          await prisma.task.update({
            where: { id: taskId },
            data: {
              status,
              completedAt: completedAt ?? undefined,
              completedBy: status === 'COMPLETED' ? advocateUserId : undefined,
              naReason: naReason ?? undefined,
              notes:
                row.notes && !existing.notes
                  ? row.notes
                  : existing.notes ?? undefined,
            },
          })
        }
        stats.tasksUpdated++
      }
    }
  } else {
    // Task doesn't exist — create it if it has completion data
    if (status === 'COMPLETED' || status === 'NA') {
      if (!DRY_RUN) {
        const newTask = await prisma.task.create({
          data: {
            title: template.name,
            department: 'CLAIMS',
            timing: 'PRE_RENEWAL',
            daysOffset: template.daysOffset,
            dueDate,
            status,
            priority: 'MEDIUM',
            clientId,
            templateId: template.id,
            completedAt: completedAt ?? undefined,
            completedBy: status === 'COMPLETED' ? advocateUserId : undefined,
            naReason: naReason ?? undefined,
            notes: row.notes || undefined,
            isAdHoc: false,
            createdBy: advocateUserId,
          },
        })
        await ensureTaskAssignment(newTask.id, advocateUserId, stats)
      } else {
        stats.tasksAssigned++ // count the would-be assignment
      }
      stats.tasksCreated++
    } else {
      // NOT_STARTED and no existing task — this shouldn't happen for SHAPE clients
      // but if it does, create the task and assign it
      if (!DRY_RUN) {
        const newTask = await prisma.task.create({
          data: {
            title: template.name,
            department: 'CLAIMS',
            timing: 'PRE_RENEWAL',
            daysOffset: template.daysOffset,
            dueDate,
            status: 'NOT_STARTED',
            priority: 'MEDIUM',
            clientId,
            templateId: template.id,
            notes: row.notes || undefined,
            isAdHoc: false,
            createdBy: advocateUserId,
          },
        })
        await ensureTaskAssignment(newTask.id, advocateUserId, stats)
      } else {
        stats.tasksAssigned++
      }
      stats.tasksCreated++
    }
  }
}

async function ensureTaskAssignment(
  taskId: string,
  userId: string,
  stats: ImportStats
): Promise<void> {
  if (DRY_RUN) {
    stats.tasksAssigned++
    return
  }
  try {
    await prisma.taskAssignment.upsert({
      where: { taskId_userId: { taskId, userId } },
      create: { taskId, userId },
      update: {},
    })
    stats.tasksAssigned++
  } catch {
    // Unique constraint violation means it already exists — fine
  }
}

async function createAdHocTask(
  clientId: string,
  text: string,
  effectiveDate: Date,
  advocateUserId: string,
  stats: ImportStats
): Promise<void> {
  const title = text.slice(0, 500)

  // Idempotency check
  const existing = await prisma.task.findFirst({
    where: { clientId, title, isAdHoc: true },
  })
  if (existing) return

  if (!DRY_RUN) {
    const task = await prisma.task.create({
      data: {
        title,
        department: 'CLAIMS',
        timing: 'PRE_RENEWAL',
        daysOffset: 0,
        dueDate: effectiveDate,
        status: 'COMPLETED',
        priority: 'MEDIUM',
        clientId,
        isAdHoc: true,
        completedAt: effectiveDate,
        completedBy: advocateUserId,
        notes: text,
        createdBy: advocateUserId,
      },
    })
    await ensureTaskAssignment(task.id, advocateUserId, stats)
  } else {
    stats.tasksAssigned++
  }
  stats.adHocCreated++
}

async function assignAdvocate(
  clientId: string,
  advocateUserId: string,
  currentAdvocateId: string | null,
  stats: ImportStats
): Promise<void> {
  if (currentAdvocateId) return // Never overwrite existing assignment
  if (!DRY_RUN) {
    await prisma.client.update({
      where: { id: clientId },
      data: { claimsAdvocateId: advocateUserId },
    })
  }
  stats.advocatesAssigned++
}

// ─── File Processing ──────────────────────────────────────────────────────────

async function processFile(
  folder: string,
  itemId: string,
  fileName: string,
  dbState: DbState,
  stats: ImportStats
): Promise<void> {
  const advocateInfo = TEAM_MEMBER_MAP[folder]!

  let buf: Buffer
  try {
    buf = await downloadFile(itemId)
    await sleep(150)
  } catch (err) {
    const msg = `${folder}/${fileName}: Download failed — ${(err as Error).message}`
    stats.errors.push(msg)
    return
  }

  let blocks: ParsedBlock[]
  try {
    blocks = parseExcelFile(buf, fileName)
  } catch (err) {
    const msg = `${folder}/${fileName}: Parse failed — ${(err as Error).message}`
    stats.errors.push(msg)
    return
  }

  if (blocks.length === 0) {
    stats.errors.push(`${folder}/${fileName}: No data blocks found`)
    return
  }

  stats.filesProcessed++

  for (const block of blocks) {
    // Match client
    const matchResult = matchClient(block.clientName, dbState)
    if (!matchResult) {
      stats.clientsUnmatched++
      stats.unmatchedClients.push({
        folder,
        file: fileName,
        excelName: block.clientName,
      })
      continue
    }

    const { client, method } = matchResult
    stats.clientsMatched++

    if (method !== 'exact') {
      stats.fuzzyMatches.push({
        folder,
        file: fileName,
        excelName: block.clientName,
        dbName: client.name,
        method,
      })
    }

    // Assign advocate at client level
    await assignAdvocate(
      client.id,
      advocateInfo.userId,
      client.claimsAdvocateId,
      stats
    )

    // Update our cached advocate status so subsequent blocks don't re-assign
    client.claimsAdvocateId = client.claimsAdvocateId ?? advocateInfo.userId

    // Choose templates based on SHAPE vs SHAPE2
    const templates = block.isShape2
      ? dbState.shape2Templates
      : dbState.shapeTemplates

    // Process each task row
    for (const row of block.rows) {
      const template = findTemplate(row, templates)
      if (!template) {
        // Unmatched template — add as additional service if it has completion data
        const { status } = interpretDateCompleted(row.dateCompleted, advocateInfo.userId)
        if (status === 'COMPLETED' || status === 'NA') {
          const adHocTitle = `[Unmatched Task] ${row.taskName}${row.notes ? ': ' + row.notes : ''}`
          await createAdHocTask(
            client.id,
            adHocTitle,
            block.effectiveDate,
            advocateInfo.userId,
            stats
          )
        }
        continue
      }

      try {
        await processTaskRow(
          client.id,
          row,
          template,
          block.effectiveDate,
          advocateInfo.userId,
          stats
        )
      } catch (err) {
        stats.errors.push(
          `${folder}/${fileName} / ${block.clientName} / "${row.taskName}": ${(err as Error).message}`
        )
      }
    }

    // Create ad-hoc tasks for additional services
    for (const svc of block.additionalServices) {
      try {
        await createAdHocTask(
          client.id,
          svc,
          block.effectiveDate,
          advocateInfo.userId,
          stats
        )
      } catch (err) {
        stats.errors.push(
          `${folder}/${fileName} / ${block.clientName} / adHoc "${svc.slice(0, 40)}": ${(err as Error).message}`
        )
      }
    }
  }
}

// ─── Report ───────────────────────────────────────────────────────────────────

function printReport(stats: ImportStats): void {
  log('')
  log('═'.repeat(60))
  log('SHAPE Historical Import Report')
  log('═'.repeat(60))
  log(`Mode: ${DRY_RUN ? 'DRY RUN (no changes written)' : 'EXECUTE (changes written to DB)'}`)
  log('')
  log(`Files processed:      ${stats.filesProcessed}`)
  log(`Clients matched:      ${stats.clientsMatched}`)
  log(`  - exact:            ${stats.clientsMatched - stats.fuzzyMatches.length}`)
  log(`  - fuzzy:            ${stats.fuzzyMatches.length}`)
  log(`Clients unmatched:    ${stats.clientsUnmatched}`)
  log(`Tasks updated:        ${stats.tasksUpdated}  (completed/NA status set)`)
  log(`Tasks assigned:       ${stats.tasksAssigned}  (advocate linked)`)
  log(`Tasks created:        ${stats.tasksCreated}  (missing from DB)`)
  log(`Duplicates deleted:   ${stats.duplicatesDeleted}`)
  log(`Ad-hoc created:       ${stats.adHocCreated}`)
  log(`Advocates assigned:   ${stats.advocatesAssigned}  (client-level)`)
  log(`Errors:               ${stats.errors.length}`)

  if (stats.fuzzyMatches.length > 0) {
    log('')
    log('── Fuzzy Client Matches (review for correctness) ──────────────')
    for (const m of stats.fuzzyMatches) {
      log(`  [${m.folder}] "${m.excelName}" → "${m.dbName}" (${m.method})`)
    }
  }

  if (stats.unmatchedClients.length > 0) {
    log('')
    log('── Unmatched Clients (manual action required) ──────────────────')
    for (const u of stats.unmatchedClients) {
      log(`  [${u.folder}] ${u.file} → "${u.excelName}"`)
    }
  }

  if (stats.errors.length > 0) {
    log('')
    log('── Errors ──────────────────────────────────────────────────────')
    for (const e of stats.errors) {
      log(`  ERROR: ${e}`)
    }
  }

  log('')
  log('═'.repeat(60))
}

// ─── Main ─────────────────────────────────────────────────────────────────────

async function main() {
  log(`SHAPE Historical Import — ${new Date().toISOString()}`)
  log(
    `Mode: ${DRY_RUN ? 'DRY RUN (pass --execute to write)' : 'EXECUTE'}`
  )

  const stats: ImportStats = {
    filesProcessed: 0,
    clientsMatched: 0,
    clientsUnmatched: 0,
    tasksUpdated: 0,
    tasksAssigned: 0,
    tasksCreated: 0,
    duplicatesDeleted: 0,
    adHocCreated: 0,
    advocatesAssigned: 0,
    errors: [],
    unmatchedClients: [],
    fuzzyMatches: [],
  }

  // Step 1: Fix template data
  const designations = await prisma.designation.findMany({
    where: { name: { in: ['Shape', 'Shape 2', 'Shape2'] } },
  })
  const shapeDes = designations.find((d) => d.name === 'Shape')
  const shape2Des = designations.find(
    (d) => d.name === 'Shape 2' || d.name === 'Shape2'
  )
  if (!shapeDes || !shape2Des) throw new Error('Shape designations not found')

  await fixTemplates(shapeDes.id, shape2Des.id)

  // Step 2: Load DB state
  const dbState = await loadDbState()

  // Step 3: Discover SharePoint files
  const files = await discoverFiles()

  // Step 4: Process each file
  log(`\nProcessing ${files.length} files...`)
  for (let i = 0; i < files.length; i++) {
    const { folder, itemId, fileName } = files[i]
    const pct = Math.round(((i + 1) / files.length) * 100)
    process.stdout.write(
      `\r  [${pct}%] ${i + 1}/${files.length} — ${folder}/${fileName.slice(0, 40)}`
    )
    await processFile(folder, itemId, fileName, dbState, stats)
  }
  process.stdout.write('\n')

  // Step 5: Print report
  printReport(stats)
  writeOutput()
}

main()
  .catch((e) => {
    console.error('\nFatal error:', e)
    process.exit(1)
  })
  .finally(async () => {
    await prisma.$disconnect()
    await pool.end()
  })