diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..c737a51 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,122 @@ +# Pulse — Repo Guide for Claude + +Pulse is an internal PSA management dashboard for Wulf Consulting. It syncs Autotask +data into Postgres and adds dashboards, workflows, and analytics around it. +Single Next.js 16 app — not a monorepo. + +`README.md` covers the human-facing overview. **Trust this file** for the details +that matter to coding decisions. + +## Stack +- Next.js 16 + React 19 (App Router, `reactCompiler: true`, `output: 'standalone'`) +- TypeScript strict, path alias `@/*` +- Postgres 16 via `pg` (no ORM), Redis (caching), Better Auth 1.4 +- Tailwind 4, shadcn/ui (`components/ui/`), recharts, sonner, lucide +- Forms: react-hook-form + Zod resolver — admin/auth forms only +- Runs on **port 3100** (Docker exposes 3100; `BETTER_AUTH_URL=http://localhost:3100`) + +## Layout +- `app/` — App Router pages + `app/api/**/route.ts` handlers +- `lib/services/` — integration clients, sync services, scheduler (~50 files) +- `lib/types/.ts` — shared types (autotask, sync, veeam, workflow, …) +- `lib/auth.ts`, `lib/auth-utils.ts`, `lib/permissions.ts` — auth wiring +- `components/ui/` — shadcn primitives; sibling dirs are feature components +- `migrations/NNN_*.sql` — numbered SQL, applied in alphabetical order on Postgres + init. Use `IF NOT EXISTS` + `ON CONFLICT DO NOTHING` for seed data. +- `docs/` — long-form integration/sync guides; reference these, don't duplicate. +- `scripts/` — one-off ops scripts, not tests. + +## Database +- All columns are **`snake_case`**. API responses are **`camelCase`** — handlers + transform manually (no ORM). +- Use the `postgresClient` singleton from `lib/services/postgres-client.ts`: + `postgresClient.query(sql, params)`, `.transaction()`, `.upsert()`, `.bulkUpsert()`. +- Audit columns convention: `created_at`, `updated_at`, `synced_at`, `is_deleted`, + `deleted_at`. +- Adding a migration: next number, `IF NOT EXISTS`, no destructive ops on existing + data without a guard. Postgres init applies them on first boot only — for an + existing DB, run via `scripts/apply-migrations` (check first; behavior varies). + +## API routes +- Pattern: `app/api//route.ts` exporting `GET`/`POST`/etc. +- No Zod validation in route handlers today. Validate inputs explicitly when it + matters; don't add a framework just to validate one field. +- Errors: `try/catch`, return `NextResponse.json({ error, message }, { status })`. + Convention: 503 for missing/bad config, 401/403 from auth helpers, 500 for runtime. +- **Auth in API routes**: import from `lib/auth-utils.ts`: + ```ts + const { session, error } = await requireAuth(); // or requireAdmin() / requireSuperAdmin() / requirePermission(resource, action) + if (error) return error; + ``` + `middleware.ts` only verifies a session cookie exists — role checks happen here. +- No `'use server'` actions in this codebase. Everything is API routes called from + client components via `fetch`. + +## Frontend +- Most pages are `'use client'` with `useState`/`useEffect`/`fetch('/api/...')`. + No SWR/react-query — don't introduce one for one-off fetches; match the + surrounding code. +- Server components are fine for static shells; data calls live on the client today. +- Toasts: `sonner`. Tables: `@tanstack/react-table` via `components/admin/DataTable.tsx`. + Modals: `components/admin/DetailModal.tsx`. Navigation: `components/navigation/app-navigation.tsx`. + +## External integrations +All clients live in `lib/services/` with a factory + `isConfigured()` helper. +Examples: `getAutotaskClient()`, `getMsgraphClient()`, `getDattoRmmClient()`, +`getVeeamClient()`. Credentials always come from env vars; clients throw if missing. + +| Service | Env prefix | +|---|---| +| Autotask | `AUTOTASK_*` (incl. `AUTOTASK_WEBHOOK_SECRET`) | +| MS Graph (app) | `MSGRAPH_*` (specific tenant, not `common`) | +| Microsoft OAuth (login) | `MICROSOFT_*` | +| Datto RMM | `DATTO_RMM_*` | +| Veeam VSPC | `VEEAM_VSPC_*` | +| Auvik / Addigy / IT Glue / Mimecast / S1 / Duo / Zoom / QBO / Zabbix / Salesbldr | `_*` | +| Anthropic | `ANTHROPIC_API_KEY` (used in `ai-triage-service.ts`, `llm-analyzer.ts`) | +| Postgres / Redis | `POSTGRES_*` or `DATABASE_URL`, `REDIS_URL` | + +## Sync & scheduling +- `lib/services/entity-sync.ts` — per-entity Autotask → Postgres sync (incremental + via `lastTrackedModificationDateTime` when supported, else full upsert). +- `lib/services/sync-scheduler.ts` — node-cron singleton. **Self-initializes on + first server-side import** (side effect at the bottom of the file). Schedules + live in DB, admin-editable at `/admin`. +- Webhooks (`/api/webhooks/...`, `/api/zabbix/webhook`) are public per + `middleware.ts`; they verify HMAC themselves. + +## Auth +- Better Auth with magic link + TOTP 2FA + Microsoft OAuth. Roles: `user`, `admin`, + `super-admin`. Tables created in migration `012`. +- Default admin bootstrapped from `DEFAULT_ADMIN_EMAIL` via `lib/bootstrap.ts`. +- `middleware.ts` redirects unauth'd page requests to `/auth/sign-in`. Public + routes (webhooks, sync, health, mobile, openclaw, kiosk, legal, qbo callbacks) + are hardcoded there — add to that list when introducing a new public endpoint. + +## Build / run / verify +- Dev: `npm run dev` → http://localhost:3100 +- Build: `npm run build` (turbopack via Next 16) +- Type check: `npx tsc --noEmit --pretty` — **this is the only automated check**; + there are no unit/integration tests and no CI. +- Docker: `docker compose up` from repo root. Postgres applies `migrations/*.sql` + on init only (existing volumes won't re-run them). + +## Conventions to follow +- Files: kebab-case. Components: `PascalCase` exports from kebab-case files. +- Don't introduce ORMs, server actions, or alternative state libraries unless asked + — match the existing pattern. +- New SQL: numbered migration; never edit a committed one. +- Long-form per-feature documentation belongs in `docs/`. Don't duplicate it here. + +## Watch out for +- A `.env` file is committed to the repo. Treat secrets as potentially real; don't + log/echo them, and flag this if it comes up. +- Duplicate migration numbers exist (002, 004, 009) — alphabetical apply order. +- Sync scheduler runs as a side effect of importing `sync-scheduler.ts` on the + server. Be careful adding eager imports of that module. + +## Useful existing docs +- `AUTOTASK_API_GUIDE.md`, `ADDIGY_API_GUIDE.md` — credential setup +- `POSTGRES_SYNC_SETUP.md`, `DOCKER_README.md` +- `PULSE_DATABASE_SKILL.md` — diagnostic queries +- `docs/` — sync behavior, webhook setup, workflow editor, per-integration guides diff --git a/README.md b/README.md index 65d70d2..5106b45 100644 --- a/README.md +++ b/README.md @@ -1,193 +1,87 @@ -# Autotask API Integration Dashboard +# Pulse -A modern Next.js application for interacting with the Autotask PSA REST API, built with React, TypeScript, shadcn/ui, and Tailwind CSS. +Internal PSA management dashboard for Wulf Consulting. Pulse syncs Autotask data +into Postgres and layers dashboards, ticket workflow automation, and analytics +across a number of MSP tooling integrations (Microsoft 365, Datto RMM, Veeam, +Auvik, Addigy, IT Glue, Mimecast, SentinelOne, Duo, Zoom, QuickBooks Online, +Zabbix, and more). -## Features +## Stack -- **Dashboard Overview** - View tickets, tasks, and company information -- **Ticket Management** - List, filter, and manage support tickets -- **Task Tracking** - Monitor and update project tasks -- **Company Selector** - Filter data by company -- **Resource Management** - Filter by assigned resources -- **Modern UI** - Beautiful interface built with shadcn/ui components -- **Dark Mode Support** - Automatic dark/light theme -- **Real-time Updates** - Fetch latest data from Autotask API -- **Secure Authentication** - API credentials stored in environment variables +- Next.js 16 (App Router) + React 19, TypeScript +- PostgreSQL 16 via `pg` (no ORM); Redis for caching +- Better Auth — magic link, TOTP 2FA, Microsoft OAuth +- Tailwind 4 + shadcn/ui, recharts, sonner, lucide +- Anthropic SDK for AI triage and analysis features +- node-cron scheduler embedded in the app process +- Docker Compose for local and prod (Traefik-fronted) -## Tech Stack +## Quick start -- **Framework**: Next.js 15 with App Router -- **Language**: TypeScript -- **Styling**: Tailwind CSS v4 -- **UI Components**: shadcn/ui -- **Icons**: Lucide React -- **Date Handling**: date-fns -- **API Integration**: Native fetch with custom client +Prerequisites: Docker + Docker Compose, or Node 20+ and a local Postgres/Redis. -## Prerequisites - -Before you begin, ensure you have: - -1. Node.js 18+ installed -2. Autotask API credentials: - - API Username (format: `apiuser@YOURDOMAIN.COM`) - - API Secret/Password - - API Integration Code - - API Base URL (e.g., `https://webservices1.autotask.net/atservicesrest/v1.0`) - -## Installation - -1. Clone the repository and navigate to the app directory: -```bash -cd /Users/lorentz/projects/PSA-Utils/autotask-app -``` - -2. Install dependencies: ```bash +cp .env.example .env.local # if present, otherwise see docker-compose.yml +docker compose up -d # Postgres applies migrations/ on first init npm install +npm run dev # http://localhost:3100 ``` -3. Create a `.env.local` file in the root directory: -```env -# Autotask API Configuration -AUTOTASK_API_URL=https://webservices1.autotask.net/atservicesrest/v1.0 -AUTOTASK_USERNAME=your-api-username@yourdomain.com -AUTOTASK_SECRET=your-api-password -AUTOTASK_API_INTEGRATION_CODE=your-tracking-code -``` +The first user to authenticate is bootstrapped as `super-admin` from +`DEFAULT_ADMIN_EMAIL`. -4. Run the development server: -```bash -npm run dev -``` +## Scripts -5. Open [http://localhost:3000](http://localhost:3000) in your browser +| Command | What it does | +|---|---| +| `npm run dev` | Next.js dev server on port 3100 | +| `npm run build` | Production build (turbopack) | +| `npm start` | Run the built app | +| `npm run lint` | ESLint | +| `npx tsc --noEmit --pretty` | Type check (the project's only automated check — there is no test suite or CI) | -## Project Structure +## Project layout ``` -autotask-app/ -├── app/ -│ ├── api/ # API route handlers -│ │ ├── tickets/ # Ticket endpoints -│ │ ├── tasks/ # Task endpoints -│ │ ├── companies/ # Company endpoints -│ │ ├── resources/ # Resource endpoints -│ │ └── picklists/ # Picklist endpoints -│ └── page.tsx # Main dashboard page -├── components/ -│ ├── tickets/ # Ticket-related components -│ ├── tasks/ # Task-related components -│ ├── companies/ # Company-related components -│ └── ui/ # shadcn/ui components -├── lib/ -│ ├── services/ # API client and services -│ │ ├── autotask-client.ts -│ │ └── autotask-factory.ts -│ ├── types/ # TypeScript type definitions -│ │ └── autotask.ts -│ └── hooks/ # Custom React hooks -│ └── use-api.ts -└── public/ # Static assets +app/ Next.js App Router — pages and app/api/**/route.ts handlers +components/ Feature components; components/ui/ is shadcn primitives +lib/services/ Integration clients, sync services, scheduler +lib/types/ Shared TypeScript types per domain +lib/auth*.ts Better Auth config and helpers +migrations/ Numbered SQL migrations applied on Postgres init +scripts/ One-off ops/diagnostic scripts (not tests) +docs/ Deep-dive guides per integration and feature ``` -## API Endpoints +## Configuration -The application provides the following API routes: +All credentials come from environment variables. The major groups: -- `GET /api/tickets` - Fetch tickets (with optional filters) -- `POST /api/tickets` - Create a new ticket -- `GET /api/tasks` - Fetch tasks (with optional filters) -- `POST /api/tasks` - Create a new task -- `GET /api/companies` - Fetch all active companies -- `GET /api/resources` - Fetch resources/users -- `GET /api/picklists` - Fetch picklist values for dropdowns +- **Database / cache**: `POSTGRES_HOST/PORT/DB/USER/PASSWORD` (or `DATABASE_URL`), `REDIS_URL` +- **Auth**: `BETTER_AUTH_SECRET`, `BETTER_AUTH_URL`, `MICROSOFT_CLIENT_ID/SECRET/TENANT_ID`, `DEFAULT_ADMIN_EMAIL` +- **Autotask**: `AUTOTASK_API_URL`, `AUTOTASK_USERNAME`, `AUTOTASK_SECRET`, `AUTOTASK_API_INTEGRATION_CODE`, `AUTOTASK_WEBHOOK_SECRET` +- **Microsoft 365 Graph (app)**: `MSGRAPH_CLIENT_ID/SECRET/TENANT_ID` +- **Other integrations**: `DATTO_RMM_*`, `VEEAM_VSPC_*`, `AUVIK_*`, `ADDIGY_*`, `ITGLUE_*`, `MIMECAST_*`, `S1_*`, `DUO_*`, `ZOOM_*`, `QBO_*`, `ZABBIX_*`, `SALESBLDR_*` +- **AI**: `ANTHROPIC_API_KEY` -## Key Features Implementation +See `AUTOTASK_API_GUIDE.md` and `ADDIGY_API_GUIDE.md` for credential setup. +`docs/` has per-integration guides for the rest. -### Rate Limiting -The API client includes built-in rate limiting (10 requests/second) to comply with Autotask API limits. +## Documentation -### Error Handling -Comprehensive error handling with user-friendly error messages and retry capabilities. +- **`CLAUDE.md`** — repo orientation for AI coding sessions; also a useful overview for new contributors +- **`AUTOTASK_API_GUIDE.md`**, **`ADDIGY_API_GUIDE.md`** — credential setup +- **`POSTGRES_SYNC_SETUP.md`** — database initialization +- **`DOCKER_README.md`** — Docker workflow +- **`PULSE_DATABASE_SKILL.md`** — diagnostic SQL queries +- **`docs/`** — sync behavior, webhook setup, workflow editor, per-integration guides -### Type Safety -Full TypeScript support with detailed type definitions for all Autotask entities. +## Deployment -### Responsive Design -Mobile-friendly interface that works on all device sizes. - -## Development - -### Adding New Features - -1. **New API Endpoints**: Add route handlers in `app/api/` -2. **New Components**: Create components in `components/` -3. **New Entity Types**: Update types in `lib/types/autotask.ts` -4. **New API Methods**: Extend `lib/services/autotask-client.ts` - -### Testing - -Run the development server and test with your Autotask sandbox environment: -```bash -npm run dev -``` - -### Building for Production - -```bash -npm run build -npm start -``` - -## Security Considerations - -- Never commit `.env.local` or any file containing API credentials -- Use environment variables for all sensitive configuration -- Implement proper authentication for production deployment -- Consider adding user authentication layer -- Use HTTPS in production - -## Common Issues & Solutions - -### API Connection Issues -- Verify your API credentials are correct -- Check the API URL matches your Autotask zone -- Ensure your API user has appropriate permissions - -### Rate Limiting -- The client automatically handles rate limiting -- If you encounter 429 errors, the client will retry - -### CORS Issues -- API routes act as a proxy to avoid CORS issues -- All Autotask API calls go through Next.js API routes - -## Future Enhancements - -- [ ] Add Redis caching for improved performance -- [ ] Implement real-time updates with WebSockets -- [ ] Add attachment upload functionality -- [ ] Create ticket/task editing forms -- [ ] Add user authentication and session management -- [ ] Implement advanced search and filtering -- [ ] Add data export functionality -- [ ] Create dashboard widgets and analytics - -## Contributing - -Feel free to submit issues and enhancement requests! +Production runs via `docker compose up -d` behind Traefik. The app is reachable +at `pulse.wulfconsulting.cloud`. There is no CI/CD pipeline — deploys are manual +(rebuild image, recreate containers). ## License -This project is for internal use. Please refer to your organization's policies. - -## Support - -For issues related to: -- **Autotask API**: Consult the [Autotask REST API Documentation](https://ww1.autotask.net/help/DeveloperHelp/Content/APIs/REST/REST_API_Home.htm) -- **Application Issues**: Create an issue in this repository - ---- - -Built with ❤️ using Next.js, React, and shadcn/ui +Internal use only. diff --git a/app/api/veeam/rpo-analyze/route.ts b/app/api/veeam/rpo-analyze/route.ts new file mode 100644 index 0000000..9b53609 --- /dev/null +++ b/app/api/veeam/rpo-analyze/route.ts @@ -0,0 +1,149 @@ +import { NextRequest, NextResponse } from 'next/server'; +import Anthropic from '@anthropic-ai/sdk'; +import postgresClient from '@/lib/services/postgres-client'; + +const HUMAN_NOTE_TYPES = [1, 2, 3]; +const MODEL = 'claude-haiku-4-5-20251001'; + +export async function POST(req: NextRequest) { + try { + const body = await req.json(); + const { at_ticket_number, hostname, org_name, hours_offline } = body as { + at_ticket_number: string; + hostname: string; + org_name: string; + hours_offline?: number; + }; + + if (!at_ticket_number) { + return NextResponse.json({ error: 'at_ticket_number required' }, { status: 400 }); + } + + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + return NextResponse.json({ error: 'ANTHROPIC_API_KEY not configured' }, { status: 503 }); + } + + // ── Fetch ticket, notes, time entries ──────────────────────────────────── + const [ticketRes, notesRes, timeRes] = await Promise.all([ + postgresClient.query(` + SELECT t.id, t.ticket_number, t.title, t.description, t.status, + t.create_date, t.completed_date, + c.company_name + FROM tickets t + LEFT JOIN companies c ON c.id = t.company_id + WHERE t.ticket_number = $1 + LIMIT 1 + `, [at_ticket_number]), + + postgresClient.query(` + SELECT tn.note_type, tn.title, tn.description, tn.create_date_time, + r.first_name || ' ' || r.last_name AS author + FROM ticket_notes tn + JOIN tickets t ON t.id = tn.ticket_id + LEFT JOIN resources r ON r.id = tn.creator_resource_id + WHERE t.ticket_number = $1 + AND tn.note_type = ANY($2) + AND tn.is_deleted = false + ORDER BY tn.create_date_time + `, [at_ticket_number, HUMAN_NOTE_TYPES]), + + postgresClient.query(` + SELECT te.hours_worked, te.notes, te.entry_date, + r.first_name || ' ' || r.last_name AS tech + FROM time_entries te + JOIN tickets t ON t.id = te.ticket_id + LEFT JOIN resources r ON r.id = te.resource_id + WHERE t.ticket_number = $1 + ORDER BY te.entry_date + `, [at_ticket_number]), + ]); + + if (ticketRes.rows.length === 0) { + return NextResponse.json({ error: 'Ticket not found' }, { status: 404 }); + } + + const ticket = ticketRes.rows[0]; + const notes = notesRes.rows; + const entries = timeRes.rows; + const totalHours = entries.reduce((s: number, e: any) => s + parseFloat(e.hours_worked ?? 0), 0); + + // ── Build prompt ───────────────────────────────────────────────────────── + const offlineContext = hours_offline != null + ? `The device (${hostname}) was last seen by RMM ${Math.round(hours_offline)} hours before this ticket was created, meaning it was offline at the time the backup alert fired.` + : `We do not have RMM last-seen data for this device.`; + + const notesText = notes.length > 0 + ? notes.map((n: any) => + `[${n.author ?? 'Unknown'} - ${new Date(n.create_date_time).toLocaleDateString()}]\n${n.title ? n.title + '\n' : ''}${(n.description ?? '').substring(0, 800)}` + ).join('\n\n---\n\n') + : '(No human-written notes on this ticket)'; + + const timeText = entries.length > 0 + ? entries.map((e: any) => + `${e.tech ?? 'Unknown'}: ${e.hours_worked}h — ${(e.notes ?? '').substring(0, 300)}` + ).join('\n') + : '(No time logged)'; + + const systemPrompt = `You are analyzing Autotask service tickets created by Datto RMM backup monitors to help determine whether an automated RPO-based suppression system would have correctly identified that a ticket was unnecessary. + +Context: Pulse RPO is a system that monitors Veeam backup jobs. When a workstation (laptop/desktop) has been offline (not seen by RMM) for longer than its backup interval (typically 24 hours), Pulse suppresses the backup alert — because if the machine is offline, it cannot run a backup, so the alert is a false positive requiring no human action. + +Your task: Analyze the ticket below and determine: +1. Was any meaningful human work performed that fixed an actual backup problem? +2. Or was the ticket resolved simply because the machine came back online / the alert self-cleared? +3. Would suppressing this ticket (never creating it) have been the correct call? + +Answer concisely in JSON with these fields: +- "would_suppress_correctly": boolean — true if suppression would have been correct (no real work needed) +- "confidence": "high" | "medium" | "low" +- "work_summary": string — 1-2 sentences describing what was actually done (or not done) +- "reasoning": string — 2-3 sentences explaining why suppression would/would not have been correct +- "recommendation": string — one actionable sentence`; + + const userPrompt = `Ticket: ${at_ticket_number} +Title: ${ticket.title} +Client: ${ticket.company_name ?? org_name} +Device: ${hostname} +Created: ${new Date(ticket.create_date).toLocaleDateString()} +Completed: ${ticket.completed_date ? new Date(ticket.completed_date).toLocaleDateString() : 'N/A'} +Total time logged: ${totalHours.toFixed(2)}h + +Offline status: ${offlineContext} + +Tech notes on this ticket: +${notesText} + +Time entries: +${timeText}`; + + // ── Call Anthropic ──────────────────────────────────────────────────────── + const client = new Anthropic({ apiKey }); + const message = await client.messages.create({ + model: MODEL, + max_tokens: 600, + system: systemPrompt, + messages: [{ role: 'user', content: userPrompt }], + }); + + const content = message.content[0].type === 'text' ? message.content[0].text : '{}'; + // Strip markdown code fences if the model wraps the JSON + const jsonText = content.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim(); + let analysis: Record = {}; + try { analysis = JSON.parse(jsonText); } catch { analysis = { raw: content }; } + + return NextResponse.json({ + ticket_number: at_ticket_number, + hostname, + org_name, + hours_offline: hours_offline ?? null, + total_hours_logged: totalHours, + note_count: notes.length, + model: MODEL, + analysis, + }); + } catch (err: any) { + console.error('[RPO-ANALYZE] Error:', err.message); + return NextResponse.json({ error: err.message }, { status: 500 }); + } +} diff --git a/app/api/veeam/rpo-comparison/route.ts b/app/api/veeam/rpo-comparison/route.ts new file mode 100644 index 0000000..762d09d --- /dev/null +++ b/app/api/veeam/rpo-comparison/route.ts @@ -0,0 +1,202 @@ +import { NextRequest, NextResponse } from 'next/server'; +import postgresClient from '@/lib/services/postgres-client'; + +const PERIOD_INTERVALS: Record = { + '1d': '1 day', + '7d': '7 days', + '14d': '14 days', + '30d': '30 days', +}; + +// Closed AT status values (Complete, Closed variants) +const CLOSED_STATUSES = [5, 29832279, 29832280]; + +export async function GET(req: NextRequest) { + try { + const { searchParams } = new URL(req.url); + const period = searchParams.get('period') ?? '7d'; + const interval = PERIOD_INTERVALS[period] ?? '7 days'; + + // ── 1. Pulse shadow tickets (open) with company_id ────────────────────── + // Pull agent name as the reliable hostname — rmm_hostname on the shadow + // ticket is only populated when the RMM resolver matched the device. + // datto_rmm_sites.autotask_company_id is not populated, so join companies + // via name match to get site_id, enabling hostname resolution via agent name. + const shadowRes = await postgresClient.query(` + SELECT + st.job_instance_uid, + st.job_name, + st.org_name, + st.priority_level, + st.hours_overdue, + st.failure_category, + st.opened_at, + vo.company_id, + -- Use agent name as the device hostname (most reliable source) + COALESCE(st.rmm_hostname, ba.name) AS rmm_hostname + FROM veeam_rpo_shadow_tickets st + JOIN veeam_backup_agent_jobs j ON j.instance_uid = st.job_instance_uid + JOIN veeam_organizations vo ON vo.instance_uid = j.organization_uid + LEFT JOIN veeam_backup_agents ba ON ba.instance_uid = j.backup_agent_uid + WHERE st.resolved_at IS NULL + `); + + // ── 2. AT Veeam tickets created by Datto (title: "Veeam:* on HOST at Site") + // Use LATERAL to pull one alert row per ticket (most recent) so the + // GROUP BY stays clean — no multi-row fanout from the alert join. + const atRes = await postgresClient.query(` + WITH ticket_with_alert AS ( + SELECT + t.ticket_number, + t.company_id, + t.title, + t.status, + t.priority, + t.create_date, + t.completed_date, + UPPER((regexp_match(t.title, ' on ([A-Za-z0-9][A-Za-z0-9._-]*) at '))[1]) AS hostname, + a.alert_uid AS datto_alert_uid, + a.resolved AS datto_resolved, + a.alert_context->>'source' AS datto_source, + a.timestamp AS datto_alert_fired + FROM tickets t + LEFT JOIN LATERAL ( + SELECT alert_uid, resolved, alert_context, timestamp + FROM datto_rmm_alerts + WHERE ticket_number = t.ticket_number + ORDER BY timestamp DESC + LIMIT 1 + ) a ON true + WHERE t.title ILIKE 'Veeam:%' + AND t.title NOT ILIKE '[Veeam RPO]%' + AND t.create_date > NOW() - $1::interval + AND (regexp_match(t.title, ' on ([A-Za-z0-9][A-Za-z0-9._-]*) at '))[1] IS NOT NULL + ) + SELECT + company_id, + hostname, + COUNT(*) AS ticket_count, + COUNT(*) FILTER ( + WHERE completed_date IS NULL + AND status NOT IN (${CLOSED_STATUSES.join(',')}) + ) AS open_count, + MAX(create_date) AS latest_ticket_at, + json_agg( + json_build_object( + 'ticket_number', ticket_number, + 'title', title, + 'status', status, + 'priority', priority, + 'created_at', create_date, + 'completed_at', completed_date, + 'datto_alert_uid', datto_alert_uid, + 'datto_resolved', datto_resolved, + 'datto_source', datto_source, + 'datto_alert_fired', datto_alert_fired + ) + ORDER BY create_date DESC + ) AS tickets + FROM ticket_with_alert + GROUP BY company_id, hostname + `, [interval]); + + // ── 3. Recent offline suppressions (latest per job) ───────────────────── + const offlineRes = await postgresClient.query(` + SELECT DISTINCT ON (job_instance_uid) + job_instance_uid, rmm_hostname, org_name, hours_offline, checked_at + FROM veeam_rpo_offline_log + WHERE checked_at > NOW() - $1::interval + ORDER BY job_instance_uid, checked_at DESC + `, [interval]); + + // ── 4. Build lookup maps ───────────────────────────────────────────────── + // Pulse: keyed by "company_id|hostname_lower". + // If no hostname is available (agent not found), use job_instance_uid as key + // so the row still appears as pulse_only in the comparison. + const shadowByKey = new Map(); + for (const row of shadowRes.rows) { + const key = row.rmm_hostname + ? `${row.company_id}|${row.rmm_hostname.toLowerCase()}` + : `pulse_only|${row.job_instance_uid}`; + shadowByKey.set(key, row); + } + + // Offline: keyed by job_instance_uid + const offlineByUid = new Map(); + for (const row of offlineRes.rows) { + offlineByUid.set(row.job_instance_uid, row); + } + + // AT/Datto: keyed by "company_id|hostname_lower" + const atByKey = new Map(); + for (const row of atRes.rows) { + if (!row.hostname) continue; + const key = `${row.company_id}|${row.hostname.toLowerCase()}`; + atByKey.set(key, row); + } + + // ── 5. Build comparison rows ───────────────────────────────────────────── + const allKeys = new Set([...shadowByKey.keys(), ...atByKey.keys()]); + const matches: any[] = []; + + for (const key of allKeys) { + const shadow = shadowByKey.get(key) ?? null; + const at = atByKey.get(key) ?? null; + const offline = shadow ? offlineByUid.get(shadow.job_instance_uid) ?? null : null; + + let status: string; + if (shadow && at) status = 'both'; + else if (shadow) status = offline ? 'offline_suppressed' : 'pulse_only'; + else status = 'datto_only'; + + const tickets = (at?.tickets ?? []).slice(0, 5); + + matches.push({ + key, + hostname: shadow?.rmm_hostname ?? at?.hostname ?? null, + org_name: shadow?.org_name ?? null, + company_id: shadow?.company_id ?? at?.company_id ?? null, + status, + pulse: shadow ? { + job_instance_uid: shadow.job_instance_uid, + job_name: shadow.job_name, + priority_level: shadow.priority_level, + hours_overdue: shadow.hours_overdue, + failure_category: shadow.failure_category, + opened_at: shadow.opened_at, + } : null, + offline: offline ? { + hours_offline: offline.hours_offline, + last_suppressed: offline.checked_at, + } : null, + // Datto/AT side + at_ticket_count: at?.ticket_count ?? 0, + at_open_count: at?.open_count ?? 0, + at_latest_at: at?.latest_ticket_at ?? null, + at_tickets: tickets, + }); + } + + // Sort: both → pulse_only → datto_only → offline_suppressed + const rank: Record = { both: 0, pulse_only: 1, datto_only: 2, offline_suppressed: 3 }; + matches.sort((a, b) => { + const r = rank[a.status] - rank[b.status]; + if (r !== 0) return r; + return (b.pulse?.hours_overdue ?? 0) - (a.pulse?.hours_overdue ?? 0); + }); + + const summary = { + pulse_open: shadowRes.rows.length, + datto_at_total: atRes.rows.reduce((s: number, r: any) => s + parseInt(r.ticket_count), 0), + datto_at_open: atRes.rows.reduce((s: number, r: any) => s + parseInt(r.open_count), 0), + both: matches.filter(m => m.status === 'both').length, + pulse_only: matches.filter(m => m.status === 'pulse_only').length, + datto_only: matches.filter(m => m.status === 'datto_only').length, + offline_suppressed: offlineRes.rows.length, + }; + + return NextResponse.json({ period, summary, matches }); + } catch (err: any) { + return NextResponse.json({ error: err.message }, { status: 500 }); + } +} diff --git a/app/api/veeam/rpo-offline-log/route.ts b/app/api/veeam/rpo-offline-log/route.ts new file mode 100644 index 0000000..38a4ba0 --- /dev/null +++ b/app/api/veeam/rpo-offline-log/route.ts @@ -0,0 +1,49 @@ +import { NextRequest, NextResponse } from 'next/server'; +import postgresClient from '@/lib/services/postgres-client'; + +export async function GET(req: NextRequest) { + try { + const { searchParams } = new URL(req.url); + const limit = Math.min(parseInt(searchParams.get('limit') ?? '100'), 500); + const offset = parseInt(searchParams.get('offset') ?? '0'); + const job = searchParams.get('job'); // filter by job_instance_uid + const host = searchParams.get('hostname'); // filter by rmm_hostname + + const conditions: string[] = []; + const params: any[] = []; + + if (job) { + params.push(job); + conditions.push(`job_instance_uid = $${params.length}`); + } + if (host) { + params.push(host.toLowerCase()); + conditions.push(`LOWER(rmm_hostname) = $${params.length}`); + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + const [rows, countRes] = await Promise.all([ + postgresClient.query(` + SELECT + id, job_instance_uid, job_name, org_name, + rmm_hostname, rmm_site_name, device_type_category, + rmm_last_seen, hours_offline, backup_interval_hours, checked_at + FROM veeam_rpo_offline_log + ${where} + ORDER BY checked_at DESC + LIMIT $${params.length + 1} OFFSET $${params.length + 2} + `, [...params, limit, offset]), + postgresClient.query(`SELECT COUNT(*) FROM veeam_rpo_offline_log ${where}`, params), + ]); + + return NextResponse.json({ + total: parseInt(countRes.rows[0].count), + limit, + offset, + rows: rows.rows, + }); + } catch (err: any) { + return NextResponse.json({ error: err.message }, { status: 500 }); + } +} diff --git a/app/api/veeam/ticket-analysis/route.ts b/app/api/veeam/ticket-analysis/route.ts new file mode 100644 index 0000000..1d5521a --- /dev/null +++ b/app/api/veeam/ticket-analysis/route.ts @@ -0,0 +1,133 @@ +import { NextRequest, NextResponse } from 'next/server'; +import postgresClient from '@/lib/services/postgres-client'; + +export async function GET(req: NextRequest) { + const { searchParams } = new URL(req.url); + const page = Math.max(1, parseInt(searchParams.get('page') ?? '1')); + const limit = 50; + const offset = (page - 1) * limit; + const category = searchParams.get('category'); + const company = searchParams.get('company'); + + // Build filter for ticket list only (aggregations always show all data) + const filterWhere: string[] = []; + const filterParams: any[] = []; + + if (category) { + filterParams.push(category); + filterWhere.push(`problem_category = $${filterParams.length}`); + } + if (company) { + filterParams.push(`%${company}%`); + filterWhere.push(`company_name ILIKE $${filterParams.length}`); + } + + const baseWhere = filterWhere.length + ? `WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) AND ${filterWhere.join(' AND ')}` + : `WHERE ticket_created_at >= DATE_TRUNC('year', NOW())`; + + const [statsRes, categoryRes, resolutionRes, skillsRes, complexityRes, ticketsRes, totalRes, ytdRes] = + await Promise.all([ + // Overall stats (unfiltered) + postgresClient.query(` + SELECT + COUNT(*) AS total_analyzed, + ROUND(AVG(hours_worked)::numeric, 2) AS avg_hours, + ROUND(100.0 * COUNT(*) FILTER (WHERE same_day_close) + / NULLIF(COUNT(*), 0), 1) AS same_day_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE preventable = true) + / NULLIF(COUNT(*) FILTER (WHERE preventable IS NOT NULL), 0), 1) AS preventable_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE device_was_offline = true) + / NULLIF(COUNT(*) FILTER (WHERE device_was_offline IS NOT NULL), 0), 1) AS offline_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE backup_completed_before_tech = true) + / NULLIF(COUNT(*) FILTER (WHERE backup_completed_before_tech IS NOT NULL), 0), 1) AS auto_resolved_pct + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + `), + + // By problem category (unfiltered) + postgresClient.query(` + SELECT problem_category, + COUNT(*) AS count, + ROUND(AVG(hours_worked)::numeric, 2) AS avg_hours, + ROUND(100.0 * COUNT(*) FILTER (WHERE same_day_close) + / NULLIF(COUNT(*), 0), 1) AS same_day_pct + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY problem_category + ORDER BY count DESC + `), + + // By resolution type (unfiltered) + postgresClient.query(` + SELECT resolution_type, COUNT(*) AS count + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY resolution_type + ORDER BY count DESC + `), + + // Skills frequency (unfiltered) + postgresClient.query(` + SELECT skill, COUNT(*) AS count + FROM veeam_ticket_analysis, unnest(skills_required) AS skill + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY skill + ORDER BY count DESC + LIMIT 15 + `), + + // Complexity breakdown (unfiltered) + postgresClient.query(` + SELECT complexity, COUNT(*) AS count + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY complexity + ORDER BY CASE complexity + WHEN 'trivial' THEN 1 WHEN 'low' THEN 2 WHEN 'medium' THEN 3 WHEN 'high' THEN 4 ELSE 5 + END + `), + + // Filtered ticket list + postgresClient.query(` + SELECT ticket_number, company_name, device_hostname, + ticket_created_at, ticket_closed_at, same_day_close, + hours_worked, problem_category, resolution_type, complexity, + device_was_offline, backup_completed_before_tech, preventable, + work_summary, recommended_procedure + FROM veeam_ticket_analysis + ${baseWhere} + ORDER BY ticket_created_at DESC + LIMIT $${filterParams.length + 1} OFFSET $${filterParams.length + 2} + `, [...filterParams, limit, offset]), + + // Filtered total + postgresClient.query( + `SELECT COUNT(*) FROM veeam_ticket_analysis ${baseWhere}`, + filterParams + ), + + // YTD total all backup tickets + postgresClient.query(` + SELECT COUNT(*) AS total FROM tickets + WHERE title ILIKE 'Veeam:%' + AND title NOT ILIKE '[Veeam RPO]%' + AND create_date >= DATE_TRUNC('year', NOW()) + `), + ]); + + return NextResponse.json({ + stats: { + ...statsRes.rows[0], + total_ytd: parseInt(ytdRes.rows[0]?.total ?? 0), + }, + by_category: categoryRes.rows, + by_resolution: resolutionRes.rows, + skills: skillsRes.rows, + by_complexity: complexityRes.rows, + tickets: ticketsRes.rows, + total: parseInt(totalRes.rows[0]?.count ?? 0), + page, + limit, + }); +} diff --git a/app/api/veeam/ticket-analysis/run/route.ts b/app/api/veeam/ticket-analysis/run/route.ts new file mode 100644 index 0000000..fae8fac --- /dev/null +++ b/app/api/veeam/ticket-analysis/run/route.ts @@ -0,0 +1,230 @@ +import { NextRequest, NextResponse } from 'next/server'; +import Anthropic from '@anthropic-ai/sdk'; +import postgresClient from '@/lib/services/postgres-client'; +import { analysisState } from '@/lib/services/veeam-analysis-state'; + +const MODEL = 'claude-haiku-4-5-20251001'; +const CONCURRENCY = 4; +const HUMAN_NOTE_TYPES = [1, 2, 3]; + +const SYSTEM_PROMPT = `You are analyzing Autotask service tickets for Veeam backup failures to help an MSP build process and procedure documentation. + +For each ticket, determine: +1. The root cause category of the backup failure +2. How it was ultimately resolved +3. What specific technical skills were required +4. Whether the device was offline/powered-off when the alert fired +5. Whether the backup completed on its own before any technician touched the ticket +6. Whether this specific issue was preventable with better process or monitoring +7. Resolution complexity + +Return JSON only — no explanation, no markdown: +{ + "problem_category": "device_offline" | "agent_issue" | "job_failed" | "storage_issue" | "network_issue" | "authentication" | "software_error" | "self_resolved" | "configuration" | "other", + "resolution_type": "no_action_needed" | "device_powered_on" | "backup_restarted" | "agent_reinstalled" | "storage_cleared" | "settings_updated" | "escalated" | "other", + "skills_required": ["array", "of", "specific", "skill", "strings"], + "complexity": "trivial" | "low" | "medium" | "high", + "device_was_offline": true | false, + "backup_completed_before_tech": true | false, + "preventable": true | false, + "work_summary": "1-2 sentence description of what happened and what was done", + "recommended_procedure": "one concrete SOP action for this class of issue" +}`; + +async function analyzeTicket(client: Anthropic, ticket: any): Promise { + const sameDayClose = !!(ticket.completed_date && + new Date(ticket.create_date).toDateString() === new Date(ticket.completed_date).toDateString()); + + const [notesRes, timeRes, offlineRes] = await Promise.all([ + postgresClient.query(` + SELECT tn.note_type, tn.title, tn.description, tn.create_date_time, + r.first_name || ' ' || r.last_name AS author + FROM ticket_notes tn + LEFT JOIN resources r ON r.id = tn.creator_resource_id + WHERE tn.ticket_id = $1 + AND tn.note_type = ANY($2) + AND tn.is_deleted = false + ORDER BY tn.create_date_time + `, [ticket.ticket_id, HUMAN_NOTE_TYPES]), + + postgresClient.query(` + SELECT te.hours_worked, te.notes, te.entry_date, + r.first_name || ' ' || r.last_name AS tech + FROM time_entries te + LEFT JOIN resources r ON r.id = te.resource_id + WHERE te.ticket_id = $1 + ORDER BY te.entry_date + `, [ticket.ticket_id]), + + ticket.device_hostname + ? postgresClient.query(` + SELECT hours_offline, rmm_last_seen + FROM veeam_rpo_offline_log + WHERE LOWER(rmm_hostname) = LOWER($1) + AND checked_at BETWEEN $2::timestamptz - INTERVAL '72 hours' + AND $2::timestamptz + INTERVAL '24 hours' + ORDER BY checked_at DESC LIMIT 1 + `, [ticket.device_hostname, ticket.create_date]) + : Promise.resolve({ rows: [] }), + ]); + + const notes = notesRes.rows; + const entries = timeRes.rows; + const offline = offlineRes.rows[0] ?? null; + const totalHours = entries.reduce((s: number, e: any) => s + parseFloat(e.hours_worked ?? 0), 0); + + const notesText = notes.length > 0 + ? notes.map((n: any) => + `[${n.author ?? 'Unknown'} — ${new Date(n.create_date_time).toLocaleDateString()}]\n` + + `${n.title ? n.title + '\n' : ''}${(n.description ?? '').substring(0, 600)}` + ).join('\n\n---\n\n') + : '(No tech notes)'; + + const timeText = entries.length > 0 + ? entries.map((e: any) => + `${e.tech ?? 'Unknown'}: ${e.hours_worked}h — ${(e.notes ?? '').substring(0, 300)}` + ).join('\n') + : '(No time entries)'; + + const offlineCtx = offline + ? `RMM offline data: device was last seen ${Math.round(offline.hours_offline)}h before ticket creation (last seen ${new Date(offline.rmm_last_seen).toLocaleDateString()}).` + : 'No RMM offline record found for this device around ticket creation time.'; + + const userPrompt = `Ticket: ${ticket.ticket_number} +Title: ${ticket.title} +Client: ${ticket.company_name ?? 'Unknown'} +Device: ${ticket.device_hostname ?? 'Unknown'} +Created: ${new Date(ticket.create_date).toLocaleDateString()} +Closed: ${ticket.completed_date ? new Date(ticket.completed_date).toLocaleDateString() : 'Still open'} +Same-day close: ${sameDayClose} +Time logged: ${totalHours.toFixed(2)}h + +${offlineCtx} + +Tech notes: +${notesText} + +Time entries: +${timeText}`; + + const message = await client.messages.create({ + model: MODEL, + max_tokens: 512, + system: SYSTEM_PROMPT, + messages: [{ role: 'user', content: userPrompt }], + }); + + const raw = message.content[0].type === 'text' ? message.content[0].text : '{}'; + const json = raw.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim(); + let a: Record = {}; + try { a = JSON.parse(json); } catch { a = {}; } + + await postgresClient.query(` + INSERT INTO veeam_ticket_analysis ( + ticket_number, ticket_id, company_id, company_name, device_hostname, + ticket_created_at, ticket_closed_at, same_day_close, + hours_worked, note_count, + problem_category, resolution_type, skills_required, complexity, + device_was_offline, backup_completed_before_tech, preventable, + work_summary, recommended_procedure, model + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20) + ON CONFLICT (ticket_number) DO UPDATE SET + problem_category = EXCLUDED.problem_category, + resolution_type = EXCLUDED.resolution_type, + skills_required = EXCLUDED.skills_required, + complexity = EXCLUDED.complexity, + device_was_offline = EXCLUDED.device_was_offline, + backup_completed_before_tech = EXCLUDED.backup_completed_before_tech, + preventable = EXCLUDED.preventable, + work_summary = EXCLUDED.work_summary, + recommended_procedure = EXCLUDED.recommended_procedure, + hours_worked = EXCLUDED.hours_worked, + note_count = EXCLUDED.note_count, + analyzed_at = NOW() + `, [ + ticket.ticket_number, ticket.ticket_id, ticket.company_id, ticket.company_name, + ticket.device_hostname, + ticket.create_date, ticket.completed_date, sameDayClose, + totalHours, notes.length, + a.problem_category ?? 'other', + a.resolution_type ?? 'other', + a.skills_required ?? [], + a.complexity ?? 'low', + a.device_was_offline ?? null, + a.backup_completed_before_tech ?? null, + a.preventable ?? null, + a.work_summary ?? null, + a.recommended_procedure ?? null, + MODEL, + ]); +} + +async function runBatch(tickets: any[]): Promise { + const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY! }); + + for (let i = 0; i < tickets.length; i += CONCURRENCY) { + const chunk = tickets.slice(i, i + CONCURRENCY); + await Promise.allSettled( + chunk.map(t => + analyzeTicket(client, t) + .then(() => { analysisState.done++; }) + .catch(err => { + console.error(`[VEEAM-ANALYSIS] ${t.ticket_number}:`, err.message); + analysisState.errors++; + analysisState.done++; + }) + ) + ); + if (i + CONCURRENCY < tickets.length) { + await new Promise(r => setTimeout(r, 150)); + } + } +} + +export async function POST(req: NextRequest) { + if (analysisState.isRunning) { + return NextResponse.json({ + error: 'Analysis already running', + progress: analysisState, + }, { status: 409 }); + } + + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + return NextResponse.json({ error: 'ANTHROPIC_API_KEY not configured' }, { status: 503 }); + } + + const body = await req.json().catch(() => ({})); + const reanalyze = body.reanalyze === true; + + const ticketsRes = await postgresClient.query(` + SELECT DISTINCT ON (t.id) + t.id AS ticket_id, t.ticket_number, t.title, t.company_id, + c.company_name, t.create_date, t.completed_date, + (regexp_match(t.title, ' on ([A-Za-z0-9][A-Za-z0-9._-]+) at '))[1] AS device_hostname + FROM tickets t + LEFT JOIN companies c ON c.id = t.company_id + INNER JOIN time_entries te ON te.ticket_id = t.id + ${reanalyze ? '' : 'LEFT JOIN veeam_ticket_analysis vta ON vta.ticket_number = t.ticket_number'} + WHERE t.title ILIKE 'Veeam:%' + AND t.title NOT ILIKE '[Veeam RPO]%' + AND t.create_date >= DATE_TRUNC('year', NOW()) + ${reanalyze ? '' : 'AND vta.ticket_number IS NULL'} + ORDER BY t.id, t.create_date DESC + `); + + const tickets = ticketsRes.rows; + if (tickets.length === 0) { + return NextResponse.json({ started: false, message: 'All eligible tickets already analyzed', total: 0 }); + } + + analysisState.isRunning = true; + analysisState.total = tickets.length; + analysisState.done = 0; + analysisState.errors = 0; + analysisState.startedAt = new Date(); + + runBatch(tickets).finally(() => { analysisState.isRunning = false; }); + + return NextResponse.json({ started: true, total: tickets.length }); +} diff --git a/app/api/veeam/ticket-analysis/status/route.ts b/app/api/veeam/ticket-analysis/status/route.ts new file mode 100644 index 0000000..27c7534 --- /dev/null +++ b/app/api/veeam/ticket-analysis/status/route.ts @@ -0,0 +1,32 @@ +import { NextResponse } from 'next/server'; +import postgresClient from '@/lib/services/postgres-client'; +import { analysisState } from '@/lib/services/veeam-analysis-state'; + +export async function GET() { + const [analyzedRes, eligibleRes] = await Promise.all([ + postgresClient.query(` + SELECT COUNT(*) AS total_analyzed, MAX(analyzed_at) AS last_analyzed_at + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + `), + postgresClient.query(` + SELECT COUNT(DISTINCT t.id) AS total_eligible + FROM tickets t + INNER JOIN time_entries te ON te.ticket_id = t.id + WHERE t.title ILIKE 'Veeam:%' + AND t.title NOT ILIKE '[Veeam RPO]%' + AND t.create_date >= DATE_TRUNC('year', NOW()) + `), + ]); + + return NextResponse.json({ + is_running: analysisState.isRunning, + run_total: analysisState.total, + run_done: analysisState.done, + run_errors: analysisState.errors, + run_started_at: analysisState.startedAt, + total_analyzed: parseInt(analyzedRes.rows[0]?.total_analyzed ?? 0), + total_eligible: parseInt(eligibleRes.rows[0]?.total_eligible ?? 0), + last_analyzed_at: analyzedRes.rows[0]?.last_analyzed_at ?? null, + }); +} diff --git a/app/api/veeam/ticket-analysis/summary/route.ts b/app/api/veeam/ticket-analysis/summary/route.ts new file mode 100644 index 0000000..978cb4a --- /dev/null +++ b/app/api/veeam/ticket-analysis/summary/route.ts @@ -0,0 +1,185 @@ +import { NextResponse } from 'next/server'; +import Anthropic from '@anthropic-ai/sdk'; +import postgresClient from '@/lib/services/postgres-client'; + +const MODEL = 'claude-sonnet-4-6'; +const SYSTEM = 'You are a senior MSP consultant. Respond with valid JSON only — no markdown fences, no prose before or after the JSON object.'; + +export async function POST() { + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + return NextResponse.json({ error: 'ANTHROPIC_API_KEY not configured' }, { status: 503 }); + } + + try { + const [statsRes, categoryRes, resolutionRes, skillsRes, complexityRes, proceduresRes] = + await Promise.all([ + postgresClient.query(` + SELECT + COUNT(*) AS total_analyzed, + ROUND(AVG(hours_worked)::numeric, 2) AS avg_hours, + ROUND(100.0 * COUNT(*) FILTER (WHERE same_day_close) + / NULLIF(COUNT(*), 0), 1) AS same_day_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE preventable = true) + / NULLIF(COUNT(*) FILTER (WHERE preventable IS NOT NULL), 0), 1) AS preventable_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE device_was_offline = true) + / NULLIF(COUNT(*) FILTER (WHERE device_was_offline IS NOT NULL), 0), 1) AS offline_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE backup_completed_before_tech = true) + / NULLIF(COUNT(*) FILTER (WHERE backup_completed_before_tech IS NOT NULL), 0), 1) AS auto_resolved_pct, + SUM(hours_worked) AS total_hours + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + `), + + postgresClient.query(` + SELECT problem_category, + COUNT(*) AS count, + ROUND(AVG(hours_worked)::numeric, 2) AS avg_hours, + ROUND(100.0 * COUNT(*) FILTER (WHERE same_day_close) + / NULLIF(COUNT(*), 0), 1) AS same_day_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE preventable = true) + / NULLIF(COUNT(*) FILTER (WHERE preventable IS NOT NULL), 0), 1) AS preventable_pct, + ROUND(100.0 * COUNT(*) FILTER (WHERE device_was_offline = true) + / NULLIF(COUNT(*) FILTER (WHERE device_was_offline IS NOT NULL), 0), 1) AS offline_pct + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY problem_category + ORDER BY count DESC + `), + + postgresClient.query(` + SELECT resolution_type, COUNT(*) AS count + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY resolution_type ORDER BY count DESC + `), + + postgresClient.query(` + SELECT skill, COUNT(*) AS count + FROM veeam_ticket_analysis, unnest(skills_required) AS skill + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY skill ORDER BY count DESC LIMIT 20 + `), + + postgresClient.query(` + SELECT complexity, COUNT(*) AS count + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + GROUP BY complexity + ORDER BY CASE complexity WHEN 'trivial' THEN 1 WHEN 'low' THEN 2 WHEN 'medium' THEN 3 WHEN 'high' THEN 4 ELSE 5 END + `), + + // Sample recommended procedures per category (up to 3 per category) + postgresClient.query(` + SELECT problem_category, recommended_procedure + FROM ( + SELECT problem_category, recommended_procedure, + ROW_NUMBER() OVER (PARTITION BY problem_category ORDER BY analyzed_at DESC) AS rn + FROM veeam_ticket_analysis + WHERE ticket_created_at >= DATE_TRUNC('year', NOW()) + AND recommended_procedure IS NOT NULL AND recommended_procedure != '' + ) ranked + WHERE rn <= 3 + ORDER BY problem_category, rn + `), + ]); + + const stats = statsRes.rows[0]; + const categories = categoryRes.rows; + const resolutions = resolutionRes.rows; + const skills = skillsRes.rows; + const complexity = complexityRes.rows; + const procedures = proceduresRes.rows; + + if (parseInt(stats.total_analyzed) === 0) { + return NextResponse.json({ error: 'No analyzed tickets yet — run the analysis first.' }, { status: 400 }); + } + + const procByCategory: Record = {}; + for (const p of procedures) { + if (!procByCategory[p.problem_category]) procByCategory[p.problem_category] = []; + procByCategory[p.problem_category].push(p.recommended_procedure); + } + + const prompt = `Review this YTD backup ticket data for a managed services provider and produce a practical operations summary. + +## Dataset +- Tickets analyzed: ${stats.total_analyzed} (YTD, all had tech time logged) +- Total tech time: ${parseFloat(stats.total_hours ?? 0).toFixed(1)}h +- Avg hours per ticket: ${stats.avg_hours}h +- Same-day close rate: ${stats.same_day_pct}% +- Device was offline at alert time: ${stats.offline_pct}% +- Backup auto-completed before tech action: ${stats.auto_resolved_pct}% +- Assessed as preventable: ${stats.preventable_pct}% + +## Problem Categories +${categories.map(c => `- ${c.problem_category}: ${c.count} tickets, avg ${c.avg_hours}h, ${c.same_day_pct}% same-day close, ${c.offline_pct ?? 0}% offline at time`).join('\n')} + +## Resolution Types +${resolutions.map(r => `- ${r.resolution_type}: ${r.count} tickets`).join('\n')} + +## Complexity Breakdown +${complexity.map(c => `- ${c.complexity}: ${c.count} tickets`).join('\n')} + +## Top Skills Required (by frequency) +${skills.map(s => `- ${s.skill}: ${s.count} tickets`).join('\n')} + +## Sample AI-Generated SOP Steps (per category) +${Object.entries(procByCategory).map(([cat, procs]) => + `${cat}:\n${procs.map(p => ` • ${p}`).join('\n')}` +).join('\n')} + +--- + +Return a JSON object with exactly this structure: +{ + "headline": "one sentence executive summary", + "key_findings": ["3-4 bullet point findings with specific numbers"], + "issue_breakdown": [ + { + "category": "exact machine key from Problem Categories above (e.g. device_offline, agent_issue)", + "insight": "1-2 sentence insight about this category", + "sop": "concrete, actionable SOP recommendation for handling this type" + } + ], + "skills_assessment": "2-3 sentences on the skills picture — what's needed most, any gaps", + "quick_wins": ["2-3 specific things the MSP could do to reduce ticket volume or time spent"], + "automation_opportunities": "1-2 sentences on what could be automated or suppressed", + "training_priority": "one sentence on the highest-value training investment" +} + +Be specific and direct. Reference actual numbers from the data. Avoid generic MSP advice.`; + + const client = new Anthropic({ apiKey }); + const message = await client.messages.create({ + model: MODEL, + max_tokens: 4000, + system: SYSTEM, + messages: [{ role: 'user', content: prompt }], + }); + + if (message.stop_reason === 'max_tokens') { + console.error('[VEEAM-SUMMARY] Response truncated at max_tokens'); + return NextResponse.json({ error: 'Model response was truncated — try again' }, { status: 502 }); + } + + const raw = message.content[0].type === 'text' ? message.content[0].text : '{}'; + + const start = raw.indexOf('{'); + const end = raw.lastIndexOf('}'); + const json = start !== -1 && end > start ? raw.slice(start, end + 1) : raw; + + let analysis: Record = {}; + try { + analysis = JSON.parse(json); + } catch (e) { + console.error('[VEEAM-SUMMARY] JSON parse failed. Raw response:', raw); + return NextResponse.json({ error: 'Model returned non-JSON response', raw }, { status: 502 }); + } + + return NextResponse.json({ analysis, model: MODEL, generated_at: new Date().toISOString() }); + } catch (e: any) { + console.error('[VEEAM-SUMMARY] Unexpected error:', e); + return NextResponse.json({ error: e.message ?? 'Internal server error' }, { status: 500 }); + } +} diff --git a/app/backup-status/page.tsx b/app/backup-status/page.tsx index 2cb271b..02371b6 100644 --- a/app/backup-status/page.tsx +++ b/app/backup-status/page.tsx @@ -9,7 +9,7 @@ import { CompanyBackupTable, CompanyBackupRow } from '@/components/backup/compan import { ComplianceSummaryCards } from '@/components/backup/compliance-summary-cards'; import { ComplianceDetailTable } from '@/components/backup/compliance-detail-table'; import { ContractCoverageTable } from '@/components/backup/contract-coverage-table'; -import { RefreshCw, CheckCircle2, AlertTriangle, XCircle, Clock } from 'lucide-react'; +import { RefreshCw, CheckCircle2, AlertTriangle, XCircle, Clock, WifiOff } from 'lucide-react'; import { Skeleton } from '@/components/ui/skeleton'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; import { RpoJobSummary } from '@/lib/services/veeam-rpo-service'; @@ -30,6 +30,7 @@ interface RpoData { total: number; healthy: number; breached: number; + offlineSuppressed: number; withOpenTicket: number; critical: number; high: number; @@ -37,6 +38,19 @@ interface RpoData { jobs: RpoJobSummary[]; } +interface OfflineLogRow { + id: number; + job_name: string; + org_name: string; + rmm_hostname: string; + rmm_site_name: string; + device_type_category: string; + rmm_last_seen: string | null; + hours_offline: number; + backup_interval_hours: number; + checked_at: string; +} + interface ComplianceData { summary: { totalContractedDevices: number; @@ -71,21 +85,24 @@ export default function BackupStatusPage() { const [companies, setCompanies] = useState([]); const [compliance, setCompliance] = useState(null); const [rpo, setRpo] = useState(null); + const [offlineLog, setOfflineLog] = useState([]); const [loading, setLoading] = useState(true); const [syncing, setSyncing] = useState(false); const fetchData = async () => { try { - const [statusRes, companiesRes, complianceRes, rpoRes] = await Promise.all([ + const [statusRes, companiesRes, complianceRes, rpoRes, offlineLogRes] = await Promise.all([ fetch('/api/veeam/backup-status').then(r => r.json()), fetch('/api/veeam/companies').then(r => r.json()), fetch('/api/veeam/compliance').then(r => r.json()), fetch('/api/veeam/rpo-check').then(r => r.json()), + fetch('/api/veeam/rpo-offline-log?limit=200').then(r => r.json()), ]); setStatus(statusRes); setCompanies(Array.isArray(companiesRes) ? companiesRes : []); setCompliance(complianceRes); setRpo(rpoRes); + setOfflineLog(offlineLogRes.rows ?? []); } catch (error) { console.error('Failed to fetch backup status:', error); } finally { @@ -156,6 +173,14 @@ export default function BackupStatusPage() { )} + + Offline Suppressed + {rpo && (rpo.summary.offlineSuppressed ?? 0) > 0 && ( + + {rpo.summary.offlineSuppressed} + + )} + Contract Compliance {compliance && (compliance.summary.contractedNotBackedUp > 0 || compliance.summary.backedUpNotContracted > 0) && ( @@ -233,6 +258,16 @@ export default function BackupStatusPage() {

{rpo.summary.high} high priority

+ + + Offline Suppressed + + + +
{rpo.summary.offlineSuppressed ?? 0}
+

breached but device offline

+ + Compliance Rate @@ -255,6 +290,7 @@ export default function BackupStatusPage() { Job Organization Last Backup + RMM Device Status Ticket Failure Reason @@ -266,8 +302,27 @@ export default function BackupStatusPage() { {job.job_name} {job.org_name} {timeAgoHours(job.hours_since_backup)} + + {job.rmm_hostname ? ( +
+ {job.rmm_hostname} + {job.is_offline_suppressed && ( +
+ + offline {timeAgo(job.rmm_last_seen)} +
+ )} +
+ ) : ( + — + )} + - {job.is_breached ? ( + {job.is_offline_suppressed ? ( + + Offline + + ) : job.is_breached ? ( Breached ) : ( Healthy @@ -292,7 +347,7 @@ export default function BackupStatusPage() { ))} {rpo.jobs.length === 0 && ( - No workstation jobs found + No workstation jobs found )} @@ -302,6 +357,52 @@ export default function BackupStatusPage() { )} + +

+ Workstation backup jobs suppressed during the last RPO check because the device was offline longer than its backup interval. + No Autotask ticket is created while the device is offline. +

+
+ + + + + + + + + + + + + + {offlineLog.map((row) => ( + + + + + + + + + + ))} + {offlineLog.length === 0 && ( + + + + )} + +
DeviceJobOrganizationTypeLast SeenOfflineChecked
{row.rmm_hostname}{row.job_name}{row.org_name} + {row.device_type_category} + {timeAgo(row.rmm_last_seen)} + {row.hours_offline >= 48 + ? `${Math.round(row.hours_offline / 24)}d` + : `${Math.round(row.hours_offline)}h`} + {timeAgo(row.checked_at)}
No offline suppressions logged yet
+
+ + {compliance && ( <> diff --git a/app/veeam-analysis/page.tsx b/app/veeam-analysis/page.tsx new file mode 100644 index 0000000..034ba30 --- /dev/null +++ b/app/veeam-analysis/page.tsx @@ -0,0 +1,733 @@ +'use client'; + +import { useState, useEffect, useCallback, useRef } from 'react'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Progress } from '@/components/ui/progress'; +import { Skeleton } from '@/components/ui/skeleton'; +import { + BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer, Cell, +} from 'recharts'; +import { + Brain, Play, RefreshCw, CheckCircle2, AlertTriangle, Clock, + WifiOff, ChevronDown, ChevronRight, Wrench, RotateCcw, Sparkles, Zap, BookOpen, GraduationCap, +} from 'lucide-react'; + +// ── Types ───────────────────────────────────────────────────────────────────── + +interface Stats { + total_analyzed: string; + total_ytd: string; + avg_hours: string; + same_day_pct: string; + preventable_pct: string; + offline_pct: string; + auto_resolved_pct: string; +} + +interface CategoryRow { problem_category: string; count: string; avg_hours: string; same_day_pct: string } +interface ResolutionRow { resolution_type: string; count: string } +interface SkillRow { skill: string; count: string } +interface ComplexityRow { complexity: string; count: string } + +interface AnalysisData { + stats: Stats; + by_category: CategoryRow[]; + by_resolution: ResolutionRow[]; + skills: SkillRow[]; + by_complexity: ComplexityRow[]; + tickets: TicketRow[]; + total: number; + limit: number; + page: number; +} + +interface TicketRow { + ticket_number: string; + company_name: string | null; + device_hostname: string | null; + ticket_created_at: string | null; + ticket_closed_at: string | null; + same_day_close: boolean; + hours_worked: string; + problem_category: string; + resolution_type: string; + complexity: string; + device_was_offline: boolean | null; + backup_completed_before_tech: boolean | null; + preventable: boolean | null; + work_summary: string | null; + recommended_procedure: string | null; +} + +interface RunStatus { + is_running: boolean; + run_total: number; + run_done: number; + run_errors: number; + total_analyzed: number; + total_eligible: number; + last_analyzed_at: string | null; +} + +// ── Config ──────────────────────────────────────────────────────────────────── + +const CATEGORY_CFG: Record = { + device_offline: { label: 'Device Offline', color: '#94a3b8' }, + agent_issue: { label: 'Agent Issue', color: '#f97316' }, + job_failed: { label: 'Job Failed', color: '#ef4444' }, + storage_issue: { label: 'Storage Issue', color: '#f59e0b' }, + network_issue: { label: 'Network Issue', color: '#3b82f6' }, + authentication: { label: 'Authentication', color: '#8b5cf6' }, + software_error: { label: 'Software Error', color: '#ec4899' }, + self_resolved: { label: 'Self-Resolved', color: '#22c55e' }, + configuration: { label: 'Configuration', color: '#06b6d4' }, + other: { label: 'Other', color: '#6b7280' }, +}; + +const RESOLUTION_CFG: Record = { + no_action_needed: 'No Action Needed', + device_powered_on: 'Device Powered On', + backup_restarted: 'Backup Restarted', + agent_reinstalled: 'Agent Reinstalled', + storage_cleared: 'Storage Cleared', + settings_updated: 'Settings Updated', + escalated: 'Escalated', + other: 'Other', +}; + +const COMPLEXITY_COLOR: Record = { + trivial: '#22c55e', + low: '#84cc16', + medium: '#f59e0b', + high: '#ef4444', +}; + +interface SummaryAnalysis { + headline: string; + key_findings: string[]; + issue_breakdown: { category: string; insight: string; sop: string }[]; + skills_assessment: string; + quick_wins: string[]; + automation_opportunities: string; + training_priority: string; +} + +interface SummaryData { + analysis: SummaryAnalysis; + model: string; + generated_at: string; +} + +function catLabel(k: string) { return CATEGORY_CFG[k]?.label ?? k; } +function catColor(k: string) { return CATEGORY_CFG[k]?.color ?? '#6b7280'; } +function resLabel(k: string) { return RESOLUTION_CFG[k] ?? k; } + +function timeAgo(d: string | null) { + if (!d) return '—'; + const h = Math.floor((Date.now() - new Date(d).getTime()) / 3_600_000); + if (h < 1) return 'Just now'; + if (h < 24) return `${h}h ago`; + return `${Math.floor(h / 24)}d ago`; +} + +// ── Expandable ticket row ───────────────────────────────────────────────────── + +function TicketRow({ t, categoryFilter, onFilter }: { + t: TicketRow; + categoryFilter: string; + onFilter: (cat: string) => void; +}) { + const [open, setOpen] = useState(false); + const catCfg = CATEGORY_CFG[t.problem_category]; + + return ( + <> + setOpen(o => !o)} + > + + {open + ? + : } + + {t.ticket_number} + {t.company_name ?? '—'} + {t.device_hostname ?? '—'} + + + {catLabel(t.problem_category)} + + + {resLabel(t.resolution_type)} + + {t.same_day_close + ? + : —} + + {parseFloat(t.hours_worked).toFixed(2)}h + + + {t.complexity} + + + {timeAgo(t.ticket_created_at)} + + {open && ( + + +
+
+ {t.work_summary && ( +
+ Summary +

{t.work_summary}

+
+ )} +
+ {t.device_was_offline != null && ( + + + {t.device_was_offline ? 'Device was offline' : 'Device was online'} + + )} + {t.backup_completed_before_tech != null && ( + + {t.backup_completed_before_tech + ? <> Backup auto-completed + : <> Tech action required} + + )} + {t.preventable != null && ( + + {t.preventable ? '⚠ Preventable' : '✓ Not preventable'} + + )} +
+
+ {t.recommended_procedure && ( +
+ Recommended SOP +

{t.recommended_procedure}

+
+ )} +
+ + + )} + + ); +} + +// ── Page ────────────────────────────────────────────────────────────────────── + +export default function VeeamAnalysisPage() { + const [data, setData] = useState(null); + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [catFilter, setCatFilter] = useState(''); + const [page, setPage] = useState(1); + const [summary, setSummary] = useState(null); + const [summaryLoading, setSummaryLoading] = useState(false); + const [summaryError, setSummaryError] = useState(null); + const pollRef = useRef | null>(null); + + const fetchData = useCallback(async (cat = catFilter, p = page) => { + const params = new URLSearchParams({ page: String(p) }); + if (cat) params.set('category', cat); + const res = await fetch(`/api/veeam/ticket-analysis?${params}`); + setData(await res.json()); + setLoading(false); + }, [catFilter, page]); + + const fetchStatus = useCallback(async () => { + const res = await fetch('/api/veeam/ticket-analysis/status'); + const s: RunStatus = await res.json(); + setStatus(s); + return s; + }, []); + + useEffect(() => { fetchData(); fetchStatus(); }, []); + + const startPolling = useCallback(() => { + if (pollRef.current) return; + pollRef.current = setInterval(async () => { + const s = await fetchStatus(); + if (!s.is_running) { + clearInterval(pollRef.current!); + pollRef.current = null; + fetchData(); + } + }, 2000); + }, [fetchStatus, fetchData]); + + useEffect(() => () => { if (pollRef.current) clearInterval(pollRef.current); }, []); + + const handleRun = async (reanalyze = false) => { + const res = await fetch('/api/veeam/ticket-analysis/run', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ reanalyze }), + }); + const json = await res.json(); + if (json.started) { + await fetchStatus(); + startPolling(); + } + }; + + const handleSummary = async () => { + setSummaryLoading(true); + setSummaryError(null); + try { + const res = await fetch('/api/veeam/ticket-analysis/summary', { method: 'POST' }); + const json = await res.json(); + if (json.error) setSummaryError(json.raw ? `${json.error}\n\n${json.raw}` : json.error); + else setSummary(json); + } catch (e: any) { + setSummaryError(e.message); + } finally { + setSummaryLoading(false); + } + }; + + const handleCatFilter = (cat: string) => { + const next = catFilter === cat ? '' : cat; + setCatFilter(next); + setPage(1); + setLoading(true); + fetchData(next, 1); + }; + + const totalPages = data ? Math.ceil(data.total / (data.limit ?? 50)) : 1; + const runPct = status?.is_running && status.run_total > 0 + ? Math.round(100 * status.run_done / status.run_total) + : null; + + return ( +
+ {/* Header */} +
+
+

+ + Veeam Backup — Ticket Analysis +

+

+ AI classification of YTD Veeam tickets to identify failure patterns, required skills, and SOP gaps. +

+
+
+ {status && !status.is_running && status.total_analyzed > 0 && ( + + )} + +
+
+ + {/* Progress bar */} + {status?.is_running && runPct !== null && ( +
+ +

+ {status.run_done} / {status.run_total} tickets analyzed + {status.run_errors > 0 && ` · ${status.run_errors} errors`} +

+
+ )} + + {/* Status line */} + {status && !status.is_running && ( +

+ {status.total_analyzed} of {status.total_eligible} eligible tickets analyzed + {status.last_analyzed_at && ` · Last run ${timeAgo(status.last_analyzed_at)}`} + {status.total_eligible === 0 && ' — no Veeam tickets with time entries found YTD'} +

+ )} + + {status?.total_analyzed === 0 && !status.is_running ? ( +
+ +

No analysis data yet.

+

Click Run Analysis to classify {status.total_eligible} tickets with time entries.

+
+ ) : ( + <> + {/* Summary cards */} + {loading && !data ? ( +
+ {[...Array(4)].map((_, i) => )} +
+ ) : data && ( +
+ {[ + { + label: 'Tickets Analyzed', + value: data.stats.total_analyzed, + sub: `of ${data.stats.total_ytd} YTD`, + icon: Brain, + cls: '', + }, + { + label: 'Same-Day Close', + value: `${data.stats.same_day_pct ?? 0}%`, + sub: 'Opened & closed same day', + icon: CheckCircle2, + cls: 'text-green-500', + }, + { + label: 'Avg Hours/Ticket', + value: `${data.stats.avg_hours ?? 0}h`, + sub: `${data.stats.auto_resolved_pct ?? 0}% auto-resolved before tech`, + icon: Clock, + cls: '', + }, + { + label: 'Preventable', + value: `${data.stats.preventable_pct ?? 0}%`, + sub: `${data.stats.offline_pct ?? 0}% device was offline`, + icon: AlertTriangle, + cls: 'text-amber-500', + }, + ].map(({ label, value, sub, icon: Icon, cls }) => ( + + + {label} + + + +
{value}
+

{sub}

+ + + ))} +
+ )} + + {/* Charts row */} + {data && ( +
+ {/* Problem categories */} + + + Problem Categories +

Click a bar to filter the ticket list

+ + + + ({ + name: catLabel(r.problem_category), + key: r.problem_category, + count: parseInt(r.count), + avg_hours: parseFloat(r.avg_hours), + }))} + layout="vertical" + margin={{ left: 10, right: 20, top: 0, bottom: 0 }} + > + + + + handleCatFilter(String(d.key ?? ''))}> + {data.by_category.map((r, i) => ( + + ))} + + + + + + + {/* Right column: Resolution + Skills */} +
+ {/* Resolution types */} + + + Resolution Types + + +
+ {data.by_resolution.map(r => { + const total = data.by_resolution.reduce((s, x) => s + parseInt(x.count), 0); + const pct = total > 0 ? Math.round(100 * parseInt(r.count) / total) : 0; + return ( +
+ {resLabel(r.resolution_type)} +
+
+
+ {r.count} +
+ ); + })} +
+ + + + {/* Skills */} + + + Skills Required + + +
+ {data.skills.map(s => ( + + {s.skill} + {s.count} + + ))} +
+ + +
+
+ )} + + {/* Sonnet Summary */} + {data && parseInt(data.stats.total_analyzed) > 0 && ( + + +
+ + + Operations Summary + +

+ Sonnet analysis of aggregate patterns — issue types, skills, and SOPs +

+
+ + + + {summaryError && ( + +

{summaryError}

+ + )} + + {summaryLoading && !summary && ( + + + + + + + )} + + {summary && ( + + {/* Headline */} +

{summary.analysis.headline}

+ + {/* Key findings */} +
+

+ Key Findings +

+
    + {summary.analysis.key_findings?.map((f, i) => ( +
  • + • + {f} +
    • + ))} +
+
+ + {/* Issue breakdown */} +
+

+ Issue Breakdown & SOPs +

+
+ {summary.analysis.issue_breakdown?.map((item, i) => ( +
+
+ + {catLabel(item.category)} + +
+

{item.insight}

+
+ + SOP: {item.sop} +
+
+ ))} +
+
+ + {/* Bottom row: skills + quick wins + automation */} +
+
+

+ Skills Assessment +

+

{summary.analysis.skills_assessment}

+ {summary.analysis.training_priority && ( +

+ {summary.analysis.training_priority} +

+ )} +
+ +
+

+ Quick Wins +

+
    + {summary.analysis.quick_wins?.map((w, i) => ( +
  • + → + {w} +
    • + ))} +
+
+ +
+

+ Automation Opportunities +

+

{summary.analysis.automation_opportunities}

+
+
+ +

+ Generated by {summary.model} · {new Date(summary.generated_at).toLocaleString()} +

+ + )} + + )} + + {/* Ticket table */} + {data && ( + + +
+ + Tickets + {catFilter && ( + + {catLabel(catFilter)} + + + )} + +

{data.total} tickets

+
+ + + +
+ + + + + + + + + + + + + + + + {data.tickets.length > 0 + ? data.tickets.map(t => ( + + )) + : ( + + + + )} + +
+ TicketClientDeviceCategoryResolutionSame-dayHoursComplexityAge
+ No analyzed tickets yet — run the analysis above. +
+
+ + {/* Pagination */} + {totalPages > 1 && ( +
+ Page {page} of {totalPages} +
+ + +
+
+ )} + + + )} + + )} +
+ ); +} diff --git a/app/veeam-comparison/page.tsx b/app/veeam-comparison/page.tsx new file mode 100644 index 0000000..1d80f01 --- /dev/null +++ b/app/veeam-comparison/page.tsx @@ -0,0 +1,575 @@ +'use client'; + +import { useEffect, useState, useCallback, useMemo } from 'react'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { + Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription, +} from '@/components/ui/dialog'; +import { + RefreshCw, CheckCircle2, AlertTriangle, WifiOff, GitCompare, + Ticket, ChevronRight, ChevronDown, Sparkles, Loader2, +} from 'lucide-react'; +import { Skeleton } from '@/components/ui/skeleton'; + +// ── Types ───────────────────────────────────────────────────────────────────── + +type MatchStatus = 'both' | 'pulse_only' | 'datto_only' | 'offline_suppressed'; + +interface AtTicket { + ticket_number: string; + title: string; + status: number | null; + priority: number | null; + created_at: string | null; + completed_at: string | null; + datto_source: string | null; +} + +interface MatchRow { + key: string; + hostname: string | null; + org_name: string | null; + company_id: number | null; + status: MatchStatus; + pulse: { + job_name: string; + priority_level: string; + hours_overdue: number; + failure_category: string | null; + opened_at: string; + } | null; + offline: { hours_offline: number; last_suppressed: string } | null; + at_ticket_count: number; + at_open_count: number; + at_tickets: AtTicket[]; +} + +interface ClientGroup { + org_name: string | null; + company_id: number | null; + rows: MatchRow[]; + counts: Record; + totalAtTickets: number; + totalAtOpen: number; +} + +interface ComparisonData { + period: string; + summary: { + pulse_open: number; + datto_at_total: number; + datto_at_open: number; + both: number; + pulse_only: number; + datto_only: number; + offline_suppressed: number; + }; + matches: MatchRow[]; +} + +interface AnalysisResult { + ticket_number: string; + hostname: string; + org_name: string; + hours_offline: number | null; + total_hours_logged: number; + note_count: number; + model: string; + analysis: { + would_suppress_correctly?: boolean; + confidence?: string; + work_summary?: string; + reasoning?: string; + recommendation?: string; + raw?: string; + }; +} + +// ── Constants ───────────────────────────────────────────────────────────────── + +const PERIODS = [ + { value: '1d', label: 'Last 24h' }, + { value: '7d', label: 'Last 7d' }, + { value: '14d', label: 'Last 14d' }, + { value: '30d', label: 'Last 30d' }, +]; + +const CLOSED_STATUSES = [5, 29832279, 29832280]; + +const STATUS_CONFIG: Record = { + both: { label: 'Both', badgeVariant: 'default', rowAccent: 'border-l-2 border-l-blue-500/50' }, + pulse_only: { label: 'Pulse Only', badgeVariant: 'destructive', rowAccent: 'border-l-2 border-l-destructive/50' }, + datto_only: { label: 'Datto/AT', badgeVariant: 'secondary', rowAccent: 'border-l-2 border-l-orange-400/50' }, + offline_suppressed: { label: 'Offline', badgeVariant: 'outline', rowAccent: 'border-l-2 border-l-muted-foreground/40' }, +}; + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +function timeAgo(dateStr: string | null | undefined): string { + if (!dateStr) return '—'; + const diff = Date.now() - new Date(dateStr).getTime(); + const h = Math.floor(diff / 3_600_000); + if (h < 1) return 'Just now'; + if (h < 24) return `${h}h ago`; + return `${Math.floor(h / 24)}d ago`; +} + +function atStatusOpen(status: number | null): boolean { + return !CLOSED_STATUSES.includes(status ?? -1); +} + +function atStatusLabel(status: number | null): string { + const map: Record = { 1: 'New', 5: 'Complete', 8: 'In Progress', 47: 'Waiting' }; + return map[status ?? -1] ?? (status != null ? `#${status}` : '?'); +} + +function PriorityBadge({ level }: { level: string }) { + const cls = level === 'critical' ? 'text-destructive border-destructive' + : level === 'high' ? 'text-orange-500 border-orange-500' + : 'text-muted-foreground border-muted-foreground/40'; + return {level}; +} + +// ── Analyze Dialog ──────────────────────────────────────────────────────────── + +function AnalyzeDialog({ + ticket, hostname, orgName, hoursOffline, onClose, +}: { + ticket: AtTicket; + hostname: string; + orgName: string; + hoursOffline?: number; + onClose: () => void; +}) { + const [loading, setLoading] = useState(true); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + fetch('/api/veeam/rpo-analyze', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + at_ticket_number: ticket.ticket_number, + hostname, + org_name: orgName, + hours_offline: hoursOffline, + }), + }) + .then(r => r.json()) + .then(d => { if (d.error) setError(d.error); else setResult(d); }) + .catch(e => setError(e.message)) + .finally(() => setLoading(false)); + }, [ticket.ticket_number, hostname, orgName, hoursOffline]); + + const a = result?.analysis; + const suppressed = a?.would_suppress_correctly; + + return ( + + + + + + RPO Suppression Analysis + + + {ticket.ticket_number} · {hostname} · {orgName} + + + + {loading && ( +
+ + Analyzing ticket work... +
+ )} + + {error && ( +
{error}
+ )} + + {result && a && ( +
+ {/* Verdict */} +
+
+ {suppressed + ? + : } +
+
+
+ {suppressed + ? 'Suppression would have been correct' + : 'Human intervention was needed'} +
+
+ Confidence: {a.confidence ?? 'unknown'} + {result.total_hours_logged > 0 && ` · ${result.total_hours_logged.toFixed(2)}h logged`} + {result.note_count > 0 && ` · ${result.note_count} note${result.note_count !== 1 ? 's' : ''}`} +
+
+
+ + {/* Work summary */} + {a.work_summary && ( +
+
Work Summary
+

{a.work_summary}

+
+ )} + + {/* Reasoning */} + {a.reasoning && ( +
+
Reasoning
+

{a.reasoning}

+
+ )} + + {/* Recommendation */} + {a.recommendation && ( +
+

{a.recommendation}

+
+ )} + + {a.raw && 
{a.raw}
} + +
Model: {result.model}
+
+ )} + + + ); +} + +// ── AT Ticket Cell ──────────────────────────────────────────────────────────── + +function AtTicketCell({ tickets, ticketCount, hostname, orgName, hoursOffline }: { + tickets: AtTicket[]; + ticketCount: number; + hostname: string | null; + orgName: string | null; + hoursOffline?: number; +}) { + const [analyzing, setAnalyzing] = useState(null); + + if (tickets.length === 0) return —; + + return ( + <> +
+ {tickets.map((t, i) => { + const isOpen = atStatusOpen(t.status); + const isClosed = !isOpen; + return ( +
+ {t.ticket_number} + + {atStatusLabel(t.status)} + + {t.datto_source && ( + {t.datto_source} + )} + {timeAgo(t.created_at)} + {isClosed && hostname && ( + + )} +
+ ); + })} + {ticketCount > 5 && ( +
+{ticketCount - 5} more
+ )} +
+ + {analyzing && ( + setAnalyzing(null)} + /> + )} + + ); +} + +// ── Client Group Row ────────────────────────────────────────────────────────── + +function ClientGroupRow({ group, defaultOpen }: { group: ClientGroup; defaultOpen: boolean }) { + const [open, setOpen] = useState(defaultOpen); + + const actionable = group.counts.both + group.counts.pulse_only; + + return ( + <> + {/* Group summary header — columns align with the detail table below */} + setOpen(o => !o)} + > + {/* Device col: chevron + org name */} + +
+ {open + ? + : } + {group.org_name ?? 'Unknown'} +
+ + {/* Match col: status pills */} + +
+ {group.counts.both > 0 && ( + {group.counts.both} Both + )} + {group.counts.pulse_only > 0 && ( + {group.counts.pulse_only} Pulse + )} + {group.counts.datto_only > 0 && ( + {group.counts.datto_only} Datto + )} + {group.counts.offline_suppressed > 0 && ( + {group.counts.offline_suppressed} Offline + )} +
+ + {/* Pulse shadow col: total devices flagged */} + + {actionable > 0 + ? {actionable} device{actionable !== 1 ? 's' : ''} need attention + : {group.rows.length} device{group.rows.length !== 1 ? 's' : ''}} + + {/* AT tickets col: ticket count */} + + {group.totalAtTickets > 0 + ? <>{group.totalAtTickets} ticket{group.totalAtTickets !== 1 ? 's' : ''}{group.totalAtOpen > 0 && · {group.totalAtOpen} open} + : —} + + + + {/* Device detail rows */} + {open && group.rows.map((row) => { + const cfg = STATUS_CONFIG[row.status]; + return ( + + + {row.hostname ?? unknown} + + + {cfg.label} + + + {row.pulse ? ( + <> +
+ + {row.pulse.hours_overdue}h overdue +
+
{row.pulse.failure_category ?? '—'}
+
since {timeAgo(row.pulse.opened_at)}
+ + ) : row.offline ? ( + + offline {Math.round(row.offline.hours_offline)}h + + ) : ( + — + )} + + + + + + ); + })} + + ); +} + +// ── Page ────────────────────────────────────────────────────────────────────── + +export default function VeeamComparisonPage() { + const [data, setData] = useState(null); + const [period, setPeriod] = useState('7d'); + const [loading, setLoading] = useState(true); + const [filter, setFilter] = useState('all'); + + const fetchData = useCallback(async () => { + setLoading(true); + try { + const res = await fetch(`/api/veeam/rpo-comparison?period=${period}`); + setData(await res.json()); + } finally { + setLoading(false); + } + }, [period]); + + useEffect(() => { fetchData(); }, [fetchData]); + + const groups = useMemo(() => { + if (!data) return []; + const filtered = data.matches.filter(m => filter === 'all' || m.status === filter); + + const byOrg = new Map(); + for (const row of filtered) { + const key = row.org_name ?? '(Unknown)'; + if (!byOrg.has(key)) { + byOrg.set(key, { + org_name: row.org_name, company_id: row.company_id, + rows: [], counts: { both: 0, pulse_only: 0, datto_only: 0, offline_suppressed: 0 }, + totalAtTickets: 0, totalAtOpen: 0, + }); + } + const g = byOrg.get(key)!; + g.rows.push(row); + g.counts[row.status]++; + g.totalAtTickets += row.at_ticket_count; + g.totalAtOpen += row.at_open_count; + } + + return Array.from(byOrg.values()).sort((a, b) => { + const aScore = (a.counts.both + a.counts.pulse_only) > 0 ? 1 : 0; + const bScore = (b.counts.both + b.counts.pulse_only) > 0 ? 1 : 0; + if (bScore !== aScore) return bScore - aScore; + return (a.org_name ?? '').localeCompare(b.org_name ?? ''); + }); + }, [data, filter]); + + const filteredTotal = data?.matches.filter(m => filter === 'all' || m.status === filter).length ?? 0; + + return ( +
+ {/* Header */} +
+
+

+ + Veeam RPO — Shadow vs Datto/AT +

+

+ What Pulse would ticket (shadow mode) vs what Datto RMM actually created in Autotask. + Click Analyze on any closed ticket to evaluate suppression accuracy. +

+
+
+ {PERIODS.map(p => ( + + ))} + +
+
+ + {loading && !data ? ( +
+ {[...Array(4)].map((_, i) => )} +
+ ) : data && ( + <> + {/* Summary cards */} +
+ {([ + { key: 'both', label: 'Both Agree', icon: CheckCircle2, iconCls: 'text-blue-500', value: data.summary.both, sub: 'Pulse + Datto both flagged', valCls: '' }, + { key: 'pulse_only', label: 'Pulse Only', icon: AlertTriangle,iconCls: 'text-destructive', value: data.summary.pulse_only, sub: 'No AT ticket from Datto', valCls: 'text-destructive' }, + { key: 'datto_only', label: 'Datto / AT Only', icon: Ticket, iconCls: 'text-orange-500', value: data.summary.datto_only, sub: `${data.summary.datto_at_total} total · ${data.summary.datto_at_open} open`, valCls: 'text-orange-500' }, + { key: 'offline_suppressed', label: 'Offline Suppressed',icon: WifiOff, iconCls: 'text-muted-foreground',value: data.summary.offline_suppressed, sub: 'Device offline — suppressed', valCls: '' }, + ] as const).map(({ key, label, icon: Icon, iconCls, value, sub, valCls }) => ( + setFilter(key as any)}> + + {label} + + + +
{value}
+

{sub}

+ + + ))} +
+ + {/* Table */} + setFilter(v as any)}> + + + All {data.matches.length} + + Both ({data.summary.both}) + Pulse Only ({data.summary.pulse_only}) + Datto/AT ({data.summary.datto_only}) + Offline ({data.summary.offline_suppressed}) + + + +
+ + + + + + + + + + + {groups.length > 0 ? groups.map(group => ( + 0} + /> + )) : ( + + + + )} + +
DeviceMatchPulse ShadowAutotask Tickets
+ {data.matches.length === 0 + ? 'No data yet — RPO check must run at least once.' + : 'No rows match this filter.'} +
+
+ {groups.length > 0 && ( +

+ {groups.length} client{groups.length !== 1 ? 's' : ''} · {filteredTotal} device{filteredTotal !== 1 ? 's' : ''} +

+ )} + + + + )} +
+ ); +} diff --git a/components/navigation/app-navigation.tsx b/components/navigation/app-navigation.tsx index 081452b..6ad5993 100644 --- a/components/navigation/app-navigation.tsx +++ b/components/navigation/app-navigation.tsx @@ -27,6 +27,8 @@ import { BarChart3, DollarSign, SlidersHorizontal, + GitCompare, + Brain, } from 'lucide-react'; import { NavigationMenu, @@ -64,9 +66,27 @@ const navigationItems: NavItem[] = [ }, { title: 'Backup Status', - href: '/backup-status', icon: HardDrive, - description: 'Veeam backup health and compliance' + children: [ + { + title: 'Backup Status', + href: '/backup-status', + icon: HardDrive, + description: 'Veeam backup health and compliance' + }, + { + title: 'RPO Comparison', + href: '/veeam-comparison', + icon: GitCompare, + description: 'Pulse RPO shadow vs Datto RMM ticket comparison' + }, + { + title: 'Ticket Analysis', + href: '/veeam-analysis', + icon: Brain, + description: 'AI classification of backup tickets — patterns, skills, SOP gaps' + }, + ] }, { title: 'Engagement', diff --git a/docker-compose.yml b/docker-compose.yml index a299122..10c10e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,6 +95,8 @@ services: # Veeam VSPC Configuration VEEAM_VSPC_URL: ${VEEAM_VSPC_URL} VEEAM_VSPC_API_KEY: ${VEEAM_VSPC_API_KEY} + VEEAM_RPO_SHADOW_MODE: "true" + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} # Zabbix API Configuration ZABBIX_API_URL: ${ZABBIX_API_URL} diff --git a/docs/mimecast-api-guide.md b/docs/mimecast-api-guide.md new file mode 100644 index 0000000..b961a82 --- /dev/null +++ b/docs/mimecast-api-guide.md @@ -0,0 +1,341 @@ +# Mimecast API Guide — Threat Dashboard Integration + +## Overview + +This guide covers authentication and data access for the Mimecast API v2, oriented toward building a threat dashboard in a client portal. All examples use OAuth 2.0 (the current recommended approach). + +--- + +## 1. Authentication (OAuth 2.0 — Client Credentials) + +### Setup in Mimecast Console + +1. Navigate to **Administration > Services > API and Platform Integrations** (or Integrations Hub in newer tenants) +2. Create a new API 2.0 application +3. Assign only the permission scopes you need (see §3 below) +4. Record the `client_id` and `client_secret` + +### Getting an Access Token + +```http +POST https://api.services.mimecast.com/oauth/token +Content-Type: application/x-www-form-urlencoded + +grant_type=client_credentials&client_id=YOUR_CLIENT_ID&client_secret=YOUR_CLIENT_SECRET +``` + +```json +{ + "access_token": "eyJ...", + "token_type": "Bearer", + "expires_in": 3600 +} +``` + +Cache the token and refresh before expiry. Plan for a 401 → refresh → retry flow. + +### Using the Token + +All API requests: + +```http +Authorization: Bearer {access_token} +Content-Type: application/json +x-mc-req-id: {uuid-per-request} +``` + +> `x-mc-req-id` should be a fresh UUID per call — used for idempotency and support tracing. + +### Region Discovery + +Mimecast routes requests by region. If you get unexpected 401s or 404s, call the discovery endpoint first to resolve the correct base URL for a given account: + +```http +GET https://api.services.mimecast.com/oauth/discover +Authorization: Bearer {access_token} +``` + +--- + +## 2. Request / Response Structure + +Nearly all Mimecast endpoints use `POST`, even for reads. The body follows a consistent envelope: + +```json +{ + "data": [ + { /* endpoint-specific params */ } + ], + "meta": { + "pagination": { + "pageToken": "next-page-token-from-previous-response" + } + } +} +``` + +Response: + +```json +{ + "data": [ /* results */ ], + "meta": { /* status info */ }, + "fail": [ /* per-item errors */ ], + "pagination": { + "next": "token_for_next_page", + "previous": "token_for_prev_page" + } +} +``` + +**Important**: HTTP 200 means the request was received. Always check `fail` array and `meta.status` for functional errors — a 200 can still contain failures. + +--- + +## 3. Threat Dashboard Endpoints + +### A. SIEM Logs (Core Email Threat Stream) + +The broadest threat feed — covers all MTA events including blocked senders, malicious URLs, attachment verdicts, spoofing attempts. + +```http +POST /api/audit/get-siem-logs + +{ + "data": [{ + "type": "MTA", + "fileFormat": "json", + "compress": false, + "token": "" + }] +} +``` + +- **Returns**: `application/octet-stream` — newline-delimited JSON events +- **Token-based cursor**: response includes `mc-siem-token` header — pass this as `token` on the next call to get the next batch +- **Retention**: 7 days only — pull at least every 6 hours, ideally hourly +- **Update frequency**: new batches available every ~30 minutes +- **Rate limit**: 300 calls/hour for this endpoint +- **Required scope**: `Gateway | Tracking | Read` +- **Prerequisite**: Enable **Enhanced Logging** in Administration Console > Account Settings + +Key fields in each event: +| Field | Description | +|-------|-------------| +| `Act` | Action taken (Acc=accepted, Rej=rejected, Hld=held) | +| `ThreatDictionary` | Threat categories triggered | +| `SpamInfo` | Spam score and verdict | +| `URL` | Rewritten URL if TTP URL protection triggered | +| `FileName` | Attachment name if scanned | +| `SHA256` | Hash of attachment | +| `SenderIP` | Originating IP | + +--- + +### B. TTP Attachment Protection Logs + +Sandboxing verdicts for email attachments — malware, ransomware, etc. + +```http +POST /api/ttp/attachment/get-logs + +{ + "data": [{ + "from": "2026-01-01T00:00:00+0000", + "to": "2026-01-02T00:00:00+0000", + "route": "inbound", + "result": "malicious", + "pageSize": 100 + }] +} +``` + +- `route`: `inbound` | `outbound` | `internal` | `all` +- `result`: `safe` | `malicious` | `timeout` | `error` | `unsafe` | `all` +- **Required scope**: `Monitoring | Attachment Protection | Read` + +Key response fields: +| Field | Description | +|-------|-------------| +| `result` | Sandbox verdict | +| `fileName` | Original attachment filename | +| `sha256` | File hash (pivot to threat intel) | +| `senderAddress` | From address | +| `recipientAddress` | Target mailbox | +| `actionTriggered` | What Mimecast did (block, sandbox, etc.) | +| `date` | ISO 8601 timestamp | + +--- + +### C. TTP URL Protection — Managed URL List + +Retrieve blocked/tracked URLs from the managed threat list: + +```http +GET /api/ttp/url/get-managed-url +``` + +Or decode a rewritten Mimecast URL back to the original for IOC extraction: + +```http +POST /api/ttp/url/decode-url + +{ + "data": [{ + "url": "https://protect-eu.mimecast.com/s/ABC..." + }] +} +``` + +--- + +### D. Audit Events (Security Configuration Changes) + +Policy changes, admin actions — useful for detecting misconfigurations or insider threats: + +```http +POST /api/audit/get-audit-events + +{ + "data": [{ + "startDateTime": "2026-01-01T00:00:00+0000", + "endDateTime": "2026-01-02T00:00:00+0000", + "categories": ["policy", "account"] + }] +} +``` + +- **Required scope**: `Account | Logs | Read` + +--- + +## 4. Pagination + +All paginated endpoints use cursor tokens: + +```typescript +async function fetchAllPages(endpoint: string, basePayload: object): Promise { + const results: any[] = []; + let pageToken: string | undefined; + + do { + const body = { + ...basePayload, + meta: pageToken ? { pagination: { pageToken } } : undefined + }; + + const res = await fetch(`https://api.services.mimecast.com${endpoint}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'x-mc-req-id': crypto.randomUUID() + }, + body: JSON.stringify(body) + }); + + const json = await res.json(); + results.push(...(json.data ?? [])); + pageToken = json.pagination?.next; + } while (pageToken); + + return results; +} +``` + +Default page size: 100. Maximum: 500. Set `pageSize` in the `data` array params. + +--- + +## 5. Rate Limiting + +| Header | Meaning | +|--------|---------| +| `X-RateLimit-Limit` | Total requests allowed in window | +| `X-RateLimit-Remaining` | Calls left | +| `X-RateLimit-Reset` | Milliseconds until reset | + +On HTTP 429, back off exponentially. SIEM logs endpoint has a hard limit of **300 calls/hour**. + +```typescript +async function fetchWithRetry(url: string, options: RequestInit, retries = 3): Promise { + const res = await fetch(url, options); + if (res.status === 429 && retries > 0) { + const resetMs = parseInt(res.headers.get('X-RateLimit-Reset') ?? '5000'); + await new Promise(r => setTimeout(r, resetMs)); + return fetchWithRetry(url, options, retries - 1); + } + return res; +} +``` + +--- + +## 6. Multi-Tenant (MSP) Access + +When accessing multiple client accounts with a single credential set, pass `accountCode` in the meta: + +```json +{ + "data": [{ /* params */ }], + "meta": { + "accountCode": "CLIENT_ACCOUNT_CODE" + } +} +``` + +Each Mimecast tenant has a unique account code visible in the administration console. + +--- + +## 7. Required Permission Scopes by Endpoint + +| Endpoint | Required Scope | +|----------|---------------| +| SIEM Logs | `Gateway > Tracking > Read` | +| Audit Events | `Account > Logs > Read` | +| TTP Attachment Logs | `Monitoring > Attachment Protection > Read` | +| TTP URL Management | `Gateway > Policies > Read` | + +Request only the scopes needed. Grant least privilege per API application. + +--- + +## 8. Dashboard Data Model Suggestion + +For a threat dashboard showing a summary per client: + +```sql +-- Suggested local cache table +CREATE TABLE mimecast_threat_events ( + id TEXT PRIMARY KEY, + account_code TEXT NOT NULL, + event_type TEXT NOT NULL, -- 'siem_mta' | 'ttp_attachment' | 'ttp_url' + event_ts TIMESTAMPTZ NOT NULL, + sender TEXT, + recipient TEXT, + sender_ip INET, + threat_type TEXT, -- malicious | spam | spoofing | etc. + action TEXT, -- blocked | sandboxed | delivered + sha256 TEXT, + url TEXT, + raw JSONB, + synced_at TIMESTAMPTZ DEFAULT now() +); + +CREATE INDEX ON mimecast_threat_events (account_code, event_ts DESC); +CREATE INDEX ON mimecast_threat_events (event_type, threat_type); +``` + +Sync strategy: poll SIEM logs hourly using stored `mc-siem-token` per account. Store the cursor token per `account_code` in a config table. + +--- + +## 9. Gotchas + +- **Enhanced Logging must be enabled** in the Mimecast console before SIEM data appears — without it the endpoint returns empty. +- **SIEM uses a streaming cursor**, not date ranges — don't skip token management or you'll re-read old events. +- **HTTP 200 ≠ success** — always check `fail[]` in the response body. +- **POST for reads** — don't expect REST conventions; almost everything is POST. +- **Token expiry** — implement proactive refresh (check `expires_in`, refresh at 80% of TTL). +- **Region routing** — if a client's tenant is in EU/AU, the base URL differs; use the discovery endpoint per account. diff --git a/docs/veeam-backup-alerting-recommendation.md b/docs/veeam-backup-alerting-recommendation.md new file mode 100644 index 0000000..9b171af --- /dev/null +++ b/docs/veeam-backup-alerting-recommendation.md @@ -0,0 +1,182 @@ +# Veeam Backup Alerting — Current State & RPO-Based Recommendation + +> **Date:** April 14, 2026 +> **Author:** Pulse / Cascade +> **Status:** Recommendation — not yet implemented + +--- + +## 1. Problem: The Current Alerting Pipeline is Noisy + +### How It Works Today + +Every Veeam backup alert that reaches Autotask passes through **Datto RMM** — Veeam does not create Autotask tickets directly. There are 6 Datto RMM monitors that watch for Veeam conditions (via Windows Event Log or script checks on each WNP/endpoint) and auto-create tickets when a condition is true: + +| RMM Monitor ID | Description | Tickets (Last 30 Days) | +|---|---|---| +| **422325** | Veeam Agent Backup Stalled | 638 | +| **770091** | Backup Copy Job Failed | 194 | +| **369602** | Veeam Agent Backup Failed | 103 | +| **369604** | Veeam Backup Job Missing or Stalled | 61 | +| **392069** | Veeam Agent Job Finished with Failed | 58 | +| **849483** | Veeam Server Agent Backup Job Stalled | 38 | + +**Total: ~1,100 backup-related tickets per month** from these 6 monitors alone. + +### Monthly Ticket Volume (Last 6 Months) + +| Month | Backup Tickets | +|---|---| +| October 2025 | 461 | +| November 2025 | 1,033 | +| December 2025 | 1,008 | +| January 2026 | 1,583 | +| February 2026 | 728 | +| March 2026 | 946 | + +### The Core Defects + +**1. No deduplication — one new ticket per check cycle.** +A workstation that misses its backup today gets a new Autotask ticket every time the RMM monitor runs. `DT061` generated **41 individual tickets in 30 days** for a single laptop. In the last 30 days, **23 devices each triggered 5+ alerts**, producing **387 redundant tickets** from a single monitor. + +**2. No schedule awareness.** +The monitors check "has Veeam run in X days?" without knowing the job's schedule. A laptop that is legitimately offline over a weekend gets stalled alerts on Saturday and Sunday even though no backup was missed relative to its RPO. + +**3. No auto-resolution.** +When the underlying backup issue is fixed and the job succeeds, the open Autotask tickets are not automatically closed. Resolution requires manual action. + +**4. No escalation logic.** +A job that has been failing for 3 days looks identical in Autotask to one that missed a single run — both generate the same priority ticket. + +**5. No failure context.** +The RMM alerts contain only the device name and a generic "stalled" or "failed" label. The root cause (VBM desync, Wasabi DNS failure, VSS error, license expiry) is not surfaced in the ticket. + +--- + +## 2. Recommendation: RPO-Based Alerting via Pulse + +### What Already Exists + +Pulse already contains a fully-built **RPO monitoring service** at `lib/services/veeam-rpo-service.ts` and an API endpoint at `POST /api/veeam/rpo-check`. The service: + +- Reads live job data from the Veeam VSPC sync in PostgreSQL (`veeam_backup_agent_jobs`) +- Computes whether each job has breached its **Recovery Point Objective (RPO)** — i.e., the acceptable maximum gap since the last successful backup — based on the job's configured schedule +- Creates **one Autotask ticket per breached job** (deduped via the `veeam_rpo_tickets` tracking table) +- **Auto-resolves** that ticket the moment a successful backup run is detected +- **Escalates** the ticket's priority as the RPO violation ages +- **Categorizes** the failure reason from the Veeam failure message + +The `veeam_rpo_tickets` table exists and is ready. It currently has 0 rows because the service has never been run. + +### RPO Threshold Logic + +| Schedule Type | RPO Window | Alert After | Escalate to High | Escalate to Critical | +|---|---|---|---|---| +| **Daily** | 24h | +4h grace = 28h | +48h | +7 days | +| **Weekly** | 168h | +4h grace = 172h | +48h | +7 days | +| **Continuous** | 1h | +1h grace = 2h | +4h | +12h | + +The **grace period** (4h for daily jobs) accounts for jobs that run slightly late due to the workstation being offline, slow networks, or queued jobs on the WNP — preventing false positives for on-time jobs with minor delays. + +### Ticket Behavior + +| Event | Current (RMM) | Proposed (RPO) | +|---|---|---| +| Job misses backup | New ticket every check cycle | One ticket opened, ticket title includes hours overdue | +| Job still failing next day | Another new ticket | Same ticket remains open, priority escalated | +| Job still failing after 2 days | Another new ticket | Ticket escalated to High | +| Job still failing after 7 days | Another new ticket | Ticket escalated to Critical | +| Backup succeeds | Tickets stay open, manual close | Ticket automatically resolved | +| Machine stale >30 days | Continuous daily alert | Skip ticket creation (likely abandoned/decommissioned machine) | + +### Ticket Content + +RPO tickets are filed to **Operations Triage** queue with: +- **Title:** `[Veeam RPO] @ — h since last backup` +- **Issue Type:** Backups / Veeam Agent for Microsoft Windows +- **Description:** Job name, org, schedule, last successful backup timestamp, hours overdue, restore points available, categorized failure reason, raw error + +**Failure categories surfaced automatically:** +- License Expired — renew via VSPC +- Cloud Gateway Unreachable — check `vcg01.wulfconsulting.com` +- Backup Repository Inaccessible +- Service Provider Maintenance +- Network/Connectivity Error +- Backup Job Timeout +- VBM desync (raw message) + +--- + +## 3. Estimated Impact + +| Metric | Current (RMM) | Projected (RPO) | +|---|---|---| +| Tickets/month (backup) | ~950–1,100 | **~50–150** (one per new breach, not per check) | +| Duplicate tickets for same device | Up to 41/month | **0** (deduped by job UID) | +| Manual ticket closures required | All | **0** (auto-resolved on success) | +| Failure root cause in ticket | No | **Yes** (categorized + raw message) | +| False positives (offline weekend) | Yes | **Minimal** (RPO + grace window) | +| Escalation based on severity | No | **Yes** (medium → high → critical) | + +--- + +## 4. Implementation Steps + +### Step 1 — Enable Veeam Sync + +The RPO service reads from the Veeam VSPC sync tables. The sync schedules already exist but are disabled: + +```sql +UPDATE sync_schedules SET is_enabled = true WHERE sync_type IN ('veeam-incremental', 'veeam-full'); +``` + +The 30-minute incremental sync keeps job status current enough for RPO evaluation. + +### Step 2 — Add RPO Check Schedule + +Add a schedule entry to run the RPO check every 2 hours: + +```sql +INSERT INTO sync_schedules (id, sync_type, cron_expression, is_enabled, description) +VALUES ('veeam-rpo-check', 'veeam-rpo-check', '0 */2 * * *', true, 'Veeam RPO check — creates/escalates/resolves backup tickets'); +``` + +The scheduler needs to handle the `veeam-rpo-check` sync type by calling `POST /api/veeam/rpo-check`. + +### Step 3 — Disable the 6 Datto RMM Backup Monitors + +In Datto RMM, disable ticket creation (or disable the monitors entirely) for the 6 monitors listed in Section 1. This eliminates the duplicate ticket stream. The monitors can remain as alerting events in RMM itself without creating Autotask tickets, if desired. + +> **Do not disable the monitors before the RPO service is confirmed working.** Run both in parallel for at least one week to validate coverage. + +### Step 4 — Dry Run Before Go-Live + +The RPO check endpoint supports a dry-run mode that shows what it *would* do without creating any tickets: + +```bash +curl -X POST https://pulse.wulfconsulting.cloud/api/veeam/rpo-check \ + -H "Content-Type: application/json" \ + -d '{"dryRun": true}' +``` + +This returns `wouldCreate`, `wouldResolve`, `wouldEscalate`, and `wouldSkipTooOld` counts along with the full job status list. + +### Step 5 — Validate & Monitor + +After enabling, confirm via Autotask that: +- RPO tickets are appearing with `[Veeam RPO]` prefix in the title +- Resolved tickets are auto-closing when backups succeed +- Ticket count is trending down from the ~1,100/month baseline + +--- + +## 5. What Is NOT Changing + +- **Backup infrastructure is unchanged** — WNPs, Veeam agents, Wasabi, VSPC all remain identical +- **Datto RMM monitoring** — RMM continues to monitor Veeam; only the ticket-creation action on those monitors is disabled +- **Server/VM backup alerting** — this recommendation is scoped to workstation jobs only; server and VM backup alerting should be evaluated separately +- **Restore workflows** — restore requests continue to be filed as standard Autotask tickets by users/staff + +--- + +*This document was produced by analyzing Autotask ticket data and the Veeam VSPC sync in the Pulse PostgreSQL database. The RPO service implementation is in `lib/services/veeam-rpo-service.ts` and `app/api/veeam/rpo-check/route.ts`.* diff --git a/docs/workstation-backup-overview.md b/docs/workstation-backup-overview.md new file mode 100644 index 0000000..27cde11 --- /dev/null +++ b/docs/workstation-backup-overview.md @@ -0,0 +1,350 @@ +# Wulf Consulting — Workstation Backup Overview + +> **Generated:** April 10, 2026 +> **Sources:** Veeam VSPC API (live sync via Pulse), Autotask PSA ticket data, IT Glue documentation +> **Scope:** Workstation (laptop and desktop) backups only — server and VM backups are excluded + +--- + +## 1. Executive Summary + +Wulf Consulting protects **1,219 workstation backup jobs** across **59 client organizations** using **Veeam Backup & Replication** managed through the **Veeam Service Provider Console (VSPC)**. Workstations are backed up via **Veeam Agent for Windows** (and 1 Mac agent) deployed to endpoints, with backup policies pushed centrally from on-premise Veeam Backup & Replication servers (WNPs) at each client site. Data flows to two tiers: **local on-prem repositories** and **Wasabi S3-compatible cloud storage** for offsite/immutable copies. + +| Metric | Value | +|---|---| +| Total workstation backup jobs | 1,219 | +| Active Veeam workstation agents | 688 | +| Client organizations covered | 59 | +| Current success rate | **83.3%** (1,016 Success) | +| Failed jobs | 84 (6.9%) | +| Warning jobs | 2 | +| Jobs with status "None" (never run / new) | 49 | +| Backup-related Autotask tickets (last 12 months) | 13,946 | + +--- + +## 2. Architecture + +### 2.1 Platform Stack + +| Component | Details | +|---|---| +| **Backup Software** | Veeam Backup & Replication v12.3 / v13.0 | +| **Management Console** | Veeam Service Provider Console (VSPC) at `vac.wulfconsulting.com:1280` | +| **Agent** | Veeam Agent for Windows (687 workstations) + Veeam Agent for Mac (1 workstation) | +| **Cloud Connect Server** | `wemvemasp02` — Veeam v13.0.1.1071, role: CloudConnect, status: Healthy | +| **Client Backup Servers** | 46 on-premise Windows servers (WNPs) at client sites, all status: Healthy | +| **Cloud Storage** | Wasabi S3 (us-east-1) via Veeam Cloud Connect — immutable object lock | +| **Monitoring** | VSPC alarms → Datto RMM alerts → Autotask ticket auto-creation | + +### 2.2 On-Premise Backup Servers (WNPs) + +Each managed client site has a dedicated **Workstation Network Proxy (WNP)** — a Windows server running Veeam Backup & Replication that acts as the local backup infrastructure. These WNPs are registered to VSPC as "Client" role backup servers. + +**Top WNPs by job count:** + +| WNP Server | Client | Workstation Jobs | +|---|---|---| +| `pitseuwnp01` | Seubert and Associates | 274 | +| `ynghynwnp01` | Hynes Industries | 254 | +| `monprewnp01` | Premier Automation Holdings | 125 | +| `geoprewnp01` | Premier Automation Holdings | 125 | +| `PITPOHWNP01` | POH+W Architects | 86 | +| `pgbvsywnp01` | V-Systems | 57 | +| `hpbhrswnp01` | Hergenroeder Rega Ewing Kennedy | 44 | + +All 46 client WNPs are running Veeam v12.3.2.4165 (two upgraded to v13.0.1.1071). All report **Healthy** status. + +### 2.3 Data Flow + +``` +Workstation (Veeam Agent) + ├── [1] Local backup → On-prem WNP repository (NAS / local disk) + └── [2] Cloud copy → WNP → Cloud Connect (wemvemasp02) → Wasabi S3 (immutable) +``` + +Each workstation runs a Veeam Agent that executes a daily backup job. Most workstations have **two jobs**: one targeting a local repository on the WNP and one replicating to Wasabi via the VSPC Cloud Connect gateway. The cloud copy is stored in per-machine immutable object-lock repositories (naming convention: `wulf-__Repository`). + +--- + +## 3. Backup Policies & Job Configuration + +### 3.1 Standard Policies + +Wulf deploys centrally managed backup policies through VSPC. The two primary workstation policies are: + +| Policy Name | Jobs | Success | Failed | Backup Mode | Target | +|---|---|---|---|---|---| +| `WulfStdWRKSTFiles` | 519 | 455 | 28 | File-level | Cloud (Wasabi) | +| `WulfStdWRKSTFilesWasabi` | 400 | 340 | 36 | File-level | Cloud (Wasabi) | +| `Premier_WulfStdWRKSTFilesD_Drive` | 48 | 36 | 5 | File-level | Cloud | +| `Premier_WulfStdWRKSTFilesD_Drive_Wasabi` | 35 | 29 | 5 | File-level | Cloud | + +Additional per-site policies exist for clients with specific requirements (e.g., `POH_ATL_Desktops`, `Blake_Desktops`, `Greco_TAR_Desktops`), each targeting the local WNP. + +### 3.2 Backup Modes + +| Mode | Jobs | Avg Backed-Up Size | Total Data | +|---|---|---|---| +| **File-level** | 1,041 (85%) | 18.88 GB | 18.96 TB | +| **Entire Computer** (volume/image) | 178 (15%) | 336.60 GB | 44.38 TB | + +- **File-level** — protects user profile data (Documents, Desktop, Downloads, AppData). This is the standard for most workstations. +- **Entire Computer** — full volume image backup. Used for specialized workstations (LOB app machines, CAD/engineering stations, machines with large local data stores). + +### 3.3 Backup Targets + +| Destination | Jobs | Total Data Protected | +|---|---|---| +| **Wasabi S3 (Cloud)** | 1,043 (86%) | 19.36 TB | +| **Local (On-Prem WNP)** | 176 (14%) | 44.38 TB | + +The cloud-targeted jobs use Wasabi S3-compatible storage via the VSPC Cloud Connect gateway (`wemvemasp02`). Local-targeted jobs write to storage attached to the client's WNP server. + +### 3.4 Schedule + +All 1,219 workstation backup jobs are configured with a **Daily** schedule. Per IT Glue documentation, typical trigger windows are: + +- **Morning run:** 6:00 AM, 9:00 AM, or 11:00 AM (varies by client) +- **Event-based:** On logoff/lock/restart events, with a minimum interval of 4–6 hours between runs +- **Backup window:** 24×7 (jobs can run any time when the workstation is available) + +> *IT Glue reference: Backup flexible asset documentation records per-client schedules. Example — Seubert and Associates desktops: "6am + log off event, not exceeding 6 hours" with Veeam Volume/Image method.* + +### 3.5 Retention + +**Per IT Glue Backup documentation (195 assets across all device types):** + +| Retention Tier | Policy | Count | +|---|---|---| +| **Local** | Veeam 4 Weeks | 62 | +| **Local** | Veeam 1 Week | 49 | +| **Local** | Veeam 2 Weeks | 33 | +| **Offsite** | GFS - 1 Year | 54 | +| **Offsite** | Veeam O365 1 year | 14 | +| **Offsite** | GFS - 6 Month | 1 | + +Workstation cloud copies average **18.9 restore points** per job (file-level) and **14.5 restore points** per job (entire computer). + +### 3.6 Immutable / Offsite Storage + +Per IT Glue documentation, **54 backup assets** reference **Wasabi Object Storage** as the offsite immutable location. Immutable object lock ensures that backup data cannot be deleted or modified by ransomware or compromised credentials for the duration of the retention period. + +Cloud repositories follow the naming convention `wulf..veeam.workstations.immutable` — e.g.: +- `wulf.seubert.veeam.workstations.immutable` +- `wulf.vsys.veeam.workstations.immutable` +- `wulf.tcg.veeam.workstations.immutable` +- `wulf.superior.veeam.workstations.immutable` + +--- + +## 4. Client Coverage + +### 4.1 Top Clients by Workstation Backup Volume + +| Client | Jobs | Success | Failed | Warning | +|---|---|---|---|---| +| Seubert and Associates | 274 | 224 | 32 | 0 | +| Hynes Industries | 254 | 225 | 14 | 0 | +| Premier Automation Holdings | 125 | 101 | 13 | 0 | +| POH+W Architects | 86 | 67 | 4 | 1 | +| ConnecTel | 58 | 52 | 3 | 0 | +| V-Systems | 57 | 53 | 2 | 0 | +| Hergenroeder Rega Ewing Kennedy | 44 | 41 | 1 | 0 | +| Superior Distributing Co | 40 | 35 | 2 | 0 | +| Greco Gas | 35 | 29 | 2 | 0 | +| Thoroughbred Construction Group | 32 | 31 | 0 | 0 | +| Commonwealth Suburban Title Agency | 25 | 9 | 3 | 0 | +| Nordmann Roofing | 17 | 17 | 0 | 0 | +| Blake Dentistry | 15 | 15 | 0 | 0 | +| TK Plastics Company | 13 | 13 | 0 | 0 | + +### 4.2 IT Glue Backup Documentation + +IT Glue contains **195 Backup flexible assets** across all managed clients, broken down by documented scope: + +| Category | Documented Assets | +|---|---| +| Server backups | 69 | +| Desktop backups | 44 | +| Other / mixed | 32 | +| Laptop backups | 30 | +| M365 backups | 18 | +| LOB application backups | 2 | + +Each asset records: backup software, method, frequency, window, local retention, offsite provider, offsite schedule, offsite retention, protected devices (tagged), and restore approval contacts. + +> *IT Glue also maintains 3 "Veeam (auto)" flexible assets that auto-document the VSPC tenant structure including sites, sub-tenants, and company mappings.* + +--- + +## 5. Current Health & Failure Analysis + +### 5.1 Job Status Breakdown + +| Status | Count | Percentage | +|---|---|---| +| ✅ Success | 1,016 | 83.3% | +| ❌ Failed | 84 | 6.9% | +| ⚠️ Warning | 2 | 0.2% | +| ⬜ None (never run) | 49 | 4.0% | +| 🔒 Disabled | 0 | 0% | +| 📉 Stale (no run in 48h) | 1,219* | — | + +*\*Note: The "stale" count includes all jobs where `last_run` is older than 48 hours or null — this is inflated by the daily schedule; many jobs last ran within 24 hours but are captured by the 48h window. The actual number of concerning stale jobs is the 49 with status "None."* + +### 5.2 Failure Root Causes + +| Failure Pattern | Failed Jobs | Description | +|---|---|---| +| **VBM Desync (needs rescan)** | 28 | Backup metadata file out of sync with DB; requires manual repository rescan on the WNP | +| **Other** | 35 | Mixed errors — agent communication failures, locked files, transient issues | +| **Wasabi DNS/Connectivity** | 14 | Cannot resolve `s3.us-east-1.wasabisys.com` — DNS or internet outage at client site | +| **VSPC Connectivity** | 6 | "Service provider's infrastructure is not responding" — VSPC gateway temporarily unavailable | +| **VSS Error** | 1 | Volume Shadow Copy failure on the workstation | + +The **dominant failure mode** is VBM (Veeam Backup Metadata) desynchronization, which requires a repository rescan on the affected WNP. This is a known Veeam issue that typically resolves after the rescan completes. + +### 5.3 Backup Job Performance + +| Backup Mode | Avg Duration (sec) | Avg Backed-Up Size (GB) | +|---|---|---| +| Entire Computer | 3,212 (~53 min) | 332.21 GB | +| File-level | 1,971 (~33 min) | 17.75 GB | + +--- + +## 6. Autotask Ticket Data — Operational Insights + +### 6.1 Backup Ticket Volume (Last 12 Months) + +Wulf's Autotask PSA logged **13,946 backup-related tickets** in the past 12 months. These are predominantly auto-generated alerts from the monitoring pipeline (Datto RMM → Autotask webhook): + +| Category | Tickets | Resolved | +|---|---|---| +| Backup Failure alerts | 7,181 | 7,177 (99.9%) | +| Veeam Agent issues | 5,378 | 5,358 (99.6%) | +| Veeam General | 1,036 | 1,031 (99.5%) | +| Restore Requests | 223 | 222 (99.6%) | +| Backup General | 128 | 125 (97.7%) | + +The vast majority of backup tickets are **automated monitoring alerts** that are triaged and resolved by the operations team. The **99.5%+ resolution rate** indicates an effective remediation workflow. + +### 6.2 Recent Ticket Examples (April 2026) + +**Automated alerts (Priority 6 — Monitoring):** +- `T20260410.0054` — *Backup Alert - SP021 (Surface Pro 9) - Veeam Agent Backup Stalled (POH+W Architects - Atlanta)* +- `T20260410.0047` — *Backup Alert - LT065 (21MV LENOVO) - Veeam Agent Backup Stalled* — ConnecTel +- `T20260410.0036` — *Backup Alert - YNGHYNLT084 (Precision 7680, Dell) - Veeam Agent Backup Stalled* — Hynes Industries +- `T20260410.0028` — *Backup Alert - pgbvsywnp01 - Veeam Backup Job Missing or Stalled* — V-Systems + +**Backup copy failures (server-side, affects workstation offsite copies):** +- `T20260410.0065` — *Backup Copy Job Failed - FOSSUPWNP01 - Superior-FOS-BackupCopyVMs-WasabiV2* — Superior Distributing Co +- `T20260410.0049` — *Backup Copy Job Failed - ynghynwnp01 - Hynes-BackupCopyVMs-WulfWasabiV2* — Hynes Industries + +### 6.3 Restore Requests (Last 6 Months — Sample) + +| Date | Ticket | Description | Client | +|---|---|---|---| +| 2026-04-06 | T20260406.0227 | Folder Restore | Thrasher Group | +| 2026-03-19 | T20260319.0350 | Restore overwritten Excel file from S:\Filestore | Insurance Restoration Consultants | +| 2026-03-16 | T20260316.0288 | Restore overwritten Photoshop file from G: drive | POH+W Architects | +| 2026-03-04 | T20260304.0210 | Device reimage — restore files via Veeam after wipe | Kuhn's Quality Foods | +| 2026-03-03 | T20260303.0189 | Veeam 365 Restore Request — OneDrive recordings | Lighthouse Electric | +| 2026-03-01 | T20260301.0108 | Restore archived Outlook calendar items | Hergenroeder Rega Ewing Kennedy | +| 2026-02-26 | T20260226.0163 | Restore emptied shared folder from file server | Blackburn's Physicians Pharmacy | + +Restore requests demonstrate the breadth of recovery scenarios handled: accidental file deletion, file overwrites, full device reimaging, and application-level restores (Outlook, OneDrive). + +--- + +## 7. IT Glue Documentation Structure + +### 7.1 Backup Flexible Asset Schema + +Each client's workstation backup is documented in IT Glue as a **Backup** flexible asset (type ID: 3791) with the following fields: + +| Field | Type | Purpose | +|---|---|---| +| Backup Software | Select | Platform (Veeam for all workstations) | +| Backup Method | Select | Files, Volume/Image, Office365 Veeam, Other | +| Backup Description | Text | What is being backed up | +| Backup Frequency | Text | Schedule description (e.g., "6am + log off event") | +| Backup Window | Select | When backups can run (typically 24×7) | +| Wulf Backup Package | Select | Service tier: All-Desktops, All-Laptops, All-Servers, Selective, etc. | +| Local Backup Server(s) | Tag | Tagged configuration items (WNP servers) | +| Local Location | Text | Repository path (e.g., `D:\Backup`) | +| Local Retention | Select | Veeam 1 Week / 2 Weeks / 4 Weeks | +| Offsite Provider | Select | Wulf, Veeam, Datto, Druva, Other | +| Offsite Replication Schedule | Select | Frequency of offsite copy | +| Offsite Retention | Select | GFS-1 Year, GFS-6 Month, Veeam O365 1 Year | +| Offsite Immutable Location | Select | Wasabi Object Storage | +| Protected Devices | Tag | Tagged workstation configuration items | +| Who Approves Restore Requests? | Tag | Contact(s) authorized to approve restores | +| Last Backup Verification | Date | Date of last manual restore test | +| Next Verification | Date | Scheduled next restore verification | + +### 7.2 Documented Backup Methods (All Device Types) + +| Method | Assets | +|---|---| +| Volume/Image | 127 | +| Files | 47 | +| Office365 Veeam | 16 | +| Other | 4 | + +### 7.3 Wulf Backup Packages + +| Package | Assets | +|---|---| +| All - Servers | 60 | +| All - Desktops | 36 | +| Selective - See Protected Devices | 34 | +| All - Laptops | 24 | +| All - O365 | 17 | +| TAM - Sell | 6 | + +--- + +## 8. Standard Operating Procedure + +Based on the data above, Wulf Consulting's standard workstation backup workflow is: + +1. **Deployment:** Veeam Agent for Windows is installed on each managed workstation during onboarding. The agent is registered to the client's on-premise WNP server. + +2. **Policy Assignment:** A VSPC backup policy (e.g., `WulfStdWRKSTFiles` or `WulfStdWRKSTFilesWasabi`) is assigned to the agent. This determines backup mode (file-level or volume), schedule, and target. + +3. **Daily Execution:** The agent runs on a daily schedule (typically early morning + logoff/lock events). File-level jobs take ~33 minutes on average; full image jobs take ~53 minutes. + +4. **Local Storage:** For clients with on-prem image backups, data is written to the WNP's local repository (NAS or direct-attached storage). Local retention is typically 1–4 weeks. + +5. **Cloud Replication:** A second job (or backup copy job on the WNP) replicates data to Wasabi S3 via the VSPC Cloud Connect gateway (`wemvemasp02`). Cloud copies are stored in per-machine **immutable** repositories with GFS retention (typically 1 year). + +6. **Monitoring:** VSPC monitors job status and raises alarms. These propagate to Datto RMM, which creates Autotask tickets automatically. The operations team triages alerts daily. + +7. **Remediation:** Failed jobs are investigated — common fixes include repository rescans (VBM desync), DNS resolution (Wasabi connectivity), and agent reinstalls. + +8. **Restores:** End users or client contacts request restores via Autotask ticket. Wulf engineers recover files from local or cloud repositories as needed. + +9. **Documentation:** Each client's backup configuration is documented in IT Glue with method, schedule, retention, protected devices, and restore approval contacts. + +--- + +## 9. Key Observations & Recommendations + +### Strengths +- **Comprehensive coverage:** 688 active workstation agents across 59 clients with centralized VSPC management +- **Immutable offsite copies:** Wasabi S3 object lock protects against ransomware and accidental deletion +- **Automated monitoring pipeline:** VSPC → Datto RMM → Autotask ensures no backup failure goes unnoticed +- **High resolution rate:** 99.5%+ of backup tickets are resolved, indicating effective operational processes +- **Standardized policies:** Two primary policies (`WulfStdWRKSTFiles`, `WulfStdWRKSTFilesWasabi`) cover 75% of jobs + +### Areas for Attention +- **VBM desync failures (28 jobs):** The most common failure mode. Consider automating repository rescans or upgrading to Veeam v13 which improved metadata handling. +- **Wasabi DNS failures (14 jobs):** Client-side DNS resolution issues. May benefit from secondary DNS or direct-IP fallback configuration. +- **49 jobs with "None" status:** Jobs that have never run — likely newly deployed agents awaiting first execution or stale configurations. +- **84 total failed jobs (6.9%):** While the alert pipeline catches these, the raw failure rate could be improved by addressing the top two root causes (VBM desync + Wasabi DNS). + +--- + +*This document was compiled from live VSPC data synced to Pulse, Autotask ticket history, and IT Glue backup documentation. Data reflects the state as of April 10, 2026.* diff --git a/docs/wulf-pulse-ticket-analyzer-prompt.md b/docs/wulf-pulse-ticket-analyzer-prompt.md new file mode 100644 index 0000000..c537813 --- /dev/null +++ b/docs/wulf-pulse-ticket-analyzer-prompt.md @@ -0,0 +1,526 @@ +# Feature: AI Ticket Analyzer (wulf-pulse) + +Add an on-demand AI-powered ticket analysis feature to the existing **wulf-pulse** app at `forgejo.wulfconsulting.cloud/lorentz/wulf-pulse`. + +This is a **feature addition**, not a new project. Conform to existing wulf-pulse conventions: React + Vite + TailwindCSS + shadcn/ui frontend, Node.js + Express backend, Entra ID OIDC auth, the existing PostgreSQL instance (the one that already has Autotask data syncing into it), Forgejo CI, Pangolin reverse proxy. Do not introduce new frameworks. Match the existing folder layout, error handling style, and route conventions. + +Before writing any code, read `claude.md`, the route registration file, the auth middleware, and the data-access layer for tickets so the feature plugs into the existing patterns. If those files don't exist, ask before guessing. + +--- + +## What this feature does + +A wulf-pulse user opens a ticket view, clicks **Analyze**, and the system produces a structured analysis covering: + +- A unified chronological timeline (with markers for source type) +- What was actually done vs. what should have been done +- Gaps — including subtle ones like the customer telling us to stop while work continued, status not matching reality, or the original ask never being directly answered +- Recommended next step with rationale +- Post-resolution analysis (if the ticket is resolved) +- Confidence score and human-review flag +- Referenced IT Glue documentation + +The analysis is stored versioned by ticket number, can be re-run when new activity arrives, and can be emailed to other Wulf users. + +--- + +## Critical: how Autotask notes/entries actually work + +The current generic prompt would mis-handle the real Autotask note structure. Here are the distinctions the analyzer **must** make: + +### Note types the analyzer must classify in pre-processing (Stage 0) + +1. **Workflow rule firings** — `Note | Autotask Administrator` with title like `Workflow Rule "..." fired.` These are **pure noise**. Filter them out entirely before any model call. Tag each as `workflow_noise`. + +2. **Service Desk Notification emails** — `Note | ` with title `Service Desk Notification` and a description that's just a list of email addresses. These are auto-generated email send confirmations. **Filter out** before model calls. Tag as `email_notification`. + +3. **Ticket Notes** — `Ticket Note | `. These are real communications, often from the customer or a forwarded email. **Keep.** Tag as `customer_communication` or `internal_communication` based on the person's email domain. + +4. **Time Entry Summary Notes** — the `Summary Notes` field of a Time Entry. **Customer-visible.** Tag as `time_entry_summary`. Keep. + +5. **Time Entry Internal Notes** — the `Internal Notes` field of the same Time Entry. **Technician-only.** Tag as `time_entry_internal`. Keep — these are usually the highest-signal entries. + +A single Time Entry can have BOTH Summary Notes and Internal Notes — they should appear as **two separate timeline events** with the same timestamp but different visibility markers, OR as a single event with both fields preserved. Choose the latter for cleaner timelines but always render them visually distinct. + +### The tagging schema + +Every retained event in the unified timeline must have: +```ts +{ + timestamp: string, // ISO 8601 + actor: string, // person name + actor_type: "wulf_tech" | "client_contact" | "vendor" | "system" | "automation", + source: "ticket_create" | "ticket_note" | "time_entry" | "status_change" | "resolution", + visibility: "customer_facing" | "internal_only" | "mixed", // mixed = time entry with both fields + summary_notes?: string, // customer-facing content if present + internal_notes?: string, // internal content if present + hours?: number, // for time entries +} +``` + +Render markers in the analysis output as: +- 🟢 customer-facing +- 🔒 internal-only +- 🔄 mixed (both) + +--- + +## Real example to test against + +Use this real ticket as a fixture for your tests. The analyzer must catch all four findings listed below — if any are missed, the analysis prompts need refinement. + +**Ticket T20260424.0045** — "Outmarket AI vendor integration request" + +The internal notes reveal: +1. The original ask was narrower than the public summary suggests — Lorentz's email said "I'm gonna need access to that for another integration with the claims department for loss run pro please let me know where that credential is in Passportal." +2. At 04/24 10:34, Lorentz posted a Ticket Note saying "I was able to access the Vertafore Developer portal and determine what is necessary - no need to reach out to Vertafore. I'll take it from here, thank you!" +3. On 04/27 (next business day), the assigned tech took a call from Vertafore anyway and logged ~20 min of additional work after the customer said to stop. +4. Status is still "Waiting Customer" three days after the requestor effectively closed the loop. + +The analyzer **must** flag: +- **Gap (high):** Work continued after the customer indicated they were taking it from here. +- **Gap (medium):** Status hasn't been updated to reflect the requestor's resolution. +- **Gap (low):** The original credential-locator ask was never directly answered before the conversation pivoted. +- **Next step:** Confirm with requestor whether the Vertafore endpoint info is still useful, then close. + +Build a test fixture from this ticket (PDF is available, transcribe the structured fields) and assert these findings appear in the analysis output. The fixture lives in `apps/api/test/fixtures/tickets/T20260424.0045.json`. + +--- + +## Database changes + +Add these tables to the wulf-pulse Postgres database. Use the project's existing migration tool. Prefix all new tables with `analyzer_` to keep them clearly scoped to this feature. + +### `analyzer_analyses` + +```sql +id uuid pk default gen_random_uuid() +ticket_number text not null +autotask_ticket_id bigint not null +analysis_version int not null -- monotonic per ticket_number +content_hash_at_analysis text not null -- sha256 of source data at analysis time +triggered_by_user_id uuid -- references existing pulse users table +triggered_at timestamptz default now() +status text not null default 'pending' -- pending|running|complete|failed +completed_at timestamptz + +-- model usage +haiku_used boolean default false +sonnet_used boolean default false +opus_used boolean default false +total_input_tokens int default 0 +total_output_tokens int default 0 +estimated_cost_usd numeric(10,4) default 0 + +-- structured output +summary text +timeline jsonb -- unified, with visibility markers +what_was_done jsonb +what_should_have_been_done jsonb +gaps jsonb -- [{description, severity, evidence_timestamps}] +next_step text +next_step_rationale text +post_resolution_analysis text +confidence_score numeric(3,2) +needs_human_review boolean default false +human_review_reasons jsonb + +-- IT Glue +itglue_docs_referenced jsonb default '[]' + +-- debugging +model_traces jsonb +filtered_noise_count int default 0 -- how many workflow/notification entries were stripped +error_message text + +unique (ticket_number, analysis_version) +``` + +Indexes: `(ticket_number, analysis_version desc)`, `(triggered_at desc)`, `(needs_human_review) where needs_human_review = true`. + +### `analyzer_shares` + +```sql +id uuid pk default gen_random_uuid() +analysis_id uuid not null references analyzer_analyses(id) on delete cascade +shared_by_user_id uuid not null +shared_with_email text not null -- validate against ALLOWED_SHARE_DOMAINS +note text +shared_at timestamptz default now() +viewed_at timestamptz +``` + +### `analyzer_jobs` + +```sql +id uuid pk default gen_random_uuid() +ticket_number text not null +queued_by_user_id uuid +status text not null default 'queued' -- queued|fetching|triaging|itglue|analyzing|deep_review|complete|failed +result_analysis_id uuid references analyzer_analyses(id) +queued_at timestamptz default now() +started_at timestamptz +finished_at timestamptz +error_message text +``` + +If wulf-pulse already uses BullMQ or another job queue, plug into it. If not, a simple Postgres-row-based queue with a worker polling every 2 seconds is acceptable for an on-demand-only feature — discuss with me before pulling in a new dependency. + +--- + +## Backend changes + +### Source-of-truth question (ask me before deciding) + +Wulf-pulse already syncs Autotask data to Postgres. **Before implementing**, look at the existing sync to determine: + +1. Does the Pulse sync include ticket notes and time entries, or just ticket headers? +2. How fresh is the sync? Real-time (webhook), minute-level, or hourly? +3. Are Internal Notes synced? (They may be excluded from some syncs for privacy reasons.) + +Based on what you find, choose one of: +- **(A) Read everything from Pulse Postgres** — preferred if notes + internal notes + time entries are all synced and fresh. +- **(B) Pull live from Autotask REST at analyze-time** — required if the sync is incomplete. +- **(C) Hybrid: Pulse for fast list/search, live REST fetch for the full payload at analyze-time** — most likely the right answer. + +Tell me which one fits before writing the data-access layer. + +### New routes (mount under existing wulf-pulse API namespace) + +``` +POST /api/analyzer/tickets/:ticketNumber/analyze + # body: { force?: boolean } + # returns: { jobId, status, existingAnalysisId? } + +GET /api/analyzer/jobs/:jobId # poll status +GET /api/analyzer/analyses/:id # fetch a specific analysis +GET /api/analyzer/tickets/:ticketNumber/analyses + # list versions +GET /api/analyzer/needs-review # filtered queue + +POST /api/analyzer/analyses/:id/share + # body: { recipientEmail, note? } +``` + +All routes require existing wulf-pulse auth middleware. + +### IT Glue client + +New module `apps/api/src/services/itglue/`: + +- `client.ts` — REST client with `x-api-key` auth, retry/backoff +- `redact.ts` — strips fields matching `/password|secret|key|token|credential|api[_-]?key/i` (case-insensitive, recursive) BEFORE any value reaches the LLM or the database. Replace with `"[REDACTED]"`. Include unit tests for nested objects and arrays. +- `search.ts` — given a client name and search hints, returns sanitized doc snippets capped at 2000 chars per doc, max 10 docs. + +The redaction is a **security-critical** code path. Add a comment explaining why and link to this prompt section. Do not log full doc bodies anywhere — only IDs and names. + +### Anthropic SDK setup + +Add `@anthropic-ai/sdk` to `apps/api/package.json`. Create `apps/api/src/services/llm/`: + +- `client.ts` — singleton SDK instance, reads `ANTHROPIC_API_KEY` from env +- `pricing.ts` — per-model input/output rates with a comment to verify against `https://docs.claude.com/en/docs/about-claude/pricing` quarterly. Don't hardcode rates without comments noting the as-of date. +- `models.ts` — exports the canonical model IDs: + - `HAIKU = "claude-haiku-4-5"` + - `SONNET = "claude-sonnet-4-6"` + - `OPUS = "claude-opus-4-7"` + +Before finalizing those constants, verify each model ID is current and available on the API. If you find newer versions or the IDs are wrong, ask me before substituting. + +--- + +## The analysis pipeline + +Implemented as `apps/api/src/services/analyzer/pipeline.ts`. Each stage is a separate function for testability. + +### Stage 0 — Fetch & Pre-process + +1. Resolve ticket via the data-access strategy chosen above. +2. **Filter noise:** strip `Note | Autotask Administrator` workflow firings and `Service Desk Notification` notes. Count them and store in `filtered_noise_count` for transparency. +3. **Tag remaining events** per the schema in the "How Autotask notes actually work" section. +4. **Sort chronologically.** +5. Compute `content_hash = sha256(canonical_json({tagged_events, ticket_status, ticket_priority, queue}))`. +6. **Idempotency check:** if `force=false` and a complete analysis exists with the same hash, short-circuit. + +### Stage 1 — Triage (Haiku) + +**Model:** `claude-haiku-4-5` + +System prompt: +``` +You are a ticket triage assistant for Wulf Consulting, an MSP. You will receive +an Autotask ticket with notes and time entries that have already been pre-filtered +to remove workflow noise and tagged by visibility (customer-facing vs internal). + +Extract structured metadata and assess complexity. Pay special attention to +internal-only notes — these often contain the real story. + +Respond ONLY with JSON: + +{ + "ticket_type": "incident" | "service_request" | "problem" | "change" | "other", + "category": string, + "entities": { + "client_name": string | null, + "site_name": string | null, + "devices": string[], + "users": string[], + "applications": string[], + "vendors": string[] // third parties involved (Vertafore, etc.) + }, + "is_resolved": boolean, + "status_matches_reality": boolean, // does the Autotask status reflect the actual state? + "complexity_tier": "low" | "medium" | "high", + "complexity_reasons": string[], + "itglue_lookup_needed": boolean, + "itglue_search_hints": string[] +} + +Complexity rubric: +- low: single straightforward issue, ≤3 retained events, clear path +- medium: multiple events, some back-and-forth, moderate ambiguity +- high: any of — bounced between techs, conflicting notes, unresolved >5 days, + customer-vs-internal narrative mismatch, multiple vendors involved, + or status appears to disagree with the actual state of the work +``` + +User message: a structured payload containing: +- Ticket header fields (title, status, priority, queue, account, contact, dates) +- Tagged event list (filtered + tagged from Stage 0) +- Counts: total events, internal-only count, customer-facing count + +Cap total payload at ~50KB. If larger, truncate oldest internal-only events first (preserving all customer-facing communications), and add a marker. + +### Stage 2 — IT Glue Retrieval (conditional) + +Run if `itglue_lookup_needed === true`. + +1. Resolve client name → IT Glue org ID. Maintain a small JSON alias map at `apps/api/src/services/itglue/aliases.json` for known fuzzy mappings (e.g. "Seubert" / "Seubert and Associates" / "S&A" → org id). Document this file in the README. +2. For each search hint, query configurations, flexible_assets, and documents endpoints. +3. Dedupe, cap at 10 docs total. +4. Run each result through `redact.ts` BEFORE adding to context. +5. Cap each doc body at 2000 chars in the LLM context. + +### Stage 3 — Deep Analysis (Sonnet) + +**Model:** `claude-sonnet-4-6` + +System prompt (verbatim): +``` +You are a senior MSP technician at Wulf Consulting reviewing a ticket. You will +receive: +1. The full ticket with all retained notes and time entries (already filtered for + workflow noise; tagged with visibility markers — customer_facing, internal_only, + or mixed) +2. Triage metadata from a previous pass +3. Optionally, sanitized IT Glue documentation snippets for the client + +Be specific and reference events by their timestamp and actor. Do not invent facts. +If something is unclear, say so explicitly. + +Pay particular attention to these patterns, which are common failure modes: +- The customer indicates they have resolved the issue or want to take it over, + but work continues afterward +- The Autotask status does not match the actual state (e.g. "Waiting Customer" + when the customer has already responded, or "In Progress" with no recent activity) +- The original ask in the requester's first message is different from what the + ticket pivoted to addressing +- Internal notes contradict or add important context missing from customer-facing + summary notes +- A vendor case was opened but the customer's direct ask could have been answered + without it +- Time was billed for work the customer didn't ultimately need + +Respond ONLY with JSON: + +{ + "summary": string, // 2-4 sentences, neutral tone + "timeline": [ + { + "timestamp": string, // ISO 8601 + "actor": string, + "actor_type": "wulf_tech" | "client_contact" | "vendor" | "system" | "automation", + "source": "ticket_create" | "ticket_note" | "time_entry" | "status_change" | "resolution", + "visibility": "customer_facing" | "internal_only" | "mixed", + "action": string // what happened, in plain language + } + ], + "what_was_done": string[], // concrete actions in order + "what_should_have_been_done": string[], // ideal actions per MSP best practice + // and any IT Glue docs provided + "gaps": [ + { + "description": string, + "severity": "low" | "medium" | "high", + "evidence_timestamps": string[] // which timeline events support this + } + ], + "next_step": string, // single concrete next action + "next_step_rationale": string, + "post_resolution_analysis": string | null, // only if is_resolved=true + "confidence_score": number, // 0.0–1.0 + "needs_human_review": boolean, + "human_review_reasons": string[], + "ambiguities_for_opus": string[], // questions a deeper-reasoning model should resolve + "itglue_docs_referenced": [ + { + "id": string, + "name": string, + "url": string, + "doc_type": string, + "relevance_reason": string + } + ] +} + +Set needs_human_review=true if any of: +- confidence_score < 0.6 +- gaps contain any "high" severity item +- ticket open >7 days with no clear resolution path +- conflicting information between notes +- billed hours appear excessive for the work performed +``` + +### Stage 4 — Deep Reasoning (Opus, conditional) + +**Model:** `claude-opus-4-7` + +Trigger conditions (any one): +- `complexity_tier === "high"` from Stage 1 +- `ambiguities_for_opus.length > 0` from Stage 3 +- `confidence_score < 0.5` from Stage 3 +- Stage 1's `status_matches_reality === false` + +System prompt: +``` +You are a principal-level MSP engineer doing a final review of a complex ticket. +You will be given: +1. The full tagged ticket +2. The Sonnet-tier analysis +3. A list of specific ambiguities or open questions + +Address each ambiguity directly with reasoning. Then produce updates ONLY for +fields that should change. + +Respond ONLY with JSON: + +{ + "opus_notes": string, // your reasoning, 1–3 paragraphs + "updates": { + // any subset of: next_step, next_step_rationale, gaps, + // confidence_score, needs_human_review, human_review_reasons, + // post_resolution_analysis + } +} +``` + +### Stage 5 — Persistence + +1. Compute next `analysis_version` for this `ticket_number`. +2. Insert into `analyzer_analyses`. +3. Sum tokens, compute estimated cost. +4. Update `analyzer_jobs` row. + +--- + +## Frontend changes + +### Routes (add to existing wulf-pulse router) + +- `/analyzer/ticket/:ticketNumber` — ticket detail with Analyze button + history of prior analyses +- `/analyzer/analysis/:id` — full analysis view, printable, with timeline visualization +- `/analyzer/queue` — needs-human-review queue + +### Analyze button behavior + +1. Click → POST to analyze endpoint. +2. If `existingAnalysisId` returned and content unchanged, navigate straight to it. +3. Otherwise show progress UI polling job status every 2s with stage labels: "Fetching..." → "Triaging..." → "Searching IT Glue..." → "Analyzing..." → "Deep review..." → "Done". +4. On completion, navigate to the analysis view. + +### Analysis view layout + +Use shadcn/ui components. Sections in order: + +1. **Header** — ticket number + title + Autotask deep link, model tier badges (Haiku/Sonnet/Opus pills), confidence score, total cost, "Share" button +2. **Summary** — paragraph +3. **Next Step** — highlighted card with rationale collapsed by default +4. **Timeline** — vertical timeline with the three visibility markers (🟢 / 🔒 / 🔄), each event clickable to expand full notes +5. **What Was Done** — bulleted list +6. **What Should Have Been Done** — bulleted list, side-by-side with #5 on wide screens +7. **Gaps** — cards colored by severity, with "Evidence:" linking back to timeline events +8. **Post-Resolution Analysis** — only if present +9. **Human Review Flags** — only if `needs_human_review = true` +10. **IT Glue References** — list of docs with external links + +### Re-analyze indicator + +On the ticket view, if cached `content_hash` differs from the latest analysis's `content_hash_at_analysis`, show a banner: "New activity since last analysis · Re-analyze". + +### Share modal + +- Recipient email field (autocomplete from existing wulf-pulse user list if available) +- Validate domain against `ALLOWED_SHARE_DOMAINS` env var +- Optional note +- Sends via M365 Graph using existing wulf-pulse mail integration if one exists; otherwise use a new module under `apps/api/src/services/mail/` and ask before adding new credentials + +--- + +## Environment variables (additions) + +``` +# IT Glue +ITGLUE_API_KEY= +ITGLUE_API_BASE=https://api.itglue.com + +# Anthropic +ANTHROPIC_API_KEY= + +# Sharing +ALLOWED_SHARE_DOMAINS=wulfconsulting.com +``` + +Add to `~/projects_env/wulf-pulse.env`. Do not commit example values. + +--- + +## Testing + +Required tests (use whatever wulf-pulse already uses for testing): + +1. **IT Glue redaction** — nested objects, arrays of objects, mixed-case field names. Must redact and never allow a password to reach the database. +2. **Note pre-processor** — given a fixture with all five note types, asserts workflow firings and notification emails are filtered, and remaining events are correctly tagged. +3. **Pipeline against the T20260424.0045 fixture** — asserts all four required findings appear in the analysis output. This is a **regression test for the prompts**, not just code. +4. **Schema validation** — every LLM response is parsed through Zod. Test with deliberately malformed responses to confirm graceful retry then failure. +5. **Idempotency** — same content_hash with `force=false` returns the existing analysis without invoking the LLM. + +--- + +## Critical correctness notes + +- **Never store IT Glue secrets/passwords in any database row, log line, or LLM context.** Redact before everything. +- **Never log full IT Glue document content.** Only doc IDs and names. +- **Validate every LLM JSON response with Zod.** On parse failure, retry once with the prior response and the parse error. After two failures, mark the job failed and store the raw response in `error_message`. +- **Token budget guard:** if any single model call would exceed 100k input tokens, truncate oldest internal-only events first while preserving all customer-facing communications, and add a marker. Log a warning. +- **Cost circuit breaker:** if estimated total cost would exceed $2.00 before the Opus call, skip Opus, set `needs_human_review = true`, and add reason "cost ceiling reached". +- **Idempotency:** the analyze endpoint must be idempotent on `(ticket_number, content_hash)` when `force=false`. +- **Verify model IDs and pricing against `https://docs.claude.com` before finalizing constants.** Models and rates change. +- **Ticket Notes from `wulfconsulting.com` addresses are internal communications, not customer communications.** Tag them by domain, not by author. + +--- + +## Delivery order + +Build and ship in this order, asking me to review between each phase: + +1. Database migrations + Zod schemas in `packages/shared` (or wulf-pulse equivalent). +2. Note pre-processor with unit tests against the T20260424.0045 fixture (PDF → JSON transcription is the first deliverable). +3. IT Glue client + redaction (security-critical, must land before LLM integration). +4. Anthropic SDK setup + per-stage prompt files. +5. Full pipeline + job worker. +6. API routes. +7. Frontend pages. +8. Share-via-email integration. +9. README updates with operator runbook (how to monitor cost, how to add IT Glue org aliases, how to triage failed analyses). + +For phase 1 specifically: confirm the data-access strategy (read from Pulse Postgres / live Autotask REST / hybrid) before writing the migration, since the schema for `tickets_cache` may or may not be needed depending on which path we take. diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 5de9890..8a770ec 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -103,7 +103,7 @@ export class EntitySyncService { // For incremental sync, filter by last sync time // Note: Companies and Resources don't support date-based filtering in Autotask API - const supportsIncremental = entity !== EntityType.COMPANIES && entity !== EntityType.RESOURCES; + const supportsIncremental = entity !== EntityType.COMPANIES && entity !== EntityType.RESOURCES && entity !== EntityType.COMPANY_TEAMS; if (isIncremental && supportsIncremental) { try { @@ -415,6 +415,41 @@ export class EntitySyncService { } } + // After contacts sync, backfill primary_contact_id and billing_contact_id on companies + if (entity === EntityType.CONTACTS) { + try { + const primaryResult = await postgresClient.query( + `UPDATE companies c + SET primary_contact_id = ( + SELECT id FROM contacts + WHERE company_id = c.id AND primary_contact = true AND is_deleted = false + ORDER BY id LIMIT 1 + ) + WHERE EXISTS ( + SELECT 1 FROM contacts + WHERE company_id = c.id AND primary_contact = true AND is_deleted = false + )` + ); + entityLogger.info('Backfilled companies.primary_contact_id', { updatedCount: (primaryResult as any).rowCount ?? 0 }); + + const billingResult = await postgresClient.query( + `UPDATE companies c + SET billing_contact_id = ( + SELECT id FROM contacts + WHERE company_id = c.id AND billing_contact = true AND is_deleted = false + ORDER BY id LIMIT 1 + ) + WHERE EXISTS ( + SELECT 1 FROM contacts + WHERE company_id = c.id AND billing_contact = true AND is_deleted = false + )` + ); + entityLogger.info('Backfilled companies.billing_contact_id', { updatedCount: (billingResult as any).rowCount ?? 0 }); + } catch (err) { + entityLogger.warn('Contact FK backfill on companies failed', { error: String(err) }); + } + } + // For full sync, soft delete records not in the fetched set // IMPORTANT: Skip soft deletes if ANY filters were applied (date, status, active, etc.) // because we cannot know what records exist outside the filter criteria. @@ -817,6 +852,13 @@ export class EntitySyncService { return await this.syncEntity(EntityType.CONTACTS, isIncremental); } + /** + * Sync Company Teams (TAMs, CSMs, co-managed resources assigned to a company) + */ + async syncCompanyTeams(isIncremental: boolean = false): Promise { + return await this.syncEntity(EntityType.COMPANY_TEAMS, isIncremental); + } + /** * Sync Contracts */ diff --git a/lib/services/rmm-device-resolver.ts b/lib/services/rmm-device-resolver.ts new file mode 100644 index 0000000..07c8a58 --- /dev/null +++ b/lib/services/rmm-device-resolver.ts @@ -0,0 +1,90 @@ +/** + * RMM Device Resolver + * Matches Veeam backup agent jobs to their Datto RMM device via: + * veeam_organizations.company_id → datto_rmm_sites.autotask_company_id (org match) + * veeam_backup_agents.name ≈ datto_rmm_devices.hostname (hostname match) + * + * Scoped to Desktop/Laptop device categories only — server jobs are handled separately. + * Reusable by any service that needs to correlate Veeam jobs with RMM device state. + */ + +import postgresClient from './postgres-client'; + +export interface RmmDeviceInfo { + hostname: string; + site_name: string; + device_type_category: string; + last_seen: Date | null; + online: boolean; +} + +/** + * Bulk-resolve Veeam backup agent jobs to their matching Datto RMM devices. + * Returns a Map keyed by job instance_uid. Jobs with no RMM match are absent. + */ +export async function resolveRmmDevicesForJobs( + jobInstanceUids: string[] +): Promise> { + if (jobInstanceUids.length === 0) return new Map(); + + // datto_rmm_sites.autotask_company_id is not reliably populated from the API, + // so join via companies.company_name = datto_rmm_sites.name instead. + // Datto sites are often named "Company - Location" while company_name is "Company", + // so match exact OR site name starts with "company - ". + // DISTINCT ON (job) prevents fanout when a company has multiple matching sites, + // preferring the device with the most recent last_seen. + const res = await postgresClient.query(` + SELECT DISTINCT ON (j.instance_uid) + j.instance_uid AS job_instance_uid, + rmm.hostname, + rs.name AS site_name, + rmm.device_type_category, + rmm.last_seen, + rmm.online + FROM veeam_backup_agent_jobs j + JOIN veeam_backup_agents ba ON ba.instance_uid = j.backup_agent_uid + JOIN veeam_organizations vo ON vo.instance_uid = j.organization_uid + JOIN companies c ON c.id = vo.company_id + JOIN datto_rmm_sites rs ON LOWER(rs.name) = LOWER(c.company_name) + OR LOWER(rs.name) LIKE LOWER(c.company_name) || ' - %' + JOIN datto_rmm_devices rmm + ON rmm.site_id = rs.id + AND LOWER(rmm.hostname) = LOWER(ba.name) + AND rmm.device_type_category IN ('Desktop', 'Laptop') + AND rmm.deleted = false + WHERE j.instance_uid = ANY($1) + ORDER BY j.instance_uid, rmm.last_seen DESC NULLS LAST + `, [jobInstanceUids]); + + const map = new Map(); + for (const row of res.rows) { + map.set(row.job_instance_uid, { + hostname: row.hostname, + site_name: row.site_name, + device_type_category: row.device_type_category, + last_seen: row.last_seen ? new Date(row.last_seen) : null, + online: row.online, + }); + } + return map; +} + +/** + * Returns true when the device should suppress RPO alerting. + * A device is considered suppressed if it has been offline longer than one full + * backup interval — meaning it was already offline when the backup was due to run. + * No last_seen → not suppressed (device exists in RMM but has never reported; alert normally). + */ +export function isDeviceOfflineSuppressed(info: RmmDeviceInfo, intervalHours: number): boolean { + if (!info.last_seen) return false; + const hoursOffline = (Date.now() - info.last_seen.getTime()) / 3_600_000; + return hoursOffline > intervalHours; +} + +/** + * Returns hours since the device was last seen by RMM, or null if never seen. + */ +export function hoursOffline(info: RmmDeviceInfo): number | null { + if (!info.last_seen) return null; + return (Date.now() - info.last_seen.getTime()) / 3_600_000; +} diff --git a/lib/services/veeam-analysis-state.ts b/lib/services/veeam-analysis-state.ts new file mode 100644 index 0000000..770db9b --- /dev/null +++ b/lib/services/veeam-analysis-state.ts @@ -0,0 +1,9 @@ +// Shared in-process state for the background ticket analysis runner. +// Works because we run in Docker (long-lived Node process), not serverless. +export const analysisState = { + isRunning: false, + total: 0, + done: 0, + errors: 0, + startedAt: null as Date | null, +}; diff --git a/lib/services/veeam-rpo-service.ts b/lib/services/veeam-rpo-service.ts index 9e57a01..2ec50b1 100644 --- a/lib/services/veeam-rpo-service.ts +++ b/lib/services/veeam-rpo-service.ts @@ -6,6 +6,12 @@ import postgresClient from './postgres-client'; import { AutotaskClient } from './autotask-client'; +import { + RmmDeviceInfo, + resolveRmmDevicesForJobs, + isDeviceOfflineSuppressed, + hoursOffline, +} from './rmm-device-resolver'; const AT_QUEUE_ID = 29832283; // Operations Triage const AT_ISSUE_TYPE = 38; // Backups @@ -16,12 +22,17 @@ const AT_PRIORITY_CRIT = 1; // Critical const AT_STATUS_NEW = 1; const AT_STATUS_DONE = 5; +// Shadow mode: log what the service *would* do without touching Autotask. +// Default true — set VEEAM_RPO_SHADOW_MODE=false in env to go live. +const SHADOW_MODE = process.env.VEEAM_RPO_SHADOW_MODE !== 'false'; + export interface RpoCheckResult { checked: number; newTickets: number; escalated: number; resolved: number; skipped: number; + offlineSuppressed: number; errors: string[]; runAt: Date; } @@ -36,8 +47,12 @@ export interface RpoJobSummary { hours_since_backup: number | null; rpo_hours: number; is_breached: boolean; + is_offline_suppressed: boolean; failure_category: string | null; failure_message: string | null; + rmm_hostname: string | null; + rmm_site_name: string | null; + rmm_last_seen: string | null; open_ticket: { at_ticket_id: number; at_ticket_number: string; @@ -138,11 +153,12 @@ export class VeeamRpoService { escalated: 0, resolved: 0, skipped: 0, + offlineSuppressed: 0, errors: [], runAt: new Date(), }; - const client = getAutotaskClient(); + const client = SHADOW_MODE ? null : getAutotaskClient(); // Fetch all enabled workstation jobs with org info const jobsRes = await postgresClient.query(` @@ -169,27 +185,32 @@ export class VeeamRpoService { const jobs = jobsRes.rows; result.checked = jobs.length; - // Fetch all currently open RPO tickets in one query - const openTicketsRes = await postgresClient.query(` - SELECT * FROM veeam_rpo_tickets WHERE resolved_at IS NULL - `); + // Fetch all currently open tracking tickets (shadow or live table) + const trackingTable = SHADOW_MODE ? 'veeam_rpo_shadow_tickets' : 'veeam_rpo_tickets'; + const openTicketsRes = await postgresClient.query( + `SELECT * FROM ${trackingTable} WHERE resolved_at IS NULL` + ); const openByJobUid: Record = {}; for (const row of openTicketsRes.rows) { openByJobUid[row.job_instance_uid] = row; } + // Bulk-resolve RMM devices for all jobs in a single query + const rmmMap = await resolveRmmDevicesForJobs(jobs.map((j: any) => j.instance_uid)); + for (const job of jobs) { try { - await this.processJob(job, openByJobUid, client, result); + const rmmDevice = rmmMap.get(job.instance_uid) ?? null; + await this.processJob(job, openByJobUid, rmmDevice, client as AutotaskClient, result); } catch (err: any) { result.errors.push(`${job.job_name}: ${err.message}`); } } - // Update last_checked_at for all processed jobs - await postgresClient.query(` - UPDATE veeam_rpo_tickets SET last_checked_at = NOW() WHERE resolved_at IS NULL - `); + // Update last_checked_at for all open tracking tickets + await postgresClient.query( + `UPDATE ${trackingTable} SET last_checked_at = NOW() WHERE resolved_at IS NULL` + ); return result; } @@ -197,6 +218,7 @@ export class VeeamRpoService { private async processJob( job: any, openByJobUid: Record, + rmmDevice: RmmDeviceInfo | null, client: AutotaskClient, result: RpoCheckResult, ): Promise { @@ -214,6 +236,24 @@ export class VeeamRpoService { : (job.schedule_type ?? '').toLowerCase().includes('continuous') ? 1 : 24; + // Suppress new tickets and escalations for Desktop/Laptop devices that have been + // offline longer than one backup interval — the machine was offline before the backup + // was due, so a missed backup is expected. Existing open tickets are left untouched, + // but we backfill rmm_hostname so the comparison page can match them correctly. + if (rmmDevice && isDeviceOfflineSuppressed(rmmDevice, intervalHours)) { + const openTicket = openByJobUid[job.instance_uid] ?? null; + if (openTicket && !openTicket.rmm_hostname) { + const trackingTable = SHADOW_MODE ? 'veeam_rpo_shadow_tickets' : 'veeam_rpo_tickets'; + await postgresClient.query( + `UPDATE ${trackingTable} SET rmm_hostname = $1, updated_at = NOW() WHERE job_instance_uid = $2`, + [rmmDevice.hostname, job.instance_uid] + ); + } + await this.logOfflineSuppression(job, rmmDevice, intervalHours); + result.offlineSuppressed++; + return; + } + // Breach rules: // - Failed/Warning: always breached — a failed backup is a failed backup regardless of recency // - None (never run): always breached @@ -226,8 +266,12 @@ export class VeeamRpoService { if (!isBreached) { if (openTicket) { - await this.resolveTicket(openTicket, client); - result.resolved++; + if (SHADOW_MODE) { + await this.shadowResolve(openTicket, job, rmmDevice, result); + } else { + await this.resolveTicket(openTicket, client!); + result.resolved++; + } } else { result.skipped++; } @@ -248,15 +292,22 @@ export class VeeamRpoService { if (!openTicket) { if (tooOldForNewTicket) { + if (SHADOW_MODE) await this.writeShadowLog(job, rmmDevice, 'would_skip_too_old', null, null, null); result.skipped++; return; } - // Create new ticket - await this.createTicket(job, hoursOverdue, failureCategory, targetPriority, client, result); + if (SHADOW_MODE) { + await this.shadowCreate(job, rmmDevice, hoursOverdue, failureCategory, targetPriority, result); + } else { + await this.createTicket(job, hoursOverdue, failureCategory, targetPriority, client!, result); + } } else { - // Escalate if needed if (openTicket.priority_level !== targetPriority && this.isPriorityHigher(targetPriority, openTicket.priority_level)) { - await this.escalateTicket(openTicket, job, hoursOverdue, failureCategory, targetPriority, client, result); + if (SHADOW_MODE) { + await this.shadowEscalate(openTicket, job, rmmDevice, hoursOverdue, failureCategory, targetPriority, result); + } else { + await this.escalateTicket(openTicket, job, hoursOverdue, failureCategory, targetPriority, client!, result); + } } else { result.skipped++; } @@ -268,6 +319,130 @@ export class VeeamRpoService { return (rank[a] ?? 0) > (rank[b] ?? 0); } + // ─── Shadow mode methods ─────────────────────────────────────────────────── + + private async writeShadowLog( + job: any, + rmmDevice: RmmDeviceInfo | null, + action: string, + priorityLevel: string | null, + hoursOverdue: number | null, + failureCategory: string | null, + ): Promise { + await postgresClient.query(` + INSERT INTO veeam_rpo_shadow_log + (job_instance_uid, job_name, org_name, rmm_hostname, rmm_site_name, + action, priority_level, hours_overdue, failure_category, checked_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,NOW()) + `, [ + job.instance_uid, job.job_name, job.org_name, + rmmDevice?.hostname ?? null, rmmDevice?.site_name ?? null, + action, priorityLevel, + hoursOverdue !== null ? Math.round(hoursOverdue) : null, + failureCategory, + ]); + } + + private async shadowCreate( + job: any, + rmmDevice: RmmDeviceInfo | null, + hoursOverdue: number, + failureCategory: string, + priorityLevel: string, + result: RpoCheckResult, + ): Promise { + await postgresClient.query(` + INSERT INTO veeam_rpo_shadow_tickets + (job_instance_uid, job_name, org_name, rmm_hostname, priority_level, + hours_overdue, failure_category, opened_at, last_checked_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,NOW(),NOW()) + ON CONFLICT (job_instance_uid) DO UPDATE SET + priority_level = EXCLUDED.priority_level, + hours_overdue = EXCLUDED.hours_overdue, + failure_category = EXCLUDED.failure_category, + rmm_hostname = COALESCE(EXCLUDED.rmm_hostname, veeam_rpo_shadow_tickets.rmm_hostname), + resolved_at = NULL, + opened_at = NOW(), + last_checked_at = NOW(), + updated_at = NOW() + `, [ + job.instance_uid, job.job_name, job.org_name, + rmmDevice?.hostname ?? null, priorityLevel, + Math.round(hoursOverdue), failureCategory, + ]); + await this.writeShadowLog(job, rmmDevice, 'would_create', priorityLevel, hoursOverdue, failureCategory); + result.newTickets++; + console.log(`[RPO-SHADOW] Would create ticket: ${job.job_name} @ ${job.org_name} (${Math.round(hoursOverdue)}h overdue, ${priorityLevel})`); + } + + private async shadowEscalate( + openTicket: any, + job: any, + rmmDevice: RmmDeviceInfo | null, + hoursOverdue: number, + failureCategory: string, + targetPriority: string, + result: RpoCheckResult, + ): Promise { + await postgresClient.query(` + UPDATE veeam_rpo_shadow_tickets SET + priority_level = $1, + hours_overdue = $2, + failure_category = $3, + last_checked_at = NOW(), + updated_at = NOW() + WHERE job_instance_uid = $4 + `, [targetPriority, Math.round(hoursOverdue), failureCategory, job.instance_uid]); + await this.writeShadowLog(job, rmmDevice, 'would_escalate', targetPriority, hoursOverdue, failureCategory); + result.escalated++; + console.log(`[RPO-SHADOW] Would escalate to ${targetPriority}: ${job.job_name} (${Math.round(hoursOverdue)}h overdue)`); + } + + private async shadowResolve( + openTicket: any, + job: any, + rmmDevice: RmmDeviceInfo | null, + result: RpoCheckResult, + ): Promise { + await postgresClient.query(` + UPDATE veeam_rpo_shadow_tickets SET + resolved_at = NOW(), + last_checked_at = NOW(), + updated_at = NOW() + WHERE job_instance_uid = $1 + `, [openTicket.job_instance_uid]); + await this.writeShadowLog(job, rmmDevice, 'would_resolve', null, null, null); + result.resolved++; + console.log(`[RPO-SHADOW] Would resolve: ${job.job_name} — backup succeeded`); + } + + // ────────────────────────────────────────────────────────────────────────── + + private async logOfflineSuppression( + job: any, + rmmDevice: RmmDeviceInfo, + intervalHours: number, + ): Promise { + const hrs = hoursOffline(rmmDevice) ?? 0; + await postgresClient.query(` + INSERT INTO veeam_rpo_offline_log + (job_instance_uid, job_name, org_name, rmm_hostname, rmm_site_name, + device_type_category, rmm_last_seen, hours_offline, backup_interval_hours, checked_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW()) + `, [ + job.instance_uid, + job.job_name, + job.org_name, + rmmDevice.hostname, + rmmDevice.site_name, + rmmDevice.device_type_category, + rmmDevice.last_seen, + Math.round(hrs * 100) / 100, + intervalHours, + ]); + console.log(`[RPO] Suppressed (offline ${Math.round(hrs)}h): ${job.job_name} @ ${rmmDevice.hostname}`); + } + private async createTicket( job: any, hoursOverdue: number, @@ -289,9 +464,8 @@ export class VeeamRpoService { // Look up Autotask company ID from org name mapping const companyRes = await postgresClient.query(` - SELECT c.id FROM companies c - JOIN veeam_organizations vo ON vo.autotask_company_id = c.id - WHERE vo.name = $1 + SELECT company_id AS id FROM veeam_organizations + WHERE name = $1 LIMIT 1 `, [job.org_name]); @@ -303,7 +477,6 @@ export class VeeamRpoService { status: AT_STATUS_NEW, queueID: AT_QUEUE_ID, issueType: AT_ISSUE_TYPE, - subIssueType: AT_SUB_ISSUE, priority: atPriority, }; if (companyId) ticketPayload.companyID = companyId; @@ -398,7 +571,7 @@ export class VeeamRpoService { console.log(`[RPO] Resolved ticket ${openTicket.at_ticket_number} — backup succeeded`); } - async getStatus(): Promise<{ summary: Record; jobs: RpoJobSummary[] }> { + async getStatus(): Promise<{ shadowMode: boolean; summary: Record; jobs: RpoJobSummary[] }> { const jobsRes = await postgresClient.query(` SELECT j.instance_uid, @@ -411,7 +584,12 @@ export class VeeamRpoService { j.failure_message, o.name as org_name, EXTRACT(EPOCH FROM (NOW() - j.last_end_time)) / 3600.0 as hours_since_backup, - EXTRACT(EPOCH FROM (j.next_run - NOW())) / 3600.0 as hours_until_next_run + EXTRACT(EPOCH FROM (j.next_run - NOW())) / 3600.0 as hours_until_next_run, + rt.at_ticket_id, + rt.at_ticket_number, + rt.priority_level, + rt.hours_overdue, + rt.opened_at FROM veeam_backup_agent_jobs j JOIN veeam_organizations o ON o.instance_uid = j.organization_uid LEFT JOIN veeam_rpo_tickets rt @@ -421,7 +599,10 @@ export class VeeamRpoService { ORDER BY hours_since_backup DESC NULLS LAST `); - const jobs: RpoJobSummary[] = jobsRes.rows.map((row) => { + const rows = jobsRes.rows; + const rmmMap = await resolveRmmDevicesForJobs(rows.map((r: any) => r.instance_uid)); + + const jobs: RpoJobSummary[] = rows.map((row: any) => { const thresholds = getRpoThresholds(row.schedule_type); const hoursAgo: number | null = row.hours_since_backup !== null ? parseFloat(row.hours_since_backup) : null; const intervalHours = (row.schedule_type ?? '').toLowerCase().includes('weekly') ? 168 @@ -432,39 +613,52 @@ export class VeeamRpoService { && (row.status === 'Failed' || row.status === 'Warning' || hoursAgo === null || hoursAgo > rpoWindowHours); + + const rmmDevice = rmmMap.get(row.instance_uid) ?? null; + const isOfflineSuppressed = isBreached + && rmmDevice !== null + && isDeviceOfflineSuppressed(rmmDevice, intervalHours); + return { - job_instance_uid: row.instance_uid, - job_name: row.job_name, - org_name: row.org_name, - status: row.status, - schedule_type: row.schedule_type, - last_end_time: row.last_end_time, - hours_since_backup: hoursAgo !== null ? Math.round(hoursAgo * 10) / 10 : null, - rpo_hours: thresholds.grace, - is_breached: isBreached, - failure_category: row.failure_message ? categorizeFailure(row.failure_message) : null, - failure_message: row.failure_message, + job_instance_uid: row.instance_uid, + job_name: row.job_name, + org_name: row.org_name, + status: row.status, + schedule_type: row.schedule_type, + last_end_time: row.last_end_time, + hours_since_backup: hoursAgo !== null ? Math.round(hoursAgo * 10) / 10 : null, + rpo_hours: thresholds.grace, + is_breached: isBreached, + is_offline_suppressed: isOfflineSuppressed, + failure_category: row.failure_message ? categorizeFailure(row.failure_message) : null, + failure_message: row.failure_message, + rmm_hostname: rmmDevice?.hostname ?? null, + rmm_site_name: rmmDevice?.site_name ?? null, + rmm_last_seen: rmmDevice?.last_seen?.toISOString() ?? null, open_ticket: row.at_ticket_id ? { - at_ticket_id: (row as any).at_ticket_id, - at_ticket_number: (row as any).at_ticket_number, - priority_level: (row as any).priority_level, - hours_overdue: (row as any).hours_overdue, - opened_at: (row as any).opened_at, + at_ticket_id: row.at_ticket_id, + at_ticket_number: row.at_ticket_number, + priority_level: row.priority_level, + hours_overdue: row.hours_overdue, + opened_at: row.opened_at, } : null, }; }); - const breached = jobs.filter(j => j.is_breached).length; - const withTicket = jobs.filter(j => j.open_ticket !== null).length; - const healthy = jobs.filter(j => !j.is_breached).length; - const critical = jobs.filter(j => j.open_ticket?.priority_level === 'critical').length; - const high = jobs.filter(j => j.open_ticket?.priority_level === 'high').length; + const breached = jobs.filter(j => j.is_breached).length; + const offlineSuppressed = jobs.filter(j => j.is_offline_suppressed).length; + const withTicket = jobs.filter(j => j.open_ticket !== null).length; + const healthy = jobs.filter(j => !j.is_breached).length; + const critical = jobs.filter(j => j.open_ticket?.priority_level === 'critical').length; + const high = jobs.filter(j => j.open_ticket?.priority_level === 'high').length; return { + shadowMode: SHADOW_MODE, summary: { total: jobs.length, healthy, breached, + offlineSuppressed, withOpenTicket: withTicket, critical, high, diff --git a/lib/services/webhook-service.ts b/lib/services/webhook-service.ts index 910e6dd..f2dc4ec 100644 --- a/lib/services/webhook-service.ts +++ b/lib/services/webhook-service.ts @@ -116,6 +116,7 @@ export class WebhookService { ); } + return { success: true, eventId: payload.eventId, diff --git a/lib/types/sync.ts b/lib/types/sync.ts index 7f9acf2..04d42ad 100644 --- a/lib/types/sync.ts +++ b/lib/types/sync.ts @@ -30,6 +30,7 @@ export enum EntityType { PROJECT_PHASES = 'project_phases', COMPANY_CATEGORIES = 'company_categories', COMPANY_TYPES = 'company_types', + COMPANY_TEAMS = 'company_teams', } // Sync operation types @@ -181,6 +182,7 @@ export const ENTITY_DEPENDENCIES: Record = { [EntityType.PROJECT_PHASES]: [EntityType.PROJECTS], // Depends on projects [EntityType.COMPANY_CATEGORIES]: [], // No dependencies — standalone lookup [EntityType.COMPANY_TYPES]: [], // No dependencies — standalone lookup + [EntityType.COMPANY_TEAMS]: [EntityType.COMPANIES, EntityType.RESOURCES], // Depends on companies and resources }; // Autotask API field names (for incremental sync) diff --git a/lib/utils/entity-mapper.ts b/lib/utils/entity-mapper.ts index 77a8955..ae3c244 100644 --- a/lib/utils/entity-mapper.ts +++ b/lib/utils/entity-mapper.ts @@ -83,6 +83,9 @@ export function mapAutotaskToDatabase( case EntityType.TAGS: mapped = mapTag(data); break; + case EntityType.COMPANY_TEAMS: + mapped = mapCompanyTeam(data); + break; default: // Fallback: auto-convert camelCase to snake_case mapped = {}; @@ -826,6 +829,19 @@ function mapPicklist(data: any): Record { }; } +/** + * Map CompanyTeam entity + */ +function mapCompanyTeam(data: any): Record { + return { + id: data.id, + company_id: data.companyID, + resource_id: data.resourceID, + is_associated_as_comanaged: data.isAssociatedAsComanaged || false, + is_deleted: false, + }; +} + /** * Batch map multiple entities */ diff --git a/lib/utils/sync-helpers.ts b/lib/utils/sync-helpers.ts index 4cbcb42..ab7f2aa 100644 --- a/lib/utils/sync-helpers.ts +++ b/lib/utils/sync-helpers.ts @@ -72,6 +72,7 @@ export function getAllEntitiesInOrder(): EntityType[] { EntityType.TIME_ENTRIES, EntityType.TAG_GROUPS, EntityType.TAGS, + EntityType.COMPANY_TEAMS, ]); } @@ -130,6 +131,7 @@ export function getAutotaskEntityName(entity: EntityType): string { [EntityType.PROJECT_PHASES]: 'Phases', [EntityType.COMPANY_CATEGORIES]: 'CompanyCategories', [EntityType.COMPANY_TYPES]: 'CompanyTypes', + [EntityType.COMPANY_TEAMS]: 'CompanyTeams', }; return mapping[entity] || entity; @@ -185,6 +187,7 @@ export function getLastModifiedField(entity: EntityType): string { [EntityType.PROJECT_PHASES]: 'lastActivityDateTime', [EntityType.COMPANY_CATEGORIES]: 'lastModifiedDate', [EntityType.COMPANY_TYPES]: 'lastModifiedDate', + [EntityType.COMPANY_TEAMS]: 'lastModifiedDate', // No date field; incremental not supported }; return mapping[entity] || 'lastModifiedDate'; @@ -222,6 +225,7 @@ export function getActiveField(entity: EntityType): string | null { [EntityType.PROJECT_PHASES]: null, // No active field on phases [EntityType.COMPANY_CATEGORIES]: 'isActive', [EntityType.COMPANY_TYPES]: 'isActive', + [EntityType.COMPANY_TEAMS]: null, }; return mapping[entity] || null; @@ -319,6 +323,7 @@ export function buildDateRangeFilter( [EntityType.PROJECT_PHASES]: null, [EntityType.COMPANY_CATEGORIES]: null, [EntityType.COMPANY_TYPES]: null, + [EntityType.COMPANY_TEAMS]: null, }; const dateField = dateFieldMapping[entity]; @@ -529,6 +534,7 @@ export function getEntityDisplayName(entity: EntityType): string { [EntityType.PROJECT_PHASES]: 'Project Phases', [EntityType.COMPANY_CATEGORIES]: 'Company Categories', [EntityType.COMPANY_TYPES]: 'Company Types', + [EntityType.COMPANY_TEAMS]: 'Company Teams', }; return mapping[entity] || entity; diff --git a/middleware.ts b/middleware.ts index 625b772..c83efd1 100644 --- a/middleware.ts +++ b/middleware.ts @@ -29,6 +29,7 @@ const publicRoutes = [ "/api/datto-rmm/sync", "/api/itglue/sync", "/api/veeam/sync", + "/api/veeam/rpo-check", "/api/sentinelone/sync", "/api/engagement/sync", "/api/zoom/sync", diff --git a/migrations/065_create_company_teams_table.sql b/migrations/065_create_company_teams_table.sql new file mode 100644 index 0000000..d07e60d --- /dev/null +++ b/migrations/065_create_company_teams_table.sql @@ -0,0 +1,42 @@ +-- Migration 065: Create company_teams table and add primary/billing contact FKs to companies + +-- Company Teams: maps resources (TAMs, CSMs, etc.) to companies +CREATE TABLE IF NOT EXISTS company_teams ( + id BIGINT PRIMARY KEY, + company_id BIGINT REFERENCES companies(id) ON DELETE CASCADE, + resource_id BIGINT, + is_associated_as_comanaged BOOLEAN DEFAULT false, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + synced_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + is_deleted BOOLEAN DEFAULT false, + deleted_at TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_company_teams_company_id ON company_teams(company_id); +CREATE INDEX IF NOT EXISTS idx_company_teams_resource_id ON company_teams(resource_id); + +-- Add primary and billing contact FK columns to companies +ALTER TABLE companies + ADD COLUMN IF NOT EXISTS primary_contact_id BIGINT REFERENCES contacts(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS billing_contact_id BIGINT REFERENCES contacts(id) ON DELETE SET NULL; + +CREATE INDEX IF NOT EXISTS idx_companies_primary_contact ON companies(primary_contact_id) WHERE primary_contact_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_companies_billing_contact ON companies(billing_contact_id) WHERE billing_contact_id IS NOT NULL; + +-- Backfill from existing contacts data +UPDATE companies c +SET primary_contact_id = ( + SELECT id FROM contacts + WHERE company_id = c.id AND primary_contact = true AND is_deleted = false + ORDER BY id LIMIT 1 +) +WHERE primary_contact_id IS NULL; + +UPDATE companies c +SET billing_contact_id = ( + SELECT id FROM contacts + WHERE company_id = c.id AND billing_contact = true AND is_deleted = false + ORDER BY id LIMIT 1 +) +WHERE billing_contact_id IS NULL; diff --git a/migrations/066_create_veeam_rpo_offline_log.sql b/migrations/066_create_veeam_rpo_offline_log.sql new file mode 100644 index 0000000..c9e7f56 --- /dev/null +++ b/migrations/066_create_veeam_rpo_offline_log.sql @@ -0,0 +1,21 @@ +-- Tracks every RPO check cycle where a workstation job was suppressed +-- because the device was offline (last_seen older than its backup interval). +-- Used for auditing, reporting, and the Claude Code skill. + +CREATE TABLE IF NOT EXISTS veeam_rpo_offline_log ( + id SERIAL PRIMARY KEY, + job_instance_uid VARCHAR(255) NOT NULL, + job_name TEXT NOT NULL, + org_name TEXT NOT NULL, + rmm_hostname TEXT NOT NULL, + rmm_site_name TEXT NOT NULL, + device_type_category TEXT NOT NULL, + rmm_last_seen TIMESTAMPTZ, + hours_offline NUMERIC(10,2) NOT NULL, + backup_interval_hours INTEGER NOT NULL, + checked_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_offline_log_job ON veeam_rpo_offline_log(job_instance_uid); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_offline_log_checked_at ON veeam_rpo_offline_log(checked_at); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_offline_log_hostname ON veeam_rpo_offline_log(rmm_hostname); diff --git a/migrations/067_create_veeam_rpo_comparison_tables.sql b/migrations/067_create_veeam_rpo_comparison_tables.sql new file mode 100644 index 0000000..1a0a123 --- /dev/null +++ b/migrations/067_create_veeam_rpo_comparison_tables.sql @@ -0,0 +1,73 @@ +-- Veeam RPO shadow-mode tables for the parallel validation period. +-- The RPO service runs without creating real Autotask tickets; all actions +-- are logged here so they can be compared against Datto RMM ticket output. + +-- Shadow log: one row per check cycle per job for every action the live service +-- *would* have taken (would_create, would_escalate, would_resolve, would_skip_too_old). +-- Offline suppressions continue to go to veeam_rpo_offline_log (migration 066). +CREATE TABLE IF NOT EXISTS veeam_rpo_shadow_log ( + id SERIAL PRIMARY KEY, + job_instance_uid VARCHAR(255) NOT NULL, + job_name TEXT NOT NULL, + org_name TEXT NOT NULL, + rmm_hostname TEXT, + rmm_site_name TEXT, + action TEXT NOT NULL, + -- 'would_create' | 'would_escalate' | 'would_resolve' | 'would_skip_too_old' + priority_level TEXT, -- medium | high | critical (null for would_resolve) + hours_overdue INTEGER, -- null for would_resolve + failure_category TEXT, + checked_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_shadow_log_job ON veeam_rpo_shadow_log(job_instance_uid); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_shadow_log_checked ON veeam_rpo_shadow_log(checked_at); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_shadow_log_hostname ON veeam_rpo_shadow_log(rmm_hostname); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_shadow_log_action ON veeam_rpo_shadow_log(action); + +-- Shadow ticket state: mirrors veeam_rpo_tickets but without Autotask ticket IDs. +-- Tracks open/resolved state so the shadow service can distinguish would_create +-- from would_escalate across check cycles. +CREATE TABLE IF NOT EXISTS veeam_rpo_shadow_tickets ( + job_instance_uid VARCHAR(255) PRIMARY KEY, + job_name TEXT NOT NULL, + org_name TEXT NOT NULL, + rmm_hostname TEXT, + priority_level TEXT NOT NULL, + hours_overdue INTEGER NOT NULL, + failure_category TEXT, + opened_at TIMESTAMPTZ DEFAULT NOW(), + resolved_at TIMESTAMPTZ, + last_checked_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_shadow_tickets_open ON veeam_rpo_shadow_tickets(resolved_at) WHERE resolved_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_shadow_tickets_hostname ON veeam_rpo_shadow_tickets(rmm_hostname); + +-- Datto RMM backup tickets captured from Autotask webhooks. +-- Populated whenever a ticket.created event arrives that matches the Datto +-- backup monitor signature (issue_type=38, Veeam-related title, not [Veeam RPO]). +-- Updated on ticket.updated events to track resolution. +CREATE TABLE IF NOT EXISTS veeam_rpo_datto_tickets ( + id SERIAL PRIMARY KEY, + at_ticket_id BIGINT UNIQUE NOT NULL, + at_ticket_number TEXT, + title TEXT NOT NULL, + company_id BIGINT, + company_name TEXT, + extracted_hostname TEXT, -- parsed from title (e.g. "DT061" from "DT061 - Veeam...") + issue_type INTEGER, + sub_issue_type INTEGER, + status INTEGER, + priority INTEGER, + ticket_created_at TIMESTAMPTZ, + ticket_resolved_at TIMESTAMPTZ, + captured_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_datto_tickets_company ON veeam_rpo_datto_tickets(company_id); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_datto_tickets_hostname ON veeam_rpo_datto_tickets(extracted_hostname); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_datto_tickets_created ON veeam_rpo_datto_tickets(ticket_created_at); +CREATE INDEX IF NOT EXISTS idx_veeam_rpo_datto_tickets_resolved ON veeam_rpo_datto_tickets(ticket_resolved_at) WHERE ticket_resolved_at IS NULL; diff --git a/migrations/068_create_veeam_ticket_analysis.sql b/migrations/068_create_veeam_ticket_analysis.sql new file mode 100644 index 0000000..8c31dd8 --- /dev/null +++ b/migrations/068_create_veeam_ticket_analysis.sql @@ -0,0 +1,31 @@ +CREATE TABLE IF NOT EXISTS veeam_ticket_analysis ( + id SERIAL PRIMARY KEY, + ticket_number TEXT NOT NULL UNIQUE, + ticket_id BIGINT, + company_id BIGINT, + company_name TEXT, + device_hostname TEXT, + ticket_created_at TIMESTAMPTZ, + ticket_closed_at TIMESTAMPTZ, + same_day_close BOOLEAN NOT NULL DEFAULT false, + hours_worked NUMERIC(10,2) NOT NULL DEFAULT 0, + note_count INTEGER NOT NULL DEFAULT 0, + -- LLM classification + problem_category TEXT NOT NULL, + resolution_type TEXT NOT NULL, + skills_required TEXT[] NOT NULL DEFAULT '{}', + complexity TEXT NOT NULL, + device_was_offline BOOLEAN, + backup_completed_before_tech BOOLEAN, + preventable BOOLEAN, + work_summary TEXT, + recommended_procedure TEXT, + -- meta + model TEXT, + analyzed_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_vta_ticket_number ON veeam_ticket_analysis(ticket_number); +CREATE INDEX IF NOT EXISTS idx_vta_category ON veeam_ticket_analysis(problem_category); +CREATE INDEX IF NOT EXISTS idx_vta_company ON veeam_ticket_analysis(company_id); +CREATE INDEX IF NOT EXISTS idx_vta_created ON veeam_ticket_analysis(ticket_created_at); diff --git a/package-lock.json b/package-lock.json index 310df5c..41f2525 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "pulse", "version": "0.1.0", "dependencies": { + "@anthropic-ai/sdk": "^0.91.1", "@better-auth/cli": "^1.4.10", "@hookform/resolvers": "^5.2.2", "@radix-ui/react-accordion": "^1.2.12", @@ -79,6 +80,26 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@aws-crypto/sha256-browser": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", @@ -1269,6 +1290,15 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.27.2", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", @@ -8974,6 +9004,19 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -11544,6 +11587,12 @@ "node": ">=8.0" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", diff --git a/package.json b/package.json index ff28730..f11afa5 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "lint": "eslint" }, "dependencies": { + "@anthropic-ai/sdk": "^0.91.1", "@better-auth/cli": "^1.4.10", "@hookform/resolvers": "^5.2.2", "@radix-ui/react-accordion": "^1.2.12", diff --git a/scripts/deactivate-cis-for-inactive-companies.ts b/scripts/deactivate-cis-for-inactive-companies.ts new file mode 100644 index 0000000..4bf622a --- /dev/null +++ b/scripts/deactivate-cis-for-inactive-companies.ts @@ -0,0 +1,253 @@ +/** + * deactivate-cis-for-inactive-companies.ts + * + * Finds all active Configuration Items whose parent company is inactive in + * Autotask and deactivates them (sets isActive = false). + * + * Usage: + * npx tsx scripts/deactivate-cis-for-inactive-companies.ts [options] + * + * Options: + * --dry-run Preview changes without writing (default) + * --commit Apply changes + * --concurrency Parallel PATCH calls (default: 5) + */ + +import { config } from 'dotenv'; +import { resolve } from 'path'; + +config({ path: resolve(__dirname, '../.env') }); + +// ─── Config ─────────────────────────────────────────────────────────────────── + +const API_BASE = process.env.AUTOTASK_API_URL!; +const USERNAME = process.env.AUTOTASK_USERNAME!; +const SECRET = process.env.AUTOTASK_SECRET!; +const INT_CODE = process.env.AUTOTASK_API_INTEGRATION_CODE!; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function authHeaders(): Record { + return { + 'Username': USERNAME, + 'Secret': SECRET, + 'APIIntegrationcode': INT_CODE, + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }; +} + +/** Paginate through all results of a POST /query endpoint */ +async function queryAll(entity: string, filter: object[]): Promise { + const all: T[] = []; + let nextUrl: string | null = null; + const requestBody = JSON.stringify({ MaxRecords: 500, filter }); + + while (true) { + const url = nextUrl ?? `${API_BASE}/${entity}/query`; + const res = await fetch(url, { + method: 'POST', + headers: authHeaders(), + body: requestBody, + }); + if (!res.ok) throw new Error(`POST ${entity}/query → ${res.status}: ${await res.text()}`); + const data: any = await res.json(); + all.push(...(data.items ?? [])); + nextUrl = data.pageDetails?.nextPageUrl ?? null; + if (!nextUrl) break; + } + + return all; +} + +async function patchCI(id: number, patch: object, retries = 5): Promise { + for (let attempt = 0; attempt <= retries; attempt++) { + const res = await fetch(`${API_BASE}/ConfigurationItems`, { + method: 'PATCH', + headers: authHeaders(), + body: JSON.stringify({ id, ...patch }), + }); + if (res.ok) return; + if (res.status === 429 && attempt < retries) { + const delay = 2000 * (attempt + 1); // 2s, 4s, 6s, 8s, 10s + await new Promise(r => setTimeout(r, delay)); + continue; + } + throw new Error(`PATCH ConfigurationItems/${id} → ${res.status}: ${await res.text()}`); + } +} + +/** Run tasks with a concurrency cap, returns results in original order */ +async function pLimit( + tasks: (() => Promise)[], + concurrency: number +): Promise<{ result?: T; error?: Error }[]> { + const results: { result?: T; error?: Error }[] = new Array(tasks.length); + let next = 0; + async function worker() { + while (next < tasks.length) { + const i = next++; + try { results[i] = { result: await tasks[i]() }; } + catch (e) { results[i] = { error: e instanceof Error ? e : new Error(String(e)) }; } + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker)); + return results; +} + +// ─── Arg parsing ────────────────────────────────────────────────────────────── + +function parseArgs() { + const args = process.argv.slice(2); + let dryRun = true; + let concurrency = 3; // Autotask hard limit is 3 concurrent API threads + for (let i = 0; i < args.length; i++) { + if (args[i] === '--commit') dryRun = false; + if (args[i] === '--dry-run') dryRun = true; + if (args[i] === '--concurrency' && args[i+1]) concurrency = parseInt(args[++i], 10); + } + return { dryRun, concurrency }; +} + +// ─── Types ──────────────────────────────────────────────────────────────────── + +interface Company { + id: number; + companyName: string; + isActive: boolean; +} + +interface CIItem { + id: number; + companyID: number; + referenceTitle: string; + configurationItemType: number; + configurationItemCategoryID: number | null; + isActive: boolean; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + const { dryRun, concurrency } = parseArgs(); + + console.log(''); + console.log('╔══════════════════════════════════════════════════════════════╗'); + console.log('║ Deactivate CIs for Inactive Companies ║'); + console.log('╚══════════════════════════════════════════════════════════════╝'); + console.log(` Mode : ${dryRun ? '🔍 DRY RUN (no changes written)' : '✏️ COMMIT (changes will be applied)'}`); + console.log(''); + + // ── 1. Fetch all inactive companies ───────────────────────────────────────── + process.stdout.write('Fetching inactive companies … '); + const inactiveCompanies = await queryAll('Companies', [ + { field: 'isActive', op: 'eq', value: false }, + ]); + console.log(`found ${inactiveCompanies.length}.`); + + if (inactiveCompanies.length === 0) { + console.log('No inactive companies found.'); + return; + } + + const companyMap = new Map( + inactiveCompanies.map(c => [c.id, c.companyName]) + ); + + // ── 2. Fetch active CIs for each inactive company ──────────────────────────── + console.log('Querying active CIs for each inactive company …'); + + const cisByCompany = new Map(); + let totalFound = 0; + + for (const company of inactiveCompanies) { + const cis = await queryAll('ConfigurationItems', [ + { field: 'companyID', op: 'eq', value: company.id }, + { field: 'isActive', op: 'eq', value: true }, + ]); + if (cis.length > 0) { + cisByCompany.set(company.id, cis); + totalFound += cis.length; + process.stdout.write(` [${company.id}] ${company.companyName}: ${cis.length} active CI(s)\n`); + } + } + + // ── 3. Report ──────────────────────────────────────────────────────────────── + console.log(''); + console.log('─── Summary ──────────────────────────────────────────────────'); + console.log(` Inactive companies checked : ${inactiveCompanies.length}`); + console.log(` Companies with active CIs : ${cisByCompany.size}`); + console.log(` Total active CIs to deactivate : ${totalFound}`); + console.log(''); + + if (totalFound === 0) { + console.log('✓ Nothing to do — no active CIs found for inactive companies.'); + return; + } + + console.log('─── Breakdown ────────────────────────────────────────────────'); + const sorted = [...cisByCompany.entries()].sort((a, b) => + (companyMap.get(a[0]) ?? '').localeCompare(companyMap.get(b[0]) ?? '') + ); + for (const [companyId, cis] of sorted) { + const name = companyMap.get(companyId) ?? `Company ${companyId}`; + console.log(`\n ${name} (${cis.length} CI${cis.length !== 1 ? 's' : ''})`); + for (const ci of cis) { + console.log(` [${ci.id}] ${(ci.referenceTitle ?? '(no title)').substring(0, 60)}`); + } + } + console.log(''); + + if (dryRun) { + console.log('─── Dry run complete ─────────────────────────────────────────'); + console.log(` ${totalFound} CI(s) across ${cisByCompany.size} company/companies would be deactivated.`); + console.log(' Run with --commit to apply.'); + return; + } + + // ── 4. Deactivate ──────────────────────────────────────────────────────────── + const allCIs = [...cisByCompany.values()].flat(); + console.log(`─── Deactivating ${allCIs.length} CIs (concurrency=${concurrency}) ──────`); + + const tasks = allCIs.map(ci => () => patchCI(ci.id, { isActive: false })); + const results = await pLimit(tasks, concurrency); + + let succeeded = 0; + let failed = 0; + const errors: { ci: CIItem; error: string }[] = []; + + for (let i = 0; i < results.length; i++) { + if (results[i].error) { + failed++; + errors.push({ ci: allCIs[i], error: results[i].error!.message }); + process.stdout.write('✗'); + } else { + succeeded++; + process.stdout.write('.'); + } + if ((i + 1) % 80 === 0) process.stdout.write('\n'); + } + console.log('\n'); + + // ── 5. Final report ─────────────────────────────────────────────────────────── + console.log('─── Results ──────────────────────────────────────────────────'); + console.log(` ✓ Deactivated : ${succeeded}`); + console.log(` ✗ Failed : ${failed}`); + + if (errors.length > 0) { + console.log('\n Failures:'); + for (const { ci, error } of errors) { + const name = companyMap.get(ci.companyID) ?? `Company ${ci.companyID}`; + console.log(` [${ci.id}] ${ci.referenceTitle ?? '(no title)'} (${name})`); + console.log(` ${error}`); + } + process.exit(1); + } + + console.log('\n✓ All done.'); +} + +main().catch(err => { + console.error('\nFatal error:', err.message); + process.exit(1); +}); diff --git a/scripts/update-workstation-desktop-category.ts b/scripts/update-workstation-desktop-category.ts new file mode 100644 index 0000000..23bac81 --- /dev/null +++ b/scripts/update-workstation-desktop-category.ts @@ -0,0 +1,365 @@ +/** + * update-workstation-desktop-category.ts + * + * Finds all active Configuration Items with type "Workstation – Desktop" (typeId=40) + * and sets their category to "Workstations". + * + * Usage: + * npx tsx scripts/update-workstation-desktop-category.ts [options] + * + * Options: + * --dry-run Preview changes without writing to Autotask (default: true) + * --commit Actually apply changes (disables dry-run) + * --company Filter to a single company (partial name match or numeric ID) + * --concurrency Parallel API update calls (default: 5) + */ + +import { config } from 'dotenv'; +import { resolve } from 'path'; + +config({ path: resolve(__dirname, '../.env') }); + +// ─── Constants ─────────────────────────────────────────────────────────────── + +const API_BASE = process.env.AUTOTASK_API_URL!; // e.g. https://webservices1.autotask.net/atservicesrest/v1.0 +const USERNAME = process.env.AUTOTASK_USERNAME!; +const SECRET = process.env.AUTOTASK_SECRET!; +const INT_CODE = process.env.AUTOTASK_API_INTEGRATION_CODE!; + +const CI_TYPE_WORKSTATION_DESKTOP = 40; // "Workstation – Desktop" picklist value +const TARGET_CATEGORY_NAME = 'Workstations'; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +function authHeaders(): Record { + return { + 'Username': USERNAME, + 'Secret': SECRET, + 'APIIntegrationcode': INT_CODE, + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }; +} + +async function apiGet(path: string): Promise { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'GET', + headers: authHeaders(), + }); + if (!res.ok) throw new Error(`GET ${path} → ${res.status}: ${await res.text()}`); + return res.json(); +} + +async function apiPost(path: string, body: object): Promise { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'POST', + headers: authHeaders(), + body: JSON.stringify(body), + }); + if (!res.ok) throw new Error(`POST ${path} → ${res.status}: ${await res.text()}`); + return res.json(); +} + +async function apiPatch(path: string, body: object, retries = 5): Promise { + for (let attempt = 0; attempt <= retries; attempt++) { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'PATCH', + headers: authHeaders(), + body: JSON.stringify(body), + }); + if (res.ok) return; + if (res.status === 429 && attempt < retries) { + await new Promise(r => setTimeout(r, 2000 * (attempt + 1))); + continue; + } + throw new Error(`PATCH ${path} → ${res.status}: ${await res.text()}`); + } +} + +/** Paginate through all results of a POST /query endpoint */ +async function queryAll(entity: string, filter: object[]): Promise { + const all: T[] = []; + let nextUrl: string | null = null; + // Autotask requires a body on every POST page request — send same body for all pages + const requestBody = JSON.stringify({ MaxRecords: 500, filter }); + + while (true) { + const url = nextUrl ?? `${API_BASE}/${entity}/query`; + + const res = await fetch(url, { + method: 'POST', + headers: authHeaders(), + body: requestBody, + }); + if (!res.ok) throw new Error(`POST ${entity}/query → ${res.status}: ${await res.text()}`); + + const data: any = await res.json(); + all.push(...(data.items ?? [])); + + nextUrl = data.pageDetails?.nextPageUrl ?? null; + if (!nextUrl) break; + } + + return all; +} + +/** Run N promises with a concurrency cap */ +async function pLimit(tasks: (() => Promise)[], concurrency: number): Promise<{ result?: T; error?: Error; index: number }[]> { + const results: { result?: T; error?: Error; index: number }[] = new Array(tasks.length); + let next = 0; + + async function worker() { + while (next < tasks.length) { + const i = next++; + try { + results[i] = { result: await tasks[i](), index: i }; + } catch (e) { + results[i] = { error: e instanceof Error ? e : new Error(String(e)), index: i }; + } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker)); + return results; +} + +// ─── Arg parsing ───────────────────────────────────────────────────────────── + +function parseArgs() { + const args = process.argv.slice(2); + let dryRun = true; + let companyArg = ''; + let concurrency = 3; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--commit') dryRun = false; + if (args[i] === '--dry-run') dryRun = true; + if (args[i] === '--company' && args[i + 1]) { companyArg = args[++i]; } + if (args[i] === '--concurrency' && args[i+1]) { concurrency = parseInt(args[++i], 10); } + } + + return { dryRun, companyArg, concurrency }; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +interface CIItem { + id: number; + referenceTitle: string; + companyID: number; + configurationItemType: number; + configurationItemCategoryID: number | null; + isActive: boolean; +} + +interface Company { + id: number; + companyName: string; +} + +interface Category { + id: number; + name: string; + isActive: boolean; +} + +async function main() { + const { dryRun, companyArg, concurrency } = parseArgs(); + + console.log(''); + console.log('╔══════════════════════════════════════════════════════════════╗'); + console.log('║ Workstation-Desktop → Workstations Category Remediation ║'); + console.log('╚══════════════════════════════════════════════════════════════╝'); + console.log(` Mode : ${dryRun ? '🔍 DRY RUN (no changes written)' : '✏️ COMMIT (changes will be applied)'}`); + if (companyArg) console.log(` Client filter: "${companyArg}"`); + console.log(''); + + // ── 1. Resolve target category ID ───────────────────────────────────────── + process.stdout.write('Fetching ConfigurationItemCategories … '); + const categories = await queryAll('ConfigurationItemCategories', [ + { field: 'isActive', op: 'eq', value: true }, + ]); + const targetCategory = categories.find(c => + c.name.toLowerCase() === TARGET_CATEGORY_NAME.toLowerCase() + ); + if (!targetCategory) { + console.error(`\n✗ Category "${TARGET_CATEGORY_NAME}" not found in Autotask. Available:\n`); + categories.forEach(c => console.error(` [${c.id}] ${c.name}`)); + process.exit(1); + } + console.log(`found ${categories.length} categories.`); + console.log(` → Target category: [${targetCategory.id}] "${targetCategory.name}"`); + + // ── 2. Resolve company filter ────────────────────────────────────────────── + let companyIdFilter: number | null = null; + let companyName = ''; + + if (companyArg) { + process.stdout.write(`Resolving company "${companyArg}" … `); + const numericId = parseInt(companyArg, 10); + + if (!isNaN(numericId)) { + // Direct ID + const resp: any = await apiGet(`Companies/${numericId}`); + const co: Company = resp.item; + if (!co) { console.error(`\n✗ Company ID ${numericId} not found.`); process.exit(1); } + companyIdFilter = co.id; + companyName = co.companyName; + } else { + // Name search — fetch all, filter locally (Autotask doesn't support contains on companyName efficiently) + const companies = await queryAll('Companies', [ + { field: 'isActive', op: 'eq', value: true }, + ]); + const matches = companies.filter(c => + c.companyName.toLowerCase().includes(companyArg.toLowerCase()) + ); + if (matches.length === 0) { + console.error(`\n✗ No active company matching "${companyArg}".`); process.exit(1); + } + if (matches.length > 1) { + console.error(`\n✗ "${companyArg}" matched ${matches.length} companies — be more specific:`); + matches.forEach(c => console.error(` [${c.id}] ${c.companyName}`)); + process.exit(1); + } + companyIdFilter = matches[0].id; + companyName = matches[0].companyName; + } + console.log(`resolved → [${companyIdFilter}] ${companyName}`); + } + + // ── 3. Fetch matching CIs from Autotask ──────────────────────────────────── + process.stdout.write('Querying ConfigurationItems (type=Workstation-Desktop, isActive=true) … '); + + // Note: configurationItemType is not queryable via API filter — fetch by isActive + // (+ optional companyID) then filter by type client-side. + const ciFilter: object[] = [ + { field: 'isActive', op: 'eq', value: true }, + ]; + if (companyIdFilter !== null) { + ciFilter.push({ field: 'companyID', op: 'eq', value: companyIdFilter }); + } + + const rawCIs = await queryAll('ConfigurationItems', ciFilter); + const allCIs = rawCIs.filter(ci => ci.configurationItemType === CI_TYPE_WORKSTATION_DESKTOP); + console.log(`found ${rawCIs.length} active items, ${allCIs.length} are type "Workstation – Desktop".`); + + // ── 4. Identify items that actually need updating ────────────────────────── + const toUpdate = allCIs.filter(ci => ci.configurationItemCategoryID !== targetCategory.id); + const alreadyCorrect = allCIs.length - toUpdate.length; + + // ── 5. Build per-company summary ────────────────────────────────────────── + const byCompany = new Map(); + for (const ci of toUpdate) { + if (!byCompany.has(ci.companyID)) byCompany.set(ci.companyID, { companyID: ci.companyID, items: [] }); + byCompany.get(ci.companyID)!.items.push(ci); + } + + // Fetch company names for the affected companies + const companyNames = new Map(); + if (byCompany.size > 0) { + process.stdout.write(`Fetching company names for ${byCompany.size} affected companies … `); + const companyIds = [...byCompany.keys()]; + // Batch in groups of 50 (Autotask OR filter limit) + const chunkSize = 50; + for (let i = 0; i < companyIds.length; i += chunkSize) { + const chunk = companyIds.slice(i, i + chunkSize); + const filter = chunk.map(id => ({ field: 'id', op: 'eq', value: id })); + const cos = await queryAll('Companies', filter); + for (const co of cos) companyNames.set(co.id, co.companyName); + } + console.log('done.'); + } + + // ── 6. Report ────────────────────────────────────────────────────────────── + console.log(''); + console.log('─── Summary ──────────────────────────────────────────────────'); + console.log(` Total "Workstation – Desktop" CIs found : ${allCIs.length}`); + console.log(` Already categorised as "Workstations" : ${alreadyCorrect}`); + console.log(` Require update : ${toUpdate.length}`); + console.log(''); + + if (toUpdate.length === 0) { + console.log('✓ Nothing to do — all items already have the correct category.'); + return; + } + + console.log('─── Breakdown by client ──────────────────────────────────────'); + const sortedCompanies = [...byCompany.values()].sort((a, b) => + (companyNames.get(a.companyID) ?? '').localeCompare(companyNames.get(b.companyID) ?? '') + ); + for (const { companyID, items } of sortedCompanies) { + const name = companyNames.get(companyID) ?? `Company ${companyID}`; + console.log(` ${name.padEnd(45)} ${items.length} item(s)`); + for (const ci of items) { + const oldCat = ci.configurationItemCategoryID ?? 'none'; + console.log(` [${ci.id}] ${(ci.referenceTitle ?? '(no title)').substring(0, 60)} (cat: ${oldCat} → ${targetCategory.id})`); + } + } + console.log(''); + + if (dryRun) { + console.log('─── Dry run complete ─────────────────────────────────────────'); + console.log(` ${toUpdate.length} item(s) would be updated.`); + console.log(' Run with --commit to apply changes.'); + return; + } + + // ── 7. Apply updates ─────────────────────────────────────────────────────── + console.log(`─── Applying ${toUpdate.length} updates (concurrency=${concurrency}) ────────`); + + let succeeded = 0; + let failed = 0; + const errors: { id: number; title: string; error: string }[] = []; + + const tasks = toUpdate.map(ci => async () => { + await apiPatch('ConfigurationItems', { + id: ci.id, + configurationItemCategoryID: targetCategory.id, + }); + return ci.id; + }); + + const results = await pLimit(tasks, concurrency); + + for (let i = 0; i < results.length; i++) { + const ci = toUpdate[i]; + const r = results[i]; + if (r.error) { + failed++; + errors.push({ id: ci.id, title: ci.referenceTitle ?? '', error: r.error.message }); + process.stdout.write('✗'); + } else { + succeeded++; + process.stdout.write('.'); + } + if ((i + 1) % 80 === 0) process.stdout.write('\n'); + } + console.log('\n'); + + // ── 8. Final report ──────────────────────────────────────────────────────── + console.log('─── Results ──────────────────────────────────────────────────'); + console.log(` ✓ Updated successfully : ${succeeded}`); + console.log(` ✗ Failed : ${failed}`); + + if (errors.length > 0) { + console.log(''); + console.log(' Failures:'); + for (const e of errors) { + console.log(` [${e.id}] ${e.title}`); + console.log(` ${e.error}`); + } + } + + console.log(''); + if (failed === 0) { + console.log('✓ All done.'); + } else { + console.log('⚠ Completed with errors — review failures above.'); + process.exit(1); + } +} + +main().catch(err => { + console.error('\nFatal error:', err.message); + process.exit(1); +}); diff --git a/scripts/update-workstation-laptop-category.ts b/scripts/update-workstation-laptop-category.ts new file mode 100644 index 0000000..ea9b36b --- /dev/null +++ b/scripts/update-workstation-laptop-category.ts @@ -0,0 +1,365 @@ +/** + * update-workstation-laptop-category.ts + * + * Finds all active Configuration Items with type "Workstation – Laptop" (typeId=40) + * and sets their category to "Workstations". + * + * Usage: + * npx tsx scripts/update-workstation-laptop-category.ts [options] + * + * Options: + * --dry-run Preview changes without writing to Autotask (default: true) + * --commit Actually apply changes (disables dry-run) + * --company Filter to a single company (partial name match or numeric ID) + * --concurrency Parallel API update calls (default: 5) + */ + +import { config } from 'dotenv'; +import { resolve } from 'path'; + +config({ path: resolve(__dirname, '../.env') }); + +// ─── Constants ─────────────────────────────────────────────────────────────── + +const API_BASE = process.env.AUTOTASK_API_URL!; // e.g. https://webservices1.autotask.net/atservicesrest/v1.0 +const USERNAME = process.env.AUTOTASK_USERNAME!; +const SECRET = process.env.AUTOTASK_SECRET!; +const INT_CODE = process.env.AUTOTASK_API_INTEGRATION_CODE!; + +const CI_TYPE_WORKSTATION_LAPTOP = 39; // "Workstation – Laptop" picklist value +const TARGET_CATEGORY_NAME = 'Workstations'; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +function authHeaders(): Record { + return { + 'Username': USERNAME, + 'Secret': SECRET, + 'APIIntegrationcode': INT_CODE, + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }; +} + +async function apiGet(path: string): Promise { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'GET', + headers: authHeaders(), + }); + if (!res.ok) throw new Error(`GET ${path} → ${res.status}: ${await res.text()}`); + return res.json(); +} + +async function apiPost(path: string, body: object): Promise { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'POST', + headers: authHeaders(), + body: JSON.stringify(body), + }); + if (!res.ok) throw new Error(`POST ${path} → ${res.status}: ${await res.text()}`); + return res.json(); +} + +async function apiPatch(path: string, body: object, retries = 5): Promise { + for (let attempt = 0; attempt <= retries; attempt++) { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'PATCH', + headers: authHeaders(), + body: JSON.stringify(body), + }); + if (res.ok) return; + if (res.status === 429 && attempt < retries) { + await new Promise(r => setTimeout(r, 2000 * (attempt + 1))); + continue; + } + throw new Error(`PATCH ${path} → ${res.status}: ${await res.text()}`); + } +} + +/** Paginate through all results of a POST /query endpoint */ +async function queryAll(entity: string, filter: object[]): Promise { + const all: T[] = []; + let nextUrl: string | null = null; + // Autotask requires a body on every POST page request — send same body for all pages + const requestBody = JSON.stringify({ MaxRecords: 500, filter }); + + while (true) { + const url = nextUrl ?? `${API_BASE}/${entity}/query`; + + const res = await fetch(url, { + method: 'POST', + headers: authHeaders(), + body: requestBody, + }); + if (!res.ok) throw new Error(`POST ${entity}/query → ${res.status}: ${await res.text()}`); + + const data: any = await res.json(); + all.push(...(data.items ?? [])); + + nextUrl = data.pageDetails?.nextPageUrl ?? null; + if (!nextUrl) break; + } + + return all; +} + +/** Run N promises with a concurrency cap */ +async function pLimit(tasks: (() => Promise)[], concurrency: number): Promise<{ result?: T; error?: Error; index: number }[]> { + const results: { result?: T; error?: Error; index: number }[] = new Array(tasks.length); + let next = 0; + + async function worker() { + while (next < tasks.length) { + const i = next++; + try { + results[i] = { result: await tasks[i](), index: i }; + } catch (e) { + results[i] = { error: e instanceof Error ? e : new Error(String(e)), index: i }; + } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker)); + return results; +} + +// ─── Arg parsing ───────────────────────────────────────────────────────────── + +function parseArgs() { + const args = process.argv.slice(2); + let dryRun = true; + let companyArg = ''; + let concurrency = 3; // Autotask hard limit is 3 concurrent API threads + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--commit') dryRun = false; + if (args[i] === '--dry-run') dryRun = true; + if (args[i] === '--company' && args[i + 1]) { companyArg = args[++i]; } + if (args[i] === '--concurrency' && args[i+1]) { concurrency = parseInt(args[++i], 10); } + } + + return { dryRun, companyArg, concurrency }; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +interface CIItem { + id: number; + referenceTitle: string; + companyID: number; + configurationItemType: number; + configurationItemCategoryID: number | null; + isActive: boolean; +} + +interface Company { + id: number; + companyName: string; +} + +interface Category { + id: number; + name: string; + isActive: boolean; +} + +async function main() { + const { dryRun, companyArg, concurrency } = parseArgs(); + + console.log(''); + console.log('╔══════════════════════════════════════════════════════════════╗'); + console.log('║ Workstation-Laptop → Workstations Category Remediation ║'); + console.log('╚══════════════════════════════════════════════════════════════╝'); + console.log(` Mode : ${dryRun ? '🔍 DRY RUN (no changes written)' : '✏️ COMMIT (changes will be applied)'}`); + if (companyArg) console.log(` Client filter: "${companyArg}"`); + console.log(''); + + // ── 1. Resolve target category ID ───────────────────────────────────────── + process.stdout.write('Fetching ConfigurationItemCategories … '); + const categories = await queryAll('ConfigurationItemCategories', [ + { field: 'isActive', op: 'eq', value: true }, + ]); + const targetCategory = categories.find(c => + c.name.toLowerCase() === TARGET_CATEGORY_NAME.toLowerCase() + ); + if (!targetCategory) { + console.error(`\n✗ Category "${TARGET_CATEGORY_NAME}" not found in Autotask. Available:\n`); + categories.forEach(c => console.error(` [${c.id}] ${c.name}`)); + process.exit(1); + } + console.log(`found ${categories.length} categories.`); + console.log(` → Target category: [${targetCategory.id}] "${targetCategory.name}"`); + + // ── 2. Resolve company filter ────────────────────────────────────────────── + let companyIdFilter: number | null = null; + let companyName = ''; + + if (companyArg) { + process.stdout.write(`Resolving company "${companyArg}" … `); + const numericId = parseInt(companyArg, 10); + + if (!isNaN(numericId)) { + // Direct ID + const resp: any = await apiGet(`Companies/${numericId}`); + const co: Company = resp.item; + if (!co) { console.error(`\n✗ Company ID ${numericId} not found.`); process.exit(1); } + companyIdFilter = co.id; + companyName = co.companyName; + } else { + // Name search — fetch all, filter locally (Autotask doesn't support contains on companyName efficiently) + const companies = await queryAll('Companies', [ + { field: 'isActive', op: 'eq', value: true }, + ]); + const matches = companies.filter(c => + c.companyName.toLowerCase().includes(companyArg.toLowerCase()) + ); + if (matches.length === 0) { + console.error(`\n✗ No active company matching "${companyArg}".`); process.exit(1); + } + if (matches.length > 1) { + console.error(`\n✗ "${companyArg}" matched ${matches.length} companies — be more specific:`); + matches.forEach(c => console.error(` [${c.id}] ${c.companyName}`)); + process.exit(1); + } + companyIdFilter = matches[0].id; + companyName = matches[0].companyName; + } + console.log(`resolved → [${companyIdFilter}] ${companyName}`); + } + + // ── 3. Fetch matching CIs from Autotask ──────────────────────────────────── + process.stdout.write('Querying ConfigurationItems (type=Workstation-Laptop, isActive=true) … '); + + // Note: configurationItemType is not queryable via API filter — fetch by isActive + // (+ optional companyID) then filter by type client-side. + const ciFilter: object[] = [ + { field: 'isActive', op: 'eq', value: true }, + ]; + if (companyIdFilter !== null) { + ciFilter.push({ field: 'companyID', op: 'eq', value: companyIdFilter }); + } + + const rawCIs = await queryAll('ConfigurationItems', ciFilter); + const allCIs = rawCIs.filter(ci => ci.configurationItemType === CI_TYPE_WORKSTATION_LAPTOP); + console.log(`found ${rawCIs.length} active items, ${allCIs.length} are type "Workstation – Laptop".`); + + // ── 4. Identify items that actually need updating ────────────────────────── + const toUpdate = allCIs.filter(ci => ci.configurationItemCategoryID !== targetCategory.id); + const alreadyCorrect = allCIs.length - toUpdate.length; + + // ── 5. Build per-company summary ────────────────────────────────────────── + const byCompany = new Map(); + for (const ci of toUpdate) { + if (!byCompany.has(ci.companyID)) byCompany.set(ci.companyID, { companyID: ci.companyID, items: [] }); + byCompany.get(ci.companyID)!.items.push(ci); + } + + // Fetch company names for the affected companies + const companyNames = new Map(); + if (byCompany.size > 0) { + process.stdout.write(`Fetching company names for ${byCompany.size} affected companies … `); + const companyIds = [...byCompany.keys()]; + // Batch in groups of 50 (Autotask OR filter limit) + const chunkSize = 50; + for (let i = 0; i < companyIds.length; i += chunkSize) { + const chunk = companyIds.slice(i, i + chunkSize); + const filter = chunk.map(id => ({ field: 'id', op: 'eq', value: id })); + const cos = await queryAll('Companies', filter); + for (const co of cos) companyNames.set(co.id, co.companyName); + } + console.log('done.'); + } + + // ── 6. Report ────────────────────────────────────────────────────────────── + console.log(''); + console.log('─── Summary ──────────────────────────────────────────────────'); + console.log(` Total "Workstation – Laptop" CIs found : ${allCIs.length}`); + console.log(` Already categorised as "Workstations" : ${alreadyCorrect}`); + console.log(` Require update : ${toUpdate.length}`); + console.log(''); + + if (toUpdate.length === 0) { + console.log('✓ Nothing to do — all items already have the correct category.'); + return; + } + + console.log('─── Breakdown by client ──────────────────────────────────────'); + const sortedCompanies = [...byCompany.values()].sort((a, b) => + (companyNames.get(a.companyID) ?? '').localeCompare(companyNames.get(b.companyID) ?? '') + ); + for (const { companyID, items } of sortedCompanies) { + const name = companyNames.get(companyID) ?? `Company ${companyID}`; + console.log(` ${name.padEnd(45)} ${items.length} item(s)`); + for (const ci of items) { + const oldCat = ci.configurationItemCategoryID ?? 'none'; + console.log(` [${ci.id}] ${(ci.referenceTitle ?? '(no title)').substring(0, 60)} (cat: ${oldCat} → ${targetCategory.id})`); + } + } + console.log(''); + + if (dryRun) { + console.log('─── Dry run complete ─────────────────────────────────────────'); + console.log(` ${toUpdate.length} item(s) would be updated.`); + console.log(' Run with --commit to apply changes.'); + return; + } + + // ── 7. Apply updates ─────────────────────────────────────────────────────── + console.log(`─── Applying ${toUpdate.length} updates (concurrency=${concurrency}) ────────`); + + let succeeded = 0; + let failed = 0; + const errors: { id: number; title: string; error: string }[] = []; + + const tasks = toUpdate.map(ci => async () => { + await apiPatch('ConfigurationItems', { + id: ci.id, + configurationItemCategoryID: targetCategory.id, + }); + return ci.id; + }); + + const results = await pLimit(tasks, concurrency); + + for (let i = 0; i < results.length; i++) { + const ci = toUpdate[i]; + const r = results[i]; + if (r.error) { + failed++; + errors.push({ id: ci.id, title: ci.referenceTitle ?? '', error: r.error.message }); + process.stdout.write('✗'); + } else { + succeeded++; + process.stdout.write('.'); + } + if ((i + 1) % 80 === 0) process.stdout.write('\n'); + } + console.log('\n'); + + // ── 8. Final report ──────────────────────────────────────────────────────── + console.log('─── Results ──────────────────────────────────────────────────'); + console.log(` ✓ Updated successfully : ${succeeded}`); + console.log(` ✗ Failed : ${failed}`); + + if (errors.length > 0) { + console.log(''); + console.log(' Failures:'); + for (const e of errors) { + console.log(` [${e.id}] ${e.title}`); + console.log(` ${e.error}`); + } + } + + console.log(''); + if (failed === 0) { + console.log('✓ All done.'); + } else { + console.log('⚠ Completed with errors — review failures above.'); + process.exit(1); + } +} + +main().catch(err => { + console.error('\nFatal error:', err.message); + process.exit(1); +});