From 07067bef1940bd1eb8762028b9bd1fdc46567d6e Mon Sep 17 00:00:00 2001 From: lorentz Date: Mon, 6 Apr 2026 09:03:19 -0400 Subject: [PATCH] feat: Display Settings UI + Company Category/Type sync - Add /admin/display-settings page with Kiosk and Mobile sections - Company category checkbox filter + excluded companies searchable multi-select - New DB tables: company_categories, company_types (migration 064) - Sync COMPANY_CATEGORIES via CompanyCategories entity (id/name/isActive) - Sync COMPANY_TYPES via Companies.companyType picklist - Add to EntityType, ENTITY_DEPENDENCIES, sync-helpers, entity-mapper, entity-sync - New API routes: /api/admin/display-settings (GET/POST), /api/data/company-categories, /api/data/companies-list - Update all 4 routes (kiosk/stats, kiosk/activity, mobile/tickets, mobile/dashboard) to filter by kiosk_settings company_category_ids + excluded_company_ids - Add Display Settings nav link (SlidersHorizontal icon) to Admin menu - Seed kiosk_settings: kiosk_company_category_ids=1, mobile_company_category_ids=1 --- app/admin/display-settings/page.tsx | 323 ++++++++++++++++++ app/api/admin/display-settings/route.ts | 65 ++++ app/api/data/companies-list/route.ts | 14 + app/api/data/company-categories/route.ts | 14 + app/api/kiosk/activity/route.ts | 55 ++- app/api/kiosk/stats/route.ts | 55 ++- app/api/mobile/dashboard/route.ts | 29 +- app/api/mobile/tickets/route.ts | 33 +- components/navigation/app-navigation.tsx | 7 + docs/AUTOTASK_SYNC.md | 156 +++++++++ lib/services/entity-sync.ts | 88 +++++ lib/types/sync.ts | 4 + lib/utils/entity-mapper.ts | 2 + lib/utils/sync-helpers.ts | 11 + ...x_tickets_company_resource_constraints.sql | 45 +++ migrations/064_create_company_picklists.sql | 31 ++ 16 files changed, 847 insertions(+), 85 deletions(-) create mode 100644 app/admin/display-settings/page.tsx create mode 100644 app/api/admin/display-settings/route.ts create mode 100644 app/api/data/companies-list/route.ts create mode 100644 app/api/data/company-categories/route.ts create mode 100644 docs/AUTOTASK_SYNC.md create mode 100644 migrations/063_relax_tickets_company_resource_constraints.sql create mode 100644 migrations/064_create_company_picklists.sql diff --git a/app/admin/display-settings/page.tsx b/app/admin/display-settings/page.tsx new file mode 100644 index 0000000..97c6789 --- /dev/null +++ b/app/admin/display-settings/page.tsx @@ -0,0 +1,323 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; +import { Loader2, Save, SlidersHorizontal, X } from "lucide-react"; +import { toast } from "sonner"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Label } from "@/components/ui/label"; + +interface CompanyCategory { + value: number; + label: string; + is_active: boolean; +} + +interface Company { + id: number; + company_name: string; +} + +interface SectionSettings { + company_category_ids: number[]; + excluded_company_ids: number[]; +} + +interface DisplaySettings { + kiosk: SectionSettings; + mobile: SectionSettings; +} + +function CompanySearch({ + selectedIds, + categoryIds, + onAdd, + onRemove, + allCompanies, +}: { + selectedIds: number[]; + categoryIds: number[]; + onAdd: (id: number) => void; + onRemove: (id: number) => void; + allCompanies: Company[]; +}) { + const [query, setQuery] = useState(""); + const [open, setOpen] = useState(false); + + const available = allCompanies.filter( + (c) => + !selectedIds.includes(c.id) && + (query === "" || c.company_name.toLowerCase().includes(query.toLowerCase())) + ); + + const selected = allCompanies.filter((c) => selectedIds.includes(c.id)); + + return ( +
+
+ {selected.map((c) => ( + + {c.company_name} + + + ))} + {selected.length === 0 && ( + No exclusions + )} +
+
+ { setQuery(e.target.value); setOpen(true); }} + onFocus={() => setOpen(true)} + onBlur={() => setTimeout(() => setOpen(false), 150)} + className="w-full rounded-md border border-input bg-background px-3 py-1.5 text-sm shadow-sm placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring" + /> + {open && available.length > 0 && ( +
+ {available.slice(0, 30).map((c) => ( + + ))} +
+ )} +
+
+ ); +} + +function Section({ + title, + description, + settingsKey, + settings, + categories, + companies, + onSaved, +}: { + title: string; + description: string; + settingsKey: "kiosk" | "mobile"; + settings: SectionSettings; + categories: CompanyCategory[]; + companies: Company[]; + onSaved: (key: "kiosk" | "mobile", updated: SectionSettings) => void; +}) { + const [local, setLocal] = useState(settings); + const [saving, setSaving] = useState(false); + + useEffect(() => { setLocal(settings); }, [settings]); + + function toggleCategory(value: number) { + setLocal((prev) => ({ + ...prev, + company_category_ids: prev.company_category_ids.includes(value) + ? prev.company_category_ids.filter((v) => v !== value) + : [...prev.company_category_ids, value], + })); + } + + function addExclusion(id: number) { + setLocal((prev) => ({ ...prev, excluded_company_ids: [...prev.excluded_company_ids, id] })); + } + + function removeExclusion(id: number) { + setLocal((prev) => ({ ...prev, excluded_company_ids: prev.excluded_company_ids.filter((v) => v !== id) })); + } + + async function save() { + setSaving(true); + try { + await Promise.all([ + fetch("/api/admin/display-settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + setting_key: `${settingsKey}_company_category_ids`, + setting_value: local.company_category_ids, + }), + }), + fetch("/api/admin/display-settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + setting_key: `${settingsKey}_excluded_company_ids`, + setting_value: local.excluded_company_ids, + }), + }), + ]); + toast.success(`${title} settings saved`); + onSaved(settingsKey, local); + } catch { + toast.error(`Failed to save ${title} settings`); + } finally { + setSaving(false); + } + } + + const filteredCompanies = companies.filter((c) => + local.company_category_ids.length === 0 + ? true + : true + ); + + return ( + + + + + {title} + + {description} + + +
+

Company Categories to Include

+ {categories.length === 0 ? ( +

+ No categories synced yet. Run a sync to populate. +

+ ) : ( +
+ {categories.map((cat) => ( +
+ toggleCategory(cat.value)} + /> + +
+ ))} +
+ )} +
+ +
+

Excluded Companies

+

+ Individual companies to hide even if they match the selected categories. +

+ +
+ + + + + ); +} + +export default function DisplaySettingsPage() { + const [settings, setSettings] = useState(null); + const [categories, setCategories] = useState([]); + const [companies, setCompanies] = useState([]); + const [loading, setLoading] = useState(true); + + const load = useCallback(async () => { + try { + const [settingsRes, catsRes, companiesRes] = await Promise.all([ + fetch("/api/admin/display-settings"), + fetch("/api/data/company-categories"), + fetch("/api/data/companies-list"), + ]); + if (settingsRes.ok) setSettings(await settingsRes.json()); + if (catsRes.ok) setCategories(await catsRes.json()); + if (companiesRes.ok) setCompanies(await companiesRes.json()); + } catch { + toast.error("Failed to load display settings"); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { load(); }, [load]); + + function handleSaved(key: "kiosk" | "mobile", updated: SectionSettings) { + setSettings((prev) => prev ? { ...prev, [key]: updated } : prev); + } + + if (loading) { + return ( +
+ +
+ ); + } + + if (!settings) { + return ( +
+

Failed to load settings.

+
+ ); + } + + return ( +
+
+

+ + Display Settings +

+

+ Configure which companies appear in the Kiosk and Mobile dashboards. +

+
+ +
+
+
+
+
+ ); +} diff --git a/app/api/admin/display-settings/route.ts b/app/api/admin/display-settings/route.ts new file mode 100644 index 0000000..ed7d46c --- /dev/null +++ b/app/api/admin/display-settings/route.ts @@ -0,0 +1,65 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { postgresClient } from '@/lib/services/postgres-client'; + +const KIOSK_KEYS = ['kiosk_company_category_ids', 'kiosk_excluded_company_ids']; +const MOBILE_KEYS = ['mobile_company_category_ids', 'mobile_excluded_company_ids']; +const ALL_KEYS = [...KIOSK_KEYS, ...MOBILE_KEYS]; + +function parseIds(value: string | null): number[] { + if (!value) return []; + return value.split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); +} + +export async function GET() { + try { + const result = await postgresClient.query( + `SELECT setting_key, setting_value FROM kiosk_settings WHERE setting_key = ANY($1)`, + [ALL_KEYS] + ); + + const map: Record = {}; + result.rows.forEach((r: any) => { map[r.setting_key] = r.setting_value || ''; }); + + return NextResponse.json({ + kiosk: { + company_category_ids: parseIds(map['kiosk_company_category_ids'] ?? '1'), + excluded_company_ids: parseIds(map['kiosk_excluded_company_ids'] ?? ''), + }, + mobile: { + company_category_ids: parseIds(map['mobile_company_category_ids'] ?? '1'), + excluded_company_ids: parseIds(map['mobile_excluded_company_ids'] ?? ''), + }, + }); + } catch (error) { + console.error('Error fetching display settings:', error); + return NextResponse.json({ error: 'Failed to fetch display settings' }, { status: 500 }); + } +} + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const { setting_key, setting_value } = body; + + if (!setting_key || !ALL_KEYS.includes(setting_key)) { + return NextResponse.json({ error: 'Invalid setting_key' }, { status: 400 }); + } + + const valueToStore = Array.isArray(setting_value) + ? setting_value.join(',') + : String(setting_value ?? ''); + + await postgresClient.query( + `INSERT INTO kiosk_settings (setting_key, setting_value, updated_at) + VALUES ($1, $2, CURRENT_TIMESTAMP) + ON CONFLICT (setting_key) + DO UPDATE SET setting_value = EXCLUDED.setting_value, updated_at = CURRENT_TIMESTAMP`, + [setting_key, valueToStore] + ); + + return NextResponse.json({ success: true }); + } catch (error) { + console.error('Error updating display settings:', error); + return NextResponse.json({ error: 'Failed to update display settings' }, { status: 500 }); + } +} diff --git a/app/api/data/companies-list/route.ts b/app/api/data/companies-list/route.ts new file mode 100644 index 0000000..0768a83 --- /dev/null +++ b/app/api/data/companies-list/route.ts @@ -0,0 +1,14 @@ +import { NextResponse } from 'next/server'; +import { postgresClient } from '@/lib/services/postgres-client'; + +export async function GET() { + try { + const result = await postgresClient.query( + `SELECT id, company_name FROM companies WHERE is_active = true AND is_deleted = false ORDER BY company_name ASC` + ); + return NextResponse.json(result.rows); + } catch (error) { + console.error('Error fetching companies list:', error); + return NextResponse.json({ error: 'Failed to fetch companies list' }, { status: 500 }); + } +} diff --git a/app/api/data/company-categories/route.ts b/app/api/data/company-categories/route.ts new file mode 100644 index 0000000..246d59f --- /dev/null +++ b/app/api/data/company-categories/route.ts @@ -0,0 +1,14 @@ +import { NextResponse } from 'next/server'; +import { postgresClient } from '@/lib/services/postgres-client'; + +export async function GET() { + try { + const result = await postgresClient.query( + `SELECT value, label, is_active FROM company_categories WHERE is_active = true ORDER BY label ASC` + ); + return NextResponse.json(result.rows); + } catch (error) { + console.error('Error fetching company categories:', error); + return NextResponse.json({ error: 'Failed to fetch company categories' }, { status: 500 }); + } +} diff --git a/app/api/kiosk/activity/route.ts b/app/api/kiosk/activity/route.ts index bd3a217..7d30f38 100644 --- a/app/api/kiosk/activity/route.ts +++ b/app/api/kiosk/activity/route.ts @@ -1,50 +1,37 @@ import { NextRequest, NextResponse } from 'next/server'; import { postgresClient } from '@/lib/services/postgres-client'; -async function getExcludedCompanyIds(): Promise { +async function getKioskCompanyFilter(): Promise { try { const result = await postgresClient.query( - `SELECT setting_value FROM kiosk_settings WHERE setting_key = 'excluded_company_ids'` + `SELECT setting_key, setting_value FROM kiosk_settings WHERE setting_key IN ('kiosk_company_category_ids', 'kiosk_excluded_company_ids')` ); - const value = result.rows[0]?.setting_value || ''; - return value ? value.split(',').map((id: string) => parseInt(id.trim())).filter((id: number) => !isNaN(id)) : []; - } catch (error) { - console.error('Error fetching excluded company IDs:', error); - return []; - } -} + const map: Record = {}; + result.rows.forEach((r: any) => { map[r.setting_key] = r.setting_value || ''; }); -async function getIncludedClassificationIds(): Promise { - try { - const result = await postgresClient.query( - `SELECT setting_value FROM kiosk_settings WHERE setting_key = 'included_classifications'` - ); - const value = result.rows[0]?.setting_value || ''; - return value ? value.split(',').map((c: string) => parseInt(c.trim(), 10)).filter((n: number) => !isNaN(n)) : []; + const catIds = (map['kiosk_company_category_ids'] || '1') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + const exclIds = (map['kiosk_excluded_company_ids'] || '') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + + const catFilter = catIds.length > 0 + ? `t.company_id IN (SELECT id FROM companies WHERE company_category_id IN (${catIds.join(',')}))` + : 'true'; + const exclFilter = exclIds.length > 0 + ? `t.company_id NOT IN (${exclIds.join(',')})` + : ''; + + return [catFilter, exclFilter].filter(Boolean).join(' AND '); } catch (error) { - console.error('Error fetching included classifications:', error); - return []; + console.error('Error fetching kiosk company filter:', error); + return 't.company_id IN (SELECT id FROM companies WHERE company_category_id = 1)'; } } export async function GET(request: NextRequest) { try { - // Get excluded company IDs (co-managed clients) and classifications - const excludedCompanyIds = await getExcludedCompanyIds(); - const includedClassificationIds = await getIncludedClassificationIds(); - - // Build company filter: only show recurring revenue classification clients - let excludeCompanyFilter = ''; - const conditions: string[] = []; - if (excludedCompanyIds.length > 0) { - conditions.push(`t.company_id NOT IN (${excludedCompanyIds.join(',')})`); - } - if (includedClassificationIds.length > 0) { - conditions.push(`t.company_id IN (SELECT id FROM companies WHERE classification::integer IN (${includedClassificationIds.join(',')}))`); - } - if (conditions.length > 0) { - excludeCompanyFilter = `AND (${conditions.join(' AND ')})`; - } + const companyFilter = await getKioskCompanyFilter(); + const excludeCompanyFilter = `AND (${companyFilter})`; // Get recent ticket activity for ticker feed - exclude RMM alerts (source = 8), co-managed clients, and excluded classifications const activityResult = await postgresClient.query( diff --git a/app/api/kiosk/stats/route.ts b/app/api/kiosk/stats/route.ts index a174571..8968aea 100644 --- a/app/api/kiosk/stats/route.ts +++ b/app/api/kiosk/stats/route.ts @@ -1,50 +1,37 @@ import { NextRequest, NextResponse } from 'next/server'; import { postgresClient } from '@/lib/services/postgres-client'; -async function getExcludedCompanyIds(): Promise { +async function getKioskCompanyFilter(): Promise { try { const result = await postgresClient.query( - `SELECT setting_value FROM kiosk_settings WHERE setting_key = 'excluded_company_ids'` + `SELECT setting_key, setting_value FROM kiosk_settings WHERE setting_key IN ('kiosk_company_category_ids', 'kiosk_excluded_company_ids')` ); - const value = result.rows[0]?.setting_value || ''; - return value ? value.split(',').map((id: string) => parseInt(id.trim())).filter((id: number) => !isNaN(id)) : []; - } catch (error) { - console.error('Error fetching excluded company IDs:', error); - return []; - } -} + const map: Record = {}; + result.rows.forEach((r: any) => { map[r.setting_key] = r.setting_value || ''; }); -async function getIncludedClassificationIds(): Promise { - try { - const result = await postgresClient.query( - `SELECT setting_value FROM kiosk_settings WHERE setting_key = 'included_classifications'` - ); - const value = result.rows[0]?.setting_value || ''; - return value ? value.split(',').map((c: string) => parseInt(c.trim(), 10)).filter((n: number) => !isNaN(n)) : []; + const catIds = (map['kiosk_company_category_ids'] || '1') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + const exclIds = (map['kiosk_excluded_company_ids'] || '') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + + const catFilter = catIds.length > 0 + ? `company_id IN (SELECT id FROM companies WHERE company_category_id IN (${catIds.join(',')}))` + : 'true'; + const exclFilter = exclIds.length > 0 + ? `company_id NOT IN (${exclIds.join(',')})` + : ''; + + return [catFilter, exclFilter].filter(Boolean).join(' AND '); } catch (error) { - console.error('Error fetching included classifications:', error); - return []; + console.error('Error fetching kiosk company filter:', error); + return 'company_id IN (SELECT id FROM companies WHERE company_category_id = 1)'; } } export async function GET(request: NextRequest) { try { - // Get excluded company IDs (co-managed clients) - const excludedCompanyIds = await getExcludedCompanyIds(); - const includedClassificationIds = await getIncludedClassificationIds(); - - // Build company filter: only show recurring revenue classification clients - let excludeCompanyFilter = ''; - const conditions: string[] = []; - if (excludedCompanyIds.length > 0) { - conditions.push(`company_id NOT IN (${excludedCompanyIds.join(',')})`); - } - if (includedClassificationIds.length > 0) { - conditions.push(`company_id IN (SELECT id FROM companies WHERE classification::integer IN (${includedClassificationIds.join(',')}))`); - } - if (conditions.length > 0) { - excludeCompanyFilter = `AND (${conditions.join(' AND ')})`; - } + const companyFilter = await getKioskCompanyFilter(); + const excludeCompanyFilter = `AND (${companyFilter})`; // Critical tickets (Priority 1-3) - exclude RMM alerts (source = 8), co-managed clients, and excluded classifications const criticalTicketsResult = await postgresClient.query( diff --git a/app/api/mobile/dashboard/route.ts b/app/api/mobile/dashboard/route.ts index e3e6317..f28a86d 100644 --- a/app/api/mobile/dashboard/route.ts +++ b/app/api/mobile/dashboard/route.ts @@ -1,17 +1,30 @@ import { NextResponse } from 'next/server'; import { postgresClient } from '@/lib/services/postgres-client'; -async function getManagedClassificationFilter(): Promise { - const result = await postgresClient.query( - `SELECT setting_value FROM kiosk_settings WHERE setting_key = 'included_classifications'` - ); - const value = result.rows[0]?.setting_value || ''; - const ids = value ? value.split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)) : []; - return ids.length > 0 ? `c.classification::integer IN (${ids.join(',')})` : 'true'; +async function getMobileClassFilter(): Promise { + try { + const result = await postgresClient.query( + `SELECT setting_key, setting_value FROM kiosk_settings WHERE setting_key IN ('mobile_company_category_ids', 'mobile_excluded_company_ids')` + ); + const map: Record = {}; + result.rows.forEach((r: any) => { map[r.setting_key] = r.setting_value || ''; }); + + const catIds = (map['mobile_company_category_ids'] || '1') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + const exclIds = (map['mobile_excluded_company_ids'] || '') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + + const catCond = catIds.length > 0 ? `c.company_category_id IN (${catIds.join(',')})` : 'true'; + const exclCond = exclIds.length > 0 ? `c.id NOT IN (${exclIds.join(',')})` : ''; + return [catCond, exclCond].filter(Boolean).join(' AND '); + } catch (error) { + console.error('Error fetching mobile company filter:', error); + return 'c.company_category_id = 1'; + } } export async function GET() { - const classFilter = await getManagedClassificationFilter(); + const classFilter = await getMobileClassFilter(); const [byStatus, byQueue, byPriority, recentActivity, sla] = await Promise.all([ postgresClient.query(` diff --git a/app/api/mobile/tickets/route.ts b/app/api/mobile/tickets/route.ts index 74f6dd9..cb88bf2 100644 --- a/app/api/mobile/tickets/route.ts +++ b/app/api/mobile/tickets/route.ts @@ -1,13 +1,28 @@ import { NextRequest, NextResponse } from 'next/server'; import { postgresClient } from '@/lib/services/postgres-client'; -async function getManagedCompanyFilter(): Promise { - const result = await postgresClient.query( - `SELECT setting_value FROM kiosk_settings WHERE setting_key = 'included_classifications'` - ); - const value = result.rows[0]?.setting_value || ''; - const ids = value ? value.split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)) : []; - return ids.length > 0 ? `c.classification::integer IN (${ids.join(',')})` : 'true'; +async function getMobileCompanyFilter(): Promise<{ join: string; condition: string }> { + try { + const result = await postgresClient.query( + `SELECT setting_key, setting_value FROM kiosk_settings WHERE setting_key IN ('mobile_company_category_ids', 'mobile_excluded_company_ids')` + ); + const map: Record = {}; + result.rows.forEach((r: any) => { map[r.setting_key] = r.setting_value || ''; }); + + const catIds = (map['mobile_company_category_ids'] || '1') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + const exclIds = (map['mobile_excluded_company_ids'] || '') + .split(',').map((s: string) => parseInt(s.trim(), 10)).filter((n: number) => !isNaN(n)); + + const catCond = catIds.length > 0 ? `c.company_category_id IN (${catIds.join(',')})` : 'true'; + const exclCond = exclIds.length > 0 ? `c.id NOT IN (${exclIds.join(',')})` : ''; + const condition = [catCond, exclCond].filter(Boolean).join(' AND '); + + return { join: 'INNER JOIN companies c ON c.id = t.company_id', condition }; + } catch (error) { + console.error('Error fetching mobile company filter:', error); + return { join: 'INNER JOIN companies c ON c.id = t.company_id', condition: 'c.company_category_id = 1' }; + } } export async function GET(request: NextRequest) { @@ -19,12 +34,12 @@ export async function GET(request: NextRequest) { const limit = 30; const offset = (page - 1) * limit; - const managedFilter = await getManagedCompanyFilter(); + const { condition: companyCondition } = await getMobileCompanyFilter(); const conditions: string[] = [ 't.status != 5', 't.is_deleted = false', - managedFilter, + companyCondition, ]; const params: unknown[] = []; diff --git a/components/navigation/app-navigation.tsx b/components/navigation/app-navigation.tsx index a86eca1..081452b 100644 --- a/components/navigation/app-navigation.tsx +++ b/components/navigation/app-navigation.tsx @@ -26,6 +26,7 @@ import { Sun, BarChart3, DollarSign, + SlidersHorizontal, } from 'lucide-react'; import { NavigationMenu, @@ -179,6 +180,12 @@ const navigationItems: NavItem[] = [ icon: DollarSign, description: 'Sync invoices, payments, deposits, transactions and financial reports' }, + { + title: 'Display Settings', + href: '/admin/display-settings', + icon: SlidersHorizontal, + description: 'Configure company filters for Kiosk and Mobile dashboards' + }, { title: 'Data Browser', href: '/admin/data-browser', diff --git a/docs/AUTOTASK_SYNC.md b/docs/AUTOTASK_SYNC.md new file mode 100644 index 0000000..5e7181a --- /dev/null +++ b/docs/AUTOTASK_SYNC.md @@ -0,0 +1,156 @@ +# Autotask Sync — Reference Guide + +## Overview + +Pulse maintains a local PostgreSQL mirror of Autotask data in the `pulse_autotask` database. Three mechanisms keep it current: **scheduled syncs**, **real-time webhooks**, and **manual/API-triggered syncs**. + +--- + +## Automatic Syncs (Scheduled) + +All schedules are managed via the **Admin → Sync Scheduler** UI or directly in the `sync_schedules` table. They can be enabled/disabled individually. + +| Schedule ID | Name | Cron (UTC) | Local (EST) | Type | Enabled | +|---|---|---|---|---|---| +| `sync-6am` | Morning Sync | `0 11 * * *` | 6:00 AM daily | incremental | ✅ | +| `sync-830am` | Morning Sync | `30 13 * * *` | 8:30 AM daily | incremental | ✅ | +| `sync-11am` | Midday Sync | `0 16 * * *` | 11:00 AM daily | incremental | ✅ | +| `sync-230pm` | Afternoon Sync | `30 19 * * *` | 2:30 PM daily | incremental | ✅ | +| `weekly-full` | Weekly Full Sync | `0 3 * * 0` | 3:00 AM Sunday | full | ✅ | +| `contract-services` | Contract Services | `0 4 * * *` | 4:00 AM daily | contract-services | ✅ | + +### Incremental vs Full + +**Incremental sync** — pulls only records modified since the last successful sync timestamp (stored in `sync_history`). Much faster. Runs 4× per day. + +**Full sync** — pulls everything regardless of modification date, going back `yearsBack` years (default: 2). Runs every Sunday at 3 AM. Also used for initial setup or repair. + +### What incremental syncs cover + +Every incremental run syncs all entities in dependency order: + +1. Companies, Resources, Statuses, Issue Types, Sub-Issue Types, Work Types, Queues, Priorities, Ticket Categories *(no dependencies)* +2. Contacts *(requires Companies)* +3. Projects *(requires Companies, Resources)* +4. Tickets *(requires Companies, Resources, Contacts)* +5. Tasks *(requires Resources, Projects, Tickets)* +6. Configuration Items *(requires Companies, Contacts)* +7. Contracts *(requires Companies, Contacts)* +8. Autotask Services *(standalone)* +9. Billing Items *(requires Companies, Tasks, Tickets, Projects)* +10. Time Entries *(requires Companies, Resources, Contacts, Projects, Tasks, Tickets)* +11. Ticket Notes *(requires Tickets)* +12. Tag Groups, Tags, Ticket Tag Associations + +> **Note:** Companies and Resources do not support date-based filtering in the Autotask API, so they always do a full pull on every run (fast — ~57 resources, ~236 companies). + +### Time-windowed entities + +Some entities are too large to sync in full and are filtered by date: + +| Entity | Filter field | Default window | +|---|---|---| +| Tickets | `createDate` | Last 2 years | +| Tasks | `createDate` | Last 2 years | +| Time Entries | `dateWorked` | Last 2 years | +| Billing Items | `itemDate` | Last 2 years | + +The `yearsBack` parameter (default `2`) controls how far back these go. The full sync and chunked sync accept a custom `yearsBack` value. + +--- + +## Real-Time Webhooks + +Autotask pushes change events to `/api/webhooks/autotask` immediately when a ticket (or other entity) is created or updated. The webhook handler fetches the full record from the Autotask API and upserts it directly — no waiting for the next scheduled sync. + +This means tickets assigned, updated, or closed in Autotask appear in Pulse within seconds. The webhook path **bypasses** the resource ID validation step, so it always writes exactly what Autotask sends. + +--- + +## Manual / API-Triggered Syncs + +### Via Admin UI + +**Admin → Sync** — the sync overview page has buttons to trigger syncs per integration. + +### Via API endpoints + +All endpoints are POST and non-blocking (fire-and-forget) — they return immediately and run in the background. + +| Endpoint | Payload | Description | +|---|---|---| +| `POST /api/sync/full` | `{ yearsBack?, triggeredBy? }` | Full sync of all entities | +| `POST /api/sync/entity` | `{ entities: [...], syncType?, yearsBack? }` | Sync specific entity types only | +| `POST /api/sync/tickets-chunked` | `{ yearsBack? }` | Chunked ticket sync with progress logging (good for large date ranges) | + +**Example — sync only tickets and resources:** +```bash +curl -X POST http://localhost:3100/api/sync/entity \ + -H "Content-Type: application/json" \ + -d '{"entities": ["tickets", "resources"], "triggeredBy": "manual"}' +``` + +**Example — full sync going back 3 years:** +```bash +curl -X POST http://localhost:3100/api/sync/full \ + -H "Content-Type: application/json" \ + -d '{"yearsBack": 3, "triggeredBy": "manual"}' +``` + +### Via OpenClaw (external agent API) + +```bash +# Incremental sync +POST /api/openclaw/sync/autotask/incremental + +# Full sync +POST /api/openclaw/sync/autotask/full { "yearsBack": 2 } + +# Specific entities +POST /api/openclaw/sync/autotask/entity { "entities": ["tickets"] } +``` +Auth: `x-openclaw-key: ` + +--- + +## Valid Entity Names + +Use these string values in the `entities` array: + +``` +companies, resources, statuses, issue_types, sub_issue_types, work_types, +queues, priorities, ticket_categories, contacts, projects, tickets, tasks, +configuration_items, contracts, contract_services, autotask_services, +billing_items, time_entries, ticket_notes, tag_groups, tags +``` + +--- + +## Sync History & Status + +Every sync run writes to `sync_history`. You can query it directly: + +```sql +-- Last sync per entity type +SELECT entity_type, sync_type, status, records_added, records_updated, completed_at +FROM sync_history +WHERE status = 'completed' +ORDER BY completed_at DESC; + +-- Check when tickets last synced successfully +SELECT completed_at FROM sync_history +WHERE entity_type = 'tickets' AND status = 'completed' +ORDER BY completed_at DESC LIMIT 1; +``` + +The incremental sync uses `completed_at` from `sync_history` to determine how far back to pull. **If a sync fails, the timestamp does not advance** — the next run re-fetches from the last successful point. + +--- + +## Known Constraints & Gotchas + +- **One sync at a time.** If a sync is already running, new requests return `409 Conflict`. +- **`assigned_resource_id` is the only resource assignment on a ticket.** There is no separate "primary resource" — `assigned_resource_id` is it. Other resource fields (`first_response_assigned_resource_id`, `last_activity_resource_id`, `creator_resource_id`) are audit/SLA tracking fields. +- **BIGINT IDs come back as strings from pg.** The sync code uses `Number()` coercion when building validation sets to avoid false-negative ID comparisons. +- **FK constraints on tickets are deferrable.** `tickets_company_id_fkey` and all resource FKs are `DEFERRABLE INITIALLY DEFERRED ON DELETE SET NULL`. This means a ticket can be inserted even if its referenced company or resource isn't in the local DB yet — the field is set to NULL rather than rejecting the row. +- **Webhook and scheduled sync can race.** If a webhook fires during a scheduled sync batch, it may be overwritten by the batch. This is harmless — both write the same Autotask data. diff --git a/lib/services/entity-sync.ts b/lib/services/entity-sync.ts index 180a888..5de9890 100644 --- a/lib/services/entity-sync.ts +++ b/lib/services/entity-sync.ts @@ -82,6 +82,12 @@ export class EntitySyncService { if (entity === EntityType.TICKET_CATEGORIES) { return await this.syncTicketCategories(isIncremental); } + if (entity === EntityType.COMPANY_CATEGORIES) { + return await this.syncCompanyCategories(isIncremental); + } + if (entity === EntityType.COMPANY_TYPES) { + return await this.syncCompanyTypes(isIncremental); + } const trackingId = syncId || `${entity}_${Date.now()}`; const entityLogger = this.logger.child({ syncId: trackingId, entityType: entity }); @@ -1136,6 +1142,88 @@ export class EntitySyncService { } } + /** + * Sync Company Categories (queried from CompanyCategories entity — id, name, isActive) + */ + async syncCompanyCategories(isIncremental: boolean = false): Promise { + const syncLogger = this.logger.child({ entityType: EntityType.COMPANY_CATEGORIES, syncType: 'entity' }); + const syncStartTime = syncLogger.start('Company Categories sync'); + + try { + const apiRecords = await this.autotaskClient.queryEntityPaginated( + 'CompanyCategories', + { filter: [{ field: 'id', op: 'gt', value: 0 }] }, + 500 + ); + + const records = apiRecords.map((r: any) => ({ + value: r.id, + label: r.name || r.nickname || String(r.id), + is_active: r.isActive !== false, + sort_order: r.id, + synced_at: new Date(), + })); + + syncLogger.info('Fetched company categories', { recordCount: records.length }); + + const tableName = getTableName(EntityType.COMPANY_CATEGORIES); + const existingResult = await postgresClient.query<{ value: number }>(`SELECT value FROM ${tableName}`); + const existingValues = new Set(existingResult.rows.map((r: any) => r.value)); + + const recordsAdded = records.filter((r: any) => !existingValues.has(r.value)).length; + const recordsUpdated = records.filter((r: any) => existingValues.has(r.value)).length; + + await postgresClient.bulkUpsert(tableName, records, ['value']); + + const stats: EntitySyncStats = { recordsAdded, recordsUpdated, recordsDeleted: 0 }; + syncLogger.complete('Company Categories sync', syncStartTime, stats); + return stats; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + syncLogger.fail('Company Categories sync', syncStartTime, err); + throw error; + } + } + + /** + * Sync Company Types (Picklist from Companies field) + */ + async syncCompanyTypes(isIncremental: boolean = false): Promise { + const picklistLogger = this.logger.child({ entityType: EntityType.COMPANY_TYPES, syncType: 'picklist' }); + const syncStartTime = picklistLogger.start('Picklist sync'); + + try { + const picklistValues = await this.autotaskClient.getPicklistValues('Companies', 'companyType'); + + const records = Object.entries(picklistValues).map(([value, label]) => ({ + value: parseInt(value), + label: label, + is_active: true, + sort_order: parseInt(value), + synced_at: new Date(), + })); + + picklistLogger.info('Found picklist values', { recordCount: records.length }); + + const tableName = getTableName(EntityType.COMPANY_TYPES); + const existingResult = await postgresClient.query<{ value: number }>(`SELECT value FROM ${tableName}`); + const existingValues = new Set(existingResult.rows.map((r: any) => r.value)); + + const recordsAdded = records.filter((r: any) => !existingValues.has(r.value)).length; + const recordsUpdated = records.filter((r: any) => existingValues.has(r.value)).length; + + await postgresClient.bulkUpsert(tableName, records, ['value']); + + const stats: EntitySyncStats = { recordsAdded, recordsUpdated, recordsDeleted: 0 }; + picklistLogger.complete('Picklist sync', syncStartTime, stats); + return stats; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + picklistLogger.fail('Picklist sync', syncStartTime, err); + throw error; + } + } + /** * Sync Work Types (Picklist from TimeEntry field) */ diff --git a/lib/types/sync.ts b/lib/types/sync.ts index 193999f..7f9acf2 100644 --- a/lib/types/sync.ts +++ b/lib/types/sync.ts @@ -28,6 +28,8 @@ export enum EntityType { TAG_GROUPS = 'autotask_tag_groups', TAGS = 'autotask_tags', PROJECT_PHASES = 'project_phases', + COMPANY_CATEGORIES = 'company_categories', + COMPANY_TYPES = 'company_types', } // Sync operation types @@ -177,6 +179,8 @@ export const ENTITY_DEPENDENCIES: Record = { [EntityType.TAG_GROUPS]: [], // No dependencies — standalone lookup [EntityType.TAGS]: [EntityType.TAG_GROUPS], // Depends on tag groups (FK) [EntityType.PROJECT_PHASES]: [EntityType.PROJECTS], // Depends on projects + [EntityType.COMPANY_CATEGORIES]: [], // No dependencies — standalone lookup + [EntityType.COMPANY_TYPES]: [], // No dependencies — standalone lookup }; // Autotask API field names (for incremental sync) diff --git a/lib/utils/entity-mapper.ts b/lib/utils/entity-mapper.ts index 1cba759..77a8955 100644 --- a/lib/utils/entity-mapper.ts +++ b/lib/utils/entity-mapper.ts @@ -73,6 +73,8 @@ export function mapAutotaskToDatabase( case EntityType.ISSUE_TYPES: case EntityType.SUB_ISSUE_TYPES: case EntityType.WORK_TYPES: + case EntityType.COMPANY_CATEGORIES: + case EntityType.COMPANY_TYPES: mapped = mapPicklist(data); break; case EntityType.TAG_GROUPS: diff --git a/lib/utils/sync-helpers.ts b/lib/utils/sync-helpers.ts index 727f1c4..4cbcb42 100644 --- a/lib/utils/sync-helpers.ts +++ b/lib/utils/sync-helpers.ts @@ -128,6 +128,8 @@ export function getAutotaskEntityName(entity: EntityType): string { [EntityType.TAG_GROUPS]: 'TagGroups', [EntityType.TAGS]: 'Tags', [EntityType.PROJECT_PHASES]: 'Phases', + [EntityType.COMPANY_CATEGORIES]: 'CompanyCategories', + [EntityType.COMPANY_TYPES]: 'CompanyTypes', }; return mapping[entity] || entity; @@ -147,6 +149,7 @@ export function isPicklistEntity(entity: EntityType): boolean { EntityType.QUEUES, EntityType.PRIORITIES, EntityType.TICKET_CATEGORIES, + EntityType.COMPANY_TYPES, ].includes(entity); } @@ -180,6 +183,8 @@ export function getLastModifiedField(entity: EntityType): string { [EntityType.TAG_GROUPS]: 'lastModifiedDate', [EntityType.TAGS]: 'lastModifiedDateTime', [EntityType.PROJECT_PHASES]: 'lastActivityDateTime', + [EntityType.COMPANY_CATEGORIES]: 'lastModifiedDate', + [EntityType.COMPANY_TYPES]: 'lastModifiedDate', }; return mapping[entity] || 'lastModifiedDate'; @@ -215,6 +220,8 @@ export function getActiveField(entity: EntityType): string | null { [EntityType.TAG_GROUPS]: 'isActive', [EntityType.TAGS]: 'isActive', [EntityType.PROJECT_PHASES]: null, // No active field on phases + [EntityType.COMPANY_CATEGORIES]: 'isActive', + [EntityType.COMPANY_TYPES]: 'isActive', }; return mapping[entity] || null; @@ -310,6 +317,8 @@ export function buildDateRangeFilter( [EntityType.PRIORITIES]: null, [EntityType.TICKET_CATEGORIES]: null, [EntityType.PROJECT_PHASES]: null, + [EntityType.COMPANY_CATEGORIES]: null, + [EntityType.COMPANY_TYPES]: null, }; const dateField = dateFieldMapping[entity]; @@ -518,6 +527,8 @@ export function getEntityDisplayName(entity: EntityType): string { [EntityType.TAG_GROUPS]: 'Tag Groups', [EntityType.TAGS]: 'Tags', [EntityType.PROJECT_PHASES]: 'Project Phases', + [EntityType.COMPANY_CATEGORIES]: 'Company Categories', + [EntityType.COMPANY_TYPES]: 'Company Types', }; return mapping[entity] || entity; diff --git a/migrations/063_relax_tickets_company_resource_constraints.sql b/migrations/063_relax_tickets_company_resource_constraints.sql new file mode 100644 index 0000000..6678e1b --- /dev/null +++ b/migrations/063_relax_tickets_company_resource_constraints.sql @@ -0,0 +1,45 @@ +-- Migration 063: Relax tickets company_id and resource FK constraints +-- Both were causing ticket sync to fail with DATABASE_CONSTRAINT_ERROR. +-- +-- Root causes: +-- 1. tickets_company_id_fkey — strict, not deferrable, ON DELETE CASCADE. +-- Tickets whose company_id references a company not yet in the local +-- companies table fail on upsert. +-- 2. tickets_assigned_resource_id_fkey — was relaxed in migration 008 but +-- got re-created as strict (condeferrable=f) by a subsequent migration. +-- +-- Fix: make both constraints deferrable INITIALLY DEFERRED with ON DELETE SET NULL. + +-- ── company_id ────────────────────────────────────────────────────────────── +ALTER TABLE tickets DROP CONSTRAINT IF EXISTS tickets_company_id_fkey; +ALTER TABLE tickets ALTER COLUMN company_id DROP NOT NULL; +ALTER TABLE tickets + ADD CONSTRAINT tickets_company_id_fkey + FOREIGN KEY (company_id) REFERENCES companies(id) + ON DELETE SET NULL + DEFERRABLE INITIALLY DEFERRED; + +-- ── assigned_resource_id (re-apply in case it was reset) ──────────────────── +ALTER TABLE tickets DROP CONSTRAINT IF EXISTS tickets_assigned_resource_id_fkey; +ALTER TABLE tickets ALTER COLUMN assigned_resource_id DROP NOT NULL; +ALTER TABLE tickets + ADD CONSTRAINT tickets_assigned_resource_id_fkey + FOREIGN KEY (assigned_resource_id) REFERENCES resources(id) + ON DELETE SET NULL + DEFERRABLE INITIALLY DEFERRED; + +-- ── first_response resource IDs ───────────────────────────────────────────── +ALTER TABLE tickets DROP CONSTRAINT IF EXISTS tickets_first_response_assigned_resource_id_fkey; +ALTER TABLE tickets DROP CONSTRAINT IF EXISTS tickets_first_response_initiating_resource_id_fkey; + +ALTER TABLE tickets + ADD CONSTRAINT tickets_first_response_assigned_resource_id_fkey + FOREIGN KEY (first_response_assigned_resource_id) REFERENCES resources(id) + ON DELETE SET NULL + DEFERRABLE INITIALLY DEFERRED; + +ALTER TABLE tickets + ADD CONSTRAINT tickets_first_response_initiating_resource_id_fkey + FOREIGN KEY (first_response_initiating_resource_id) REFERENCES resources(id) + ON DELETE SET NULL + DEFERRABLE INITIALLY DEFERRED; diff --git a/migrations/064_create_company_picklists.sql b/migrations/064_create_company_picklists.sql new file mode 100644 index 0000000..a422298 --- /dev/null +++ b/migrations/064_create_company_picklists.sql @@ -0,0 +1,31 @@ +-- Company Categories picklist table (synced from Autotask Companies.companyCategoryID field) +CREATE TABLE IF NOT EXISTS company_categories ( + value INTEGER PRIMARY KEY, + label VARCHAR(200) NOT NULL, + is_active BOOLEAN DEFAULT true, + sort_order INTEGER DEFAULT 0, + synced_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Company Types picklist table (synced from Autotask Companies.companyType field) +CREATE TABLE IF NOT EXISTS company_types ( + value INTEGER PRIMARY KEY, + label VARCHAR(200) NOT NULL, + is_active BOOLEAN DEFAULT true, + sort_order INTEGER DEFAULT 0, + synced_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_company_categories_active ON company_categories(is_active); +CREATE INDEX IF NOT EXISTS idx_company_types_active ON company_types(is_active); + +-- Seed new display settings keys into kiosk_settings +INSERT INTO kiosk_settings (setting_key, setting_value, description) VALUES + ('kiosk_company_category_ids', '1', 'Comma-separated company category IDs to include in kiosk (default: 1=Recurring Revenue Customer)'), + ('mobile_company_category_ids', '1', 'Comma-separated company category IDs to include in mobile (default: 1=Recurring Revenue Customer)'), + ('mobile_excluded_company_ids', '', 'Comma-separated company IDs to exclude from mobile dashboard') +ON CONFLICT (setting_key) DO NOTHING;