lorentz/wulf-pulse
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

feat: RMM Overshell, IT Glue audit/write-back, LogLift, link-aware bundles, dashboard overhaul

Browse source 
- RMM Overshell (migration 077): admin page, dispatch UI, executor/worker, target
  resolver, script registry (AD/DHCP/DNS/event-log/services/software/network/loglift)
- LogLift evidence pipeline (migration 078): upload webhook, B2 storage client,
  receiver/matcher, EventLogCollector PowerShell script
- IT Glue audit + write-back (migrations 075, 076): asset-audit runner, ticket
  xrefs, applications/configurations browse pages + apply/revert/audit endpoints
- Link-aware analyzer bundles (migration 073) + provider toggle (migration 074):
  link-discovery service, OpenRouter LLM provider, related-tickets/itglue-suggestion
  panels, analyze-bundle endpoint
- Endpoint data model + device-link reconciliation (migrations 079, 080): conflicts
  admin page, reconciler service, resolve endpoints
- Dashboard overhaul: integration-health service + alerts, overview/health endpoints
- Permissions: add itglue + rmm scopes; middleware: public /api/rmm/loglift route

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
lorentz 2026-05-03 07:13:18 -04:00
parent 378e68ad8a
commit 1112a06afe
132 changed files with 21352 additions and 743 deletions
Download patch file Download diff file Expand all files Collapse all files

44
migrations/073_analyzer_link_aware_bundles.sql Normal file
View file

@ -0,0 +1,44 @@
-- AI Ticket Analyzer — link-aware bundle support.
-- See docs/wulf-pulse-ticket-analyzer-build-notes.md (Phase 3).
--
-- A "bundle" is an aggregate report that was triggered from a single ticket
-- analysis page (typically a master/problem ticket) and is waiting for the
-- per-ticket analyses of its referenced tickets to complete before it can run.
--
-- Adds:
-- 1. expected_ticket_numbers TEXT[] — the full set the bundle is waiting on
-- 2. triggered_by_ticket_number TEXT — the master ticket the user clicked from
-- 3. New status value 'pending_analyses' — set while underlying analyses are
-- still running. Worker chain-trigger flips it to 'pending' once all
-- expected tickets have a complete analysis, then runAggregateReport runs.
ALTER TABLE analyzer_aggregate_reports
ADD COLUMN IF NOT EXISTS expected_ticket_numbers TEXT[],
ADD COLUMN IF NOT EXISTS triggered_by_ticket_number TEXT;
ALTER TABLE analyzer_aggregate_reports
DROP CONSTRAINT IF EXISTS analyzer_aggregate_reports_status_check;
ALTER TABLE analyzer_aggregate_reports
ADD CONSTRAINT analyzer_aggregate_reports_status_check
CHECK (status IN ('pending','pending_analyses','running','complete','failed'));
-- Replace the existing partial index so chain-trigger queries are still fast.
DROP INDEX IF EXISTS idx_analyzer_aggregate_reports_pending;
CREATE INDEX IF NOT EXISTS idx_analyzer_aggregate_reports_pending
ON analyzer_aggregate_reports (status, generated_at)
WHERE status IN ('pending','pending_analyses','running');
-- Worker chain-trigger looks up reports waiting on a specific ticket number.
CREATE INDEX IF NOT EXISTS idx_analyzer_aggregate_reports_expected_tickets
ON analyzer_aggregate_reports USING gin (expected_ticket_numbers)
WHERE status = 'pending_analyses';
COMMENT ON COLUMN analyzer_aggregate_reports.expected_ticket_numbers IS
'For bundles created from a master ticket: the full set of ticket numbers whose '
'analyses must complete before the aggregate-reduce step runs. NULL for legacy '
'aggregate reports created by manual multi-select.';
COMMENT ON COLUMN analyzer_aggregate_reports.triggered_by_ticket_number IS
'The ticket number the bundle was launched from (the master in a problem-ticket '
'flow). NULL for legacy reports.';

38
migrations/074_analyzer_provider.sql Normal file
View file

@ -0,0 +1,38 @@
-- AI Ticket Analyzer — provider column for per-run LLM provider tracking.
--
-- Adds a `provider` field so multiple analyses can coexist for the same
-- ticket from different LLM providers (Claude vs DeepSeek-via-OpenRouter)
-- without colliding on the (ticket_number, analysis_version) uniqueness.
--
-- analysis_version is now monotonic *within (ticket_number, provider)*, not
-- globally per ticket. This means:
-- - The first Anthropic run is v1, first OpenRouter run is also v1.
-- - Re-runs of Anthropic produce v2, v3, …; OpenRouter likewise.
ALTER TABLE analyzer_analyses
ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 'anthropic'
CHECK (provider IN ('anthropic', 'openrouter'));
ALTER TABLE analyzer_jobs
ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 'anthropic'
CHECK (provider IN ('anthropic', 'openrouter'));
-- Replace the old unique constraint with a provider-scoped one.
ALTER TABLE analyzer_analyses
DROP CONSTRAINT IF EXISTS analyzer_analyses_version_unique;
ALTER TABLE analyzer_analyses
ADD CONSTRAINT analyzer_analyses_version_unique
UNIQUE (ticket_number, provider, analysis_version);
-- Useful covering index for the listing path.
DROP INDEX IF EXISTS idx_analyzer_analyses_ticket_version;
CREATE INDEX IF NOT EXISTS idx_analyzer_analyses_ticket_provider_version
ON analyzer_analyses (ticket_number, provider, analysis_version DESC);
COMMENT ON COLUMN analyzer_analyses.provider IS
'Which LLM provider produced this analysis. anthropic=Claude direct; '
'openrouter=DeepSeek (V4 Pro/Flash + R1) via OpenRouter.';
COMMENT ON COLUMN analyzer_jobs.provider IS
'Provider the worker should use when running this job. Set by the analyze '
'endpoint based on the user''s pick.';

87
migrations/075_itglue_audit.sql Normal file
View file

@ -0,0 +1,87 @@
-- IT Glue asset audit + write-back tables.
-- See docs/wulf-pulse-ticket-analyzer-build-notes.md → Phase 4.
--
-- Two tables:
-- 1. itglue_asset_audits — one row per audit run (LLM gap analysis).
-- 2. itglue_writes — one row per write attempt (one field, one moment),
-- with provenance back to the audit that prompted it.
--
-- A separate generic audit_log row is also written by the API layer
-- (lib/services/audit.ts) so /admin/audit-log surfaces every IT Glue change
-- alongside other admin actions. The two domain-specific tables here exist
-- because:
-- - audits accumulate full LLM context snapshots (heavy, infrequent),
-- - writes are atomic per-field decisions with before/after diffs that the
-- generic audit_log's free-form JSONB doesn't capture cleanly.
CREATE TABLE IF NOT EXISTS itglue_asset_audits (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
asset_type TEXT NOT NULL CHECK (asset_type IN ('flexible_asset')),
asset_id BIGINT NOT NULL,
asset_type_id BIGINT,
organization_id BIGINT,
generated_by_user_id TEXT REFERENCES "user"(id) ON DELETE SET NULL,
generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
provider TEXT NOT NULL DEFAULT 'anthropic'
CHECK (provider IN ('anthropic','openrouter')),
model_used TEXT,
asset_snapshot JSONB NOT NULL,
ticket_count INT NOT NULL,
field_gaps JSONB NOT NULL,
notes_promotions JSONB NOT NULL,
contradictions JSONB NOT NULL,
overall_score NUMERIC(3,2),
estimated_cost_usd NUMERIC(10,4),
total_input_tokens INT,
total_output_tokens INT,
status TEXT NOT NULL DEFAULT 'complete'
CHECK (status IN ('pending','running','complete','failed')),
error_message TEXT
);
CREATE INDEX IF NOT EXISTS ix_itglue_asset_audits_asset
ON itglue_asset_audits (asset_type, asset_id, generated_at DESC);
CREATE INDEX IF NOT EXISTS ix_itglue_asset_audits_org
ON itglue_asset_audits (organization_id, generated_at DESC);
COMMENT ON TABLE itglue_asset_audits IS
'One row per LLM-driven audit of an IT Glue asset against ticket history. '
'Captures the full input snapshot (asset traits at the time, redacted), the '
'extracted gaps/promotions/contradictions, and cost. Forever-retained.';
COMMENT ON COLUMN itglue_asset_audits.field_gaps IS
'JSON array: [{ field_name, why_missing_matters, suggested_value, '
'evidence_ticket_numbers, confidence }]';
COMMENT ON COLUMN itglue_asset_audits.notes_promotions IS
'JSON array: [{ quoted_note_text, target_field, suggested_value, confidence }]';
COMMENT ON COLUMN itglue_asset_audits.contradictions IS
'JSON array: [{ description, evidence }]';
CREATE TABLE IF NOT EXISTS itglue_writes (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
audit_id UUID REFERENCES itglue_asset_audits(id) ON DELETE SET NULL,
asset_type TEXT NOT NULL CHECK (asset_type IN ('flexible_asset')),
asset_id BIGINT NOT NULL,
field_name TEXT NOT NULL,
before_value JSONB,
after_value JSONB NOT NULL,
performed_by_user_id TEXT REFERENCES "user"(id) ON DELETE SET NULL,
performed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
status TEXT NOT NULL
CHECK (status IN ('pending','committed','failed','reverted')),
itglue_response JSONB,
error_message TEXT,
source_evidence JSONB
);
CREATE INDEX IF NOT EXISTS ix_itglue_writes_asset
ON itglue_writes (asset_type, asset_id, performed_at DESC);
CREATE INDEX IF NOT EXISTS ix_itglue_writes_user
ON itglue_writes (performed_by_user_id, performed_at DESC);
CREATE INDEX IF NOT EXISTS ix_itglue_writes_audit
ON itglue_writes (audit_id);
COMMENT ON TABLE itglue_writes IS
'One row per write attempt to IT Glue. Pending → committed | failed | reverted. '
'Reverts produce a new row whose before_value/after_value swap the original; '
'the original row gets status=reverted. Forever-retained.';

83
migrations/076_itglue_ticket_xrefs.sql Normal file
View file

@ -0,0 +1,83 @@
-- Phase 4.1 — ticket-first IT Glue audit + Configuration support + cross-reference index.
--
-- Three things:
-- 1. Add ticket linkage to existing audit + write tables (denormalized for
-- direct queries; the audit row is the source of truth).
-- 2. Extend asset_type CHECK constraints to include 'configuration'.
-- 3. New itglue_ticket_xrefs table — ticket↔asset linkage with relationship
-- type. Powers "tickets that touched this asset" + "docs this ticket
-- touched" views, and is the lookup index for the future RAG automation.
-- 1. Ticket linkage on audits + writes
ALTER TABLE itglue_asset_audits
ADD COLUMN IF NOT EXISTS triggered_by_ticket_number TEXT,
ADD COLUMN IF NOT EXISTS triggered_by_analysis_id UUID
REFERENCES analyzer_analyses(id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS ix_itglue_asset_audits_ticket
ON itglue_asset_audits (triggered_by_ticket_number)
WHERE triggered_by_ticket_number IS NOT NULL;
ALTER TABLE itglue_writes
ADD COLUMN IF NOT EXISTS triggered_by_ticket_number TEXT;
CREATE INDEX IF NOT EXISTS ix_itglue_writes_ticket
ON itglue_writes (triggered_by_ticket_number)
WHERE triggered_by_ticket_number IS NOT NULL;
-- 2. Extend asset_type to include 'configuration'
ALTER TABLE itglue_asset_audits
DROP CONSTRAINT IF EXISTS itglue_asset_audits_asset_type_check;
ALTER TABLE itglue_asset_audits
ADD CONSTRAINT itglue_asset_audits_asset_type_check
CHECK (asset_type IN ('flexible_asset', 'configuration'));
ALTER TABLE itglue_writes
DROP CONSTRAINT IF EXISTS itglue_writes_asset_type_check;
ALTER TABLE itglue_writes
ADD CONSTRAINT itglue_writes_asset_type_check
CHECK (asset_type IN ('flexible_asset', 'configuration'));
-- 3. Cross-reference table
CREATE TABLE IF NOT EXISTS itglue_ticket_xrefs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
ticket_number TEXT NOT NULL,
analysis_id UUID REFERENCES analyzer_analyses(id) ON DELETE CASCADE,
asset_type TEXT NOT NULL
CHECK (asset_type IN ('flexible_asset','configuration','document')),
asset_id BIGINT NOT NULL,
relationship TEXT NOT NULL
CHECK (relationship IN ('referenced','updated','should_have_referenced')),
source TEXT NOT NULL
CHECK (source IN ('analyzer_referenced','audit_write','manual')),
confidence TEXT
CHECK (confidence IS NULL OR confidence IN ('high','medium','low')),
details JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS ix_xrefs_ticket
ON itglue_ticket_xrefs (ticket_number, created_at DESC);
CREATE INDEX IF NOT EXISTS ix_xrefs_asset
ON itglue_ticket_xrefs (asset_type, asset_id, created_at DESC);
CREATE INDEX IF NOT EXISTS ix_xrefs_rel
ON itglue_ticket_xrefs (relationship);
-- Dedup: same logical xref (ticket, analysis, asset, relationship, source)
-- inserted once. analysis_id is nullable, so coalesce to a sentinel UUID for
-- the unique-index purposes.
CREATE UNIQUE INDEX IF NOT EXISTS ux_xrefs_unique
ON itglue_ticket_xrefs (
ticket_number,
COALESCE(analysis_id, '00000000-0000-0000-0000-000000000000'::uuid),
asset_type,
asset_id,
relationship,
source
);
COMMENT ON TABLE itglue_ticket_xrefs IS
'Ticket↔IT-Glue-asset linkage. relationship=referenced (analyzer cited the doc), '
'updated (a ticket-driven audit produced a write), should_have_referenced '
'(gap text indicates the asset should exist or be tagged). Powers asset-side '
'"tickets that touched me" and ticket-side "docs this ticket touched" views.';

88
migrations/077_rmm_overshell.sql Normal file
View file

@ -0,0 +1,88 @@
-- Phase 4.2 — Datto RMM Overshell evidence pipeline.
--
-- Two tables:
-- 1. rmm_settings — singleton row caching the discovered Overshell
-- component_uid + variable_name. Avoids re-discovering on every dispatch.
-- 2. rmm_executions — full lifecycle of every Overshell job Pulse triggers.
-- Forever-retained: stdout/stderr (redacted), parsed evidence, audit
-- linkage, target/asset linkage.
CREATE TABLE IF NOT EXISTS rmm_settings (
id BOOLEAN PRIMARY KEY DEFAULT true CHECK (id),
overshell_component_uid TEXT,
overshell_component_name TEXT,
-- The variable name the Overshell component expects the script body in.
-- Default 'CommandLine' matches the Datto-provided "Run Command" component;
-- Wulf's custom Overshell may use a different name (e.g. 'Script') —
-- adjustable via /admin/rmm-overshell.
overshell_variable_name TEXT NOT NULL DEFAULT 'CommandLine',
discovered_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO rmm_settings (id) VALUES (true) ON CONFLICT DO NOTHING;
CREATE TABLE IF NOT EXISTS rmm_executions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-- Script library identifiers (script bodies are version-controlled in code).
script_id TEXT NOT NULL,
script_version INT NOT NULL DEFAULT 1,
-- Target resolution.
target_type TEXT NOT NULL
CHECK (target_type IN ('site_anchor','asset_self')),
target_device_uid TEXT NOT NULL,
target_hostname TEXT,
target_company_id BIGINT,
-- Optional links back to the audit / asset the user clicked from.
triggered_by_audit_id UUID REFERENCES itglue_asset_audits(id) ON DELETE SET NULL,
asset_type TEXT
CHECK (asset_type IS NULL OR asset_type IN ('flexible_asset','configuration')),
asset_id BIGINT,
-- Datto job tracking.
job_uid TEXT,
job_name TEXT NOT NULL,
variables JSONB NOT NULL,
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued','running','complete','failed','timeout')),
exit_code INT,
raw_stdout TEXT,
raw_stderr TEXT,
parsed_evidence JSONB,
parse_error TEXT,
error_message TEXT,
performed_by_user_id TEXT REFERENCES "user"(id) ON DELETE SET NULL,
queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
-- Hard timeout marker. The poller will mark a still-running execution as
-- timeout once NOW() > timeout_at and stop polling.
timeout_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + INTERVAL '5 minutes'
);
CREATE INDEX IF NOT EXISTS ix_rmm_executions_status
ON rmm_executions (status, queued_at);
CREATE INDEX IF NOT EXISTS ix_rmm_executions_target_device
ON rmm_executions (target_device_uid, queued_at DESC);
CREATE INDEX IF NOT EXISTS ix_rmm_executions_company
ON rmm_executions (target_company_id, queued_at DESC);
CREATE INDEX IF NOT EXISTS ix_rmm_executions_audit
ON rmm_executions (triggered_by_audit_id);
CREATE INDEX IF NOT EXISTS ix_rmm_executions_asset
ON rmm_executions (asset_type, asset_id, queued_at DESC)
WHERE asset_type IS NOT NULL;
-- The audit-context loader pulls the latest successful execution per
-- (company, script). Partial index keeps the lookup fast even as the table
-- grows.
CREATE INDEX IF NOT EXISTS ix_rmm_executions_latest_success
ON rmm_executions (target_company_id, script_id, completed_at DESC)
WHERE status = 'complete';
CREATE INDEX IF NOT EXISTS ix_rmm_executions_user_window
ON rmm_executions (performed_by_user_id, queued_at)
WHERE performed_by_user_id IS NOT NULL;
COMMENT ON TABLE rmm_executions IS
'Every Overshell job Pulse dispatched to Datto RMM. status: queued (we '
'inserted the row but have not yet called runQuickJob), running (job_uid '
'returned by Datto), complete (poller saw stdout), failed (exit_code != 0 '
'or HTTP failure), timeout (still running past timeout_at).';

43
migrations/078_loglift_uploads.sql Normal file
View file

@ -0,0 +1,43 @@
-- Phase 4.3 — LogLift event-log ingestion pipeline.
--
-- Extends rmm_executions to support B2-uploaded payloads (separate transport
-- from Overshell stdout) and rmm_settings to cache the LogLift component uid.
--
-- transport='overshell_stdout' (default for existing rows) — payload comes
-- back via Datto getJobResults stdout. The Phase 4.2 path.
-- transport='b2_upload' — collector script uploads gzipped JSON to B2 and
-- POSTs a webhook; Pulse downloads + parses asynchronously.
ALTER TABLE rmm_executions
ADD COLUMN IF NOT EXISTS transport TEXT NOT NULL DEFAULT 'overshell_stdout'
CHECK (transport IN ('overshell_stdout','b2_upload')),
ADD COLUMN IF NOT EXISTS evidence_object_key TEXT,
ADD COLUMN IF NOT EXISTS run_id TEXT;
-- Used to correlate inbound webhooks to a Pulse-dispatched execution.
CREATE UNIQUE INDEX IF NOT EXISTS ux_rmm_executions_run_id
ON rmm_executions (run_id) WHERE run_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_rmm_executions_object_key
ON rmm_executions (evidence_object_key) WHERE evidence_object_key IS NOT NULL;
ALTER TABLE rmm_settings
ADD COLUMN IF NOT EXISTS loglift_component_uid TEXT,
ADD COLUMN IF NOT EXISTS loglift_component_name TEXT,
ADD COLUMN IF NOT EXISTS loglift_discovered_at TIMESTAMPTZ;
COMMENT ON COLUMN rmm_executions.transport IS
'overshell_stdout (default) — Phase 4.2 path; payload arrives in Datto job stdout. '
'b2_upload — Phase 4.3 LogLift; collector uploads gzipped JSON to Backblaze B2 '
'and POSTs a webhook. Pulse downloads + parses async.';
COMMENT ON COLUMN rmm_executions.evidence_object_key IS
'B2 object key for transport=b2_upload rows. Format: '
'{datto_site_uid}/{computer_name}/eventlogs_{timestamp}.json.gz. '
'Full payload stays in B2 (encrypted, signed-URL access only); '
'parsed_evidence holds a slimmed view.';
COMMENT ON COLUMN rmm_executions.run_id IS
'Collector-supplied correlation token. Used by the receiver to match an '
'inbound webhook to a Pulse-dispatched execution. NULL for out-of-band '
'collector runs.';

303
migrations/079_endpoint_data_model.sql Normal file
View file

@ -0,0 +1,303 @@
-- Endpoint data model — anchor on configuration_items, sidecar everything else.
--
-- Adds three tables:
-- 1. device_external_ids — cross-reference: one row per (tool, device-in-tool).
-- 2. device_observations — time-series script/telemetry results (JSONB),
-- with optional B2 pointer for blobs that don't belong inline.
-- 3. endpoint_audits — LLM-produced audits anchored to a configuration_item.
-- Replaces itglue_asset_audits rows where asset_type='configuration'.
-- Flexible-asset audits stay in itglue_asset_audits (different lifecycle).
--
-- Backfill strategy is conservative: write source rows for every tool, but
-- only auto-link to a configuration_item when the match is deterministic
-- (Autotask's own rmm_device_uid for Datto). Hostname/serial fuzzy linking
-- is the reconciliation cron's job — it logs conflicts instead of merging.
-- ---------------------------------------------------------------------------
-- 1. device_external_ids
-- ---------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS device_external_ids (
id BIGSERIAL PRIMARY KEY,
configuration_item_id BIGINT NULL REFERENCES configuration_items(id) ON DELETE SET NULL,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
hostname TEXT,
serial TEXT,
mac TEXT,
-- No FK on company_id: configuration_items already tolerates dangling
-- company_ids (Autotask retains CIs after company hard-deletes), and the
-- xref shouldn't be stricter than its anchor.
company_id BIGINT NULL,
last_seen_at TIMESTAMPTZ,
link_confidence TEXT,
linked_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT device_external_ids_source_check
CHECK (source IN ('autotask','datto_rmm','itglue','s1','veeam','addigy','auvik')),
CONSTRAINT device_external_ids_link_confidence_check
CHECK (link_confidence IS NULL OR link_confidence IN
('canonical','exact_uid','exact_serial','hostname_in_company','mac','manual')),
CONSTRAINT device_external_ids_unique UNIQUE (source, source_id)
);
CREATE INDEX IF NOT EXISTS ix_device_external_ids_ci
ON device_external_ids(configuration_item_id) WHERE configuration_item_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_device_external_ids_unlinked
ON device_external_ids(source, last_seen_at) WHERE configuration_item_id IS NULL;
CREATE INDEX IF NOT EXISTS ix_device_external_ids_hostname
ON device_external_ids(LOWER(hostname)) WHERE hostname IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_device_external_ids_serial
ON device_external_ids(serial) WHERE serial IS NOT NULL;
COMMENT ON TABLE device_external_ids IS
'Cross-reference from per-tool device IDs to a canonical Autotask configuration_item. configuration_item_id NULL = seen in tool X but no Autotask CI matched yet; reconciliation cron tries to link.';
-- ---------------------------------------------------------------------------
-- 2. device_observations
-- ---------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS device_observations (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
configuration_item_id BIGINT NULL REFERENCES configuration_items(id) ON DELETE SET NULL,
source TEXT NOT NULL,
kind TEXT NOT NULL,
collected_at TIMESTAMPTZ NOT NULL,
payload JSONB NOT NULL,
evidence_object_key TEXT,
run_id TEXT,
supersedes_id UUID NULL REFERENCES device_observations(id) ON DELETE SET NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT device_observations_source_check
CHECK (source IN ('autotask','datto_rmm','itglue','s1','veeam','addigy','auvik','loglift','overshell','manual'))
);
CREATE INDEX IF NOT EXISTS ix_device_observations_ci_kind_time
ON device_observations(configuration_item_id, kind, collected_at DESC)
WHERE configuration_item_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_device_observations_kind_time
ON device_observations(kind, collected_at DESC);
CREATE INDEX IF NOT EXISTS ix_device_observations_run_id
ON device_observations(run_id) WHERE run_id IS NOT NULL;
COMMENT ON TABLE device_observations IS
'Time-series telemetry / script results per endpoint. Payload is JSONB for direct querying; large blobs (raw event-log gzips, screenshots) live in B2 referenced by evidence_object_key.';
-- ---------------------------------------------------------------------------
-- 3. endpoint_audits
-- ---------------------------------------------------------------------------
-- An audit anchors on an Autotask configuration_item when one is known, but
-- IT Glue is also a valid anchor for devices we track there but not in
-- Autotask (orphans, IT-Glue-first onboarding). At least one must be set —
-- the CHECK below enforces it.
CREATE TABLE IF NOT EXISTS endpoint_audits (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
configuration_item_id BIGINT NULL REFERENCES configuration_items(id) ON DELETE SET NULL,
itglue_configuration_id BIGINT NULL,
organization_id BIGINT NULL,
generated_by_user_id TEXT NULL REFERENCES "user"(id) ON DELETE SET NULL,
generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
provider TEXT NOT NULL DEFAULT 'anthropic',
model_used TEXT,
asset_snapshot JSONB NOT NULL,
observations_consumed UUID[] NOT NULL DEFAULT '{}',
ticket_count INTEGER NOT NULL DEFAULT 0,
field_gaps JSONB NOT NULL DEFAULT '[]'::jsonb,
notes_promotions JSONB NOT NULL DEFAULT '[]'::jsonb,
contradictions JSONB NOT NULL DEFAULT '[]'::jsonb,
overall_score NUMERIC(3,2),
estimated_cost_usd NUMERIC(10,4),
total_input_tokens INTEGER,
total_output_tokens INTEGER,
status TEXT NOT NULL DEFAULT 'complete',
error_message TEXT,
triggered_by_ticket_number TEXT,
triggered_by_analysis_id UUID NULL REFERENCES analyzer_analyses(id) ON DELETE SET NULL,
triggered_by_observation_id UUID NULL REFERENCES device_observations(id) ON DELETE SET NULL,
legacy_itglue_audit_id UUID NULL,
CONSTRAINT endpoint_audits_provider_check
CHECK (provider IN ('anthropic','openrouter')),
CONSTRAINT endpoint_audits_status_check
CHECK (status IN ('pending','running','complete','failed')),
CONSTRAINT endpoint_audits_anchor_check
CHECK (configuration_item_id IS NOT NULL OR itglue_configuration_id IS NOT NULL)
);
-- Idempotent guard for re-runs against an already-created table.
ALTER TABLE endpoint_audits ADD COLUMN IF NOT EXISTS itglue_configuration_id BIGINT NULL;
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint WHERE conname = 'endpoint_audits_anchor_check'
) THEN
ALTER TABLE endpoint_audits
ADD CONSTRAINT endpoint_audits_anchor_check
CHECK (configuration_item_id IS NOT NULL OR itglue_configuration_id IS NOT NULL);
END IF;
END $$;
CREATE INDEX IF NOT EXISTS ix_endpoint_audits_ci_time
ON endpoint_audits(configuration_item_id, generated_at DESC) WHERE configuration_item_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_endpoint_audits_org_time
ON endpoint_audits(organization_id, generated_at DESC) WHERE organization_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_endpoint_audits_ticket
ON endpoint_audits(triggered_by_ticket_number) WHERE triggered_by_ticket_number IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_endpoint_audits_legacy
ON endpoint_audits(legacy_itglue_audit_id) WHERE legacy_itglue_audit_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS ix_endpoint_audits_itglue_config
ON endpoint_audits(itglue_configuration_id, generated_at DESC) WHERE itglue_configuration_id IS NOT NULL;
COMMENT ON TABLE endpoint_audits IS
'LLM-produced audits anchored to a configuration_item. The configuration variant of itglue_asset_audits — flexible-asset audits stay in itglue_asset_audits since their lifecycle is different.';
COMMENT ON COLUMN endpoint_audits.legacy_itglue_audit_id IS
'Backfill pointer to the original itglue_asset_audits row this was migrated from. Drop after cutover is complete.';
-- ---------------------------------------------------------------------------
-- Backfill: device_external_ids
-- ---------------------------------------------------------------------------
-- Autotask: every CI is its own canonical row (the anchor).
INSERT INTO device_external_ids
(configuration_item_id, source, source_id, hostname, serial, mac, company_id,
last_seen_at, link_confidence, linked_at)
SELECT
ci.id,
'autotask',
ci.id::text,
ci.reference_title,
ci.serial_number,
ci.rmm_device_audit_mac_address,
ci.company_id,
ci.synced_at AT TIME ZONE 'UTC',
'canonical',
NOW()
FROM configuration_items ci
WHERE NOT EXISTS (
SELECT 1 FROM device_external_ids dx
WHERE dx.source = 'autotask' AND dx.source_id = ci.id::text
)
ON CONFLICT (source, source_id) DO NOTHING;
-- Datto RMM: link via Autotask's stored rmm_device_uid (deterministic).
-- Rows that don't match stay unlinked; reconciliation cron will retry by hostname.
INSERT INTO device_external_ids
(configuration_item_id, source, source_id, hostname, serial, mac, company_id,
last_seen_at, link_confidence, linked_at)
SELECT
ci.id,
'datto_rmm',
drd.uid,
drd.hostname,
NULL,
NULL,
ci.company_id,
drd.last_seen,
CASE WHEN ci.id IS NOT NULL THEN 'exact_uid' ELSE NULL END,
CASE WHEN ci.id IS NOT NULL THEN NOW() ELSE NULL END
FROM datto_rmm_devices drd
LEFT JOIN configuration_items ci ON ci.rmm_device_uid = drd.uid
ON CONFLICT (source, source_id) DO NOTHING;
-- IT Glue: stays unlinked initially. Reconciliation matches by serial → hostname.
INSERT INTO device_external_ids
(source, source_id, hostname, serial, mac, last_seen_at)
SELECT
'itglue',
igc.id::text,
COALESCE(igc.hostname, igc.name),
igc.serial_number,
igc.mac_address,
igc.synced_at
FROM itg_configurations igc
ON CONFLICT (source, source_id) DO NOTHING;
-- SentinelOne: stays unlinked initially. Reconciliation matches by computer_name + company.
INSERT INTO device_external_ids
(source, source_id, hostname, last_seen_at)
SELECT
's1',
sa.id::text,
sa.computer_name,
sa.last_active_date AT TIME ZONE 'UTC'
FROM s1_agents sa
ON CONFLICT (source, source_id) DO NOTHING;
-- Veeam: stays unlinked initially. Reconciliation matches by hostname (the
-- agent's `name` field). No serial / MAC available.
INSERT INTO device_external_ids
(source, source_id, hostname, last_seen_at)
SELECT
'veeam',
vba.instance_uid,
vba.name,
vba.synced_at
FROM veeam_backup_agents vba
ON CONFLICT (source, source_id) DO NOTHING;
-- ---------------------------------------------------------------------------
-- Backfill: endpoint_audits from itglue_asset_audits (configuration only)
-- ---------------------------------------------------------------------------
-- We snapshot the configuration audits into endpoint_audits but keep the
-- originals in itglue_asset_audits until the receiver flips. legacy_itglue_audit_id
-- preserves the link both ways during the dual-source window.
-- itglue_asset_audits.asset_id (when asset_type='configuration') is an IT Glue
-- Configuration ID, not an Autotask CI. We anchor primarily on IT Glue here
-- and let the reconciler later fill in configuration_item_id via the xref.
INSERT INTO endpoint_audits (
configuration_item_id, itglue_configuration_id,
organization_id, generated_by_user_id, generated_at,
provider, model_used, asset_snapshot, ticket_count,
field_gaps, notes_promotions, contradictions,
overall_score, estimated_cost_usd, total_input_tokens, total_output_tokens,
status, error_message, triggered_by_ticket_number, triggered_by_analysis_id,
legacy_itglue_audit_id
)
SELECT
dx.configuration_item_id,
ia.asset_id,
ia.organization_id,
ia.generated_by_user_id,
ia.generated_at,
ia.provider,
ia.model_used,
ia.asset_snapshot,
ia.ticket_count,
ia.field_gaps,
ia.notes_promotions,
ia.contradictions,
ia.overall_score,
ia.estimated_cost_usd,
ia.total_input_tokens,
ia.total_output_tokens,
ia.status,
ia.error_message,
ia.triggered_by_ticket_number,
ia.triggered_by_analysis_id,
ia.id
FROM itglue_asset_audits ia
LEFT JOIN device_external_ids dx
ON dx.source = 'itglue' AND dx.source_id = ia.asset_id::text
WHERE ia.asset_type = 'configuration'
AND NOT EXISTS (
SELECT 1 FROM endpoint_audits ea WHERE ea.legacy_itglue_audit_id = ia.id
);
-- ---------------------------------------------------------------------------
-- updated_at trigger for device_external_ids
-- ---------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION device_external_ids_set_updated_at() RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_device_external_ids_updated_at ON device_external_ids;
CREATE TRIGGER trg_device_external_ids_updated_at
BEFORE UPDATE ON device_external_ids
FOR EACH ROW EXECUTE FUNCTION device_external_ids_set_updated_at();

86
migrations/080_device_xref_company_id.sql Normal file
View file

@ -0,0 +1,86 @@
-- Backfill device_external_ids.company_id for sources that have a known
-- per-tool → Autotask-company mapping table. Populating company_id unlocks
-- the reconciler's hostname-in-company strategy for s1 + veeam + datto_rmm
-- rows that had no serial.
--
-- Mappings used:
-- datto_rmm → datto_rmm_devices.site_uid → rmm_site_mappings.company_id
-- s1 → s1_agents.site_id → s1_company_mappings.company_id
-- itglue → itg_configurations.organization_id → itg_organizations.psa_id (cast)
-- where psa_integration = 'autotask'
-- veeam → no clean mapping today; left null.
--
-- Also adds device_link_review to capture reconciler conflicts (multiple
-- candidate CIs) for admin resolution.
-- ---------------------------------------------------------------------------
-- 1. company_id backfill — datto_rmm
-- ---------------------------------------------------------------------------
UPDATE device_external_ids dx
SET company_id = m.company_id
FROM datto_rmm_devices d
JOIN rmm_site_mappings m ON m.rmm_site_uid = d.site_uid
WHERE dx.source = 'datto_rmm'
AND dx.source_id = d.uid
AND dx.company_id IS NULL;
-- ---------------------------------------------------------------------------
-- 2. company_id backfill — s1
-- ---------------------------------------------------------------------------
UPDATE device_external_ids dx
SET company_id = m.company_id
FROM s1_agents sa
JOIN s1_company_mappings m ON m.s1_site_id = sa.site_id
WHERE dx.source = 's1'
AND dx.source_id = sa.id::text
AND dx.company_id IS NULL;
-- ---------------------------------------------------------------------------
-- 3. company_id backfill — itglue
-- ---------------------------------------------------------------------------
-- itg_organizations.psa_id is unpopulated in this environment. Fall back to
-- fuzzy name match against companies — the same approach asset-matcher.ts
-- and itglue-redact.ts use elsewhere in the codebase. Imperfect but consistent.
UPDATE device_external_ids dx
SET company_id = c.id
FROM itg_configurations igc
JOIN itg_organizations ig ON ig.id = igc.organization_id
JOIN companies c ON LOWER(c.company_name) = LOWER(ig.name)
WHERE dx.source = 'itglue'
AND dx.source_id = igc.id::text
AND dx.company_id IS NULL;
-- ---------------------------------------------------------------------------
-- 4. device_link_review — conflicts queue for admin resolution
-- ---------------------------------------------------------------------------
-- One row per (xref-row, batch) when the reconciler finds 2+ CI candidates.
-- Resolving = the admin picks one CI; the reconciler's match logic doesn't
-- get to decide on its own.
CREATE TABLE IF NOT EXISTS device_link_review (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
device_external_id BIGINT NOT NULL REFERENCES device_external_ids(id) ON DELETE CASCADE,
candidate_ci_ids BIGINT[] NOT NULL,
match_confidences TEXT[] NOT NULL,
detected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
resolved_at TIMESTAMPTZ,
resolved_by_user_id TEXT NULL REFERENCES "user"(id) ON DELETE SET NULL,
resolved_to_ci_id BIGINT NULL REFERENCES configuration_items(id) ON DELETE SET NULL,
resolution_note TEXT
);
CREATE INDEX IF NOT EXISTS ix_device_link_review_unresolved
ON device_link_review(detected_at DESC) WHERE resolved_at IS NULL;
CREATE INDEX IF NOT EXISTS ix_device_link_review_xref
ON device_link_review(device_external_id);
-- Don't keep stale rows: when the underlying xref row gets linked some other
-- way (admin manually fixes the source-of-truth), a unique index would block
-- new reviews. Instead we use a partial index for "open reviews per xref".
CREATE UNIQUE INDEX IF NOT EXISTS uq_device_link_review_open_per_xref
ON device_link_review(device_external_id) WHERE resolved_at IS NULL;
COMMENT ON TABLE device_link_review IS
'Reconciler conflicts queue: one row per unlinked device_external_ids row that matches 2+ configuration_items. Admin picks the right CI; reconciler does not auto-merge.';