377 lines
14 KiB
TypeScript
377 lines
14 KiB
TypeScript
|
/**
|
||
|
|
* AppGate sync orchestrator.
|
||
|
|
*
|
||
|
|
* Two modes:
|
||
|
|
*
|
||
|
|
* • sessions — pulls /stats/active-sessions/dn only, replaces the table.
|
||
|
|
* Cheap; safe to run every few minutes.
|
||
|
|
*
|
||
|
|
* • daily — pulls everything else (devices, appliances, license,
|
||
|
|
* license users, 24h login histogram), upserts, and tombstones
|
||
|
|
* devices/appliances QBO no longer returns (same pattern as
|
||
|
|
* qbo_invoices).
|
||
|
|
*
|
||
|
|
* Sessions are stored as a *snapshot* — Pulse truncates the table and
|
||
|
|
* re-inserts the current set. Devices and appliances persist with a
|
||
|
|
* soft-delete flag. Login totals are written into a daily series.
|
||
|
|
*/
|
||
|
|
|
||
|
|
import postgresClient from './postgres-client';
|
||
|
|
import { AppgateClient } from './appgate-client';
|
||
|
|
import { getAppgateClient } from './appgate-factory';
|
||
|
|
import type {
|
||
|
|
AppgateActiveSession,
|
||
|
|
AppgateAppliance,
|
||
|
|
AppgateHourlyLogins,
|
||
|
|
AppgateOnBoardedDevice,
|
||
|
|
AppgateSyncResult,
|
||
|
|
} from '@/lib/types/appgate';
|
||
|
|
|
||
|
|
export class AppgateSyncService {
|
||
|
|
private client: AppgateClient;
|
||
|
|
private syncing = false;
|
||
|
|
|
||
|
|
constructor(client?: AppgateClient) {
|
||
|
|
this.client = client ?? getAppgateClient();
|
||
|
|
}
|
||
|
|
|
||
|
|
isSyncInProgress(): boolean {
|
||
|
|
return this.syncing;
|
||
|
|
}
|
||
|
|
|
||
|
|
async sessionsSync(triggeredBy = 'system'): Promise<AppgateSyncResult> {
|
||
|
|
return this.run('sessions', triggeredBy);
|
||
|
|
}
|
||
|
|
|
||
|
|
async dailySync(triggeredBy = 'system'): Promise<AppgateSyncResult> {
|
||
|
|
return this.run('daily', triggeredBy);
|
||
|
|
}
|
||
|
|
|
||
|
|
private emptyResult(syncId: string, syncType: 'sessions' | 'daily' | 'full', startedAt: Date): AppgateSyncResult {
|
||
|
|
return {
|
||
|
|
syncId,
|
||
|
|
syncType,
|
||
|
|
startedAt,
|
||
|
|
completedAt: new Date(),
|
||
|
|
durationMs: 0,
|
||
|
|
status: 'completed',
|
||
|
|
sessions: 0,
|
||
|
|
devices: { upserted: 0, tombstoned: 0 },
|
||
|
|
appliances: { upserted: 0, tombstoned: 0 },
|
||
|
|
licenseUsers: 0,
|
||
|
|
loginsCaptured: 0,
|
||
|
|
errors: [],
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
private async run(syncType: 'sessions' | 'daily', triggeredBy: string): Promise<AppgateSyncResult> {
|
||
|
|
if (this.syncing) throw new Error('AppGate sync already in progress');
|
||
|
|
this.syncing = true;
|
||
|
|
const syncId = `appgate_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||
|
|
const startedAt = new Date();
|
||
|
|
const result = this.emptyResult(syncId, syncType, startedAt);
|
||
|
|
|
||
|
|
console.log(`[AppgateSync] Starting ${syncType} sync (${syncId}) — triggered by ${triggeredBy}`);
|
||
|
|
|
||
|
|
try {
|
||
|
|
result.sessions = await this.syncSessions();
|
||
|
|
if (syncType === 'daily') {
|
||
|
|
result.devices = await this.syncDevices();
|
||
|
|
result.appliances = await this.syncAppliances();
|
||
|
|
result.licenseUsers = await this.syncLicense();
|
||
|
|
result.loginsCaptured = await this.syncUserLogins();
|
||
|
|
}
|
||
|
|
} catch (err) {
|
||
|
|
const msg = err instanceof Error ? err.message : String(err);
|
||
|
|
result.errors.push(msg);
|
||
|
|
result.status = 'failed';
|
||
|
|
console.error(`[AppgateSync] ${syncType} sync failed:`, msg);
|
||
|
|
} finally {
|
||
|
|
result.completedAt = new Date();
|
||
|
|
result.durationMs = result.completedAt.getTime() - startedAt.getTime();
|
||
|
|
await this.persistHistory(result, triggeredBy);
|
||
|
|
this.syncing = false;
|
||
|
|
}
|
||
|
|
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── Per-entity syncs ─────────────────────────────────────────────────
|
||
|
|
|
||
|
|
private async syncSessions(): Promise<number> {
|
||
|
|
const sessions = await this.client.getActiveSessions();
|
||
|
|
await postgresClient.transaction(async (tx) => {
|
||
|
|
await tx.query('TRUNCATE appgate_active_sessions');
|
||
|
|
for (const s of sessions) {
|
||
|
|
await tx.query(
|
||
|
|
`INSERT INTO appgate_active_sessions
|
||
|
|
(distinguished_name, device_id, username, provider_name,
|
||
|
|
hostname, os_family, os_name, os_parent,
|
||
|
|
client_version, client_type, client_support,
|
||
|
|
geo_ip_latitude, geo_ip_longitude, gateways)
|
||
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`,
|
||
|
|
[
|
||
|
|
s.distinguishedName,
|
||
|
|
s.deviceId ?? null,
|
||
|
|
s.username ?? null,
|
||
|
|
s.providerName ?? null,
|
||
|
|
s.hostname ?? null,
|
||
|
|
s.osFamily ?? null,
|
||
|
|
s.osName ?? null,
|
||
|
|
s.osParent ?? null,
|
||
|
|
s.clientVersion ?? null,
|
||
|
|
s.clientType ?? null,
|
||
|
|
s.clientSupport ?? null,
|
||
|
|
s.geoIpLatitude ?? null,
|
||
|
|
s.geoIpLongitude ?? null,
|
||
|
|
JSON.stringify(s.gateways ?? []),
|
||
|
|
],
|
||
|
|
);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
return sessions.length;
|
||
|
|
}
|
||
|
|
|
||
|
|
private async syncDevices(): Promise<{ upserted: number; tombstoned: number }> {
|
||
|
|
const devices = await this.client.getOnBoardedDevices();
|
||
|
|
let upserted = 0;
|
||
|
|
const seen: string[] = [];
|
||
|
|
for (const d of devices) {
|
||
|
|
if (!d.distinguishedName) continue;
|
||
|
|
seen.push(d.distinguishedName);
|
||
|
|
await postgresClient.query(
|
||
|
|
`INSERT INTO appgate_devices
|
||
|
|
(distinguished_name, device_id, username, provider_name,
|
||
|
|
device_type, hostname, on_boarded_at, last_seen_at,
|
||
|
|
synced_at, is_deleted, deleted_at)
|
||
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8, NOW(), false, NULL)
|
||
|
|
ON CONFLICT (distinguished_name) DO UPDATE SET
|
||
|
|
device_id = EXCLUDED.device_id,
|
||
|
|
username = EXCLUDED.username,
|
||
|
|
provider_name = EXCLUDED.provider_name,
|
||
|
|
device_type = EXCLUDED.device_type,
|
||
|
|
hostname = EXCLUDED.hostname,
|
||
|
|
on_boarded_at = EXCLUDED.on_boarded_at,
|
||
|
|
last_seen_at = EXCLUDED.last_seen_at,
|
||
|
|
synced_at = NOW(),
|
||
|
|
is_deleted = false,
|
||
|
|
deleted_at = NULL`,
|
||
|
|
[
|
||
|
|
d.distinguishedName,
|
||
|
|
d.deviceId ?? null,
|
||
|
|
d.username ?? null,
|
||
|
|
d.providerName ?? null,
|
||
|
|
(d as AppgateOnBoardedDevice & { deviceType?: string }).device_type
|
||
|
|
?? (d as AppgateOnBoardedDevice & { deviceType?: string }).deviceType
|
||
|
|
?? null,
|
||
|
|
d.hostname ?? null,
|
||
|
|
d.onBoardedAt ?? null,
|
||
|
|
d.lastSeenAt ?? null,
|
||
|
|
],
|
||
|
|
);
|
||
|
|
upserted++;
|
||
|
|
}
|
||
|
|
const tomb = seen.length === 0
|
||
|
|
? 0
|
||
|
|
: (await postgresClient.query(
|
||
|
|
`UPDATE appgate_devices SET is_deleted=true, deleted_at=NOW()
|
||
|
|
WHERE is_deleted=false AND distinguished_name <> ALL($1::text[])`,
|
||
|
|
[seen],
|
||
|
|
)).rowCount ?? 0;
|
||
|
|
if (tomb > 0) console.log(`[AppgateSync] Tombstoned ${tomb} device(s)`);
|
||
|
|
return { upserted, tombstoned: tomb };
|
||
|
|
}
|
||
|
|
|
||
|
|
private async syncAppliances(): Promise<{ upserted: number; tombstoned: number }> {
|
||
|
|
const items = await this.client.getAppliances();
|
||
|
|
let upserted = 0;
|
||
|
|
const seen: string[] = [];
|
||
|
|
for (const a of items) {
|
||
|
|
if (!a.id) continue;
|
||
|
|
seen.push(a.id);
|
||
|
|
const roles = {
|
||
|
|
controller: Boolean(a.controller?.enabled),
|
||
|
|
gateway: Boolean(a.gateway?.enabled),
|
||
|
|
logServer: Boolean(a.logServer?.enabled),
|
||
|
|
logForwarder: Boolean(a.logForwarder?.enabled),
|
||
|
|
metricsAggregator: Boolean(a.metricsAggregator?.enabled),
|
||
|
|
connector: Boolean(a.connector?.enabled),
|
||
|
|
portal: Boolean(a.portal?.enabled),
|
||
|
|
};
|
||
|
|
await postgresClient.query(
|
||
|
|
`INSERT INTO appgate_appliances
|
||
|
|
(id, name, hostname, notes, version, site, site_name,
|
||
|
|
activated, pending_certificate_renewal, tags, roles, raw,
|
||
|
|
created_at, updated_at, synced_at, is_deleted, deleted_at)
|
||
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14, NOW(), false, NULL)
|
||
|
|
ON CONFLICT (id) DO UPDATE SET
|
||
|
|
name = EXCLUDED.name,
|
||
|
|
hostname = EXCLUDED.hostname,
|
||
|
|
notes = EXCLUDED.notes,
|
||
|
|
version = EXCLUDED.version,
|
||
|
|
site = EXCLUDED.site,
|
||
|
|
site_name = EXCLUDED.site_name,
|
||
|
|
activated = EXCLUDED.activated,
|
||
|
|
pending_certificate_renewal = EXCLUDED.pending_certificate_renewal,
|
||
|
|
tags = EXCLUDED.tags,
|
||
|
|
roles = EXCLUDED.roles,
|
||
|
|
raw = EXCLUDED.raw,
|
||
|
|
created_at = EXCLUDED.created_at,
|
||
|
|
updated_at = EXCLUDED.updated_at,
|
||
|
|
synced_at = NOW(),
|
||
|
|
is_deleted = false,
|
||
|
|
deleted_at = NULL`,
|
||
|
|
[
|
||
|
|
a.id, a.name, a.hostname ?? null, a.notes ?? null,
|
||
|
|
a.version ?? null, a.site ?? null, a.siteName ?? null,
|
||
|
|
a.activated ?? null, a.pendingCertificateRenewal ?? null,
|
||
|
|
JSON.stringify(a.tags ?? []), JSON.stringify(roles),
|
||
|
|
JSON.stringify(a),
|
||
|
|
a.created ?? null, a.updated ?? null,
|
||
|
|
],
|
||
|
|
);
|
||
|
|
upserted++;
|
||
|
|
}
|
||
|
|
const tomb = seen.length === 0
|
||
|
|
? 0
|
||
|
|
: (await postgresClient.query(
|
||
|
|
`UPDATE appgate_appliances SET is_deleted=true, deleted_at=NOW()
|
||
|
|
WHERE is_deleted=false AND id <> ALL($1::uuid[])`,
|
||
|
|
[seen],
|
||
|
|
)).rowCount ?? 0;
|
||
|
|
if (tomb > 0) console.log(`[AppgateSync] Tombstoned ${tomb} appliance(s)`);
|
||
|
|
return { upserted, tombstoned: tomb };
|
||
|
|
}
|
||
|
|
|
||
|
|
private async syncLicense(): Promise<number> {
|
||
|
|
const [license, users] = await Promise.all([
|
||
|
|
this.client.getLicense().catch(() => null),
|
||
|
|
this.client.getLicenseUsers().catch(() => [] as Awaited<ReturnType<AppgateClient['getLicenseUsers']>>),
|
||
|
|
]);
|
||
|
|
|
||
|
|
// Snapshot capacity / today's used count.
|
||
|
|
if (license) {
|
||
|
|
await postgresClient.query(
|
||
|
|
`INSERT INTO appgate_license_snapshots
|
||
|
|
(snapshot_date, max_users, max_portal_users, max_service_users,
|
||
|
|
used_users, expiration, license_type, license_version,
|
||
|
|
risk_engine_enabled, app_discovery_enabled)
|
||
|
|
VALUES (CURRENT_DATE, $1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||
|
|
ON CONFLICT (snapshot_date) DO UPDATE SET
|
||
|
|
max_users = EXCLUDED.max_users,
|
||
|
|
max_portal_users = EXCLUDED.max_portal_users,
|
||
|
|
max_service_users = EXCLUDED.max_service_users,
|
||
|
|
used_users = EXCLUDED.used_users,
|
||
|
|
expiration = EXCLUDED.expiration,
|
||
|
|
license_type = EXCLUDED.license_type,
|
||
|
|
license_version = EXCLUDED.license_version,
|
||
|
|
risk_engine_enabled = EXCLUDED.risk_engine_enabled,
|
||
|
|
app_discovery_enabled = EXCLUDED.app_discovery_enabled,
|
||
|
|
captured_at = NOW()`,
|
||
|
|
[
|
||
|
|
license.maxUsers ?? null,
|
||
|
|
license.maxPortalUsers ?? null,
|
||
|
|
license.maxServiceUsers ?? null,
|
||
|
|
users.length,
|
||
|
|
license.expiration ?? null,
|
||
|
|
license.type ?? null,
|
||
|
|
license.version ?? null,
|
||
|
|
license.riskEngine ?? null,
|
||
|
|
license.applicationDiscovery ?? null,
|
||
|
|
],
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Per-user license consumption — replace.
|
||
|
|
await postgresClient.transaction(async (tx) => {
|
||
|
|
await tx.query('TRUNCATE appgate_license_users');
|
||
|
|
for (const u of users) {
|
||
|
|
if (!u.userDistinguishedName) continue;
|
||
|
|
await tx.query(
|
||
|
|
`INSERT INTO appgate_license_users
|
||
|
|
(user_distinguished_name, username, provider_name,
|
||
|
|
license_type, profile_name, created_at, last_seen_at)
|
||
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7)`,
|
||
|
|
[
|
||
|
|
u.userDistinguishedName,
|
||
|
|
u.username ?? null,
|
||
|
|
u.providerName ?? null,
|
||
|
|
u.type ?? null,
|
||
|
|
u.profileName ?? null,
|
||
|
|
u.created ?? null,
|
||
|
|
u.lastSeenAt ?? null,
|
||
|
|
],
|
||
|
|
);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
return users.length;
|
||
|
|
}
|
||
|
|
|
||
|
|
private async syncUserLogins(): Promise<number> {
|
||
|
|
const hourly = await this.client.getUserLoginsLast24h();
|
||
|
|
const totals = aggregateLogins(hourly);
|
||
|
|
let captured = 0;
|
||
|
|
for (const [date, total] of Object.entries(totals)) {
|
||
|
|
await postgresClient.query(
|
||
|
|
`INSERT INTO appgate_user_logins_daily (login_date, total_logins, captured_at)
|
||
|
|
VALUES ($1, $2, NOW())
|
||
|
|
ON CONFLICT (login_date) DO UPDATE SET
|
||
|
|
total_logins = EXCLUDED.total_logins,
|
||
|
|
captured_at = NOW()`,
|
||
|
|
[date, total],
|
||
|
|
);
|
||
|
|
captured++;
|
||
|
|
}
|
||
|
|
return captured;
|
||
|
|
}
|
||
|
|
|
||
|
|
// ─── History ──────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
private async persistHistory(r: AppgateSyncResult, triggeredBy: string): Promise<void> {
|
||
|
|
await postgresClient.query(
|
||
|
|
`INSERT INTO appgate_sync_history
|
||
|
|
(sync_id, sync_type, triggered_by, started_at, completed_at, status,
|
||
|
|
sessions_synced, devices_synced, devices_tombstoned,
|
||
|
|
appliances_synced, appliances_tombstoned,
|
||
|
|
license_users_synced, last_error)
|
||
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)`,
|
||
|
|
[
|
||
|
|
r.syncId, r.syncType, triggeredBy, r.startedAt, r.completedAt, r.status,
|
||
|
|
r.sessions, r.devices.upserted, r.devices.tombstoned,
|
||
|
|
r.appliances.upserted, r.appliances.tombstoned,
|
||
|
|
r.licenseUsers, r.errors[0] ?? null,
|
||
|
|
],
|
||
|
|
).catch((e) => console.error('[AppgateSync] history insert failed:', e));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
let _instance: AppgateSyncService | null = null;
|
||
|
|
export function getAppgateSyncService(): AppgateSyncService {
|
||
|
|
if (!_instance) _instance = new AppgateSyncService();
|
||
|
|
return _instance;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Reduce the rolling-24h hourly login map to per-date totals. Hours that
|
||
|
|
* lie in the future (relative to UTC now) reflect yesterday's logins; the
|
||
|
|
* rest reflect today. Pulse only persists "today" and "yesterday".
|
||
|
|
*/
|
||
|
|
function aggregateLogins(hourly: AppgateHourlyLogins): Record<string, number> {
|
||
|
|
const nowUtc = new Date();
|
||
|
|
const todayDate = nowUtc.toISOString().slice(0, 10);
|
||
|
|
const yesterdayDate = new Date(nowUtc.getTime() - 86_400_000).toISOString().slice(0, 10);
|
||
|
|
const currentHourUtc = nowUtc.getUTCHours();
|
||
|
|
const totals: Record<string, number> = {};
|
||
|
|
for (const [key, val] of Object.entries(hourly)) {
|
||
|
|
const hour = parseInt(key, 10);
|
||
|
|
if (Number.isNaN(hour)) continue;
|
||
|
|
const bucket = hour > currentHourUtc ? yesterdayDate : todayDate;
|
||
|
|
const count = typeof val === 'number'
|
||
|
|
? val
|
||
|
|
: Number(val?.total ?? 0);
|
||
|
|
if (!Number.isFinite(count)) continue;
|
||
|
|
totals[bucket] = (totals[bucket] ?? 0) + count;
|
||
|
|
}
|
||
|
|
return totals;
|
||
|
|
}
|