diff --git a/.planning/phases/11-company-catalog-subscription-sync/11-01-SUMMARY.md b/.planning/phases/11-company-catalog-subscription-sync/11-01-SUMMARY.md new file mode 100644 index 0000000..e70bb6e --- /dev/null +++ b/.planning/phases/11-company-catalog-subscription-sync/11-01-SUMMARY.md @@ -0,0 +1,120 @@ +--- +phase: 11-company-catalog-subscription-sync +plan: 01 +subsystem: database, api-client +tags: [postgres, migration, typescript, pax8, vitest, tdd] + +# Dependency graph +requires: + - phase: 10-pax8-client-auth-foundation + provides: Pax8Client OAuth2 client-credentials auth, pax8-factory, migrations/091_pax8_tables.sql schema, lib/types/pax8.ts base types +provides: + - migrations/092_pax8_subscription_costs.sql adding price/partner_cost/currency to pax8_subscriptions (applied to dev DB) + - Extended lib/types/pax8.ts with cost fields on Pax8Subscription, vendorName/shortDescription on Pax8Product, and new Pax8EntitySyncResult/Pax8SyncResult shapes + - Pax8Client.listAllCompanies()/listAllSubscriptions()/listAllProducts() read-only pagination helpers with mocked-fetch test coverage +affects: [11-02-company-catalog-subscription-sync-service, 12-historical-sync-company-matching] + +# Tech tracking +tech-stack: + added: [] + patterns: ["paginateAll() private helper looping page.number until page.totalPages - 1, reusing existing fetchJson() 429 backoff"] + +key-files: + created: + - migrations/092_pax8_subscription_costs.sql + modified: + - lib/types/pax8.ts + - lib/services/pax8-client.ts + - lib/services/pax8-client.test.ts + +key-decisions: + - "Cost columns are additive-only NUMERIC(12,2) price/partner_cost + CHAR(3) currency on pax8_subscriptions, matching migration 091's currency convention on pax8_orders/pax8_order_items — no monthly-normalization column (D-04 deferred to read-time)" + - "Pagination ownership: the client (not the future sync service) owns the loop-until-exhausted logic via a shared private paginateAll() helper, since all three entity types need identical page-walking logic" + +patterns-established: + - "Pattern: paginateAll(fetchPage) — generic pagination loop reused by listAllCompanies/listAllSubscriptions/listAllProducts, stops at page.number >= page.totalPages - 1" + +requirements-completed: [PAX8-04, PAX8-05, PAX8-08] + +# Metrics +duration: ~20min +completed: 2026-07-10 +--- + +# Phase 11 Plan 01: Company Catalog & Subscription Sync — Data Contract Layer Summary + +**Migration 092 adds price/partner_cost/currency to pax8_subscriptions, lib/types/pax8.ts gains cost fields plus Pax8SyncResult/Pax8EntitySyncResult, and Pax8Client gains read-only listAllCompanies/listAllSubscriptions/listAllProducts pagination helpers (TDD, 9 passing tests).** + +## Performance + +- **Duration:** ~20 min +- **Completed:** 2026-07-10 +- **Tasks:** 3 completed (Task 3 was a full TDD RED→GREEN cycle) +- **Files modified:** 4 (1 created, 3 modified) + +## Accomplishments +- Added `migrations/092_pax8_subscription_costs.sql` (3 additive `ADD COLUMN IF NOT EXISTS` clauses) and applied it to the running dev Postgres container (`pulse-postgres` / `pulse_autotask` db, `pulse_user` role) — confirmed via `\d pax8_subscriptions` before and after +- Extended `Pax8Subscription` (price, partnerCost, currencyCode, productName, endDate, updatedDate) and `Pax8Product` (vendorName, shortDescription) in `lib/types/pax8.ts`, plus new exported `Pax8EntitySyncResult`/`Pax8SyncResult` interfaces mirroring the AppGate sync-result shape +- Added `listAllCompanies()`, `listAllSubscriptions()`, `listAllProducts()` to `Pax8Client` via a shared private `paginateAll()` helper, requesting `size=200` and looping through `fetchJson()` (inheriting existing 429/Retry-After backoff); existing `listCompanies(page, size)` signature untouched +- Full TDD cycle for Task 3: 5 new tests written and confirmed failing (RED) before implementation, then all 9 tests (4 existing + 5 new) passed after implementation (GREEN) + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Add subscription cost columns (migration 092)** - `b0f6de0` (feat) +2. **Task 2: Extend PAX8 types with cost fields and sync-result shapes** - `0d819ba` (feat) +3. **Task 3: Add read-only pagination helpers to the PAX8 client** - `19fe788` (test, RED) then `c3a0432` (feat, GREEN) + +_TDD task (Task 3) has two commits per the RED→GREEN cycle; no REFACTOR commit was needed._ + +## Files Created/Modified +- `migrations/092_pax8_subscription_costs.sql` - Additive migration: `price NUMERIC(12,2)`, `partner_cost NUMERIC(12,2)`, `currency CHAR(3) NOT NULL DEFAULT 'USD'` on `pax8_subscriptions`; applied to dev DB via `docker exec -i pulse-postgres psql -U pulse_user -d pulse_autotask` +- `lib/types/pax8.ts` - Cost/lifecycle fields on `Pax8Subscription`, `vendorName`/`shortDescription` on `Pax8Product`, new `Pax8EntitySyncResult`/`Pax8SyncResult` interfaces +- `lib/services/pax8-client.ts` - New `paginateAll()` private helper plus `listAllCompanies()`/`listAllSubscriptions()`/`listAllProducts()` public methods, all GET-only +- `lib/services/pax8-client.test.ts` - New `makeMultiPageFetchMock()` helper and 5 new tests covering multi-page concatenation, `size=200` assertion, single-page no-infinite-loop, and GET-only/Bearer-header assertion across all three new methods + +## Decisions Made +- Followed the plan's explicit column/type/method specs verbatim — no open decisions required beyond what the plan already resolved (D-03/D-04 already settled at plan-authoring time) +- Chose to implement pagination via one shared private `paginateAll()` helper rather than duplicating the loop three times, since all three entity types share identical page-walking semantics (`page.number >= page.totalPages - 1`) + +## Deviations from Plan + +None - plan executed exactly as written. One pre-existing, out-of-scope issue was noted (see below) but not modified. + +### Out-of-Scope Discovery (logged, not fixed) + +`npx tsc --noEmit --pretty` surfaces 2 pre-existing errors in `lib/services/sync-scheduler.ts` (lines 446, 450) referencing `@/lib/services/appgate-factory` and `@/lib/services/appgate-sync-service`. These files exist as untracked (`??`) files in the main repo checkout but were never committed, so they are absent from this git worktree's history — a worktree/commit-state artifact unrelated to this plan's changes to `lib/types/pax8.ts`, `pax8-client.ts`, or the new migration. Confirmed identical before and after this plan's edits. Logged to `.planning/phases/11-company-catalog-subscription-sync/deferred-items.md` per the executor's scope-boundary rule; no action taken. + +## Issues Encountered + +None - the RED phase confirmed all 5 new tests failed for the expected reason (`TypeError: client.listAllX is not a function`), and the GREEN phase confirmed all 9 tests (4 existing + 5 new) passed with no regressions. + +## User Setup Required + +None - no external service configuration required. The migration was applied directly to the existing `pulse-postgres` dev container as part of Task 1 (per CLAUDE.md's manual-apply-on-existing-volume caveat). + +## Next Phase Readiness + +Plan 02 (the sync service) can now consume: +- `pax8_subscriptions.price` / `partner_cost` / `currency` columns (confirmed present in dev DB) +- `Pax8Subscription.price`/`partnerCost`/`currencyCode`/`productName`/`endDate`/`updatedDate`, `Pax8Product.vendorName`/`shortDescription`, and `Pax8SyncResult`/`Pax8EntitySyncResult` from `lib/types/pax8.ts` +- `Pax8Client.listAllCompanies()`/`listAllSubscriptions()`/`listAllProducts()` for full-catalog reads + +No blockers. The unrelated `sync-scheduler.ts` tsc error (see Deviations) is a worktree artifact that should resolve itself once the AppGate work referenced there is committed to the shared base — it does not block Plan 02's PAX8 work. + +## Self-Check: PASSED + +- FOUND: migrations/092_pax8_subscription_costs.sql +- FOUND: lib/types/pax8.ts (partnerCost, Pax8SyncResult present) +- FOUND: lib/services/pax8-client.ts (listAllCompanies/listAllSubscriptions/listAllProducts present) +- FOUND: lib/services/pax8-client.test.ts (9 passing tests) +- FOUND commit b0f6de0 +- FOUND commit 0d819ba +- FOUND commit 19fe788 +- FOUND commit c3a0432 + +--- +*Phase: 11-company-catalog-subscription-sync* +*Plan: 01* +*Completed: 2026-07-10* diff --git a/.planning/phases/11-company-catalog-subscription-sync/deferred-items.md b/.planning/phases/11-company-catalog-subscription-sync/deferred-items.md new file mode 100644 index 0000000..df8ae9e --- /dev/null +++ b/.planning/phases/11-company-catalog-subscription-sync/deferred-items.md @@ -0,0 +1,16 @@ +# Deferred Items — Phase 11 + +Items discovered during execution that are out of scope for the current +plan/task and are not auto-fixed per the executor's scope boundary rule. + +## Plan 01 + +- **Pre-existing `tsc` errors unrelated to this plan**: `lib/services/sync-scheduler.ts:446,450` + reference `@/lib/services/appgate-factory` and `@/lib/services/appgate-sync-service`, + which are untracked files present in the main repo checkout (`git status` shows them + as `??`) but were never committed — so they do not exist in this git worktree's + history. This is a worktree/commit-state artifact, not something introduced by + Plan 01's changes (migration 092, `lib/types/pax8.ts` extensions). Confirmed via + `npx tsc --noEmit --pretty` before and after this plan's edits — same 2 errors, + same file, unrelated to `lib/types/pax8.ts`. No action taken; will resolve itself + once the AppGate work is committed to the main branch/worktree base. diff --git a/lib/services/pax8-client.test.ts b/lib/services/pax8-client.test.ts index 8adc001..10f781b 100644 --- a/lib/services/pax8-client.test.ts +++ b/lib/services/pax8-client.test.ts @@ -41,6 +41,62 @@ function makeFetchMock(opts: { return { fetchMock, calls }; } +/** + * Multi-page fetch mock for the listAll* pagination helpers. Keys page + * bodies by URL substring (e.g. '/subscriptions') and by the `page=N` query + * param, returning one page envelope per (resource, page) pair. + */ +function makeMultiPageFetchMock(opts: { + resource: string; // e.g. 'subscriptions', 'products', 'companies' + pages: Array<{ content: unknown[]; totalPages: number }>; + tokenBody?: unknown; +}) { + const { + resource, + pages, + tokenBody = { access_token: 'tok', token_type: 'Bearer', expires_in: 86400 }, + } = opts; + + const calls: Array<{ url: string; init?: RequestInit }> = []; + + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + calls.push({ url, init }); + + if (url.includes('/token')) { + return { + ok: true, + status: 200, + json: async () => tokenBody, + text: async () => '', + } as unknown as Response; + } + + if (url.includes(`/${resource}`)) { + const match = url.match(/page=(\d+)/); + const pageNum = match ? parseInt(match[1], 10) : 0; + const p = pages[pageNum]; + return { + ok: true, + status: 200, + json: async () => ({ + content: p.content, + page: { size: 200, totalElements: pages.reduce((n, pg) => n + pg.content.length, 0), totalPages: p.totalPages, number: pageNum }, + }), + text: async () => '', + } as unknown as Response; + } + + return { + ok: true, + status: 200, + json: async () => ({ content: [], page: { size: 200, totalElements: 0, totalPages: 1, number: 0 } }), + text: async () => '', + } as unknown as Response; + }); + + return { fetchMock, calls }; +} + describe('Pax8Client', () => { beforeEach(() => { vi.unstubAllGlobals(); @@ -105,4 +161,81 @@ describe('Pax8Client', () => { expect(result).toEqual(companiesBody); }); + + it('listAllSubscriptions() concatenates content across all pages in order', async () => { + const pages = [ + { content: [{ id: 's1' }, { id: 's2' }], totalPages: 2 }, + { content: [{ id: 's3' }], totalPages: 2 }, + ]; + const { fetchMock, calls } = makeMultiPageFetchMock({ resource: 'subscriptions', pages }); + vi.stubGlobal('fetch', fetchMock); + + const client = new Pax8Client({ clientId: 'id1', clientSecret: SECRET }); + const result = await client.listAllSubscriptions(); + + expect(result).toEqual([{ id: 's1' }, { id: 's2' }, { id: 's3' }]); + + const subCalls = calls.filter(c => c.url.includes('/subscriptions')); + expect(subCalls).toHaveLength(2); + for (const c of subCalls) { + expect(c.url).toContain('size=200'); + } + }); + + it('listAllSubscriptions() stops after a single page when totalPages is 1 (no infinite loop)', async () => { + const pages = [{ content: [{ id: 's1' }], totalPages: 1 }]; + const { fetchMock, calls } = makeMultiPageFetchMock({ resource: 'subscriptions', pages }); + vi.stubGlobal('fetch', fetchMock); + + const client = new Pax8Client({ clientId: 'id1', clientSecret: SECRET }); + const result = await client.listAllSubscriptions(); + + expect(result).toEqual([{ id: 's1' }]); + expect(calls.filter(c => c.url.includes('/subscriptions'))).toHaveLength(1); + }); + + it('listAllProducts() concatenates content across all pages in order', async () => { + const pages = [ + { content: [{ id: 'p1' }], totalPages: 2 }, + { content: [{ id: 'p2' }], totalPages: 2 }, + ]; + const { fetchMock } = makeMultiPageFetchMock({ resource: 'products', pages }); + vi.stubGlobal('fetch', fetchMock); + + const client = new Pax8Client({ clientId: 'id1', clientSecret: SECRET }); + const result = await client.listAllProducts(); + + expect(result).toEqual([{ id: 'p1' }, { id: 'p2' }]); + }); + + it('listAllCompanies() concatenates content across all pages in order', async () => { + const pages = [ + { content: [{ id: 'c1' }], totalPages: 2 }, + { content: [{ id: 'c2' }], totalPages: 2 }, + ]; + const { fetchMock } = makeMultiPageFetchMock({ resource: 'companies', pages }); + vi.stubGlobal('fetch', fetchMock); + + const client = new Pax8Client({ clientId: 'id1', clientSecret: SECRET }); + const result = await client.listAllCompanies(); + + expect(result).toEqual([{ id: 'c1' }, { id: 'c2' }]); + }); + + it('every request the listAll* helpers issue is a GET with an Authorization: Bearer header; none use a mutating method', async () => { + const pages = [{ content: [{ id: 's1' }], totalPages: 1 }]; + const { fetchMock, calls } = makeMultiPageFetchMock({ resource: 'subscriptions', pages }); + vi.stubGlobal('fetch', fetchMock); + + const client = new Pax8Client({ clientId: 'id1', clientSecret: SECRET }); + await client.listAllSubscriptions(); + + const dataCalls = calls.filter(c => !c.url.includes('/token')); + expect(dataCalls.length).toBeGreaterThan(0); + for (const c of dataCalls) { + expect(c.init?.method === undefined || c.init?.method === 'GET').toBe(true); + const authHeader = (c.init!.headers as Record)['Authorization']; + expect(authHeader).toMatch(/^Bearer /); + } + }); }); diff --git a/lib/services/pax8-client.ts b/lib/services/pax8-client.ts index e578d3a..316a68f 100644 --- a/lib/services/pax8-client.ts +++ b/lib/services/pax8-client.ts @@ -4,7 +4,7 @@ * https://devx.pax8.com */ -import type { Pax8Company, Pax8PageEnvelope } from '@/lib/types/pax8'; +import type { Pax8Company, Pax8PageEnvelope, Pax8Subscription, Pax8Product } from '@/lib/types/pax8'; export interface Pax8ClientConfig { clientId: string; @@ -76,4 +76,48 @@ export class Pax8Client { async listCompanies(page = 0, size = 10): Promise> { return this.fetchJson>(`/companies?page=${page}&size=${size}`); } + + /** + * Generic read-only paginate-until-exhausted helper. Loops the given + * page-fetcher (each call inherits fetchJson's 429/Retry-After backoff), + * requesting size=200, and stops once page.number >= page.totalPages - 1. + * GET-only — no method override is ever set, satisfying PAX8-08. + */ + private async paginateAll( + fetchPage: (page: number, size: number) => Promise>, + ): Promise { + const size = 200; + const items: T[] = []; + let page = 0; + + while (true) { + const envelope = await fetchPage(page, size); + items.push(...envelope.content); + if (envelope.page.number >= envelope.page.totalPages - 1) break; + page++; + } + + return items; + } + + /** Read-only: page through every company (PAX8-08 — GET only). */ + async listAllCompanies(): Promise { + return this.paginateAll((page, size) => + this.fetchJson>(`/companies?page=${page}&size=${size}`), + ); + } + + /** Read-only: page through every subscription (PAX8-08 — GET only). */ + async listAllSubscriptions(): Promise { + return this.paginateAll((page, size) => + this.fetchJson>(`/subscriptions?page=${page}&size=${size}`), + ); + } + + /** Read-only: page through every product (PAX8-08 — GET only). */ + async listAllProducts(): Promise { + return this.paginateAll((page, size) => + this.fetchJson>(`/products?page=${page}&size=${size}`), + ); + } } diff --git a/lib/types/pax8.ts b/lib/types/pax8.ts index c62e6d8..ca6c9de 100644 --- a/lib/types/pax8.ts +++ b/lib/types/pax8.ts @@ -41,6 +41,12 @@ export interface Pax8Subscription { billingTerm: string | null; status: string | null; startDate: string | null; + price: number | null; // customer/list price (D-03) + partnerCost: number | null; // partner/reseller cost (D-03) + currencyCode: string | null; + productName: string | null; + endDate: string | null; + updatedDate: string | null; [key: string]: unknown; // escape hatch for fields not yet modeled } @@ -50,6 +56,8 @@ export interface Pax8Product { vendorSku: string | null; name: string; category: string | null; + vendorName: string | null; + shortDescription: string | null; [key: string]: unknown; // escape hatch for fields not yet modeled // NOTE: the deprecated alternate-vendor-SKU field per PAX8's own schema is intentionally omitted. } @@ -78,3 +86,27 @@ export interface Pax8OrderItem { currencyCode: string | null; [key: string]: unknown; // escape hatch for fields not yet modeled } + +// ─── Sync result shapes ─────────────────────────────────────────────────── +// Mirrors lib/types/appgate.ts's AppgateSyncResult/AppgateEntity shape — +// the sync service (Plan 02) reports one Pax8EntitySyncResult per entity +// type (companies/products/subscriptions), rolled up into a Pax8SyncResult. + +export interface Pax8EntitySyncResult { + entity: string; + success: boolean; + upserted: number; + tombstoned: number; + durationMs: number; + error?: string; +} + +export interface Pax8SyncResult { + syncId: string; + status: 'completed' | 'failed'; + startedAt: Date; + completedAt: Date | null; + durationMs: number; + entities: Pax8EntitySyncResult[]; + errors: string[]; +} diff --git a/migrations/092_pax8_subscription_costs.sql b/migrations/092_pax8_subscription_costs.sql new file mode 100644 index 0000000..c30ed41 --- /dev/null +++ b/migrations/092_pax8_subscription_costs.sql @@ -0,0 +1,20 @@ +-- PAX8 subscription cost columns. +-- +-- PAX8's Subscription object (GET /subscriptions?page=&size=, Phase 11 +-- research, devx.pax8.com findsubscriptions) exposes both a customer/list +-- `price` and a partner/reseller `partnerCost` per subscription, plus a +-- `currencyCode`. Migration 091 did not model these — this migration adds +-- them as additive columns on the existing pax8_subscriptions table (D-03). +-- +-- Raw per-billing-period amounts are stored exactly as PAX8 returns them, +-- paired with the existing billing_term column. Monthly normalization is a +-- read-time concern, not modeled here (D-04). + +ALTER TABLE pax8_subscriptions + ADD COLUMN IF NOT EXISTS price NUMERIC(12,2); + +ALTER TABLE pax8_subscriptions + ADD COLUMN IF NOT EXISTS partner_cost NUMERIC(12,2); + +ALTER TABLE pax8_subscriptions + ADD COLUMN IF NOT EXISTS currency CHAR(3) NOT NULL DEFAULT 'USD';