From 415ae52143122930968288e0c6356caa8b96b997 Mon Sep 17 00:00:00 2001 From: Nathnael Date: Tue, 4 Aug 2026 11:48:59 +0000 Subject: [PATCH] test(integration): boot freight API in-process across parallel shards The suite drove a containerized freight API, so every code change needed an image rebuild before a test could see it, and there was no way to attach a debugger. Files also ran strictly in sequence against one shared database, which is the root of the warm-stack gotchas the README documents: stowaway paid bookings climbing back aboard, a short consist on the fifth file. The freight app now boots inside each vitest worker from dist/, and each worker owns a whole shard of the topology - its own database, payment API, gateway mock and broker vhost - so nothing mutable is shared and files run in parallel. Full suite drops from roughly 20 minutes to 196s at 4 shards. - main.ts exports createFreightApp() so the harness applies the same prefix, pipes, filters and interceptors as production instead of replaying them by hand; self-start is guarded by require.main so the Dockerfile CMD still boots - booking-window tick cadence is env-driven (BOOKING_WINDOW_TICK_CRON), */1 in the suite, */10 unchanged in production - prepare-shards.mjs seeds a template database (boot seeders, then the SQL fixtures that depend on them) and clones it per shard; it.mjs re-clones on every run, so each run is hermetic - gateway mock and payment API are generated per shard: the mock keeps modes and orders process-global and 20 of 25 specs reset it in beforeAll, and the inbound CBE bill query has to reach one specific shard's app - poll() samples every 250ms instead of 2000ms, keeping the caller's deadline - authz.it.ts seeds its own invoice; it previously read another spec's leftover and returned early, which silently passed on a pristine database Known: an unlocked MAX(sequence_no)+1 in train-scheduling.service.ts races under concurrent allocation and leaves a short consist, so 1-3 specs fail intermittently. Pre-existing and reproduces at the production tick cadence. --- .gitignore | 3 + apps/edr-freight-api/src/main.ts | 42 ++- .../booking-window.service.ts | 11 +- docker-compose.it.yaml | 176 ++------- integration/README.md | 109 ++++-- integration/scripts/gen-shards.mjs | 182 ++++++++++ integration/scripts/it.mjs | 338 +++++++++++++++--- integration/scripts/prepare-shards.mjs | 134 +++++++ integration/src/app.ts | 133 +++++++ integration/src/authz.it.ts | 71 +++- integration/src/client.ts | 73 ++-- integration/src/global-setup.ts | 76 ++-- integration/vitest.config.ts | 26 +- 13 files changed, 1045 insertions(+), 329 deletions(-) create mode 100644 integration/scripts/gen-shards.mjs create mode 100644 integration/scripts/prepare-shards.mjs create mode 100644 integration/src/app.ts diff --git a/.gitignore b/.gitignore index b9dc16c3c..18fdae9c2 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ test-results/ playwright-report/ blob-report/ RUNNING_LOCALLY.md + +# Generated per-shard compose file for the integration suite (it.mjs). +integration/.it-shards.yaml diff --git a/apps/edr-freight-api/src/main.ts b/apps/edr-freight-api/src/main.ts index 6a51fecb6..a4fbefdd2 100644 --- a/apps/edr-freight-api/src/main.ts +++ b/apps/edr-freight-api/src/main.ts @@ -19,7 +19,7 @@ import { AppModule } from "./app.module"; * ~13.4MB on the wire. Express defaults to 100kb, which rejected any real stamp * image with a 413 "request entity too large". */ -const JSON_BODY_LIMIT = '20mb'; +const JSON_BODY_LIMIT = "20mb"; /** * Static /etc/hosts-style overrides from `DNS_HOST_OVERRIDES`, formatted as @@ -44,7 +44,9 @@ function applyDnsHostOverrides(): void { } if (overrides.size === 0) return; - const dns = createRequire(__filename)("node:dns") as typeof import("node:dns"); + const dns = createRequire(__filename)( + "node:dns", + ) as typeof import("node:dns"); const originalLookup = dns.lookup.bind(dns); // `dns.lookup` is overloaded (options optional, all/family variants); the // cast keeps that surface intact while we intercept only mapped hostnames. @@ -63,7 +65,9 @@ function applyDnsHostOverrides(): void { ) => void; const family = ip.includes(":") ? 6 : 4; const wantsAll = - typeof options === "object" && options !== null && (options as { all?: boolean }).all; + typeof options === "object" && + options !== null && + (options as { all?: boolean }).all; process.nextTick(() => wantsAll ? done(null, [{ address: ip, family }]) : done(null, ip, family), @@ -77,7 +81,15 @@ function applyDnsHostOverrides(): void { applyDnsHostOverrides(); -async function bootstrap() { +/** + * Build the app with every global the production process applies, but do NOT + * listen. Exported so a test harness can boot the REAL app in its own process + * (integration/src/app.ts) and get the same prefix, pipe, filter, interceptor + * and body-parser configuration — replaying this list by hand is how an e2e + * harness silently drifts from production (routes 404 without the "api" + * prefix, responses lose the transform envelope). + */ +export async function createFreightApp(): Promise { const app = await NestFactory.create(AppModule); // Nest's own body-parser API, NOT `app.use(json(...))` from express: express @@ -86,14 +98,14 @@ async function bootstrap() { // pnpm's hoisted dev store and died as MODULE_NOT_FOUND in the production // image, where `pnpm deploy --prod` installs declared dependencies only. // This also RECONFIGURES the default parsers rather than racing them. - app.useBodyParser('json', { limit: JSON_BODY_LIMIT }); - app.useBodyParser('urlencoded', { limit: JSON_BODY_LIMIT, extended: true }); + app.useBodyParser("json", { limit: JSON_BODY_LIMIT }); + app.useBodyParser("urlencoded", { limit: JSON_BODY_LIMIT, extended: true }); // Dev CORS: reflect any localhost origin and allow credentials so the // freight portal (5173), passenger portal (5174), backoffices (5183/5184) // and any other dev port can call the API with cookies + Authorization. // For production, restrict `origin` to known FQDNs. - + app.enableCors({ origin: true, // reflect request origin credentials: true, @@ -157,13 +169,21 @@ async function bootstrap() { const document = SwaggerModule.createDocument(app, config); SwaggerModule.setup("api/docs", app, document); + return app; +} + +async function bootstrap() { + const app = await createFreightApp(); const port = parseInt(process.env.PORT ?? "3001", 10); // await app.listen(port, "0.0.0.0"); - await app.listen( - - port) + await app.listen(port); // eslint-disable-next-line no-console console.log(`[freight-api] listening on port ${port}`); } -bootstrap(); +// Only self-start when this file IS the entrypoint. The Dockerfile's +// `CMD ["node", "dist/main.js"]` still boots; importers get `createFreightApp` +// without the process binding a port behind their back. +if (require.main === module) { + bootstrap(); +} diff --git a/apps/edr-freight-api/src/modules/train-scheduling/booking-window.service.ts b/apps/edr-freight-api/src/modules/train-scheduling/booking-window.service.ts index 6cdef8c05..c60dd14a6 100644 --- a/apps/edr-freight-api/src/modules/train-scheduling/booking-window.service.ts +++ b/apps/edr-freight-api/src/modules/train-scheduling/booking-window.service.ts @@ -91,7 +91,16 @@ export class BookingWindowService implements OnModuleInit { // 10-second cadence: every transition is derived from persisted timestamps // and applied idempotently, so a finer tick only shrinks the lag between a // deadline passing and the phase actually moving (was a full minute). - @Cron('*/10 * * * * *', { name: 'booking-window-tick', timeZone: BATCH_TIMEZONE }) + // + // Overridable because that lag is the integration suite's pacing floor: every + // window phase, wagon allocation and expiry it waits on lands on this tick, so + // a 10s cadence costs ~5s of pure latency per wait across a few hundred waits. + // The suite runs it at `*/1 * * * * *`. Read at class-definition time, so the + // env var must be set before the module is imported. + @Cron(process.env.BOOKING_WINDOW_TICK_CRON ?? '*/10 * * * * *', { + name: 'booking-window-tick', + timeZone: BATCH_TIMEZONE, + }) async tick(): Promise { if (this.ticking) return; this.ticking = true; diff --git a/docker-compose.it.yaml b/docker-compose.it.yaml index 8bb341f0b..33c6281af 100644 --- a/docker-compose.it.yaml +++ b/docker-compose.it.yaml @@ -1,12 +1,24 @@ -# EDR Freight — API integration stack (headless). +# EDR Freight — API integration stack (headless, sharded). # -# Overlay on docker-compose.e2e.yaml. Same base services (postgres, minio, -# mocks, freight-api), except the payment microservice is REAL here instead of -# `payment-mock-e2e`, and only the bank gateways are stubbed: +# Overlay on docker-compose.e2e.yaml, plus a GENERATED third file holding the +# per-shard services (integration/.it-shards.yaml, written by gen-shards.mjs). # -# freight-api-it ──HTTP──> payment-api-it ──HTTP──> gateway-mock-it -# ^ │ -# └────── RabbitMQ ───────┘ (outbox → payment.events → consumer) +# The freight API is NOT a container here — it boots inside each vitest worker, +# on the host, so a code change needs no image rebuild and a breakpoint works. +# Each worker is a full shard of the topology; nothing mutable is shared: +# +# shard i: +# freight app (in vitest worker, host :3111+i) +# │ ▲ +# │ └──── HTTP ─── payment-api-it-{i} :3131+i (inbound CBE bill query, +# │ │ via host.docker.internal) +# └── HTTP ──────────────► │ ──HTTP──> gateway-mock-it-{i} :4600+i +# ▲ │ +# └──── RabbitMQ vhost payment_s{i} ──┘ (outbox → payment.events → consumer) +# +# Shared by every shard: postgres (one container, one database per shard cloned +# from a seeded template), rabbitmq (one container, one vhost per shard), minio, +# and the one-shot migration. # # Its own compose project (`name:` below overrides the base) and its own host # ports, so it can run side by side with the Cypress e2e stack. @@ -15,13 +27,16 @@ # # Never start it with plain `docker compose -f docker-compose.it.yaml` — it is # an OVERLAY and needs the base file first: -# docker compose -f docker-compose.e2e.yaml -f docker-compose.it.yaml ... +# docker compose -f docker-compose.e2e.yaml -f docker-compose.it.yaml \ +# -f integration/.it-shards.yaml ... name: edr-freight-it services: # Outbox transport. The payment API publishes payment.succeeded/failed here # and freight consumes it — the production path. Copied from the passenger - # harness (e2e/docker-compose.yml). + # harness (e2e/docker-compose.yml). One broker, one vhost per shard + # (payment_s0, payment_s1, …) created by it.mjs — a shared vhost would let one + # shard's freight consumer eat another shard's settlement event. rabbitmq-it: image: rabbitmq:3-management environment: @@ -32,139 +47,14 @@ services: - "${IT_RABBIT_PORT:-5772}:5672" - "${IT_RABBIT_UI_PORT:-15772}:15672" healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + # check_running, NOT ping: ping only proves the Erlang node answers, and + # it.mjs runs `rabbitmqctl add_vhost` the moment this goes healthy — which + # on a cold boot failed with "this command requires the 'rabbit' app to be + # running on the target node". check_running waits for the application. + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] interval: 5s - timeout: 5s + timeout: 10s retries: 20 - - # Stand-in for every bank/wallet gateway the payment API talks to, plus a - # control plane the tests drive (force a provider to fail/hang, fire a - # correctly-signed webhook, read back what was called). See - # integration/gateway-mock/server.js. - gateway-mock-it: - image: node:20-alpine - volumes: - - ./integration/gateway-mock:/app:ro - working_dir: /app - environment: - PORT: "4600" - # Same secrets the payment API gets — so webhooks the mock signs pass the - # API's REAL signature verification instead of bypassing it. - CBE_SECRET_KEY: it-cbe-secret - CBE_MERCHANT_ID: it-cbe-merchant - PAYMENT_API_URL: http://payment-api-it:3003 - command: ["node", "server.js"] - ports: - - "${IT_GATEWAY_PORT:-4600}:4600" - healthcheck: - test: - [ - "CMD", - "node", - "-e", - "fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", - ] - interval: 3s - timeout: 3s - retries: 10 - - payment-api-it: - build: - context: . - dockerfile: apps/edr-payment-api/Dockerfile - secrets: - - npmrc - depends_on: - postgres-freight-e2e: - condition: service_healthy - rabbitmq-it: - condition: service_healthy - gateway-mock-it: - condition: service_healthy - environment: - PORT: "3003" - NODE_ENV: test - # Payment tables live in their own schema of the same throwaway DB; - # main.ts ensurePaymentSchema() creates it, migrationsRun does the rest. - DB_HOST: postgres-freight-e2e - DB_PORT: "5432" - DB_USER: edr_e2e - DB_PASSWORD: edr_e2e - DB_NAME: edr_freight_e2e - DB_SCHEMA: edr_payment - # Same token freight already uses in the base stack. - SERVICE_AUTH_TOKEN: e2e-service-token - PUBLISHER_TRANSPORT: rabbitmq - PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/payment - # HTTP fallback targets (only used with PUBLISHER_TRANSPORT=http). - PAYMENT_NOTIFY_FREIGHT_URL: http://freight-api-e2e:3001/api/internal/payments/mark-paid - # Fast relay + sweep so retry/reconciliation are observable inside a test - # rather than a minute later. - OUTBOX_RELAY_INTERVAL_MS: "1000" - RECONCILE_STALE_AFTER_MS: "5000" - # Every gateway points at the one mock. Paths are per-provider prefixes. - CBE_BASE_URL: http://gateway-mock-it:4600/cbe-birr - CBE_MERCHANT_ID: it-cbe-merchant - CBE_SECRET_KEY: it-cbe-secret - CBE_NOTIFY_URL: http://payment-api-it:3003/webhooks/cbe-birr - CBE_RETURN_URL: http://localhost/return - TELEBIRR_BASE_URL: http://gateway-mock-it:4600/telebirr - TELEBIRR_WEB_BASE_URL: http://gateway-mock-it:4600/telebirr/web - TELEBIRR_FABRIC_APP_ID: it-fabric - TELEBIRR_APP_SECRET: it-secret - TELEBIRR_MERCHANT_APP_ID: it-merchant-app - TELEBIRR_MERCHANT_CODE: "999999" - TELEBIRR_NOTIFY_URL: http://payment-api-it:3003/webhooks/telebirr - # Telebirr PSS-signs every request object — a throwaway key generated per - # launch by it.mjs (nothing key-shaped lives in git). - TELEBIRR_PRIVATE_KEY: ${IT_TELEBIRR_PRIVATE_KEY} - EBIRR_BASE_URL: http://gateway-mock-it:4600/ebirr - DMONEY_BASE_URL: http://gateway-mock-it:4600/dmoney - CARD_BASE_URL: http://gateway-mock-it:4600/card - WAAFI_BASE_URL: http://gateway-mock-it:4600/waafi - CAC_BASE_URL: http://gateway-mock-it:4600/cac - CAC_USERNAME: it-cac - CAC_PASSWORD: it-cac - CAC_APP_KEY: it-cac-key - CAC_API_KEY: it-cac-api - CAC_COMPANY_SERVICES_ID: "1" - # Inbound CBE Unified Bill — we are the biller; bill-query hops back into - # the freight API, so this direction runs real code on both sides. - CBE_BILL_ENABLED: "true" - CBE_BILL_CLIENT_ID: it-cbe-bill - CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret - CBE_BILL_JWT_SECRET: it-cbe-bill-jwt - FREIGHT_API_BASE_URL: http://freight-api-e2e:3001/api - ports: - - "${IT_PAYMENT_PORT:-3113}:3003" - healthcheck: - test: - [ - "CMD", - "node", - "-e", - "fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", - ] - interval: 5s - timeout: 5s - retries: 12 - start_period: 40s - - # Base-stack service, re-pointed at the real payment API. - freight-api-e2e: - depends_on: - payment-api-it: - condition: service_healthy - environment: - PAYMENT_API_URL: http://payment-api-it:3003 - # Freight's payment module skips RabbitMQModule entirely when this is - # unset (payment.module.ts) — without it, outbox events never arrive. - PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/payment - # RABBITMQ_ENABLED stays "false" (base stack) — it only gates the SMS/email - # clients, which must remain off. The payment consumer is wired by - # PAYMENT_RABBITMQ_URL alone. - # Drain tail on every pay window. Production defaults to 5 minutes; a - # reservation here lives ~60s, so 5 would push every natural expiry past - # the suite's 180s timeouts. One minute keeps the tail real and observable - # (src/expired-invoice-late-settle.it.ts asserts both sides of it). - FREIGHT_PAYMENT_DRAIN_MINUTES: "1" +# The per-shard `gateway-mock-it-{i}` and `payment-api-it-{i}` services live in +# integration/.it-shards.yaml. The base file's `freight-api-e2e` is never +# started — the app runs in-process (integration/src/app.ts). diff --git a/integration/README.md b/integration/README.md index 9798fd7f8..bbf7b144a 100644 --- a/integration/README.md +++ b/integration/README.md @@ -3,29 +3,77 @@ Headless, API-level tests for the freight API running against the **real** `edr-payment-api`. Only the bank/wallet gateways are stubbed. +The freight API is **not containerized** here — it boots inside each vitest +worker from `apps/edr-freight-api/dist`, which `it:test` rebuilds every run. A +code change needs no image rebuild, and a breakpoint in the API is hit by the +test that provoked it. + ``` pnpm it:up # build + start the stack (first run ~5 min) pnpm it:test # vitest run (auto-ups the stack if needed) pnpm it:test -- --reporter=basic src/payment-happy.it.ts -pnpm it:logs payment-api-it -pnpm it:down # -v, wipes the throwaway DB +pnpm it:test --no-reset -- src/payment-happy.it.ts # skip the DB re-clone +pnpm it:logs payment-api-it-0 +pnpm it:down # -v, wipes the throwaway DBs and the generated shard file + +IT_SHARDS=1 pnpm it:up # 1 shard when RAM is tight or output must be readable ``` ## Stack -`docker-compose.it.yaml` is an **overlay** on `docker-compose.e2e.yaml` — same -freight API, Postgres (tmpfs), MinIO, Fayda/eTrade mocks; plus RabbitMQ, the -real payment API, and one gateway mock. It is a separate compose project -(`edr-freight-it`) on offset ports, so the Cypress e2e stack can run alongside. +`docker-compose.it.yaml` is an **overlay** on `docker-compose.e2e.yaml`, plus a +third **generated** file (`integration/.it-shards.yaml`, written by +`scripts/gen-shards.mjs`, gitignored) holding the per-shard services. It is a +separate compose project (`edr-freight-it`) on offset ports, so the Cypress e2e +stack can run alongside. + +Each vitest worker is a whole shard of the topology. Nothing mutable is shared +between shards, so spec files run in parallel: ``` -freight-api-e2e :3111 ──HTTP──> payment-api-it :3113 ──HTTP──> gateway-mock-it :4600 - ^ │ - └────────── RabbitMQ :5772 ─────┘ (outbox → payment.events → consumer) +shard i (vitest worker i, VITEST_POOL_ID) + + freight app in-process, host :3111+i + │ ▲ + │ └──── HTTP ──── payment-api-it-{i} :3131+i inbound CBE bill query, + │ │ via host.docker.internal + └── HTTP ──────────────► │ ──HTTP──> gateway-mock-it-{i} :4600+i + ▲ │ + └── RabbitMQ vhost payment_s{i} ──┘ (outbox → payment.events → consumer) + + database edr_it_s{i} ← CREATE DATABASE … TEMPLATE edr_freight_e2e + freight tables AND the payment API's edr_payment + schema live in it, so `db()` and `paymentDb()` are + one connection ``` +Shared by every shard: Postgres (one container, one database each), RabbitMQ +(one container, one vhost each), MinIO, and the one-shot migration. + +`IT_SHARDS` defaults to **4**. RAM is the ceiling, not cores — a shard is an +in-process Nest app plus two containers. + Never `docker compose -f docker-compose.it.yaml` on its own — it needs the base -file first. Use `it.mjs`. +file first, and the generated shard file last. Use `it.mjs`. + +## Setup order + +The sequence is load-bearing, which is why `scripts/prepare-shards.mjs` exists +rather than vitest's `globalSetup`: + +1. `freight-migration-e2e` migrates the **template** database `edr_freight_e2e`. +2. `prepare-shards.mjs` boots the freight app once against the template and + calls `app.init()` — that fires `onApplicationBootstrap`, the always-on + seeders (org, units, positions, permissions). +3. …then applies the SQL fixtures, which declare exactly those as prerequisites + (`seed-users.sql`: *"created by the API's always-on boot seeders"*). +4. `CREATE DATABASE edr_it_s{i} TEMPLATE edr_freight_e2e` per shard — a file + copy on the tmpfs Postgres, not a re-run of migrations and five seeders. +5. The per-shard payment APIs start; each creates its own `edr_payment` schema + inside its shard database. + +`it:test` re-runs step 4 on **every run** (stop payment APIs → drop → clone → +start), so each run is hermetic. `--no-reset` skips it for a quick rerun. ## Gateway mock @@ -46,7 +94,15 @@ negative test. The suite drives **CBE Birr** end to end (plain HMAC, no key material); other providers answer a generic stub until a scenario needs them. Editing `server.js` needs a container restart (`docker compose … restart -gateway-mock-it`) — the code is a read-only mount, not baked into an image. +gateway-mock-it-0`) — the code is a read-only mount, not baked into an image. + +**One mock per shard.** `modes`, `calls` and `orders` are process-global in +`server.js`, and 20 of 25 specs call `POST /__control/reset` in `beforeAll`. On a +shared mock every starting spec would wipe every running spec's live orders — +and that fails as a wrong *assertion*, not an error: an unknown order still gets +a correctly-signed webhook, just carrying the mock's default amount. Per-shard +mocks are why the suite can run `payment-failure` (which forces providers into +`fail`/`timeout`) beside a spec that settles normally. ## Files @@ -151,21 +207,34 @@ what it actually does and says so in a comment, so a fix fails loudly: ignores `deleted_at`, so a soft-unlinked booking can never be re-batched). - **Arrange is slow.** Contract → booking → clearance → ops accept → batch is 30–60s of real API work per booking, so files share one schedule day. -- Files run sequentially (one DB); concurrency is exercised inside a test with - `Promise.all`. +- **Files run in parallel across shards, sequentially within one.** A shard owns + its whole topology, so two files never contend; two files on the *same* shard + still run one after the other, which is what the day-offset partitioning and + the per-file `releaseUnpaidHolds()` / `resetCorridorDay()` hygiene still + assume. Concurrency inside a single test is exercised with `Promise.all`. +- **Day offsets are the de-facto partition key** (`departureAt(n)` per file). + `payment-happy` and `expired-invoice-late-settle` both claim day 4 — harmless + now that a shard has its own database, but do not read the offsets as unique. - **A retired fixture keeps its shipment day at its peril.** `rescueStrandedPaidForDay` sweeps every unlinked booking whose `payment_status` is PAID and whose `scheduled_date` falls on the day being - filled, and re-places it on the fresh train. A previous run's paid bookings - therefore climb back aboard — 18 stowaway wagons on a 28-wagon day, until - `releaseUnpaidHolds` / `resetCorridorDay` started nulling `scheduled_date`. + filled, and re-places it on the fresh train. Within a run this still bites + across files on one shard; *between* runs it no longer can — `it:test` + re-clones every shard database from the template first. - **Only a FULL train rests at DONE.** An under-filled day CONCLUDES and REOPENS (`window_phase` back to OPEN, `booking_cycle_no` 2), so waiting for DONE there waits forever — use `pollCycleConcluded`. -- **Wagon stock is finite and shared.** Paid bookings keep their wagons, so - `releaseUnpaidHolds()` also frees every earlier `CTR-IT-%` allocation — - without it the fifth or sixth file on a warm stack silently gets a short - consist. +- **Wagon stock is finite and shared** *within a shard*. Paid bookings keep + their wagons, so `releaseUnpaidHolds()` also frees every earlier `CTR-IT-%` + allocation — without it the fifth or sixth file on one shard silently gets a + short consist. +- **`poll()` samples every 250ms, but keeps the caller's deadline.** Callers pass + `attempts × intervalMs`; that product is the timeout, and only the sampling + rate changed. Override with `IT_POLL_INTERVAL_MS`. +- **The tick cadence is env-driven.** `BOOKING_WINDOW_TICK_CRON` is `*/1` here + and `*/10` in production. Every window phase, allocation and expiry the suite + waits on lands on that tick. A spec that proves sensitive to it should pin its + own value rather than the whole suite reverting. - **One tenant per booking.** `seedTenantContracts` mints a company per booking because a company may hold only one unpaid reservation at a time; staff book and pay on their behalf, which is also the real Path B flow. diff --git a/integration/scripts/gen-shards.mjs b/integration/scripts/gen-shards.mjs new file mode 100644 index 000000000..93e520480 --- /dev/null +++ b/integration/scripts/gen-shards.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +/** + * Generate `integration/.it-shards.yaml` — one payment API + one gateway mock + * per shard. Compose has no loops, so the per-shard services are written out. + * + * Every shard is the real topology in miniature; nothing crosses between them: + * + * gateway-mock-it-{i} its own `modes`/`calls`/`orders` — the mock keeps those + * process-global, and 20 of 25 specs call + * POST /__control/reset in beforeAll, so a shared mock + * would have each starting spec wipe every running spec's + * live orders (which fails as a wrong *assertion*: an + * unknown order still gets a correctly-signed webhook, + * just with the mock's default amount). + * payment-api-it-{i} its own DB_NAME (the shard's database, `edr_payment` + * schema inside it), its own broker vhost, and its own + * FREIGHT_API_BASE_URL — the inbound CBE-bill query has + * to land on THIS shard's in-process app, and a single + * container could only ever point at one of them. + * + * Written by it.mjs, gitignored. Do not edit by hand. + */ +import { writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const itDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +/** + * All shards share one build and one image tag, so the payment API image is + * built once and reused rather than N times. + */ +const PAYMENT_IMAGE = "edr-payment-api-it:local"; + +export function renderShards({ + shards, + apiPortBase, + paymentPortBase, + gatewayPortBase, + dbPrefix, + telebirrKey, +}) { + const services = []; + + for (let i = 0; i < shards; i++) { + const gw = `gateway-mock-it-${i}`; + const pay = `payment-api-it-${i}`; + const db = `${dbPrefix}${i}`; + const vhost = `payment_s${i}`; + // The freight app for this shard runs on the HOST, inside the vitest worker. + const freightBase = `http://host.docker.internal:${apiPortBase + i}/api`; + + services.push(` + ${gw}: + image: node:20-alpine + volumes: + - ./integration/gateway-mock:/app:ro + working_dir: /app + environment: + PORT: "4600" + # Same secrets the payment API gets — so webhooks the mock signs pass the + # API's REAL signature verification instead of bypassing it. + CBE_SECRET_KEY: it-cbe-secret + CBE_MERCHANT_ID: it-cbe-merchant + PAYMENT_API_URL: http://${pay}:3003 + command: ["node", "server.js"] + ports: + - "${gatewayPortBase + i}:4600" + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 3s + timeout: 3s + retries: 10 + + ${pay}: + image: ${PAYMENT_IMAGE} + build: + context: . + dockerfile: apps/edr-payment-api/Dockerfile + secrets: + - npmrc + depends_on: + postgres-freight-e2e: + condition: service_healthy + rabbitmq-it: + condition: service_healthy + ${gw}: + condition: service_healthy + # The freight app is on the host now, not in this network. + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + PORT: "3003" + NODE_ENV: test + # Payment tables live in their own schema of THIS SHARD's database; + # main.ts ensurePaymentSchema() creates it, migrationsRun does the rest. + # Same database as freight, so every raw edr_payment.* query in the specs + # reads through the one connection. + DB_HOST: postgres-freight-e2e + DB_PORT: "5432" + DB_USER: edr_e2e + DB_PASSWORD: edr_e2e + DB_NAME: ${db} + DB_SCHEMA: edr_payment + SERVICE_AUTH_TOKEN: e2e-service-token + PUBLISHER_TRANSPORT: rabbitmq + PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/${vhost} + PAYMENT_NOTIFY_FREIGHT_URL: ${freightBase}/internal/payments/mark-paid + # Fast relay + sweep so retry/reconciliation are observable inside a test + # rather than a minute later. + OUTBOX_RELAY_INTERVAL_MS: "1000" + RECONCILE_STALE_AFTER_MS: "5000" + # Every gateway points at this shard's mock. Paths are per-provider prefixes. + CBE_BASE_URL: http://${gw}:4600/cbe-birr + CBE_MERCHANT_ID: it-cbe-merchant + CBE_SECRET_KEY: it-cbe-secret + CBE_NOTIFY_URL: http://${pay}:3003/webhooks/cbe-birr + CBE_RETURN_URL: http://localhost/return + TELEBIRR_BASE_URL: http://${gw}:4600/telebirr + TELEBIRR_WEB_BASE_URL: http://${gw}:4600/telebirr/web + TELEBIRR_FABRIC_APP_ID: it-fabric + TELEBIRR_APP_SECRET: it-secret + TELEBIRR_MERCHANT_APP_ID: it-merchant-app + TELEBIRR_MERCHANT_CODE: "999999" + TELEBIRR_NOTIFY_URL: http://${pay}:3003/webhooks/telebirr + # Telebirr PSS-signs every request object — a throwaway key generated per + # launch by it.mjs (nothing key-shaped lives in git). + TELEBIRR_PRIVATE_KEY: ${JSON.stringify(telebirrKey)} + EBIRR_BASE_URL: http://${gw}:4600/ebirr + DMONEY_BASE_URL: http://${gw}:4600/dmoney + CARD_BASE_URL: http://${gw}:4600/card + WAAFI_BASE_URL: http://${gw}:4600/waafi + CAC_BASE_URL: http://${gw}:4600/cac + CAC_USERNAME: it-cac + CAC_PASSWORD: it-cac + CAC_APP_KEY: it-cac-key + CAC_API_KEY: it-cac-api + CAC_COMPANY_SERVICES_ID: "1" + # Inbound CBE Unified Bill — we are the biller; bill-query hops back into + # the freight API, so this direction runs real code on both sides. + CBE_BILL_ENABLED: "true" + CBE_BILL_CLIENT_ID: it-cbe-bill + CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret + CBE_BILL_JWT_SECRET: it-cbe-bill-jwt + FREIGHT_API_BASE_URL: ${freightBase} + ports: + - "${paymentPortBase + i}:3003" + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 5s + timeout: 5s + retries: 12 + start_period: 40s`); + } + + return `# GENERATED by integration/scripts/gen-shards.mjs — do not edit. +# ${shards} shard(s). Overlaid on docker-compose.e2e.yaml + docker-compose.it.yaml. +services:${services.join("\n")} +`; +} + +export const SHARD_FILE = join(itDir, ".it-shards.yaml"); + +export function writeShards(opts) { + writeFileSync(SHARD_FILE, renderShards(opts)); + return SHARD_FILE; +} + +export const shardServices = (shards) => + Array.from({ length: shards }, (_, i) => [`gateway-mock-it-${i}`, `payment-api-it-${i}`]).flat(); diff --git a/integration/scripts/it.mjs b/integration/scripts/it.mjs index e3e2b3075..c2153e63d 100644 --- a/integration/scripts/it.mjs +++ b/integration/scripts/it.mjs @@ -3,10 +3,13 @@ * Freight integration-suite launcher. * * node integration/scripts/it.mjs [vitest args...] + * IT_SHARDS=4 node integration/scripts/it.mjs test + * node integration/scripts/it.mjs test --no-reset -- src/payment-happy.it.ts * - * Overlays docker-compose.it.yaml on docker-compose.e2e.yaml: same freight - * stack, but the payment microservice is real and only the bank gateways are - * stubbed. Web/Cypress containers are never started — this suite is HTTP only. + * The freight API is NOT containerized here: it boots inside each vitest worker + * from `apps/edr-freight-api/dist`, which `test` rebuilds every run. Each worker + * is a whole shard — own database, own payment API, own gateway mock, own broker + * vhost — so spec files run in parallel without sharing anything mutable. * * Ports are fixed (and distinct from the Cypress e2e defaults) so both stacks * can be up at once; they are separate compose projects. @@ -16,21 +19,24 @@ import { execFileSync, spawnSync } from "node:child_process"; import { generateKeyPairSync } from "node:crypto"; -import { existsSync } from "node:fs"; +import { existsSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { SHARD_FILE, shardServices, writeShards } from "./gen-shards.mjs"; + const itDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const repoRoot = resolve(itDir, ".."); -const composeBase = [ - "compose", - "-f", - join(repoRoot, "docker-compose.e2e.yaml"), - "-f", - join(repoRoot, "docker-compose.it.yaml"), -]; -/** Deliberately offset from the Cypress stack's defaults (3101/5533/9310…). */ +/** + * Deliberately offset from the Cypress stack's defaults (3101/5533/9310…). + * + * Each per-shard service takes a CONTIGUOUS BLOCK of `IT_SHARDS` ports starting + * at its base (freight :3111+i, payment :3131+i, gateway :4600+i), so the bases + * must be at least MAX_SHARDS apart. They were originally 3111 and 3113 — two + * apart — which silently worked at 1–2 shards and then had shard 2's freight app + * try to bind :3113, the port payment-api-it-0 was already published on. + */ const PORTS = { E2E_API_PORT: 3111, E2E_DB_PORT: 5543, @@ -39,32 +45,39 @@ const PORTS = { // Unused here (no web containers) but referenced by the base file's build args. E2E_PORTAL_PORT: 5393, E2E_BACKOFFICE_PORT: 5394, - IT_PAYMENT_PORT: 3113, + IT_PAYMENT_PORT: 3131, IT_GATEWAY_PORT: 4600, IT_RABBIT_PORT: 5772, IT_RABBIT_UI_PORT: 15772, }; -/** Everything the suite needs up — web + cypress are deliberately absent. */ -const SERVICES = [ - "postgres-freight-e2e", - "minio-e2e", - "minio-init-e2e", - "freight-migration-e2e", - "fayda-mock-e2e", - "etrade-mock-e2e", - // Still a base-stack dependency of freight-api-e2e (reconcile-before-expire - // has its own client); cheap to run alongside the real payment API. - "payment-mock-e2e", - "gateway-mock-it", - "rabbitmq-it", - "payment-api-it", - "freight-api-e2e", -]; +/** Beyond this the freight block (3111+) would run into the payment block (3131+). */ +const MAX_SHARDS = 16; -const RUNNING = SERVICES.filter( - (s) => !["minio-init-e2e", "freight-migration-e2e"].includes(s), -); +/** + * How many shards. RAM is the ceiling, not cores: each shard is an in-process + * Nest app plus two containers. + */ +const SHARDS = Math.max(1, Number(process.env.IT_SHARDS ?? 4)); +if (SHARDS > MAX_SHARDS) { + console.error( + `\nit: IT_SHARDS=${SHARDS} exceeds ${MAX_SHARDS} — the per-service port blocks would overlap.` + + `\n Raise IT_PAYMENT_PORT/IT_GATEWAY_PORT in it.mjs first.`, + ); + process.exit(1); +} +const TEMPLATE_DB = "edr_freight_e2e"; +const DB_PREFIX = "edr_it_s"; + +/** Long-running services shared by every shard. Web + Cypress are absent. */ +const INFRA = ["postgres-freight-e2e", "minio-e2e", "rabbitmq-it"]; + +/** + * One-shots: they run, then exit 0. They cannot go in the `up --wait` set — + * compose reports an exited container as a failed wait, so a healthy stack + * looks broken. Run them with `compose run`, which returns their exit code. + */ +const ONE_SHOT = ["minio-init-e2e", "freight-migration-e2e"]; function fail(msg) { console.error(`\nit: ${msg}`); @@ -97,64 +110,271 @@ function fakeFaydaPrivateKeyBase64() { return Buffer.from(JSON.stringify(jwk)).toString("base64"); } +const telebirrKey = process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey(); + +/** + * The freight app's own environment. This is docker-compose.e2e.yaml's + * `freight-api-e2e` block with container hostnames swapped for published host + * ports — the app runs on the host now. Per-shard values (DB_NAME, PORT, + * PAYMENT_API_URL, PAYMENT_RABBITMQ_URL) are derived per worker in src/app.ts. + */ +const freightEnv = { + NODE_ENV: "test", + DB_HOST: "localhost", + DB_PORT: String(PORTS.E2E_DB_PORT), + DB_USER: "edr_e2e", + DB_PASSWORD: "edr_e2e", + DB_NAME: TEMPLATE_DB, + + // e2e-only secrets — never reuse outside this stack + JWT_SECRET: "e2e-jwt-secret", + JWT_ACCESS_TOKEN_SECRET: "e2e-access-secret", + JWT_REFRESH_TOKEN_SECRET: "e2e-refresh-secret", + JWT_EXPIRES_IN: "1d", + JWT_ACCESS_TOKEN_EXPIRES: "1d", + JWT_REFRESH_TOKEN_EXPIRES: "7d", + // Mandatory: ServiceAuthGuard returns TRUE when this is unset, and + // authz.it.ts asserts that the internal surface rejects an unsigned caller. + SERVICE_AUTH_TOKEN: "e2e-service-token", + + SEED_EDR_ORG: "true", + SUPER_ADMIN_EMAIL: "superadmin@tria.com", + SUPER_ADMIN_PHONE: "+251900000000", + + MINIO_ENDPOINT: "localhost", + MINIO_PORT: String(PORTS.E2E_MINIO_PORT), + MINIO_USE_SSL: "false", + MINIO_ACCESS_KEY: "e2e-minio", + MINIO_SECRET_KEY: "e2e-minio-secret", + MINIO_REGION: "us-east-1", + + // Only gates the SMS/email clients. The payment consumer is wired by + // PAYMENT_RABBITMQ_URL alone (payment.module.ts). + RABBITMQ_ENABLED: "false", + // fayda.config.ts THROWS at load if this is "true" without the full var set, + // and the fayda/etrade mocks publish no host ports. No IT spec touches them. + FAYDA_ENABLED: "false", + // SMS strategy has no kill switch and defaults to a real dev endpoint. + OZIKING_SMS_URL: "http://127.0.0.1:9/sms", + // app.config.ts otherwise live-scrapes https://ethio.forex on first booking. + CBE_EXCHANGE_SCRAPE_URL: "http://127.0.0.1:9/fx", + CBE_EXCHANGE_API_URL: "http://127.0.0.1:9/fx", + CBE_EXCHANGE_FALLBACK_RATE: "130", + FREIGHT_PORTAL_URL: `http://localhost:${PORTS.E2E_PORTAL_PORT}`, + + // Drain tail on every pay window. Production defaults to 5 minutes; a + // reservation here lives ~60s, so 5 would push every natural expiry past the + // suite's timeouts. One minute keeps the tail real and observable + // (src/expired-invoice-late-settle.it.ts asserts both sides of it, and + // hardcodes DRAIN_MS = 60_000 to match). + FREIGHT_PAYMENT_DRAIN_MINUTES: "1", + // The suite's pacing floor: every window phase, allocation and expiry it waits + // on lands on this tick. Production stays at */10. Overridable from the + // environment so a cadence-sensitive spec (or a bisect) can pin it back. + BOOKING_WINDOW_TICK_CRON: process.env.BOOKING_WINDOW_TICK_CRON ?? "*/1 * * * * *", +}; + const env = { ...process.env, ...Object.fromEntries(Object.entries(PORTS).map(([k, v]) => [k, String(v)])), - IT_API_URL: `http://localhost:${PORTS.E2E_API_PORT}`, - IT_PAYMENT_URL: `http://localhost:${PORTS.IT_PAYMENT_PORT}`, - IT_GATEWAY_URL: `http://localhost:${PORTS.IT_GATEWAY_PORT}`, - IT_DB_URL: `postgres://edr_e2e:edr_e2e@localhost:${PORTS.E2E_DB_PORT}/edr_freight_e2e`, + ...freightEnv, + IT_SHARDS: String(SHARDS), + IT_TEMPLATE_DB: TEMPLATE_DB, + IT_DB_PREFIX: DB_PREFIX, + IT_API_PORT_BASE: String(PORTS.E2E_API_PORT), + IT_PAYMENT_PORT_BASE: String(PORTS.IT_PAYMENT_PORT), + IT_GATEWAY_PORT_BASE: String(PORTS.IT_GATEWAY_PORT), FAYDA_PRIVATE_KEY_BASE64: process.env.FAYDA_PRIVATE_KEY_BASE64 ?? fakeFaydaPrivateKeyBase64(), - IT_TELEBIRR_PRIVATE_KEY: - process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey(), + IT_TELEBIRR_PRIVATE_KEY: telebirrKey, }; +function generateShardFile() { + writeShards({ + shards: SHARDS, + apiPortBase: PORTS.E2E_API_PORT, + paymentPortBase: PORTS.IT_PAYMENT_PORT, + gatewayPortBase: PORTS.IT_GATEWAY_PORT, + dbPrefix: DB_PREFIX, + telebirrKey, + }); +} + +/** The shard file must exist before compose is invoked — it is one of the -f's. */ +function composeBase() { + if (!existsSync(SHARD_FILE)) generateShardFile(); + return [ + "compose", + "-f", + join(repoRoot, "docker-compose.e2e.yaml"), + "-f", + join(repoRoot, "docker-compose.it.yaml"), + "-f", + SHARD_FILE, + ]; +} + function compose(args) { - const { status } = spawnSync("docker", [...composeBase, ...args], { stdio: "inherit", env }); + const { status } = spawnSync("docker", [...composeBase(), ...args], { + stdio: "inherit", + env, + }); return status ?? 1; } +function composeQuiet(args) { + return spawnSync("docker", [...composeBase(), ...args], { + encoding: "utf8", + env, + stdio: ["ignore", "pipe", "pipe"], + }); +} + function stackRunning() { try { - const out = execFileSync("docker", [...composeBase, "ps", "--services", "--status", "running"], { - encoding: "utf8", - env, - stdio: ["ignore", "pipe", "ignore"], - }); + const out = execFileSync( + "docker", + [...composeBase(), "ps", "--services", "--status", "running"], + { encoding: "utf8", env, stdio: ["ignore", "pipe", "ignore"] }, + ); const running = new Set(out.split("\n").filter(Boolean)); - return RUNNING.every((s) => running.has(s)); + const want = [...INFRA, ...shardServices(SHARDS)]; + return want.every((s) => running.has(s)); } catch { return false; } } +function node(script, args = []) { + const { status } = spawnSync("node", [join(itDir, "scripts", script), ...args], { + cwd: repoRoot, + stdio: "inherit", + env, + }); + return status ?? 1; +} + +/** + * One broker, one vhost per shard — a shared vhost would let one shard's freight + * consumer eat another shard's settlement event. + * + * Retried: the healthcheck now waits for the rabbit *application*, but a cold + * boot can still land between "healthy" and "accepting rabbitmqctl", and a + * half-provisioned broker fails later as an unexplained missing settlement. + */ +function createVhosts() { + for (let i = 0; i < SHARDS; i++) { + const vhost = `payment_s${i}`; + let last = ""; + let ok = false; + for (let attempt = 1; attempt <= 10 && !ok; attempt++) { + // Idempotent: add_vhost on an existing vhost exits non-zero, which is fine. + composeQuiet(["exec", "-T", "rabbitmq-it", "rabbitmqctl", "add_vhost", vhost]); + const perm = composeQuiet([ + "exec", "-T", "rabbitmq-it", + "rabbitmqctl", "set_permissions", "-p", vhost, "edr", ".*", ".*", ".*", + ]); + ok = perm.status === 0; + last = perm.stderr ?? ""; + if (!ok) execFileSync("sleep", ["3"]); + } + if (!ok) fail(`could not grant on rabbit vhost ${vhost} after 10 tries:\n${last}`); + } + console.log(`it: rabbit vhosts payment_s0…payment_s${SHARDS - 1} ready`); +} + +function buildFreight() { + console.log("it: building @edr/freight-api (the suite runs dist/, never stale)"); + const { status } = spawnSync("pnpm", ["--filter", "@edr/freight-api", "run", "build"], { + cwd: repoRoot, + stdio: "inherit", + env, + }); + if (status !== 0) fail("freight API build failed — fix it before running the suite."); +} + function up() { preflight(); + generateShardFile(); console.log( - `it: starting stack — freight :${PORTS.E2E_API_PORT} payment :${PORTS.IT_PAYMENT_PORT} ` + - `gateway :${PORTS.IT_GATEWAY_PORT} db :${PORTS.E2E_DB_PORT}`, + `it: starting ${SHARDS} shard(s) — freight :${PORTS.E2E_API_PORT}..${ + PORTS.E2E_API_PORT + SHARDS - 1 + } (in-process) payment :${PORTS.IT_PAYMENT_PORT}.. gateway :${ + PORTS.IT_GATEWAY_PORT + }.. db :${PORTS.E2E_DB_PORT}`, ); - if (compose(["up", "-d", "--build", "--wait", ...SERVICES]) !== 0) { + + // 1. Shared infrastructure. + if (compose(["up", "-d", "--build", "--wait", "--remove-orphans", ...INFRA]) !== 0) { fail( - "stack failed to become healthy. Inspect with:\n" + - " node integration/scripts/it.mjs logs payment-api-it", + "shared services failed to become healthy. Inspect with:\n" + + " node integration/scripts/it.mjs logs postgres-freight-e2e", + ); + } + createVhosts(); + + // 1b. One-shots: the MinIO bucket, then freight's migrations into the + // TEMPLATE database every shard is cloned from. + for (const svc of ONE_SHOT) { + if (compose(["run", "--rm", "--no-deps", "--build", svc]) !== 0) { + fail(`${svc} failed — inspect with:\n node integration/scripts/it.mjs logs ${svc}`); + } + } + + // 2. The app must exist as dist/ before we can boot it to seed the template. + buildFreight(); + + // 3. Seed the template (boot seeders + SQL fixtures) and clone it per shard. + // Must precede the payment APIs: they connect to the shard databases. + if (node("prepare-shards.mjs") !== 0) fail("template seed / shard clone failed."); + + // 4. Per-shard payment API + gateway mock. + if (compose(["up", "-d", "--build", "--wait", ...shardServices(SHARDS)]) !== 0) { + fail( + "shard services failed to become healthy. Inspect with:\n" + + " node integration/scripts/it.mjs logs payment-api-it-0", ); } } -const [cmd, ...rawExtra] = process.argv.slice(2); +/** + * Re-clone every shard database from the template so each run is hermetic. The + * payment APIs hold connections to those databases, so they come down first — + * their `edr_payment` schema is recreated by their own boot migrations. + * + * This is what retires the warm-stack failure modes the suite used to document: + * stowaway paid bookings climbing back aboard, a short consist on the fifth + * file, invoice numbers continuing from a previous run. + */ +function resetShardDbs() { + const svc = shardServices(SHARDS); + console.log("it: re-cloning shard databases from the template"); + compose(["stop", ...svc]); + if (node("prepare-shards.mjs", ["--clone"]) !== 0) fail("shard database reset failed."); + if (compose(["up", "-d", "--wait", ...svc]) !== 0) { + fail("shard services failed to become healthy after the reset."); + } +} + +const [cmd, ...rawArgs] = process.argv.slice(2); +const noReset = rawArgs.includes("--no-reset"); // `pnpm it:test -- src/foo.it.ts` hands us a literal "--" first. Forwarding it // makes vitest treat everything after it as CLI options and ignore the file // filter — the "one file" run silently becomes the whole suite. -const extra = rawExtra[0] === "--" ? rawExtra.slice(1) : rawExtra; +const passthrough = rawArgs.filter((a) => a !== "--no-reset"); +const extra = passthrough[0] === "--" ? passthrough.slice(1) : passthrough; switch (cmd) { case "up": up(); break; case "test": { - if (!stackRunning()) up(); + if (!stackRunning()) { + up(); + } else { + buildFreight(); + if (!noReset) resetShardDbs(); + } const { status } = spawnSync( "pnpm", ["--filter", "@edr/freight-integration", "run", "test", ...extra], @@ -162,12 +382,14 @@ switch (cmd) { ); process.exit(status ?? 1); } - case "logs": + case "logs": { process.exit(compose(["logs", "--tail", "200", ...extra])); - break; - case "down": - process.exit(compose(["down", "-v", "--remove-orphans"])); - break; + } + case "down": { + const status = compose(["down", "-v", "--remove-orphans"]); + rmSync(SHARD_FILE, { force: true }); + process.exit(status); + } default: fail(`unknown command "${cmd ?? ""}" — use up | test | logs | down`); } diff --git a/integration/scripts/prepare-shards.mjs b/integration/scripts/prepare-shards.mjs new file mode 100644 index 000000000..29968c85f --- /dev/null +++ b/integration/scripts/prepare-shards.mjs @@ -0,0 +1,134 @@ +#!/usr/bin/env node +/** + * Build the seeded template database, then clone it once per shard. + * + * node integration/scripts/prepare-shards.mjs # seed template + clone + * node integration/scripts/prepare-shards.mjs --clone # clone only (per-run reset) + * + * Seeding has to happen here, not in vitest's globalSetup, because the order is + * load-bearing and half of it is the app itself: + * + * 1. freight migrations (freight-migration-e2e container, before us) + * 2. the app's always-on boot seeders — org, units, positions, permissions + * (app.module.ts onApplicationBootstrap) + * 3. the SQL fixtures, which declare those as prerequisites + * (seed-users.sql: "created by the API's always-on boot seeders") + * + * Step 2 needs a real Nest boot, and the app now lives inside the vitest workers + * — which globalSetup runs in a different process from. So we boot it once here + * against the template and every shard is a `CREATE DATABASE … TEMPLATE` copy of + * the result: a file copy on the tmpfs postgres, versus re-running migrations + * and five seeders per shard. + * + * Run by it.mjs, which owns the env. No dependencies beyond `pg`. + */ +import { createRequire } from "node:module"; +import { readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Client } from "pg"; + +const itDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolve(itDir, ".."); +const freightDir = join(repoRoot, "apps", "edr-freight-api"); + +const TEMPLATE = process.env.IT_TEMPLATE_DB ?? "edr_freight_e2e"; +const PREFIX = process.env.IT_DB_PREFIX ?? "edr_it_s"; +const SHARDS = Number(process.env.IT_SHARDS ?? 1); + +const admin = () => + new Client({ + host: process.env.DB_HOST ?? "localhost", + port: Number(process.env.DB_PORT ?? 5543), + user: process.env.DB_USER ?? "edr_e2e", + password: process.env.DB_PASSWORD ?? "edr_e2e", + // Never the template itself: CREATE DATABASE … TEMPLATE refuses while any + // session is connected to the source. + database: "postgres", + }); + +const CYPRESS_FIXTURES = join(repoRoot, "e2e", "freight", "cypress", "fixtures"); +const OWN_FIXTURES = join(itDir, "sql"); + +/** Order matters — company needs the users, everything needs the corridor. */ +const SEEDS = [ + [CYPRESS_FIXTURES, "seed-users.sql"], + [CYPRESS_FIXTURES, "seed-company.sql"], + [CYPRESS_FIXTURES, "seed-import-corridor.sql"], + [CYPRESS_FIXTURES, "seed-bulk-items.sql"], + [CYPRESS_FIXTURES, "seed-g1-train.sql"], + [CYPRESS_FIXTURES, "seed-g2-weight.sql"], + [CYPRESS_FIXTURES, "seed-government.sql"], + [OWN_FIXTURES, "seed-company-b.sql"], + [OWN_FIXTURES, "seed-customs-service-type.sql"], +]; + +async function seedTemplate() { + console.log(`it: seeding template ${TEMPLATE}`); + + process.env.DB_NAME = TEMPLATE; + // No broker for the template boot: payment.module.ts skips RabbitMQModule + // entirely when this is unset, and nothing here publishes. + delete process.env.PAYMENT_RABBITMQ_URL; + + const { createFreightApp } = createRequire(join(freightDir, "package.json"))("./dist/main.js"); + + // createFreightApp() only builds the graph. `init()` is what fires + // onApplicationBootstrap — the seeders we are here for. We never listen. + const app = await createFreightApp(); + try { + await app.init(); + console.log("it: boot seeders done (org, units, positions, permissions)"); + } finally { + await app.close(); + } + + const client = new Client({ + host: process.env.DB_HOST ?? "localhost", + port: Number(process.env.DB_PORT ?? 5543), + user: process.env.DB_USER ?? "edr_e2e", + password: process.env.DB_PASSWORD ?? "edr_e2e", + database: TEMPLATE, + }); + await client.connect(); + try { + for (const [dir, file] of SEEDS) { + await client.query(readFileSync(join(dir, file), "utf8")); + console.log(`it: seeded ${file}`); + } + } finally { + await client.end(); + } +} + +async function cloneShards() { + const pg = admin(); + await pg.connect(); + try { + // The app's pool and any stray psql hold the template open; without this the + // CREATE below fails with "source database is being accessed by other users". + await pg.query( + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid()`, + [TEMPLATE], + ); + + for (let i = 0; i < SHARDS; i++) { + const db = `${PREFIX}${i}`; + await pg.query( + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid()`, + [db], + ); + await pg.query(`DROP DATABASE IF EXISTS ${db}`); + await pg.query(`CREATE DATABASE ${db} TEMPLATE ${TEMPLATE}`); + console.log(`it: shard ${i} → ${db}`); + } + } finally { + await pg.end(); + } +} + +const cloneOnly = process.argv.includes("--clone"); +if (!cloneOnly) await seedTemplate(); +await cloneShards(); diff --git a/integration/src/app.ts b/integration/src/app.ts new file mode 100644 index 000000000..f34c2fa52 --- /dev/null +++ b/integration/src/app.ts @@ -0,0 +1,133 @@ +/** + * The freight API, booted IN THIS PROCESS — one instance per vitest worker. + * + * Each worker is a self-contained shard of the whole topology, so spec files can + * run in parallel without sharing anything mutable: + * + * shard i = this in-process freight app on :3111+i + * + database edr_it_s{i} (cloned from the seeded template) + * + payment-api-it-{i} container on :3113+i, DB_NAME=edr_it_s{i} + * + gateway-mock-it-{i} container on :4600+i + * + RabbitMQ vhost payment_s{i} + * + * The payment schema lives INSIDE the shard's own database under the usual + * `edr_payment` name, so every raw `edr_payment.*` query in the specs works + * unchanged and `paymentDb()` stays an alias for `db()`. + * + * Why the app is loaded from `dist/` and not from TypeScript: the freight source + * is CJS-flavoured — `src/config/database.config.ts` uses `__dirname`, + * `require.resolve` and an entity glob, none of which survive vitest's ESM + * transform. `createRequire` anchored at the freight app's own package.json + * keeps all Nest / typeorm / @tria-plc resolution inside its node_modules, and + * `dist` is rebuilt by `it.mjs test` on every run so it can never go stale. + */ +import { createRequire } from "node:module"; +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Server } from "node:http"; + +const here = dirname(fileURLToPath(import.meta.url)); +const freightDir = join(here, "..", "..", "apps", "edr-freight-api"); +const distMain = join(freightDir, "dist", "main.js"); + +/** + * `VITEST_POOL_ID` is 1-based and stable per worker for the life of the run. + * Absent outside a worker (global-setup, scripts) — that context gets shard 0. + */ +export const SHARD = Math.max(0, Number(process.env.VITEST_POOL_ID ?? "1") - 1); + +/** + * Each service takes a contiguous block of `IT_SHARDS` ports from its base, so + * the bases must stay far apart — see the PORTS comment in scripts/it.mjs. + */ +const port = (base: string, fallback: number) => + Number(process.env[base] ?? fallback) + SHARD; + +export const API_PORT = port("IT_API_PORT_BASE", 3111); +export const PAYMENT_API = `http://localhost:${port("IT_PAYMENT_PORT_BASE", 3131)}`; +export const GATEWAY = `http://localhost:${port("IT_GATEWAY_PORT_BASE", 4600)}`; + +export const SHARD_DB = `${process.env.IT_DB_PREFIX ?? "edr_it_s"}${SHARD}`; + +export const DB_URL = + `postgres://${process.env.DB_USER ?? "edr_e2e"}:${process.env.DB_PASSWORD ?? "edr_e2e"}` + + `@${process.env.DB_HOST ?? "localhost"}:${process.env.DB_PORT ?? "5543"}/${SHARD_DB}`; + +const RABBIT_URL = + `amqp://edr:edr_secret@localhost:${process.env.IT_RABBIT_PORT ?? "5772"}` + + `/payment_s${SHARD}`; + +/** Prefix every diagnostic — parallel worker output interleaves. */ +export const tag = (msg: string) => `[shard ${SHARD}] ${msg}`; + +interface FreightMain { + createFreightApp: () => Promise<{ + listen: (port: number) => Promise; + getHttpServer: () => Server; + close: () => Promise; + }>; +} + +let booted: Promise | undefined; + +/** + * The shard's HTTP server, booted once per worker. supertest takes this + * directly in place of a base URL, so every call site in client.ts is a + * one-token change. + */ +export function freightServer(): Promise { + return (booted ??= boot()); +} + +async function boot(): Promise { + if (!existsSync(distMain)) { + throw new Error( + tag( + `${distMain} is missing — build the API first ` + + `(\`pnpm --filter @edr/freight-api run build\`, which \`it.mjs test\` does for you).`, + ), + ); + } + + // Must be set BEFORE the require: payment.module.ts reads + // PAYMENT_RABBITMQ_URL at module-definition time to decide whether + // RabbitMQModule is in the graph at all, and database.config.ts reads DB_NAME + // when ConfigModule loads it. + process.env.DB_NAME = SHARD_DB; + process.env.PORT = String(API_PORT); + process.env.PAYMENT_API_URL = PAYMENT_API; + process.env.PAYMENT_RABBITMQ_URL = RABBIT_URL; + + const { createFreightApp } = createRequire(join(freightDir, "package.json"))( + "./dist/main.js", + ) as FreightMain; + + // Postgres and the broker are up before vitest starts, but a shard's payment + // API may still be finishing its own migrations when the first worker boots. + // Retry the whole app rather than a connection: a half-initialised Nest app + // cannot be resumed, only closed and rebuilt. + let lastErr: unknown; + for (let attempt = 1; attempt <= 3; attempt++) { + const app = await createFreightApp().catch((err) => { + lastErr = err; + return undefined; + }); + if (app) { + try { + // A real port, not an ephemeral one: payment-api-it-{SHARD} calls back + // in here for the inbound CBE Unified Bill query (cbe-bill.it.ts). + await app.listen(API_PORT); + console.log(tag(`freight-api in-process on :${API_PORT} → ${SHARD_DB}`)); + return app.getHttpServer(); + } catch (err) { + lastErr = err; + await app.close().catch(() => {}); + } + } + if (attempt < 3) await new Promise((r) => setTimeout(r, 2000 * attempt)); + } + + console.error(tag(`freight-api failed to boot on :${API_PORT} (${SHARD_DB})`), lastErr); + throw lastErr; +} diff --git a/integration/src/authz.it.ts b/integration/src/authz.it.ts index 8af26f720..5f86b0e10 100644 --- a/integration/src/authz.it.ts +++ b/integration/src/authz.it.ts @@ -3,44 +3,75 @@ * failures here are the expensive kind: a tenant reading another tenant's * invoice, or an unauthenticated caller marking one paid. */ -import { afterAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; import request from "supertest"; import { - API, PAYMENT_API, api, closeDb, customerA, customerB, db, + freightServer, login, payment, } from "./client"; +/** Company A, from e2e/freight/cypress/fixtures/seed-company.sql. */ +const TENANT_A_TIN = "0102030405"; + +/** + * This file used to read "the newest invoice of company A" and `return` early + * when there wasn't one — which passed silently on a database no payment spec + * had run against yet. Now that every shard starts from a pristine clone, that + * would be *every* run. It bills itself instead. + * + * Straight SQL, not the booking chain: the point of this file is that it is + * cheap, and a cross-tenant read is refused on ownership alone — nothing here + * cares how the invoice came to exist. + */ +async function seedTenantAInvoice(): Promise { + const rows = await db<{ id: string }>( + `INSERT INTO freight.invoices ( + id, invoice_number, company_id, company_profile_id, + subtotal_amount, tax_amount, total_amount, paid_amount, balance_amount, + currency, status, source, source_id, type, issued_at, due_at, payments + ) + SELECT gen_random_uuid(), $1, c.id, p.id, + 1000, 0, 1000, 0, 1000, + 'ETB', 'ISSUED', 'booking', gen_random_uuid()::text, 'PREPAID', + now(), now() + interval '7 days', '[]'::jsonb + FROM freight.companies c + JOIN freight.company_profiles p ON p.company_id = c.id AND p.deleted_at IS NULL + WHERE c.tin = $2 AND c.deleted_at IS NULL + LIMIT 1 + RETURNING id`, + [`INV-AUTHZ-${Date.now()}`, TENANT_A_TIN], + ); + const id = rows[0]?.id; + if (!id) { + throw new Error( + `authz: could not bill company ${TENANT_A_TIN} — seed-company.sql missing from this shard?`, + ); + } + return id; +} + describe("payment authorization boundaries", () => { + let invoiceId: string; + + beforeAll(async () => { + invoiceId = await seedTenantAInvoice(); + }); afterAll(closeDb); it("hides one tenant's invoice from the other", async () => { - const rows = await db<{ id: string; company_id: string }>( - `SELECT i.id, i.company_id FROM freight.invoices i - JOIN freight.companies c ON c.id = i.company_id - WHERE c.tin = '0102030405' AND i.deleted_at IS NULL - ORDER BY i.created_at DESC LIMIT 1`, - ); - if (!rows[0]) return; // nothing billed yet in this run — payment files cover it - const res = await api(customerB, "get", `/api/billing/my-invoices/${rows[0].id}`); + const res = await api(customerB, "get", `/api/billing/my-invoices/${invoiceId}`); expect([403, 404]).toContain(res.status); }); it("refuses to let one tenant pay the other's invoice", async () => { - const rows = await db<{ id: string }>( - `SELECT i.id FROM freight.invoices i - JOIN freight.companies c ON c.id = i.company_id - WHERE c.tin = '0102030405' AND i.status <> 'PAID' AND i.deleted_at IS NULL - ORDER BY i.created_at DESC LIMIT 1`, - ); - if (!rows[0]) return; - const res = await api(customerB, "post", `/api/billing/my-invoices/${rows[0].id}/pay`, { + const res = await api(customerB, "post", `/api/billing/my-invoices/${invoiceId}/pay`, { method: "CBE_BIRR", platform: "web", }); @@ -71,7 +102,9 @@ describe("payment authorization boundaries", () => { amountMinor: 1, currency: "ETB", }; - const res = await request(API).post("/api/internal/payments/mark-paid").send(body); + const res = await request(await freightServer()) + .post("/api/internal/payments/mark-paid") + .send(body); expect([401, 403]).toContain(res.status); }); diff --git a/integration/src/client.ts b/integration/src/client.ts index c9af9ec87..e4cd74232 100644 --- a/integration/src/client.ts +++ b/integration/src/client.ts @@ -1,7 +1,11 @@ /** - * Plumbing for the integration suite: HTTP against the containerized freight - * and payment APIs, SQL against their shared throwaway Postgres, and the - * gateway mock's control plane. + * Plumbing for the integration suite: HTTP against this worker's IN-PROCESS + * freight API and its containerized payment API, SQL against the worker's own + * throwaway database, and its own gateway mock's control plane. + * + * Everything here is shard-scoped — see app.ts for the topology. supertest takes + * an `http.Server` exactly where it takes a base URL, so pointing the suite at + * the in-process app is a one-token change at each call site. * * This is the Cypress-free port of e2e/freight/cypress/e2e/flows/import-utils.ts — * same request sequences, same SQL, `pg.Pool` instead of `cy.task`. @@ -9,13 +13,11 @@ import request from "supertest"; import { Pool, type QueryResultRow } from "pg"; -export const API = process.env.IT_API_URL ?? "http://localhost:3111"; -export const PAYMENT_API = process.env.IT_PAYMENT_URL ?? "http://localhost:3113"; -export const GATEWAY = process.env.IT_GATEWAY_URL ?? "http://localhost:4600"; -export const SERVICE_TOKEN = process.env.IT_SERVICE_TOKEN ?? "e2e-service-token"; +import { DB_URL, GATEWAY, PAYMENT_API, SHARD, freightServer, tag } from "./app"; -const DB_URL = - process.env.IT_DB_URL ?? "postgres://edr_e2e:edr_e2e@localhost:5543/edr_freight_e2e"; +export { GATEWAY, PAYMENT_API, SHARD, freightServer }; + +export const SERVICE_TOKEN = process.env.IT_SERVICE_TOKEN ?? "e2e-service-token"; // --------------------------------------------------------------------------- // users (e2e/freight/cypress/fixtures/seed-users.sql + users.json) @@ -45,11 +47,23 @@ export async function db>( return res.rows; } -export async function closeDb(): Promise { - await pool.end(); -} +/** + * No-op. The pool is per-WORKER, not per-file: with `isolate: false` the module + * registry survives across every spec a worker runs, so the first `afterAll` to + * call this would break every later file on that shard. The pool dies with the + * worker process. + */ +export async function closeDb(): Promise {} + +/** + * Poll a query until `check` passes. Async settlement here is broker-driven and + * tick-driven, so the interval is pure overshoot: at the old 2000ms every wait + * in the suite finished up to two seconds after the state it wanted was already + * in the database, several hundred times over. The ceiling + * (`attempts × intervalMs`) is unchanged. + */ +const POLL_INTERVAL_MS = Number(process.env.IT_POLL_INTERVAL_MS ?? 250); -/** Poll a query until `check` passes. Async settlement here is broker-driven. */ export async function poll>( label: string, sql: string, @@ -57,14 +71,24 @@ export async function poll>( check: (row: T | undefined) => boolean, { attempts = 40, intervalMs = 2000 } = {}, ): Promise { + // Callers express patience as attempts × their own interval; keep that + // deadline and just sample it finely. + const deadlineMs = attempts * intervalMs; + const tries = Math.ceil(deadlineMs / POLL_INTERVAL_MS); + let last: T | undefined; - for (let i = 0; i < attempts; i++) { + for (let i = 0; i < tries; i++) { last = (await db(sql, params))[0]; if (check(last)) return last as T; - await sleep(intervalMs); + await sleep(POLL_INTERVAL_MS); } throw new Error( - `timed out waiting for ${label} after ${attempts} attempts — last row: ${JSON.stringify(last)}`, + tag( + `timed out waiting for ${label} after ${Math.round(deadlineMs / 1000)}s\n` + + ` sql: ${sql.replace(/\s+/g, " ").trim()}\n` + + ` params: ${JSON.stringify(params)}\n` + + ` last row: ${JSON.stringify(last)}`, + ), ); } @@ -86,7 +110,7 @@ export async function tokenFor(email: string): Promise { if (cached) return cached; const portal = email.endsWith("@gmail.com"); - const res = await request(API) + const res = await request(await freightServer()) .post("/api/auth/login") .set("x-client-app", portal ? "portal" : "backoffice") .send({ email, password: portal ? CUSTOMER_PASSWORD : STAFF_PASSWORD }); @@ -101,8 +125,11 @@ export async function tokenFor(email: string): Promise { } /** Login without caching, so audience-rejection can be asserted. */ -export function login(email: string, password: string, app: "portal" | "backoffice") { - return request(API).post("/api/auth/login").set("x-client-app", app).send({ email, password }); +export async function login(email: string, password: string, app: "portal" | "backoffice") { + return request(await freightServer()) + .post("/api/auth/login") + .set("x-client-app", app) + .send({ email, password }); } // --------------------------------------------------------------------------- @@ -119,7 +146,9 @@ export async function api( body?: unknown, ): Promise { const token = await tokenFor(email); - const req = request(API)[method](path).set("Authorization", `Bearer ${token}`); + const req = request(await freightServer()) + [method](path) + .set("Authorization", `Bearer ${token}`); return method === "get" || method === "delete" ? req.send() : req.send(body ?? {}); } @@ -148,7 +177,9 @@ export async function upload( fields: Record = {}, ): Promise { const token = await tokenFor(email); - const req = request(API).post(path).set("Authorization", `Bearer ${token}`); + const req = request(await freightServer()) + .post(path) + .set("Authorization", `Bearer ${token}`); for (const [k, v] of Object.entries(fields)) req.field(k, v); return req.attach(field, filePath); } diff --git a/integration/src/global-setup.ts b/integration/src/global-setup.ts index 2dec8ba49..45c52de06 100644 --- a/integration/src/global-setup.ts +++ b/integration/src/global-setup.ts @@ -1,41 +1,17 @@ /** - * Runs once before any spec: wait for both APIs, then seed. + * Runs once, in its own process, before any spec: confirm every shard's + * containerized half is answering, then clear its gateway mock. * - * Seeds are the Cypress suite's fixtures, reused verbatim (they are idempotent - * `insert … where not exists`), plus one of our own for the second tenant: - * seed-users.sql → seed-company.sql (order matters; company needs the users) - * seed-import-corridor.sql (yards, locos, wagons, rates, distances) - * seed-bulk-items.sql (PER_ITEM break-bulk cargo types) - * seed-g1-train.sql (the 53-wagon BUILT container train) - * seed-g2-weight.sql (the two 3 500 T weight-bound trains) - * seed-government.sql (the kind='government' company) - * seed-company-b.sql (user2@gmail.com's company — this suite) - * seed-customs-service-type.sql (a service type that bundles customs) + * Seeding is NOT here any more. The order is load-bearing — + * `seed-users.sql` names its prerequisites as "created by the API's always-on + * boot seeders" (iam.organizations / units / positions) — and the freight app + * now boots inside the vitest *workers*, which this process cannot reach. So the + * template database is seeded once by `scripts/prepare-shards.mjs` (boot the app + * → boot seeders → SQL fixtures) and each shard is a clone of it. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { Client } from "pg"; - -const API = process.env.IT_API_URL ?? "http://localhost:3111"; -const PAYMENT_API = process.env.IT_PAYMENT_URL ?? "http://localhost:3113"; -const GATEWAY = process.env.IT_GATEWAY_URL ?? "http://localhost:4600"; -const DB_URL = - process.env.IT_DB_URL ?? "postgres://edr_e2e:edr_e2e@localhost:5543/edr_freight_e2e"; - -const CYPRESS_FIXTURES = join(process.cwd(), "..", "e2e", "freight", "cypress", "fixtures"); -const OWN_FIXTURES = join(process.cwd(), "sql"); - -const SEEDS: Array<[dir: string, file: string]> = [ - [CYPRESS_FIXTURES, "seed-users.sql"], - [CYPRESS_FIXTURES, "seed-company.sql"], - [CYPRESS_FIXTURES, "seed-import-corridor.sql"], - [CYPRESS_FIXTURES, "seed-bulk-items.sql"], - [CYPRESS_FIXTURES, "seed-g1-train.sql"], - [CYPRESS_FIXTURES, "seed-g2-weight.sql"], - [CYPRESS_FIXTURES, "seed-government.sql"], - [OWN_FIXTURES, "seed-company-b.sql"], - [OWN_FIXTURES, "seed-customs-service-type.sql"], -]; +const SHARDS = Number(process.env.IT_SHARDS ?? 1); +const PAYMENT_BASE = Number(process.env.IT_PAYMENT_PORT_BASE ?? 3131); +const GATEWAY_BASE = Number(process.env.IT_GATEWAY_PORT_BASE ?? 4600); async function waitFor(label: string, url: string, attempts = 60): Promise { for (let i = 0; i < attempts; i++) { @@ -51,22 +27,20 @@ async function waitFor(label: string, url: string, attempts = 60): Promise } export async function setup(): Promise { - await Promise.all([ - waitFor("freight-api", `${API}/api/health`), - waitFor("payment-api", `${PAYMENT_API}/health`), - waitFor("gateway-mock", `${GATEWAY}/__control/health`), - ]); + const shards = Array.from({ length: SHARDS }, (_, i) => i); - const client = new Client({ connectionString: DB_URL }); - await client.connect(); - try { - for (const [dir, file] of SEEDS) { - await client.query(readFileSync(join(dir, file), "utf8")); - console.log(`it: seeded ${file}`); - } - } finally { - await client.end(); - } + await Promise.all( + shards.flatMap((i) => [ + waitFor(`payment-api (shard ${i})`, `http://localhost:${PAYMENT_BASE + i}/health`), + waitFor(`gateway-mock (shard ${i})`, `http://localhost:${GATEWAY_BASE + i}/__control/health`), + ]), + ); - await fetch(`${GATEWAY}/__control/reset`, { method: "POST" }); + await Promise.all( + shards.map((i) => + fetch(`http://localhost:${GATEWAY_BASE + i}/__control/reset`, { method: "POST" }), + ), + ); + + console.log(`it: ${SHARDS} shard(s) ready`); } diff --git a/integration/vitest.config.ts b/integration/vitest.config.ts index 348a1303d..6241f091b 100644 --- a/integration/vitest.config.ts +++ b/integration/vitest.config.ts @@ -1,18 +1,34 @@ import { defineConfig } from "vitest/config"; +/** + * One worker per shard. A shard owns its whole topology — in-process freight + * app, database, payment API, gateway mock, broker vhost — so files can run in + * parallel without sharing anything mutable. See src/app.ts. + */ +const SHARDS = Number(process.env.IT_SHARDS ?? 1); + export default defineConfig({ test: { include: ["src/**/*.it.ts"], globalSetup: ["src/global-setup.ts"], - // One shared containerized stack and one shared database: files run in - // sequence. Concurrency is exercised INSIDE tests (Promise.all), which is - // what the guards under test actually see in production. - fileParallelism: false, + pool: "forks", + // Pinned min = max: VITEST_POOL_ID is the shard key, and a worker that vitest + // declines to spawn is a shard whose containers are idling. + poolOptions: { forks: { minForks: SHARDS, maxForks: SHARDS } }, + fileParallelism: SHARDS > 1, + // The freight app boots ONCE per worker and is memoised in module scope. + // With isolation on, every file would pay a fresh Nest boot (60 modules, + // TypeORM, 5 seeders) — 25 boots instead of `SHARDS`. + isolate: false, testTimeout: 180_000, hookTimeout: 180_000, // Booking/scheduling steps are not idempotent — a retry would assert - // against a half-advanced booking. + // against a half-advanced booking. (A whole-RUN retry is now safe, because + // `it.mjs test` re-clones every shard database first.) retry: 0, + // Workers hold a pg pool, three Socket.IO gateways, a RabbitMQ channel and + // live cron timers; don't wait on those handles to unwind. + teardownTimeout: 20_000, reporters: ["verbose"], }, });