test(integration): boot freight API in-process across parallel shards

The suite drove a containerized freight API, so every code change needed an
image rebuild before a test could see it, and there was no way to attach a
debugger. Files also ran strictly in sequence against one shared database,
which is the root of the warm-stack gotchas the README documents: stowaway
paid bookings climbing back aboard, a short consist on the fifth file.

The freight app now boots inside each vitest worker from dist/, and each
worker owns a whole shard of the topology - its own database, payment API,
gateway mock and broker vhost - so nothing mutable is shared and files run
in parallel. Full suite drops from roughly 20 minutes to 196s at 4 shards.

- main.ts exports createFreightApp() so the harness applies the same prefix,
  pipes, filters and interceptors as production instead of replaying them by
  hand; self-start is guarded by require.main so the Dockerfile CMD still boots
- booking-window tick cadence is env-driven (BOOKING_WINDOW_TICK_CRON), */1 in
  the suite, */10 unchanged in production
- prepare-shards.mjs seeds a template database (boot seeders, then the SQL
  fixtures that depend on them) and clones it per shard; it.mjs re-clones on
  every run, so each run is hermetic
- gateway mock and payment API are generated per shard: the mock keeps modes
  and orders process-global and 20 of 25 specs reset it in beforeAll, and the
  inbound CBE bill query has to reach one specific shard's app
- poll() samples every 250ms instead of 2000ms, keeping the caller's deadline
- authz.it.ts seeds its own invoice; it previously read another spec's leftover
  and returned early, which silently passed on a pristine database

Known: an unlocked MAX(sequence_no)+1 in train-scheduling.service.ts races
under concurrent allocation and leaves a short consist, so 1-3 specs fail
intermittently. Pre-existing and reproduces at the production tick cadence.
This commit is contained in:
Nathnael
2026-08-04 11:48:59 +00:00
parent 3a69b961d4
commit 415ae52143
13 changed files with 1045 additions and 329 deletions

3
.gitignore vendored
View File

@@ -49,3 +49,6 @@ test-results/
playwright-report/
blob-report/
RUNNING_LOCALLY.md
# Generated per-shard compose file for the integration suite (it.mjs).
integration/.it-shards.yaml

View File

@@ -19,7 +19,7 @@ import { AppModule } from "./app.module";
* ~13.4MB on the wire. Express defaults to 100kb, which rejected any real stamp
* image with a 413 "request entity too large".
*/
const JSON_BODY_LIMIT = '20mb';
const JSON_BODY_LIMIT = "20mb";
/**
* Static /etc/hosts-style overrides from `DNS_HOST_OVERRIDES`, formatted as
@@ -44,7 +44,9 @@ function applyDnsHostOverrides(): void {
}
if (overrides.size === 0) return;
const dns = createRequire(__filename)("node:dns") as typeof import("node:dns");
const dns = createRequire(__filename)(
"node:dns",
) as typeof import("node:dns");
const originalLookup = dns.lookup.bind(dns);
// `dns.lookup` is overloaded (options optional, all/family variants); the
// cast keeps that surface intact while we intercept only mapped hostnames.
@@ -63,7 +65,9 @@ function applyDnsHostOverrides(): void {
) => void;
const family = ip.includes(":") ? 6 : 4;
const wantsAll =
typeof options === "object" && options !== null && (options as { all?: boolean }).all;
typeof options === "object" &&
options !== null &&
(options as { all?: boolean }).all;
process.nextTick(() =>
wantsAll ? done(null, [{ address: ip, family }]) : done(null, ip, family),
@@ -77,7 +81,15 @@ function applyDnsHostOverrides(): void {
applyDnsHostOverrides();
async function bootstrap() {
/**
* Build the app with every global the production process applies, but do NOT
* listen. Exported so a test harness can boot the REAL app in its own process
* (integration/src/app.ts) and get the same prefix, pipe, filter, interceptor
* and body-parser configuration — replaying this list by hand is how an e2e
* harness silently drifts from production (routes 404 without the "api"
* prefix, responses lose the transform envelope).
*/
export async function createFreightApp(): Promise<NestExpressApplication> {
const app = await NestFactory.create<NestExpressApplication>(AppModule);
// Nest's own body-parser API, NOT `app.use(json(...))` from express: express
@@ -86,14 +98,14 @@ async function bootstrap() {
// pnpm's hoisted dev store and died as MODULE_NOT_FOUND in the production
// image, where `pnpm deploy --prod` installs declared dependencies only.
// This also RECONFIGURES the default parsers rather than racing them.
app.useBodyParser('json', { limit: JSON_BODY_LIMIT });
app.useBodyParser('urlencoded', { limit: JSON_BODY_LIMIT, extended: true });
app.useBodyParser("json", { limit: JSON_BODY_LIMIT });
app.useBodyParser("urlencoded", { limit: JSON_BODY_LIMIT, extended: true });
// Dev CORS: reflect any localhost origin and allow credentials so the
// freight portal (5173), passenger portal (5174), backoffices (5183/5184)
// and any other dev port can call the API with cookies + Authorization.
// For production, restrict `origin` to known FQDNs.
app.enableCors({
origin: true, // reflect request origin
credentials: true,
@@ -157,13 +169,21 @@ async function bootstrap() {
const document = SwaggerModule.createDocument(app, config);
SwaggerModule.setup("api/docs", app, document);
return app;
}
async function bootstrap() {
const app = await createFreightApp();
const port = parseInt(process.env.PORT ?? "3001", 10);
// await app.listen(port, "0.0.0.0");
await app.listen(
port)
await app.listen(port);
// eslint-disable-next-line no-console
console.log(`[freight-api] listening on port ${port}`);
}
bootstrap();
// Only self-start when this file IS the entrypoint. The Dockerfile's
// `CMD ["node", "dist/main.js"]` still boots; importers get `createFreightApp`
// without the process binding a port behind their back.
if (require.main === module) {
bootstrap();
}

View File

@@ -91,7 +91,16 @@ export class BookingWindowService implements OnModuleInit {
// 10-second cadence: every transition is derived from persisted timestamps
// and applied idempotently, so a finer tick only shrinks the lag between a
// deadline passing and the phase actually moving (was a full minute).
@Cron('*/10 * * * * *', { name: 'booking-window-tick', timeZone: BATCH_TIMEZONE })
//
// Overridable because that lag is the integration suite's pacing floor: every
// window phase, wagon allocation and expiry it waits on lands on this tick, so
// a 10s cadence costs ~5s of pure latency per wait across a few hundred waits.
// The suite runs it at `*/1 * * * * *`. Read at class-definition time, so the
// env var must be set before the module is imported.
@Cron(process.env.BOOKING_WINDOW_TICK_CRON ?? '*/10 * * * * *', {
name: 'booking-window-tick',
timeZone: BATCH_TIMEZONE,
})
async tick(): Promise<void> {
if (this.ticking) return;
this.ticking = true;

View File

@@ -1,12 +1,24 @@
# EDR Freight — API integration stack (headless).
# EDR Freight — API integration stack (headless, sharded).
#
# Overlay on docker-compose.e2e.yaml. Same base services (postgres, minio,
# mocks, freight-api), except the payment microservice is REAL here instead of
# `payment-mock-e2e`, and only the bank gateways are stubbed:
# Overlay on docker-compose.e2e.yaml, plus a GENERATED third file holding the
# per-shard services (integration/.it-shards.yaml, written by gen-shards.mjs).
#
# freight-api-it ──HTTP──> payment-api-it ──HTTP──> gateway-mock-it
# ^ │
# └────── RabbitMQ ───────┘ (outbox → payment.events → consumer)
# The freight API is NOT a container here — it boots inside each vitest worker,
# on the host, so a code change needs no image rebuild and a breakpoint works.
# Each worker is a full shard of the topology; nothing mutable is shared:
#
# shard i:
# freight app (in vitest worker, host :3111+i)
# │ ▲
# │ └──── HTTP ─── payment-api-it-{i} :3131+i (inbound CBE bill query,
# │ │ via host.docker.internal)
# └── HTTP ──────────────► │ ──HTTP──> gateway-mock-it-{i} :4600+i
# ▲ │
# └──── RabbitMQ vhost payment_s{i} ──┘ (outbox → payment.events → consumer)
#
# Shared by every shard: postgres (one container, one database per shard cloned
# from a seeded template), rabbitmq (one container, one vhost per shard), minio,
# and the one-shot migration.
#
# Its own compose project (`name:` below overrides the base) and its own host
# ports, so it can run side by side with the Cypress e2e stack.
@@ -15,13 +27,16 @@
#
# Never start it with plain `docker compose -f docker-compose.it.yaml` — it is
# an OVERLAY and needs the base file first:
# docker compose -f docker-compose.e2e.yaml -f docker-compose.it.yaml ...
# docker compose -f docker-compose.e2e.yaml -f docker-compose.it.yaml \
# -f integration/.it-shards.yaml ...
name: edr-freight-it
services:
# Outbox transport. The payment API publishes payment.succeeded/failed here
# and freight consumes it — the production path. Copied from the passenger
# harness (e2e/docker-compose.yml).
# harness (e2e/docker-compose.yml). One broker, one vhost per shard
# (payment_s0, payment_s1, …) created by it.mjs — a shared vhost would let one
# shard's freight consumer eat another shard's settlement event.
rabbitmq-it:
image: rabbitmq:3-management
environment:
@@ -32,139 +47,14 @@ services:
- "${IT_RABBIT_PORT:-5772}:5672"
- "${IT_RABBIT_UI_PORT:-15772}:15672"
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
# check_running, NOT ping: ping only proves the Erlang node answers, and
# it.mjs runs `rabbitmqctl add_vhost` the moment this goes healthy — which
# on a cold boot failed with "this command requires the 'rabbit' app to be
# running on the target node". check_running waits for the application.
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
interval: 5s
timeout: 5s
timeout: 10s
retries: 20
# Stand-in for every bank/wallet gateway the payment API talks to, plus a
# control plane the tests drive (force a provider to fail/hang, fire a
# correctly-signed webhook, read back what was called). See
# integration/gateway-mock/server.js.
gateway-mock-it:
image: node:20-alpine
volumes:
- ./integration/gateway-mock:/app:ro
working_dir: /app
environment:
PORT: "4600"
# Same secrets the payment API gets — so webhooks the mock signs pass the
# API's REAL signature verification instead of bypassing it.
CBE_SECRET_KEY: it-cbe-secret
CBE_MERCHANT_ID: it-cbe-merchant
PAYMENT_API_URL: http://payment-api-it:3003
command: ["node", "server.js"]
ports:
- "${IT_GATEWAY_PORT:-4600}:4600"
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
]
interval: 3s
timeout: 3s
retries: 10
payment-api-it:
build:
context: .
dockerfile: apps/edr-payment-api/Dockerfile
secrets:
- npmrc
depends_on:
postgres-freight-e2e:
condition: service_healthy
rabbitmq-it:
condition: service_healthy
gateway-mock-it:
condition: service_healthy
environment:
PORT: "3003"
NODE_ENV: test
# Payment tables live in their own schema of the same throwaway DB;
# main.ts ensurePaymentSchema() creates it, migrationsRun does the rest.
DB_HOST: postgres-freight-e2e
DB_PORT: "5432"
DB_USER: edr_e2e
DB_PASSWORD: edr_e2e
DB_NAME: edr_freight_e2e
DB_SCHEMA: edr_payment
# Same token freight already uses in the base stack.
SERVICE_AUTH_TOKEN: e2e-service-token
PUBLISHER_TRANSPORT: rabbitmq
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/payment
# HTTP fallback targets (only used with PUBLISHER_TRANSPORT=http).
PAYMENT_NOTIFY_FREIGHT_URL: http://freight-api-e2e:3001/api/internal/payments/mark-paid
# Fast relay + sweep so retry/reconciliation are observable inside a test
# rather than a minute later.
OUTBOX_RELAY_INTERVAL_MS: "1000"
RECONCILE_STALE_AFTER_MS: "5000"
# Every gateway points at the one mock. Paths are per-provider prefixes.
CBE_BASE_URL: http://gateway-mock-it:4600/cbe-birr
CBE_MERCHANT_ID: it-cbe-merchant
CBE_SECRET_KEY: it-cbe-secret
CBE_NOTIFY_URL: http://payment-api-it:3003/webhooks/cbe-birr
CBE_RETURN_URL: http://localhost/return
TELEBIRR_BASE_URL: http://gateway-mock-it:4600/telebirr
TELEBIRR_WEB_BASE_URL: http://gateway-mock-it:4600/telebirr/web
TELEBIRR_FABRIC_APP_ID: it-fabric
TELEBIRR_APP_SECRET: it-secret
TELEBIRR_MERCHANT_APP_ID: it-merchant-app
TELEBIRR_MERCHANT_CODE: "999999"
TELEBIRR_NOTIFY_URL: http://payment-api-it:3003/webhooks/telebirr
# Telebirr PSS-signs every request object — a throwaway key generated per
# launch by it.mjs (nothing key-shaped lives in git).
TELEBIRR_PRIVATE_KEY: ${IT_TELEBIRR_PRIVATE_KEY}
EBIRR_BASE_URL: http://gateway-mock-it:4600/ebirr
DMONEY_BASE_URL: http://gateway-mock-it:4600/dmoney
CARD_BASE_URL: http://gateway-mock-it:4600/card
WAAFI_BASE_URL: http://gateway-mock-it:4600/waafi
CAC_BASE_URL: http://gateway-mock-it:4600/cac
CAC_USERNAME: it-cac
CAC_PASSWORD: it-cac
CAC_APP_KEY: it-cac-key
CAC_API_KEY: it-cac-api
CAC_COMPANY_SERVICES_ID: "1"
# Inbound CBE Unified Bill — we are the biller; bill-query hops back into
# the freight API, so this direction runs real code on both sides.
CBE_BILL_ENABLED: "true"
CBE_BILL_CLIENT_ID: it-cbe-bill
CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret
CBE_BILL_JWT_SECRET: it-cbe-bill-jwt
FREIGHT_API_BASE_URL: http://freight-api-e2e:3001/api
ports:
- "${IT_PAYMENT_PORT:-3113}:3003"
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
]
interval: 5s
timeout: 5s
retries: 12
start_period: 40s
# Base-stack service, re-pointed at the real payment API.
freight-api-e2e:
depends_on:
payment-api-it:
condition: service_healthy
environment:
PAYMENT_API_URL: http://payment-api-it:3003
# Freight's payment module skips RabbitMQModule entirely when this is
# unset (payment.module.ts) — without it, outbox events never arrive.
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/payment
# RABBITMQ_ENABLED stays "false" (base stack) — it only gates the SMS/email
# clients, which must remain off. The payment consumer is wired by
# PAYMENT_RABBITMQ_URL alone.
# Drain tail on every pay window. Production defaults to 5 minutes; a
# reservation here lives ~60s, so 5 would push every natural expiry past
# the suite's 180s timeouts. One minute keeps the tail real and observable
# (src/expired-invoice-late-settle.it.ts asserts both sides of it).
FREIGHT_PAYMENT_DRAIN_MINUTES: "1"
# The per-shard `gateway-mock-it-{i}` and `payment-api-it-{i}` services live in
# integration/.it-shards.yaml. The base file's `freight-api-e2e` is never
# started — the app runs in-process (integration/src/app.ts).

View File

@@ -3,29 +3,77 @@
Headless, API-level tests for the freight API running against the **real**
`edr-payment-api`. Only the bank/wallet gateways are stubbed.
The freight API is **not containerized** here — it boots inside each vitest
worker from `apps/edr-freight-api/dist`, which `it:test` rebuilds every run. A
code change needs no image rebuild, and a breakpoint in the API is hit by the
test that provoked it.
```
pnpm it:up # build + start the stack (first run ~5 min)
pnpm it:test # vitest run (auto-ups the stack if needed)
pnpm it:test -- --reporter=basic src/payment-happy.it.ts
pnpm it:logs payment-api-it
pnpm it:down # -v, wipes the throwaway DB
pnpm it:test --no-reset -- src/payment-happy.it.ts # skip the DB re-clone
pnpm it:logs payment-api-it-0
pnpm it:down # -v, wipes the throwaway DBs and the generated shard file
IT_SHARDS=1 pnpm it:up # 1 shard when RAM is tight or output must be readable
```
## Stack
`docker-compose.it.yaml` is an **overlay** on `docker-compose.e2e.yaml` — same
freight API, Postgres (tmpfs), MinIO, Fayda/eTrade mocks; plus RabbitMQ, the
real payment API, and one gateway mock. It is a separate compose project
(`edr-freight-it`) on offset ports, so the Cypress e2e stack can run alongside.
`docker-compose.it.yaml` is an **overlay** on `docker-compose.e2e.yaml`, plus a
third **generated** file (`integration/.it-shards.yaml`, written by
`scripts/gen-shards.mjs`, gitignored) holding the per-shard services. It is a
separate compose project (`edr-freight-it`) on offset ports, so the Cypress e2e
stack can run alongside.
Each vitest worker is a whole shard of the topology. Nothing mutable is shared
between shards, so spec files run in parallel:
```
freight-api-e2e :3111 ──HTTP──> payment-api-it :3113 ──HTTP──> gateway-mock-it :4600
^ │
└────────── RabbitMQ :5772 ─────┘ (outbox → payment.events → consumer)
shard i (vitest worker i, VITEST_POOL_ID)
freight app in-process, host :3111+i
│ ▲
│ └──── HTTP ──── payment-api-it-{i} :3131+i inbound CBE bill query,
│ │ via host.docker.internal
└── HTTP ──────────────► │ ──HTTP──> gateway-mock-it-{i} :4600+i
▲ │
└── RabbitMQ vhost payment_s{i} ──┘ (outbox → payment.events → consumer)
database edr_it_s{i} ← CREATE DATABASE … TEMPLATE edr_freight_e2e
freight tables AND the payment API's edr_payment
schema live in it, so `db()` and `paymentDb()` are
one connection
```
Shared by every shard: Postgres (one container, one database each), RabbitMQ
(one container, one vhost each), MinIO, and the one-shot migration.
`IT_SHARDS` defaults to **4**. RAM is the ceiling, not cores — a shard is an
in-process Nest app plus two containers.
Never `docker compose -f docker-compose.it.yaml` on its own — it needs the base
file first. Use `it.mjs`.
file first, and the generated shard file last. Use `it.mjs`.
## Setup order
The sequence is load-bearing, which is why `scripts/prepare-shards.mjs` exists
rather than vitest's `globalSetup`:
1. `freight-migration-e2e` migrates the **template** database `edr_freight_e2e`.
2. `prepare-shards.mjs` boots the freight app once against the template and
calls `app.init()` — that fires `onApplicationBootstrap`, the always-on
seeders (org, units, positions, permissions).
3. …then applies the SQL fixtures, which declare exactly those as prerequisites
(`seed-users.sql`: *"created by the API's always-on boot seeders"*).
4. `CREATE DATABASE edr_it_s{i} TEMPLATE edr_freight_e2e` per shard — a file
copy on the tmpfs Postgres, not a re-run of migrations and five seeders.
5. The per-shard payment APIs start; each creates its own `edr_payment` schema
inside its shard database.
`it:test` re-runs step 4 on **every run** (stop payment APIs → drop → clone →
start), so each run is hermetic. `--no-reset` skips it for a quick rerun.
## Gateway mock
@@ -46,7 +94,15 @@ negative test. The suite drives **CBE Birr** end to end (plain HMAC, no key
material); other providers answer a generic stub until a scenario needs them.
Editing `server.js` needs a container restart (`docker compose … restart
gateway-mock-it`) — the code is a read-only mount, not baked into an image.
gateway-mock-it-0`) — the code is a read-only mount, not baked into an image.
**One mock per shard.** `modes`, `calls` and `orders` are process-global in
`server.js`, and 20 of 25 specs call `POST /__control/reset` in `beforeAll`. On a
shared mock every starting spec would wipe every running spec's live orders —
and that fails as a wrong *assertion*, not an error: an unknown order still gets
a correctly-signed webhook, just carrying the mock's default amount. Per-shard
mocks are why the suite can run `payment-failure` (which forces providers into
`fail`/`timeout`) beside a spec that settles normally.
## Files
@@ -151,21 +207,34 @@ what it actually does and says so in a comment, so a fix fails loudly:
ignores `deleted_at`, so a soft-unlinked booking can never be re-batched).
- **Arrange is slow.** Contract → booking → clearance → ops accept → batch is
3060s of real API work per booking, so files share one schedule day.
- Files run sequentially (one DB); concurrency is exercised inside a test with
`Promise.all`.
- **Files run in parallel across shards, sequentially within one.** A shard owns
its whole topology, so two files never contend; two files on the *same* shard
still run one after the other, which is what the day-offset partitioning and
the per-file `releaseUnpaidHolds()` / `resetCorridorDay()` hygiene still
assume. Concurrency inside a single test is exercised with `Promise.all`.
- **Day offsets are the de-facto partition key** (`departureAt(n)` per file).
`payment-happy` and `expired-invoice-late-settle` both claim day 4 — harmless
now that a shard has its own database, but do not read the offsets as unique.
- **A retired fixture keeps its shipment day at its peril.**
`rescueStrandedPaidForDay` sweeps every unlinked booking whose
`payment_status` is PAID and whose `scheduled_date` falls on the day being
filled, and re-places it on the fresh train. A previous run's paid bookings
therefore climb back aboard — 18 stowaway wagons on a 28-wagon day, until
`releaseUnpaidHolds` / `resetCorridorDay` started nulling `scheduled_date`.
filled, and re-places it on the fresh train. Within a run this still bites
across files on one shard; *between* runs it no longer can — `it:test`
re-clones every shard database from the template first.
- **Only a FULL train rests at DONE.** An under-filled day CONCLUDES and
REOPENS (`window_phase` back to OPEN, `booking_cycle_no` 2), so waiting for
DONE there waits forever — use `pollCycleConcluded`.
- **Wagon stock is finite and shared.** Paid bookings keep their wagons, so
`releaseUnpaidHolds()` also frees every earlier `CTR-IT-%` allocation —
without it the fifth or sixth file on a warm stack silently gets a short
consist.
- **Wagon stock is finite and shared** *within a shard*. Paid bookings keep
their wagons, so `releaseUnpaidHolds()` also frees every earlier `CTR-IT-%`
allocation — without it the fifth or sixth file on one shard silently gets a
short consist.
- **`poll()` samples every 250ms, but keeps the caller's deadline.** Callers pass
`attempts × intervalMs`; that product is the timeout, and only the sampling
rate changed. Override with `IT_POLL_INTERVAL_MS`.
- **The tick cadence is env-driven.** `BOOKING_WINDOW_TICK_CRON` is `*/1` here
and `*/10` in production. Every window phase, allocation and expiry the suite
waits on lands on that tick. A spec that proves sensitive to it should pin its
own value rather than the whole suite reverting.
- **One tenant per booking.** `seedTenantContracts` mints a company per booking
because a company may hold only one unpaid reservation at a time; staff book
and pay on their behalf, which is also the real Path B flow.

View File

@@ -0,0 +1,182 @@
#!/usr/bin/env node
/**
* Generate `integration/.it-shards.yaml` — one payment API + one gateway mock
* per shard. Compose has no loops, so the per-shard services are written out.
*
* Every shard is the real topology in miniature; nothing crosses between them:
*
* gateway-mock-it-{i} its own `modes`/`calls`/`orders` — the mock keeps those
* process-global, and 20 of 25 specs call
* POST /__control/reset in beforeAll, so a shared mock
* would have each starting spec wipe every running spec's
* live orders (which fails as a wrong *assertion*: an
* unknown order still gets a correctly-signed webhook,
* just with the mock's default amount).
* payment-api-it-{i} its own DB_NAME (the shard's database, `edr_payment`
* schema inside it), its own broker vhost, and its own
* FREIGHT_API_BASE_URL — the inbound CBE-bill query has
* to land on THIS shard's in-process app, and a single
* container could only ever point at one of them.
*
* Written by it.mjs, gitignored. Do not edit by hand.
*/
import { writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
/**
* All shards share one build and one image tag, so the payment API image is
* built once and reused rather than N times.
*/
const PAYMENT_IMAGE = "edr-payment-api-it:local";
export function renderShards({
shards,
apiPortBase,
paymentPortBase,
gatewayPortBase,
dbPrefix,
telebirrKey,
}) {
const services = [];
for (let i = 0; i < shards; i++) {
const gw = `gateway-mock-it-${i}`;
const pay = `payment-api-it-${i}`;
const db = `${dbPrefix}${i}`;
const vhost = `payment_s${i}`;
// The freight app for this shard runs on the HOST, inside the vitest worker.
const freightBase = `http://host.docker.internal:${apiPortBase + i}/api`;
services.push(`
${gw}:
image: node:20-alpine
volumes:
- ./integration/gateway-mock:/app:ro
working_dir: /app
environment:
PORT: "4600"
# Same secrets the payment API gets — so webhooks the mock signs pass the
# API's REAL signature verification instead of bypassing it.
CBE_SECRET_KEY: it-cbe-secret
CBE_MERCHANT_ID: it-cbe-merchant
PAYMENT_API_URL: http://${pay}:3003
command: ["node", "server.js"]
ports:
- "${gatewayPortBase + i}:4600"
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
]
interval: 3s
timeout: 3s
retries: 10
${pay}:
image: ${PAYMENT_IMAGE}
build:
context: .
dockerfile: apps/edr-payment-api/Dockerfile
secrets:
- npmrc
depends_on:
postgres-freight-e2e:
condition: service_healthy
rabbitmq-it:
condition: service_healthy
${gw}:
condition: service_healthy
# The freight app is on the host now, not in this network.
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
PORT: "3003"
NODE_ENV: test
# Payment tables live in their own schema of THIS SHARD's database;
# main.ts ensurePaymentSchema() creates it, migrationsRun does the rest.
# Same database as freight, so every raw edr_payment.* query in the specs
# reads through the one connection.
DB_HOST: postgres-freight-e2e
DB_PORT: "5432"
DB_USER: edr_e2e
DB_PASSWORD: edr_e2e
DB_NAME: ${db}
DB_SCHEMA: edr_payment
SERVICE_AUTH_TOKEN: e2e-service-token
PUBLISHER_TRANSPORT: rabbitmq
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/${vhost}
PAYMENT_NOTIFY_FREIGHT_URL: ${freightBase}/internal/payments/mark-paid
# Fast relay + sweep so retry/reconciliation are observable inside a test
# rather than a minute later.
OUTBOX_RELAY_INTERVAL_MS: "1000"
RECONCILE_STALE_AFTER_MS: "5000"
# Every gateway points at this shard's mock. Paths are per-provider prefixes.
CBE_BASE_URL: http://${gw}:4600/cbe-birr
CBE_MERCHANT_ID: it-cbe-merchant
CBE_SECRET_KEY: it-cbe-secret
CBE_NOTIFY_URL: http://${pay}:3003/webhooks/cbe-birr
CBE_RETURN_URL: http://localhost/return
TELEBIRR_BASE_URL: http://${gw}:4600/telebirr
TELEBIRR_WEB_BASE_URL: http://${gw}:4600/telebirr/web
TELEBIRR_FABRIC_APP_ID: it-fabric
TELEBIRR_APP_SECRET: it-secret
TELEBIRR_MERCHANT_APP_ID: it-merchant-app
TELEBIRR_MERCHANT_CODE: "999999"
TELEBIRR_NOTIFY_URL: http://${pay}:3003/webhooks/telebirr
# Telebirr PSS-signs every request object — a throwaway key generated per
# launch by it.mjs (nothing key-shaped lives in git).
TELEBIRR_PRIVATE_KEY: ${JSON.stringify(telebirrKey)}
EBIRR_BASE_URL: http://${gw}:4600/ebirr
DMONEY_BASE_URL: http://${gw}:4600/dmoney
CARD_BASE_URL: http://${gw}:4600/card
WAAFI_BASE_URL: http://${gw}:4600/waafi
CAC_BASE_URL: http://${gw}:4600/cac
CAC_USERNAME: it-cac
CAC_PASSWORD: it-cac
CAC_APP_KEY: it-cac-key
CAC_API_KEY: it-cac-api
CAC_COMPANY_SERVICES_ID: "1"
# Inbound CBE Unified Bill — we are the biller; bill-query hops back into
# the freight API, so this direction runs real code on both sides.
CBE_BILL_ENABLED: "true"
CBE_BILL_CLIENT_ID: it-cbe-bill
CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret
CBE_BILL_JWT_SECRET: it-cbe-bill-jwt
FREIGHT_API_BASE_URL: ${freightBase}
ports:
- "${paymentPortBase + i}:3003"
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
]
interval: 5s
timeout: 5s
retries: 12
start_period: 40s`);
}
return `# GENERATED by integration/scripts/gen-shards.mjs — do not edit.
# ${shards} shard(s). Overlaid on docker-compose.e2e.yaml + docker-compose.it.yaml.
services:${services.join("\n")}
`;
}
export const SHARD_FILE = join(itDir, ".it-shards.yaml");
export function writeShards(opts) {
writeFileSync(SHARD_FILE, renderShards(opts));
return SHARD_FILE;
}
export const shardServices = (shards) =>
Array.from({ length: shards }, (_, i) => [`gateway-mock-it-${i}`, `payment-api-it-${i}`]).flat();

View File

@@ -3,10 +3,13 @@
* Freight integration-suite launcher.
*
* node integration/scripts/it.mjs <up|test|down|logs> [vitest args...]
* IT_SHARDS=4 node integration/scripts/it.mjs test
* node integration/scripts/it.mjs test --no-reset -- src/payment-happy.it.ts
*
* Overlays docker-compose.it.yaml on docker-compose.e2e.yaml: same freight
* stack, but the payment microservice is real and only the bank gateways are
* stubbed. Web/Cypress containers are never started — this suite is HTTP only.
* The freight API is NOT containerized here: it boots inside each vitest worker
* from `apps/edr-freight-api/dist`, which `test` rebuilds every run. Each worker
* is a whole shard — own database, own payment API, own gateway mock, own broker
* vhost — so spec files run in parallel without sharing anything mutable.
*
* Ports are fixed (and distinct from the Cypress e2e defaults) so both stacks
* can be up at once; they are separate compose projects.
@@ -16,21 +19,24 @@
import { execFileSync, spawnSync } from "node:child_process";
import { generateKeyPairSync } from "node:crypto";
import { existsSync } from "node:fs";
import { existsSync, rmSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { SHARD_FILE, shardServices, writeShards } from "./gen-shards.mjs";
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const repoRoot = resolve(itDir, "..");
const composeBase = [
"compose",
"-f",
join(repoRoot, "docker-compose.e2e.yaml"),
"-f",
join(repoRoot, "docker-compose.it.yaml"),
];
/** Deliberately offset from the Cypress stack's defaults (3101/5533/9310…). */
/**
* Deliberately offset from the Cypress stack's defaults (3101/5533/9310…).
*
* Each per-shard service takes a CONTIGUOUS BLOCK of `IT_SHARDS` ports starting
* at its base (freight :3111+i, payment :3131+i, gateway :4600+i), so the bases
* must be at least MAX_SHARDS apart. They were originally 3111 and 3113 — two
* apart — which silently worked at 12 shards and then had shard 2's freight app
* try to bind :3113, the port payment-api-it-0 was already published on.
*/
const PORTS = {
E2E_API_PORT: 3111,
E2E_DB_PORT: 5543,
@@ -39,32 +45,39 @@ const PORTS = {
// Unused here (no web containers) but referenced by the base file's build args.
E2E_PORTAL_PORT: 5393,
E2E_BACKOFFICE_PORT: 5394,
IT_PAYMENT_PORT: 3113,
IT_PAYMENT_PORT: 3131,
IT_GATEWAY_PORT: 4600,
IT_RABBIT_PORT: 5772,
IT_RABBIT_UI_PORT: 15772,
};
/** Everything the suite needs up — web + cypress are deliberately absent. */
const SERVICES = [
"postgres-freight-e2e",
"minio-e2e",
"minio-init-e2e",
"freight-migration-e2e",
"fayda-mock-e2e",
"etrade-mock-e2e",
// Still a base-stack dependency of freight-api-e2e (reconcile-before-expire
// has its own client); cheap to run alongside the real payment API.
"payment-mock-e2e",
"gateway-mock-it",
"rabbitmq-it",
"payment-api-it",
"freight-api-e2e",
];
/** Beyond this the freight block (3111+) would run into the payment block (3131+). */
const MAX_SHARDS = 16;
const RUNNING = SERVICES.filter(
(s) => !["minio-init-e2e", "freight-migration-e2e"].includes(s),
);
/**
* How many shards. RAM is the ceiling, not cores: each shard is an in-process
* Nest app plus two containers.
*/
const SHARDS = Math.max(1, Number(process.env.IT_SHARDS ?? 4));
if (SHARDS > MAX_SHARDS) {
console.error(
`\nit: IT_SHARDS=${SHARDS} exceeds ${MAX_SHARDS} — the per-service port blocks would overlap.` +
`\n Raise IT_PAYMENT_PORT/IT_GATEWAY_PORT in it.mjs first.`,
);
process.exit(1);
}
const TEMPLATE_DB = "edr_freight_e2e";
const DB_PREFIX = "edr_it_s";
/** Long-running services shared by every shard. Web + Cypress are absent. */
const INFRA = ["postgres-freight-e2e", "minio-e2e", "rabbitmq-it"];
/**
* One-shots: they run, then exit 0. They cannot go in the `up --wait` set —
* compose reports an exited container as a failed wait, so a healthy stack
* looks broken. Run them with `compose run`, which returns their exit code.
*/
const ONE_SHOT = ["minio-init-e2e", "freight-migration-e2e"];
function fail(msg) {
console.error(`\nit: ${msg}`);
@@ -97,64 +110,271 @@ function fakeFaydaPrivateKeyBase64() {
return Buffer.from(JSON.stringify(jwk)).toString("base64");
}
const telebirrKey = process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey();
/**
* The freight app's own environment. This is docker-compose.e2e.yaml's
* `freight-api-e2e` block with container hostnames swapped for published host
* ports — the app runs on the host now. Per-shard values (DB_NAME, PORT,
* PAYMENT_API_URL, PAYMENT_RABBITMQ_URL) are derived per worker in src/app.ts.
*/
const freightEnv = {
NODE_ENV: "test",
DB_HOST: "localhost",
DB_PORT: String(PORTS.E2E_DB_PORT),
DB_USER: "edr_e2e",
DB_PASSWORD: "edr_e2e",
DB_NAME: TEMPLATE_DB,
// e2e-only secrets — never reuse outside this stack
JWT_SECRET: "e2e-jwt-secret",
JWT_ACCESS_TOKEN_SECRET: "e2e-access-secret",
JWT_REFRESH_TOKEN_SECRET: "e2e-refresh-secret",
JWT_EXPIRES_IN: "1d",
JWT_ACCESS_TOKEN_EXPIRES: "1d",
JWT_REFRESH_TOKEN_EXPIRES: "7d",
// Mandatory: ServiceAuthGuard returns TRUE when this is unset, and
// authz.it.ts asserts that the internal surface rejects an unsigned caller.
SERVICE_AUTH_TOKEN: "e2e-service-token",
SEED_EDR_ORG: "true",
SUPER_ADMIN_EMAIL: "superadmin@tria.com",
SUPER_ADMIN_PHONE: "+251900000000",
MINIO_ENDPOINT: "localhost",
MINIO_PORT: String(PORTS.E2E_MINIO_PORT),
MINIO_USE_SSL: "false",
MINIO_ACCESS_KEY: "e2e-minio",
MINIO_SECRET_KEY: "e2e-minio-secret",
MINIO_REGION: "us-east-1",
// Only gates the SMS/email clients. The payment consumer is wired by
// PAYMENT_RABBITMQ_URL alone (payment.module.ts).
RABBITMQ_ENABLED: "false",
// fayda.config.ts THROWS at load if this is "true" without the full var set,
// and the fayda/etrade mocks publish no host ports. No IT spec touches them.
FAYDA_ENABLED: "false",
// SMS strategy has no kill switch and defaults to a real dev endpoint.
OZIKING_SMS_URL: "http://127.0.0.1:9/sms",
// app.config.ts otherwise live-scrapes https://ethio.forex on first booking.
CBE_EXCHANGE_SCRAPE_URL: "http://127.0.0.1:9/fx",
CBE_EXCHANGE_API_URL: "http://127.0.0.1:9/fx",
CBE_EXCHANGE_FALLBACK_RATE: "130",
FREIGHT_PORTAL_URL: `http://localhost:${PORTS.E2E_PORTAL_PORT}`,
// Drain tail on every pay window. Production defaults to 5 minutes; a
// reservation here lives ~60s, so 5 would push every natural expiry past the
// suite's timeouts. One minute keeps the tail real and observable
// (src/expired-invoice-late-settle.it.ts asserts both sides of it, and
// hardcodes DRAIN_MS = 60_000 to match).
FREIGHT_PAYMENT_DRAIN_MINUTES: "1",
// The suite's pacing floor: every window phase, allocation and expiry it waits
// on lands on this tick. Production stays at */10. Overridable from the
// environment so a cadence-sensitive spec (or a bisect) can pin it back.
BOOKING_WINDOW_TICK_CRON: process.env.BOOKING_WINDOW_TICK_CRON ?? "*/1 * * * * *",
};
const env = {
...process.env,
...Object.fromEntries(Object.entries(PORTS).map(([k, v]) => [k, String(v)])),
IT_API_URL: `http://localhost:${PORTS.E2E_API_PORT}`,
IT_PAYMENT_URL: `http://localhost:${PORTS.IT_PAYMENT_PORT}`,
IT_GATEWAY_URL: `http://localhost:${PORTS.IT_GATEWAY_PORT}`,
IT_DB_URL: `postgres://edr_e2e:edr_e2e@localhost:${PORTS.E2E_DB_PORT}/edr_freight_e2e`,
...freightEnv,
IT_SHARDS: String(SHARDS),
IT_TEMPLATE_DB: TEMPLATE_DB,
IT_DB_PREFIX: DB_PREFIX,
IT_API_PORT_BASE: String(PORTS.E2E_API_PORT),
IT_PAYMENT_PORT_BASE: String(PORTS.IT_PAYMENT_PORT),
IT_GATEWAY_PORT_BASE: String(PORTS.IT_GATEWAY_PORT),
FAYDA_PRIVATE_KEY_BASE64:
process.env.FAYDA_PRIVATE_KEY_BASE64 ?? fakeFaydaPrivateKeyBase64(),
IT_TELEBIRR_PRIVATE_KEY:
process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey(),
IT_TELEBIRR_PRIVATE_KEY: telebirrKey,
};
function generateShardFile() {
writeShards({
shards: SHARDS,
apiPortBase: PORTS.E2E_API_PORT,
paymentPortBase: PORTS.IT_PAYMENT_PORT,
gatewayPortBase: PORTS.IT_GATEWAY_PORT,
dbPrefix: DB_PREFIX,
telebirrKey,
});
}
/** The shard file must exist before compose is invoked — it is one of the -f's. */
function composeBase() {
if (!existsSync(SHARD_FILE)) generateShardFile();
return [
"compose",
"-f",
join(repoRoot, "docker-compose.e2e.yaml"),
"-f",
join(repoRoot, "docker-compose.it.yaml"),
"-f",
SHARD_FILE,
];
}
function compose(args) {
const { status } = spawnSync("docker", [...composeBase, ...args], { stdio: "inherit", env });
const { status } = spawnSync("docker", [...composeBase(), ...args], {
stdio: "inherit",
env,
});
return status ?? 1;
}
function composeQuiet(args) {
return spawnSync("docker", [...composeBase(), ...args], {
encoding: "utf8",
env,
stdio: ["ignore", "pipe", "pipe"],
});
}
function stackRunning() {
try {
const out = execFileSync("docker", [...composeBase, "ps", "--services", "--status", "running"], {
encoding: "utf8",
env,
stdio: ["ignore", "pipe", "ignore"],
});
const out = execFileSync(
"docker",
[...composeBase(), "ps", "--services", "--status", "running"],
{ encoding: "utf8", env, stdio: ["ignore", "pipe", "ignore"] },
);
const running = new Set(out.split("\n").filter(Boolean));
return RUNNING.every((s) => running.has(s));
const want = [...INFRA, ...shardServices(SHARDS)];
return want.every((s) => running.has(s));
} catch {
return false;
}
}
function node(script, args = []) {
const { status } = spawnSync("node", [join(itDir, "scripts", script), ...args], {
cwd: repoRoot,
stdio: "inherit",
env,
});
return status ?? 1;
}
/**
* One broker, one vhost per shard — a shared vhost would let one shard's freight
* consumer eat another shard's settlement event.
*
* Retried: the healthcheck now waits for the rabbit *application*, but a cold
* boot can still land between "healthy" and "accepting rabbitmqctl", and a
* half-provisioned broker fails later as an unexplained missing settlement.
*/
function createVhosts() {
for (let i = 0; i < SHARDS; i++) {
const vhost = `payment_s${i}`;
let last = "";
let ok = false;
for (let attempt = 1; attempt <= 10 && !ok; attempt++) {
// Idempotent: add_vhost on an existing vhost exits non-zero, which is fine.
composeQuiet(["exec", "-T", "rabbitmq-it", "rabbitmqctl", "add_vhost", vhost]);
const perm = composeQuiet([
"exec", "-T", "rabbitmq-it",
"rabbitmqctl", "set_permissions", "-p", vhost, "edr", ".*", ".*", ".*",
]);
ok = perm.status === 0;
last = perm.stderr ?? "";
if (!ok) execFileSync("sleep", ["3"]);
}
if (!ok) fail(`could not grant on rabbit vhost ${vhost} after 10 tries:\n${last}`);
}
console.log(`it: rabbit vhosts payment_s0…payment_s${SHARDS - 1} ready`);
}
function buildFreight() {
console.log("it: building @edr/freight-api (the suite runs dist/, never stale)");
const { status } = spawnSync("pnpm", ["--filter", "@edr/freight-api", "run", "build"], {
cwd: repoRoot,
stdio: "inherit",
env,
});
if (status !== 0) fail("freight API build failed — fix it before running the suite.");
}
function up() {
preflight();
generateShardFile();
console.log(
`it: starting stack — freight :${PORTS.E2E_API_PORT} payment :${PORTS.IT_PAYMENT_PORT} ` +
`gateway :${PORTS.IT_GATEWAY_PORT} db :${PORTS.E2E_DB_PORT}`,
`it: starting ${SHARDS} shard(s) — freight :${PORTS.E2E_API_PORT}..${
PORTS.E2E_API_PORT + SHARDS - 1
} (in-process) payment :${PORTS.IT_PAYMENT_PORT}.. gateway :${
PORTS.IT_GATEWAY_PORT
}.. db :${PORTS.E2E_DB_PORT}`,
);
if (compose(["up", "-d", "--build", "--wait", ...SERVICES]) !== 0) {
// 1. Shared infrastructure.
if (compose(["up", "-d", "--build", "--wait", "--remove-orphans", ...INFRA]) !== 0) {
fail(
"stack failed to become healthy. Inspect with:\n" +
" node integration/scripts/it.mjs logs payment-api-it",
"shared services failed to become healthy. Inspect with:\n" +
" node integration/scripts/it.mjs logs postgres-freight-e2e",
);
}
createVhosts();
// 1b. One-shots: the MinIO bucket, then freight's migrations into the
// TEMPLATE database every shard is cloned from.
for (const svc of ONE_SHOT) {
if (compose(["run", "--rm", "--no-deps", "--build", svc]) !== 0) {
fail(`${svc} failed — inspect with:\n node integration/scripts/it.mjs logs ${svc}`);
}
}
// 2. The app must exist as dist/ before we can boot it to seed the template.
buildFreight();
// 3. Seed the template (boot seeders + SQL fixtures) and clone it per shard.
// Must precede the payment APIs: they connect to the shard databases.
if (node("prepare-shards.mjs") !== 0) fail("template seed / shard clone failed.");
// 4. Per-shard payment API + gateway mock.
if (compose(["up", "-d", "--build", "--wait", ...shardServices(SHARDS)]) !== 0) {
fail(
"shard services failed to become healthy. Inspect with:\n" +
" node integration/scripts/it.mjs logs payment-api-it-0",
);
}
}
const [cmd, ...rawExtra] = process.argv.slice(2);
/**
* Re-clone every shard database from the template so each run is hermetic. The
* payment APIs hold connections to those databases, so they come down first —
* their `edr_payment` schema is recreated by their own boot migrations.
*
* This is what retires the warm-stack failure modes the suite used to document:
* stowaway paid bookings climbing back aboard, a short consist on the fifth
* file, invoice numbers continuing from a previous run.
*/
function resetShardDbs() {
const svc = shardServices(SHARDS);
console.log("it: re-cloning shard databases from the template");
compose(["stop", ...svc]);
if (node("prepare-shards.mjs", ["--clone"]) !== 0) fail("shard database reset failed.");
if (compose(["up", "-d", "--wait", ...svc]) !== 0) {
fail("shard services failed to become healthy after the reset.");
}
}
const [cmd, ...rawArgs] = process.argv.slice(2);
const noReset = rawArgs.includes("--no-reset");
// `pnpm it:test -- src/foo.it.ts` hands us a literal "--" first. Forwarding it
// makes vitest treat everything after it as CLI options and ignore the file
// filter — the "one file" run silently becomes the whole suite.
const extra = rawExtra[0] === "--" ? rawExtra.slice(1) : rawExtra;
const passthrough = rawArgs.filter((a) => a !== "--no-reset");
const extra = passthrough[0] === "--" ? passthrough.slice(1) : passthrough;
switch (cmd) {
case "up":
up();
break;
case "test": {
if (!stackRunning()) up();
if (!stackRunning()) {
up();
} else {
buildFreight();
if (!noReset) resetShardDbs();
}
const { status } = spawnSync(
"pnpm",
["--filter", "@edr/freight-integration", "run", "test", ...extra],
@@ -162,12 +382,14 @@ switch (cmd) {
);
process.exit(status ?? 1);
}
case "logs":
case "logs": {
process.exit(compose(["logs", "--tail", "200", ...extra]));
break;
case "down":
process.exit(compose(["down", "-v", "--remove-orphans"]));
break;
}
case "down": {
const status = compose(["down", "-v", "--remove-orphans"]);
rmSync(SHARD_FILE, { force: true });
process.exit(status);
}
default:
fail(`unknown command "${cmd ?? ""}" — use up | test | logs | down`);
}

View File

@@ -0,0 +1,134 @@
#!/usr/bin/env node
/**
* Build the seeded template database, then clone it once per shard.
*
* node integration/scripts/prepare-shards.mjs # seed template + clone
* node integration/scripts/prepare-shards.mjs --clone # clone only (per-run reset)
*
* Seeding has to happen here, not in vitest's globalSetup, because the order is
* load-bearing and half of it is the app itself:
*
* 1. freight migrations (freight-migration-e2e container, before us)
* 2. the app's always-on boot seeders — org, units, positions, permissions
* (app.module.ts onApplicationBootstrap)
* 3. the SQL fixtures, which declare those as prerequisites
* (seed-users.sql: "created by the API's always-on boot seeders")
*
* Step 2 needs a real Nest boot, and the app now lives inside the vitest workers
* — which globalSetup runs in a different process from. So we boot it once here
* against the template and every shard is a `CREATE DATABASE … TEMPLATE` copy of
* the result: a file copy on the tmpfs postgres, versus re-running migrations
* and five seeders per shard.
*
* Run by it.mjs, which owns the env. No dependencies beyond `pg`.
*/
import { createRequire } from "node:module";
import { readFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { Client } from "pg";
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const repoRoot = resolve(itDir, "..");
const freightDir = join(repoRoot, "apps", "edr-freight-api");
const TEMPLATE = process.env.IT_TEMPLATE_DB ?? "edr_freight_e2e";
const PREFIX = process.env.IT_DB_PREFIX ?? "edr_it_s";
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
const admin = () =>
new Client({
host: process.env.DB_HOST ?? "localhost",
port: Number(process.env.DB_PORT ?? 5543),
user: process.env.DB_USER ?? "edr_e2e",
password: process.env.DB_PASSWORD ?? "edr_e2e",
// Never the template itself: CREATE DATABASE … TEMPLATE refuses while any
// session is connected to the source.
database: "postgres",
});
const CYPRESS_FIXTURES = join(repoRoot, "e2e", "freight", "cypress", "fixtures");
const OWN_FIXTURES = join(itDir, "sql");
/** Order matters — company needs the users, everything needs the corridor. */
const SEEDS = [
[CYPRESS_FIXTURES, "seed-users.sql"],
[CYPRESS_FIXTURES, "seed-company.sql"],
[CYPRESS_FIXTURES, "seed-import-corridor.sql"],
[CYPRESS_FIXTURES, "seed-bulk-items.sql"],
[CYPRESS_FIXTURES, "seed-g1-train.sql"],
[CYPRESS_FIXTURES, "seed-g2-weight.sql"],
[CYPRESS_FIXTURES, "seed-government.sql"],
[OWN_FIXTURES, "seed-company-b.sql"],
[OWN_FIXTURES, "seed-customs-service-type.sql"],
];
async function seedTemplate() {
console.log(`it: seeding template ${TEMPLATE}`);
process.env.DB_NAME = TEMPLATE;
// No broker for the template boot: payment.module.ts skips RabbitMQModule
// entirely when this is unset, and nothing here publishes.
delete process.env.PAYMENT_RABBITMQ_URL;
const { createFreightApp } = createRequire(join(freightDir, "package.json"))("./dist/main.js");
// createFreightApp() only builds the graph. `init()` is what fires
// onApplicationBootstrap — the seeders we are here for. We never listen.
const app = await createFreightApp();
try {
await app.init();
console.log("it: boot seeders done (org, units, positions, permissions)");
} finally {
await app.close();
}
const client = new Client({
host: process.env.DB_HOST ?? "localhost",
port: Number(process.env.DB_PORT ?? 5543),
user: process.env.DB_USER ?? "edr_e2e",
password: process.env.DB_PASSWORD ?? "edr_e2e",
database: TEMPLATE,
});
await client.connect();
try {
for (const [dir, file] of SEEDS) {
await client.query(readFileSync(join(dir, file), "utf8"));
console.log(`it: seeded ${file}`);
}
} finally {
await client.end();
}
}
async function cloneShards() {
const pg = admin();
await pg.connect();
try {
// The app's pool and any stray psql hold the template open; without this the
// CREATE below fails with "source database is being accessed by other users".
await pg.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = $1 AND pid <> pg_backend_pid()`,
[TEMPLATE],
);
for (let i = 0; i < SHARDS; i++) {
const db = `${PREFIX}${i}`;
await pg.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = $1 AND pid <> pg_backend_pid()`,
[db],
);
await pg.query(`DROP DATABASE IF EXISTS ${db}`);
await pg.query(`CREATE DATABASE ${db} TEMPLATE ${TEMPLATE}`);
console.log(`it: shard ${i}${db}`);
}
} finally {
await pg.end();
}
}
const cloneOnly = process.argv.includes("--clone");
if (!cloneOnly) await seedTemplate();
await cloneShards();

133
integration/src/app.ts Normal file
View File

@@ -0,0 +1,133 @@
/**
* The freight API, booted IN THIS PROCESS — one instance per vitest worker.
*
* Each worker is a self-contained shard of the whole topology, so spec files can
* run in parallel without sharing anything mutable:
*
* shard i = this in-process freight app on :3111+i
* + database edr_it_s{i} (cloned from the seeded template)
* + payment-api-it-{i} container on :3113+i, DB_NAME=edr_it_s{i}
* + gateway-mock-it-{i} container on :4600+i
* + RabbitMQ vhost payment_s{i}
*
* The payment schema lives INSIDE the shard's own database under the usual
* `edr_payment` name, so every raw `edr_payment.*` query in the specs works
* unchanged and `paymentDb()` stays an alias for `db()`.
*
* Why the app is loaded from `dist/` and not from TypeScript: the freight source
* is CJS-flavoured — `src/config/database.config.ts` uses `__dirname`,
* `require.resolve` and an entity glob, none of which survive vitest's ESM
* transform. `createRequire` anchored at the freight app's own package.json
* keeps all Nest / typeorm / @tria-plc resolution inside its node_modules, and
* `dist` is rebuilt by `it.mjs test` on every run so it can never go stale.
*/
import { createRequire } from "node:module";
import { existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import type { Server } from "node:http";
const here = dirname(fileURLToPath(import.meta.url));
const freightDir = join(here, "..", "..", "apps", "edr-freight-api");
const distMain = join(freightDir, "dist", "main.js");
/**
* `VITEST_POOL_ID` is 1-based and stable per worker for the life of the run.
* Absent outside a worker (global-setup, scripts) — that context gets shard 0.
*/
export const SHARD = Math.max(0, Number(process.env.VITEST_POOL_ID ?? "1") - 1);
/**
* Each service takes a contiguous block of `IT_SHARDS` ports from its base, so
* the bases must stay far apart — see the PORTS comment in scripts/it.mjs.
*/
const port = (base: string, fallback: number) =>
Number(process.env[base] ?? fallback) + SHARD;
export const API_PORT = port("IT_API_PORT_BASE", 3111);
export const PAYMENT_API = `http://localhost:${port("IT_PAYMENT_PORT_BASE", 3131)}`;
export const GATEWAY = `http://localhost:${port("IT_GATEWAY_PORT_BASE", 4600)}`;
export const SHARD_DB = `${process.env.IT_DB_PREFIX ?? "edr_it_s"}${SHARD}`;
export const DB_URL =
`postgres://${process.env.DB_USER ?? "edr_e2e"}:${process.env.DB_PASSWORD ?? "edr_e2e"}` +
`@${process.env.DB_HOST ?? "localhost"}:${process.env.DB_PORT ?? "5543"}/${SHARD_DB}`;
const RABBIT_URL =
`amqp://edr:edr_secret@localhost:${process.env.IT_RABBIT_PORT ?? "5772"}` +
`/payment_s${SHARD}`;
/** Prefix every diagnostic — parallel worker output interleaves. */
export const tag = (msg: string) => `[shard ${SHARD}] ${msg}`;
interface FreightMain {
createFreightApp: () => Promise<{
listen: (port: number) => Promise<unknown>;
getHttpServer: () => Server;
close: () => Promise<void>;
}>;
}
let booted: Promise<Server> | undefined;
/**
* The shard's HTTP server, booted once per worker. supertest takes this
* directly in place of a base URL, so every call site in client.ts is a
* one-token change.
*/
export function freightServer(): Promise<Server> {
return (booted ??= boot());
}
async function boot(): Promise<Server> {
if (!existsSync(distMain)) {
throw new Error(
tag(
`${distMain} is missing — build the API first ` +
`(\`pnpm --filter @edr/freight-api run build\`, which \`it.mjs test\` does for you).`,
),
);
}
// Must be set BEFORE the require: payment.module.ts reads
// PAYMENT_RABBITMQ_URL at module-definition time to decide whether
// RabbitMQModule is in the graph at all, and database.config.ts reads DB_NAME
// when ConfigModule loads it.
process.env.DB_NAME = SHARD_DB;
process.env.PORT = String(API_PORT);
process.env.PAYMENT_API_URL = PAYMENT_API;
process.env.PAYMENT_RABBITMQ_URL = RABBIT_URL;
const { createFreightApp } = createRequire(join(freightDir, "package.json"))(
"./dist/main.js",
) as FreightMain;
// Postgres and the broker are up before vitest starts, but a shard's payment
// API may still be finishing its own migrations when the first worker boots.
// Retry the whole app rather than a connection: a half-initialised Nest app
// cannot be resumed, only closed and rebuilt.
let lastErr: unknown;
for (let attempt = 1; attempt <= 3; attempt++) {
const app = await createFreightApp().catch((err) => {
lastErr = err;
return undefined;
});
if (app) {
try {
// A real port, not an ephemeral one: payment-api-it-{SHARD} calls back
// in here for the inbound CBE Unified Bill query (cbe-bill.it.ts).
await app.listen(API_PORT);
console.log(tag(`freight-api in-process on :${API_PORT}${SHARD_DB}`));
return app.getHttpServer();
} catch (err) {
lastErr = err;
await app.close().catch(() => {});
}
}
if (attempt < 3) await new Promise((r) => setTimeout(r, 2000 * attempt));
}
console.error(tag(`freight-api failed to boot on :${API_PORT} (${SHARD_DB})`), lastErr);
throw lastErr;
}

View File

@@ -3,44 +3,75 @@
* failures here are the expensive kind: a tenant reading another tenant's
* invoice, or an unauthenticated caller marking one paid.
*/
import { afterAll, describe, expect, it } from "vitest";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import request from "supertest";
import {
API,
PAYMENT_API,
api,
closeDb,
customerA,
customerB,
db,
freightServer,
login,
payment,
} from "./client";
/** Company A, from e2e/freight/cypress/fixtures/seed-company.sql. */
const TENANT_A_TIN = "0102030405";
/**
* This file used to read "the newest invoice of company A" and `return` early
* when there wasn't one — which passed silently on a database no payment spec
* had run against yet. Now that every shard starts from a pristine clone, that
* would be *every* run. It bills itself instead.
*
* Straight SQL, not the booking chain: the point of this file is that it is
* cheap, and a cross-tenant read is refused on ownership alone — nothing here
* cares how the invoice came to exist.
*/
async function seedTenantAInvoice(): Promise<string> {
const rows = await db<{ id: string }>(
`INSERT INTO freight.invoices (
id, invoice_number, company_id, company_profile_id,
subtotal_amount, tax_amount, total_amount, paid_amount, balance_amount,
currency, status, source, source_id, type, issued_at, due_at, payments
)
SELECT gen_random_uuid(), $1, c.id, p.id,
1000, 0, 1000, 0, 1000,
'ETB', 'ISSUED', 'booking', gen_random_uuid()::text, 'PREPAID',
now(), now() + interval '7 days', '[]'::jsonb
FROM freight.companies c
JOIN freight.company_profiles p ON p.company_id = c.id AND p.deleted_at IS NULL
WHERE c.tin = $2 AND c.deleted_at IS NULL
LIMIT 1
RETURNING id`,
[`INV-AUTHZ-${Date.now()}`, TENANT_A_TIN],
);
const id = rows[0]?.id;
if (!id) {
throw new Error(
`authz: could not bill company ${TENANT_A_TIN} — seed-company.sql missing from this shard?`,
);
}
return id;
}
describe("payment authorization boundaries", () => {
let invoiceId: string;
beforeAll(async () => {
invoiceId = await seedTenantAInvoice();
});
afterAll(closeDb);
it("hides one tenant's invoice from the other", async () => {
const rows = await db<{ id: string; company_id: string }>(
`SELECT i.id, i.company_id FROM freight.invoices i
JOIN freight.companies c ON c.id = i.company_id
WHERE c.tin = '0102030405' AND i.deleted_at IS NULL
ORDER BY i.created_at DESC LIMIT 1`,
);
if (!rows[0]) return; // nothing billed yet in this run — payment files cover it
const res = await api(customerB, "get", `/api/billing/my-invoices/${rows[0].id}`);
const res = await api(customerB, "get", `/api/billing/my-invoices/${invoiceId}`);
expect([403, 404]).toContain(res.status);
});
it("refuses to let one tenant pay the other's invoice", async () => {
const rows = await db<{ id: string }>(
`SELECT i.id FROM freight.invoices i
JOIN freight.companies c ON c.id = i.company_id
WHERE c.tin = '0102030405' AND i.status <> 'PAID' AND i.deleted_at IS NULL
ORDER BY i.created_at DESC LIMIT 1`,
);
if (!rows[0]) return;
const res = await api(customerB, "post", `/api/billing/my-invoices/${rows[0].id}/pay`, {
const res = await api(customerB, "post", `/api/billing/my-invoices/${invoiceId}/pay`, {
method: "CBE_BIRR",
platform: "web",
});
@@ -71,7 +102,9 @@ describe("payment authorization boundaries", () => {
amountMinor: 1,
currency: "ETB",
};
const res = await request(API).post("/api/internal/payments/mark-paid").send(body);
const res = await request(await freightServer())
.post("/api/internal/payments/mark-paid")
.send(body);
expect([401, 403]).toContain(res.status);
});

View File

@@ -1,7 +1,11 @@
/**
* Plumbing for the integration suite: HTTP against the containerized freight
* and payment APIs, SQL against their shared throwaway Postgres, and the
* gateway mock's control plane.
* Plumbing for the integration suite: HTTP against this worker's IN-PROCESS
* freight API and its containerized payment API, SQL against the worker's own
* throwaway database, and its own gateway mock's control plane.
*
* Everything here is shard-scoped — see app.ts for the topology. supertest takes
* an `http.Server` exactly where it takes a base URL, so pointing the suite at
* the in-process app is a one-token change at each call site.
*
* This is the Cypress-free port of e2e/freight/cypress/e2e/flows/import-utils.ts —
* same request sequences, same SQL, `pg.Pool` instead of `cy.task`.
@@ -9,13 +13,11 @@
import request from "supertest";
import { Pool, type QueryResultRow } from "pg";
export const API = process.env.IT_API_URL ?? "http://localhost:3111";
export const PAYMENT_API = process.env.IT_PAYMENT_URL ?? "http://localhost:3113";
export const GATEWAY = process.env.IT_GATEWAY_URL ?? "http://localhost:4600";
export const SERVICE_TOKEN = process.env.IT_SERVICE_TOKEN ?? "e2e-service-token";
import { DB_URL, GATEWAY, PAYMENT_API, SHARD, freightServer, tag } from "./app";
const DB_URL =
process.env.IT_DB_URL ?? "postgres://edr_e2e:edr_e2e@localhost:5543/edr_freight_e2e";
export { GATEWAY, PAYMENT_API, SHARD, freightServer };
export const SERVICE_TOKEN = process.env.IT_SERVICE_TOKEN ?? "e2e-service-token";
// ---------------------------------------------------------------------------
// users (e2e/freight/cypress/fixtures/seed-users.sql + users.json)
@@ -45,11 +47,23 @@ export async function db<T extends QueryResultRow = Record<string, unknown>>(
return res.rows;
}
export async function closeDb(): Promise<void> {
await pool.end();
}
/**
* No-op. The pool is per-WORKER, not per-file: with `isolate: false` the module
* registry survives across every spec a worker runs, so the first `afterAll` to
* call this would break every later file on that shard. The pool dies with the
* worker process.
*/
export async function closeDb(): Promise<void> {}
/**
* Poll a query until `check` passes. Async settlement here is broker-driven and
* tick-driven, so the interval is pure overshoot: at the old 2000ms every wait
* in the suite finished up to two seconds after the state it wanted was already
* in the database, several hundred times over. The ceiling
* (`attempts × intervalMs`) is unchanged.
*/
const POLL_INTERVAL_MS = Number(process.env.IT_POLL_INTERVAL_MS ?? 250);
/** Poll a query until `check` passes. Async settlement here is broker-driven. */
export async function poll<T extends QueryResultRow = Record<string, unknown>>(
label: string,
sql: string,
@@ -57,14 +71,24 @@ export async function poll<T extends QueryResultRow = Record<string, unknown>>(
check: (row: T | undefined) => boolean,
{ attempts = 40, intervalMs = 2000 } = {},
): Promise<T> {
// Callers express patience as attempts × their own interval; keep that
// deadline and just sample it finely.
const deadlineMs = attempts * intervalMs;
const tries = Math.ceil(deadlineMs / POLL_INTERVAL_MS);
let last: T | undefined;
for (let i = 0; i < attempts; i++) {
for (let i = 0; i < tries; i++) {
last = (await db<T>(sql, params))[0];
if (check(last)) return last as T;
await sleep(intervalMs);
await sleep(POLL_INTERVAL_MS);
}
throw new Error(
`timed out waiting for ${label} after ${attempts} attempts — last row: ${JSON.stringify(last)}`,
tag(
`timed out waiting for ${label} after ${Math.round(deadlineMs / 1000)}s\n` +
` sql: ${sql.replace(/\s+/g, " ").trim()}\n` +
` params: ${JSON.stringify(params)}\n` +
` last row: ${JSON.stringify(last)}`,
),
);
}
@@ -86,7 +110,7 @@ export async function tokenFor(email: string): Promise<string> {
if (cached) return cached;
const portal = email.endsWith("@gmail.com");
const res = await request(API)
const res = await request(await freightServer())
.post("/api/auth/login")
.set("x-client-app", portal ? "portal" : "backoffice")
.send({ email, password: portal ? CUSTOMER_PASSWORD : STAFF_PASSWORD });
@@ -101,8 +125,11 @@ export async function tokenFor(email: string): Promise<string> {
}
/** Login without caching, so audience-rejection can be asserted. */
export function login(email: string, password: string, app: "portal" | "backoffice") {
return request(API).post("/api/auth/login").set("x-client-app", app).send({ email, password });
export async function login(email: string, password: string, app: "portal" | "backoffice") {
return request(await freightServer())
.post("/api/auth/login")
.set("x-client-app", app)
.send({ email, password });
}
// ---------------------------------------------------------------------------
@@ -119,7 +146,9 @@ export async function api(
body?: unknown,
): Promise<request.Response> {
const token = await tokenFor(email);
const req = request(API)[method](path).set("Authorization", `Bearer ${token}`);
const req = request(await freightServer())
[method](path)
.set("Authorization", `Bearer ${token}`);
return method === "get" || method === "delete" ? req.send() : req.send(body ?? {});
}
@@ -148,7 +177,9 @@ export async function upload(
fields: Record<string, string> = {},
): Promise<request.Response> {
const token = await tokenFor(email);
const req = request(API).post(path).set("Authorization", `Bearer ${token}`);
const req = request(await freightServer())
.post(path)
.set("Authorization", `Bearer ${token}`);
for (const [k, v] of Object.entries(fields)) req.field(k, v);
return req.attach(field, filePath);
}

View File

@@ -1,41 +1,17 @@
/**
* Runs once before any spec: wait for both APIs, then seed.
* Runs once, in its own process, before any spec: confirm every shard's
* containerized half is answering, then clear its gateway mock.
*
* Seeds are the Cypress suite's fixtures, reused verbatim (they are idempotent
* `insert … where not exists`), plus one of our own for the second tenant:
* seed-users.sql → seed-company.sql (order matters; company needs the users)
* seed-import-corridor.sql (yards, locos, wagons, rates, distances)
* seed-bulk-items.sql (PER_ITEM break-bulk cargo types)
* seed-g1-train.sql (the 53-wagon BUILT container train)
* seed-g2-weight.sql (the two 3 500 T weight-bound trains)
* seed-government.sql (the kind='government' company)
* seed-company-b.sql (user2@gmail.com's company — this suite)
* seed-customs-service-type.sql (a service type that bundles customs)
* Seeding is NOT here any more. The order is load-bearing —
* `seed-users.sql` names its prerequisites as "created by the API's always-on
* boot seeders" (iam.organizations / units / positions) — and the freight app
* now boots inside the vitest *workers*, which this process cannot reach. So the
* template database is seeded once by `scripts/prepare-shards.mjs` (boot the app
* → boot seeders → SQL fixtures) and each shard is a clone of it.
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { Client } from "pg";
const API = process.env.IT_API_URL ?? "http://localhost:3111";
const PAYMENT_API = process.env.IT_PAYMENT_URL ?? "http://localhost:3113";
const GATEWAY = process.env.IT_GATEWAY_URL ?? "http://localhost:4600";
const DB_URL =
process.env.IT_DB_URL ?? "postgres://edr_e2e:edr_e2e@localhost:5543/edr_freight_e2e";
const CYPRESS_FIXTURES = join(process.cwd(), "..", "e2e", "freight", "cypress", "fixtures");
const OWN_FIXTURES = join(process.cwd(), "sql");
const SEEDS: Array<[dir: string, file: string]> = [
[CYPRESS_FIXTURES, "seed-users.sql"],
[CYPRESS_FIXTURES, "seed-company.sql"],
[CYPRESS_FIXTURES, "seed-import-corridor.sql"],
[CYPRESS_FIXTURES, "seed-bulk-items.sql"],
[CYPRESS_FIXTURES, "seed-g1-train.sql"],
[CYPRESS_FIXTURES, "seed-g2-weight.sql"],
[CYPRESS_FIXTURES, "seed-government.sql"],
[OWN_FIXTURES, "seed-company-b.sql"],
[OWN_FIXTURES, "seed-customs-service-type.sql"],
];
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
const PAYMENT_BASE = Number(process.env.IT_PAYMENT_PORT_BASE ?? 3131);
const GATEWAY_BASE = Number(process.env.IT_GATEWAY_PORT_BASE ?? 4600);
async function waitFor(label: string, url: string, attempts = 60): Promise<void> {
for (let i = 0; i < attempts; i++) {
@@ -51,22 +27,20 @@ async function waitFor(label: string, url: string, attempts = 60): Promise<void>
}
export async function setup(): Promise<void> {
await Promise.all([
waitFor("freight-api", `${API}/api/health`),
waitFor("payment-api", `${PAYMENT_API}/health`),
waitFor("gateway-mock", `${GATEWAY}/__control/health`),
]);
const shards = Array.from({ length: SHARDS }, (_, i) => i);
const client = new Client({ connectionString: DB_URL });
await client.connect();
try {
for (const [dir, file] of SEEDS) {
await client.query(readFileSync(join(dir, file), "utf8"));
console.log(`it: seeded ${file}`);
}
} finally {
await client.end();
}
await Promise.all(
shards.flatMap((i) => [
waitFor(`payment-api (shard ${i})`, `http://localhost:${PAYMENT_BASE + i}/health`),
waitFor(`gateway-mock (shard ${i})`, `http://localhost:${GATEWAY_BASE + i}/__control/health`),
]),
);
await fetch(`${GATEWAY}/__control/reset`, { method: "POST" });
await Promise.all(
shards.map((i) =>
fetch(`http://localhost:${GATEWAY_BASE + i}/__control/reset`, { method: "POST" }),
),
);
console.log(`it: ${SHARDS} shard(s) ready`);
}

View File

@@ -1,18 +1,34 @@
import { defineConfig } from "vitest/config";
/**
* One worker per shard. A shard owns its whole topology — in-process freight
* app, database, payment API, gateway mock, broker vhost — so files can run in
* parallel without sharing anything mutable. See src/app.ts.
*/
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
export default defineConfig({
test: {
include: ["src/**/*.it.ts"],
globalSetup: ["src/global-setup.ts"],
// One shared containerized stack and one shared database: files run in
// sequence. Concurrency is exercised INSIDE tests (Promise.all), which is
// what the guards under test actually see in production.
fileParallelism: false,
pool: "forks",
// Pinned min = max: VITEST_POOL_ID is the shard key, and a worker that vitest
// declines to spawn is a shard whose containers are idling.
poolOptions: { forks: { minForks: SHARDS, maxForks: SHARDS } },
fileParallelism: SHARDS > 1,
// The freight app boots ONCE per worker and is memoised in module scope.
// With isolation on, every file would pay a fresh Nest boot (60 modules,
// TypeORM, 5 seeders) — 25 boots instead of `SHARDS`.
isolate: false,
testTimeout: 180_000,
hookTimeout: 180_000,
// Booking/scheduling steps are not idempotent — a retry would assert
// against a half-advanced booking.
// against a half-advanced booking. (A whole-RUN retry is now safe, because
// `it.mjs test` re-clones every shard database first.)
retry: 0,
// Workers hold a pg pool, three Socket.IO gateways, a RabbitMQ channel and
// live cron timers; don't wait on those handles to unwind.
teardownTimeout: 20_000,
reporters: ["verbose"],
},
});