mirror of
https://github.com/Tria-plc/edr-platform.git
synced 2026-08-26 12:41:04 +00:00
test(integration): boot freight API in-process across parallel shards
The suite drove a containerized freight API, so every code change needed an image rebuild before a test could see it, and there was no way to attach a debugger. Files also ran strictly in sequence against one shared database, which is the root of the warm-stack gotchas the README documents: stowaway paid bookings climbing back aboard, a short consist on the fifth file. The freight app now boots inside each vitest worker from dist/, and each worker owns a whole shard of the topology - its own database, payment API, gateway mock and broker vhost - so nothing mutable is shared and files run in parallel. Full suite drops from roughly 20 minutes to 196s at 4 shards. - main.ts exports createFreightApp() so the harness applies the same prefix, pipes, filters and interceptors as production instead of replaying them by hand; self-start is guarded by require.main so the Dockerfile CMD still boots - booking-window tick cadence is env-driven (BOOKING_WINDOW_TICK_CRON), */1 in the suite, */10 unchanged in production - prepare-shards.mjs seeds a template database (boot seeders, then the SQL fixtures that depend on them) and clones it per shard; it.mjs re-clones on every run, so each run is hermetic - gateway mock and payment API are generated per shard: the mock keeps modes and orders process-global and 20 of 25 specs reset it in beforeAll, and the inbound CBE bill query has to reach one specific shard's app - poll() samples every 250ms instead of 2000ms, keeping the caller's deadline - authz.it.ts seeds its own invoice; it previously read another spec's leftover and returned early, which silently passed on a pristine database Known: an unlocked MAX(sequence_no)+1 in train-scheduling.service.ts races under concurrent allocation and leaves a short consist, so 1-3 specs fail intermittently. Pre-existing and reproduces at the production tick cadence.
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -49,3 +49,6 @@ test-results/
|
||||
playwright-report/
|
||||
blob-report/
|
||||
RUNNING_LOCALLY.md
|
||||
|
||||
# Generated per-shard compose file for the integration suite (it.mjs).
|
||||
integration/.it-shards.yaml
|
||||
|
||||
@@ -19,7 +19,7 @@ import { AppModule } from "./app.module";
|
||||
* ~13.4MB on the wire. Express defaults to 100kb, which rejected any real stamp
|
||||
* image with a 413 "request entity too large".
|
||||
*/
|
||||
const JSON_BODY_LIMIT = '20mb';
|
||||
const JSON_BODY_LIMIT = "20mb";
|
||||
|
||||
/**
|
||||
* Static /etc/hosts-style overrides from `DNS_HOST_OVERRIDES`, formatted as
|
||||
@@ -44,7 +44,9 @@ function applyDnsHostOverrides(): void {
|
||||
}
|
||||
if (overrides.size === 0) return;
|
||||
|
||||
const dns = createRequire(__filename)("node:dns") as typeof import("node:dns");
|
||||
const dns = createRequire(__filename)(
|
||||
"node:dns",
|
||||
) as typeof import("node:dns");
|
||||
const originalLookup = dns.lookup.bind(dns);
|
||||
// `dns.lookup` is overloaded (options optional, all/family variants); the
|
||||
// cast keeps that surface intact while we intercept only mapped hostnames.
|
||||
@@ -63,7 +65,9 @@ function applyDnsHostOverrides(): void {
|
||||
) => void;
|
||||
const family = ip.includes(":") ? 6 : 4;
|
||||
const wantsAll =
|
||||
typeof options === "object" && options !== null && (options as { all?: boolean }).all;
|
||||
typeof options === "object" &&
|
||||
options !== null &&
|
||||
(options as { all?: boolean }).all;
|
||||
|
||||
process.nextTick(() =>
|
||||
wantsAll ? done(null, [{ address: ip, family }]) : done(null, ip, family),
|
||||
@@ -77,7 +81,15 @@ function applyDnsHostOverrides(): void {
|
||||
|
||||
applyDnsHostOverrides();
|
||||
|
||||
async function bootstrap() {
|
||||
/**
|
||||
* Build the app with every global the production process applies, but do NOT
|
||||
* listen. Exported so a test harness can boot the REAL app in its own process
|
||||
* (integration/src/app.ts) and get the same prefix, pipe, filter, interceptor
|
||||
* and body-parser configuration — replaying this list by hand is how an e2e
|
||||
* harness silently drifts from production (routes 404 without the "api"
|
||||
* prefix, responses lose the transform envelope).
|
||||
*/
|
||||
export async function createFreightApp(): Promise<NestExpressApplication> {
|
||||
const app = await NestFactory.create<NestExpressApplication>(AppModule);
|
||||
|
||||
// Nest's own body-parser API, NOT `app.use(json(...))` from express: express
|
||||
@@ -86,8 +98,8 @@ async function bootstrap() {
|
||||
// pnpm's hoisted dev store and died as MODULE_NOT_FOUND in the production
|
||||
// image, where `pnpm deploy --prod` installs declared dependencies only.
|
||||
// This also RECONFIGURES the default parsers rather than racing them.
|
||||
app.useBodyParser('json', { limit: JSON_BODY_LIMIT });
|
||||
app.useBodyParser('urlencoded', { limit: JSON_BODY_LIMIT, extended: true });
|
||||
app.useBodyParser("json", { limit: JSON_BODY_LIMIT });
|
||||
app.useBodyParser("urlencoded", { limit: JSON_BODY_LIMIT, extended: true });
|
||||
|
||||
// Dev CORS: reflect any localhost origin and allow credentials so the
|
||||
// freight portal (5173), passenger portal (5174), backoffices (5183/5184)
|
||||
@@ -157,13 +169,21 @@ async function bootstrap() {
|
||||
const document = SwaggerModule.createDocument(app, config);
|
||||
SwaggerModule.setup("api/docs", app, document);
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await createFreightApp();
|
||||
const port = parseInt(process.env.PORT ?? "3001", 10);
|
||||
// await app.listen(port, "0.0.0.0");
|
||||
await app.listen(
|
||||
|
||||
port)
|
||||
await app.listen(port);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`[freight-api] listening on port ${port}`);
|
||||
}
|
||||
|
||||
bootstrap();
|
||||
// Only self-start when this file IS the entrypoint. The Dockerfile's
|
||||
// `CMD ["node", "dist/main.js"]` still boots; importers get `createFreightApp`
|
||||
// without the process binding a port behind their back.
|
||||
if (require.main === module) {
|
||||
bootstrap();
|
||||
}
|
||||
|
||||
@@ -91,7 +91,16 @@ export class BookingWindowService implements OnModuleInit {
|
||||
// 10-second cadence: every transition is derived from persisted timestamps
|
||||
// and applied idempotently, so a finer tick only shrinks the lag between a
|
||||
// deadline passing and the phase actually moving (was a full minute).
|
||||
@Cron('*/10 * * * * *', { name: 'booking-window-tick', timeZone: BATCH_TIMEZONE })
|
||||
//
|
||||
// Overridable because that lag is the integration suite's pacing floor: every
|
||||
// window phase, wagon allocation and expiry it waits on lands on this tick, so
|
||||
// a 10s cadence costs ~5s of pure latency per wait across a few hundred waits.
|
||||
// The suite runs it at `*/1 * * * * *`. Read at class-definition time, so the
|
||||
// env var must be set before the module is imported.
|
||||
@Cron(process.env.BOOKING_WINDOW_TICK_CRON ?? '*/10 * * * * *', {
|
||||
name: 'booking-window-tick',
|
||||
timeZone: BATCH_TIMEZONE,
|
||||
})
|
||||
async tick(): Promise<void> {
|
||||
if (this.ticking) return;
|
||||
this.ticking = true;
|
||||
|
||||
@@ -1,12 +1,24 @@
|
||||
# EDR Freight — API integration stack (headless).
|
||||
# EDR Freight — API integration stack (headless, sharded).
|
||||
#
|
||||
# Overlay on docker-compose.e2e.yaml. Same base services (postgres, minio,
|
||||
# mocks, freight-api), except the payment microservice is REAL here instead of
|
||||
# `payment-mock-e2e`, and only the bank gateways are stubbed:
|
||||
# Overlay on docker-compose.e2e.yaml, plus a GENERATED third file holding the
|
||||
# per-shard services (integration/.it-shards.yaml, written by gen-shards.mjs).
|
||||
#
|
||||
# freight-api-it ──HTTP──> payment-api-it ──HTTP──> gateway-mock-it
|
||||
# ^ │
|
||||
# └────── RabbitMQ ───────┘ (outbox → payment.events → consumer)
|
||||
# The freight API is NOT a container here — it boots inside each vitest worker,
|
||||
# on the host, so a code change needs no image rebuild and a breakpoint works.
|
||||
# Each worker is a full shard of the topology; nothing mutable is shared:
|
||||
#
|
||||
# shard i:
|
||||
# freight app (in vitest worker, host :3111+i)
|
||||
# │ ▲
|
||||
# │ └──── HTTP ─── payment-api-it-{i} :3131+i (inbound CBE bill query,
|
||||
# │ │ via host.docker.internal)
|
||||
# └── HTTP ──────────────► │ ──HTTP──> gateway-mock-it-{i} :4600+i
|
||||
# ▲ │
|
||||
# └──── RabbitMQ vhost payment_s{i} ──┘ (outbox → payment.events → consumer)
|
||||
#
|
||||
# Shared by every shard: postgres (one container, one database per shard cloned
|
||||
# from a seeded template), rabbitmq (one container, one vhost per shard), minio,
|
||||
# and the one-shot migration.
|
||||
#
|
||||
# Its own compose project (`name:` below overrides the base) and its own host
|
||||
# ports, so it can run side by side with the Cypress e2e stack.
|
||||
@@ -15,13 +27,16 @@
|
||||
#
|
||||
# Never start it with plain `docker compose -f docker-compose.it.yaml` — it is
|
||||
# an OVERLAY and needs the base file first:
|
||||
# docker compose -f docker-compose.e2e.yaml -f docker-compose.it.yaml ...
|
||||
# docker compose -f docker-compose.e2e.yaml -f docker-compose.it.yaml \
|
||||
# -f integration/.it-shards.yaml ...
|
||||
name: edr-freight-it
|
||||
|
||||
services:
|
||||
# Outbox transport. The payment API publishes payment.succeeded/failed here
|
||||
# and freight consumes it — the production path. Copied from the passenger
|
||||
# harness (e2e/docker-compose.yml).
|
||||
# harness (e2e/docker-compose.yml). One broker, one vhost per shard
|
||||
# (payment_s0, payment_s1, …) created by it.mjs — a shared vhost would let one
|
||||
# shard's freight consumer eat another shard's settlement event.
|
||||
rabbitmq-it:
|
||||
image: rabbitmq:3-management
|
||||
environment:
|
||||
@@ -32,139 +47,14 @@ services:
|
||||
- "${IT_RABBIT_PORT:-5772}:5672"
|
||||
- "${IT_RABBIT_UI_PORT:-15772}:15672"
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
# check_running, NOT ping: ping only proves the Erlang node answers, and
|
||||
# it.mjs runs `rabbitmqctl add_vhost` the moment this goes healthy — which
|
||||
# on a cold boot failed with "this command requires the 'rabbit' app to be
|
||||
# running on the target node". check_running waits for the application.
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
|
||||
# Stand-in for every bank/wallet gateway the payment API talks to, plus a
|
||||
# control plane the tests drive (force a provider to fail/hang, fire a
|
||||
# correctly-signed webhook, read back what was called). See
|
||||
# integration/gateway-mock/server.js.
|
||||
gateway-mock-it:
|
||||
image: node:20-alpine
|
||||
volumes:
|
||||
- ./integration/gateway-mock:/app:ro
|
||||
working_dir: /app
|
||||
environment:
|
||||
PORT: "4600"
|
||||
# Same secrets the payment API gets — so webhooks the mock signs pass the
|
||||
# API's REAL signature verification instead of bypassing it.
|
||||
CBE_SECRET_KEY: it-cbe-secret
|
||||
CBE_MERCHANT_ID: it-cbe-merchant
|
||||
PAYMENT_API_URL: http://payment-api-it:3003
|
||||
command: ["node", "server.js"]
|
||||
ports:
|
||||
- "${IT_GATEWAY_PORT:-4600}:4600"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
payment-api-it:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: apps/edr-payment-api/Dockerfile
|
||||
secrets:
|
||||
- npmrc
|
||||
depends_on:
|
||||
postgres-freight-e2e:
|
||||
condition: service_healthy
|
||||
rabbitmq-it:
|
||||
condition: service_healthy
|
||||
gateway-mock-it:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PORT: "3003"
|
||||
NODE_ENV: test
|
||||
# Payment tables live in their own schema of the same throwaway DB;
|
||||
# main.ts ensurePaymentSchema() creates it, migrationsRun does the rest.
|
||||
DB_HOST: postgres-freight-e2e
|
||||
DB_PORT: "5432"
|
||||
DB_USER: edr_e2e
|
||||
DB_PASSWORD: edr_e2e
|
||||
DB_NAME: edr_freight_e2e
|
||||
DB_SCHEMA: edr_payment
|
||||
# Same token freight already uses in the base stack.
|
||||
SERVICE_AUTH_TOKEN: e2e-service-token
|
||||
PUBLISHER_TRANSPORT: rabbitmq
|
||||
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/payment
|
||||
# HTTP fallback targets (only used with PUBLISHER_TRANSPORT=http).
|
||||
PAYMENT_NOTIFY_FREIGHT_URL: http://freight-api-e2e:3001/api/internal/payments/mark-paid
|
||||
# Fast relay + sweep so retry/reconciliation are observable inside a test
|
||||
# rather than a minute later.
|
||||
OUTBOX_RELAY_INTERVAL_MS: "1000"
|
||||
RECONCILE_STALE_AFTER_MS: "5000"
|
||||
# Every gateway points at the one mock. Paths are per-provider prefixes.
|
||||
CBE_BASE_URL: http://gateway-mock-it:4600/cbe-birr
|
||||
CBE_MERCHANT_ID: it-cbe-merchant
|
||||
CBE_SECRET_KEY: it-cbe-secret
|
||||
CBE_NOTIFY_URL: http://payment-api-it:3003/webhooks/cbe-birr
|
||||
CBE_RETURN_URL: http://localhost/return
|
||||
TELEBIRR_BASE_URL: http://gateway-mock-it:4600/telebirr
|
||||
TELEBIRR_WEB_BASE_URL: http://gateway-mock-it:4600/telebirr/web
|
||||
TELEBIRR_FABRIC_APP_ID: it-fabric
|
||||
TELEBIRR_APP_SECRET: it-secret
|
||||
TELEBIRR_MERCHANT_APP_ID: it-merchant-app
|
||||
TELEBIRR_MERCHANT_CODE: "999999"
|
||||
TELEBIRR_NOTIFY_URL: http://payment-api-it:3003/webhooks/telebirr
|
||||
# Telebirr PSS-signs every request object — a throwaway key generated per
|
||||
# launch by it.mjs (nothing key-shaped lives in git).
|
||||
TELEBIRR_PRIVATE_KEY: ${IT_TELEBIRR_PRIVATE_KEY}
|
||||
EBIRR_BASE_URL: http://gateway-mock-it:4600/ebirr
|
||||
DMONEY_BASE_URL: http://gateway-mock-it:4600/dmoney
|
||||
CARD_BASE_URL: http://gateway-mock-it:4600/card
|
||||
WAAFI_BASE_URL: http://gateway-mock-it:4600/waafi
|
||||
CAC_BASE_URL: http://gateway-mock-it:4600/cac
|
||||
CAC_USERNAME: it-cac
|
||||
CAC_PASSWORD: it-cac
|
||||
CAC_APP_KEY: it-cac-key
|
||||
CAC_API_KEY: it-cac-api
|
||||
CAC_COMPANY_SERVICES_ID: "1"
|
||||
# Inbound CBE Unified Bill — we are the biller; bill-query hops back into
|
||||
# the freight API, so this direction runs real code on both sides.
|
||||
CBE_BILL_ENABLED: "true"
|
||||
CBE_BILL_CLIENT_ID: it-cbe-bill
|
||||
CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret
|
||||
CBE_BILL_JWT_SECRET: it-cbe-bill-jwt
|
||||
FREIGHT_API_BASE_URL: http://freight-api-e2e:3001/api
|
||||
ports:
|
||||
- "${IT_PAYMENT_PORT:-3113}:3003"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 40s
|
||||
|
||||
# Base-stack service, re-pointed at the real payment API.
|
||||
freight-api-e2e:
|
||||
depends_on:
|
||||
payment-api-it:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PAYMENT_API_URL: http://payment-api-it:3003
|
||||
# Freight's payment module skips RabbitMQModule entirely when this is
|
||||
# unset (payment.module.ts) — without it, outbox events never arrive.
|
||||
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/payment
|
||||
# RABBITMQ_ENABLED stays "false" (base stack) — it only gates the SMS/email
|
||||
# clients, which must remain off. The payment consumer is wired by
|
||||
# PAYMENT_RABBITMQ_URL alone.
|
||||
# Drain tail on every pay window. Production defaults to 5 minutes; a
|
||||
# reservation here lives ~60s, so 5 would push every natural expiry past
|
||||
# the suite's 180s timeouts. One minute keeps the tail real and observable
|
||||
# (src/expired-invoice-late-settle.it.ts asserts both sides of it).
|
||||
FREIGHT_PAYMENT_DRAIN_MINUTES: "1"
|
||||
# The per-shard `gateway-mock-it-{i}` and `payment-api-it-{i}` services live in
|
||||
# integration/.it-shards.yaml. The base file's `freight-api-e2e` is never
|
||||
# started — the app runs in-process (integration/src/app.ts).
|
||||
|
||||
@@ -3,29 +3,77 @@
|
||||
Headless, API-level tests for the freight API running against the **real**
|
||||
`edr-payment-api`. Only the bank/wallet gateways are stubbed.
|
||||
|
||||
The freight API is **not containerized** here — it boots inside each vitest
|
||||
worker from `apps/edr-freight-api/dist`, which `it:test` rebuilds every run. A
|
||||
code change needs no image rebuild, and a breakpoint in the API is hit by the
|
||||
test that provoked it.
|
||||
|
||||
```
|
||||
pnpm it:up # build + start the stack (first run ~5 min)
|
||||
pnpm it:test # vitest run (auto-ups the stack if needed)
|
||||
pnpm it:test -- --reporter=basic src/payment-happy.it.ts
|
||||
pnpm it:logs payment-api-it
|
||||
pnpm it:down # -v, wipes the throwaway DB
|
||||
pnpm it:test --no-reset -- src/payment-happy.it.ts # skip the DB re-clone
|
||||
pnpm it:logs payment-api-it-0
|
||||
pnpm it:down # -v, wipes the throwaway DBs and the generated shard file
|
||||
|
||||
IT_SHARDS=1 pnpm it:up # 1 shard when RAM is tight or output must be readable
|
||||
```
|
||||
|
||||
## Stack
|
||||
|
||||
`docker-compose.it.yaml` is an **overlay** on `docker-compose.e2e.yaml` — same
|
||||
freight API, Postgres (tmpfs), MinIO, Fayda/eTrade mocks; plus RabbitMQ, the
|
||||
real payment API, and one gateway mock. It is a separate compose project
|
||||
(`edr-freight-it`) on offset ports, so the Cypress e2e stack can run alongside.
|
||||
`docker-compose.it.yaml` is an **overlay** on `docker-compose.e2e.yaml`, plus a
|
||||
third **generated** file (`integration/.it-shards.yaml`, written by
|
||||
`scripts/gen-shards.mjs`, gitignored) holding the per-shard services. It is a
|
||||
separate compose project (`edr-freight-it`) on offset ports, so the Cypress e2e
|
||||
stack can run alongside.
|
||||
|
||||
Each vitest worker is a whole shard of the topology. Nothing mutable is shared
|
||||
between shards, so spec files run in parallel:
|
||||
|
||||
```
|
||||
freight-api-e2e :3111 ──HTTP──> payment-api-it :3113 ──HTTP──> gateway-mock-it :4600
|
||||
^ │
|
||||
└────────── RabbitMQ :5772 ─────┘ (outbox → payment.events → consumer)
|
||||
shard i (vitest worker i, VITEST_POOL_ID)
|
||||
|
||||
freight app in-process, host :3111+i
|
||||
│ ▲
|
||||
│ └──── HTTP ──── payment-api-it-{i} :3131+i inbound CBE bill query,
|
||||
│ │ via host.docker.internal
|
||||
└── HTTP ──────────────► │ ──HTTP──> gateway-mock-it-{i} :4600+i
|
||||
▲ │
|
||||
└── RabbitMQ vhost payment_s{i} ──┘ (outbox → payment.events → consumer)
|
||||
|
||||
database edr_it_s{i} ← CREATE DATABASE … TEMPLATE edr_freight_e2e
|
||||
freight tables AND the payment API's edr_payment
|
||||
schema live in it, so `db()` and `paymentDb()` are
|
||||
one connection
|
||||
```
|
||||
|
||||
Shared by every shard: Postgres (one container, one database each), RabbitMQ
|
||||
(one container, one vhost each), MinIO, and the one-shot migration.
|
||||
|
||||
`IT_SHARDS` defaults to **4**. RAM is the ceiling, not cores — a shard is an
|
||||
in-process Nest app plus two containers.
|
||||
|
||||
Never `docker compose -f docker-compose.it.yaml` on its own — it needs the base
|
||||
file first. Use `it.mjs`.
|
||||
file first, and the generated shard file last. Use `it.mjs`.
|
||||
|
||||
## Setup order
|
||||
|
||||
The sequence is load-bearing, which is why `scripts/prepare-shards.mjs` exists
|
||||
rather than vitest's `globalSetup`:
|
||||
|
||||
1. `freight-migration-e2e` migrates the **template** database `edr_freight_e2e`.
|
||||
2. `prepare-shards.mjs` boots the freight app once against the template and
|
||||
calls `app.init()` — that fires `onApplicationBootstrap`, the always-on
|
||||
seeders (org, units, positions, permissions).
|
||||
3. …then applies the SQL fixtures, which declare exactly those as prerequisites
|
||||
(`seed-users.sql`: *"created by the API's always-on boot seeders"*).
|
||||
4. `CREATE DATABASE edr_it_s{i} TEMPLATE edr_freight_e2e` per shard — a file
|
||||
copy on the tmpfs Postgres, not a re-run of migrations and five seeders.
|
||||
5. The per-shard payment APIs start; each creates its own `edr_payment` schema
|
||||
inside its shard database.
|
||||
|
||||
`it:test` re-runs step 4 on **every run** (stop payment APIs → drop → clone →
|
||||
start), so each run is hermetic. `--no-reset` skips it for a quick rerun.
|
||||
|
||||
## Gateway mock
|
||||
|
||||
@@ -46,7 +94,15 @@ negative test. The suite drives **CBE Birr** end to end (plain HMAC, no key
|
||||
material); other providers answer a generic stub until a scenario needs them.
|
||||
|
||||
Editing `server.js` needs a container restart (`docker compose … restart
|
||||
gateway-mock-it`) — the code is a read-only mount, not baked into an image.
|
||||
gateway-mock-it-0`) — the code is a read-only mount, not baked into an image.
|
||||
|
||||
**One mock per shard.** `modes`, `calls` and `orders` are process-global in
|
||||
`server.js`, and 20 of 25 specs call `POST /__control/reset` in `beforeAll`. On a
|
||||
shared mock every starting spec would wipe every running spec's live orders —
|
||||
and that fails as a wrong *assertion*, not an error: an unknown order still gets
|
||||
a correctly-signed webhook, just carrying the mock's default amount. Per-shard
|
||||
mocks are why the suite can run `payment-failure` (which forces providers into
|
||||
`fail`/`timeout`) beside a spec that settles normally.
|
||||
|
||||
## Files
|
||||
|
||||
@@ -151,21 +207,34 @@ what it actually does and says so in a comment, so a fix fails loudly:
|
||||
ignores `deleted_at`, so a soft-unlinked booking can never be re-batched).
|
||||
- **Arrange is slow.** Contract → booking → clearance → ops accept → batch is
|
||||
30–60s of real API work per booking, so files share one schedule day.
|
||||
- Files run sequentially (one DB); concurrency is exercised inside a test with
|
||||
`Promise.all`.
|
||||
- **Files run in parallel across shards, sequentially within one.** A shard owns
|
||||
its whole topology, so two files never contend; two files on the *same* shard
|
||||
still run one after the other, which is what the day-offset partitioning and
|
||||
the per-file `releaseUnpaidHolds()` / `resetCorridorDay()` hygiene still
|
||||
assume. Concurrency inside a single test is exercised with `Promise.all`.
|
||||
- **Day offsets are the de-facto partition key** (`departureAt(n)` per file).
|
||||
`payment-happy` and `expired-invoice-late-settle` both claim day 4 — harmless
|
||||
now that a shard has its own database, but do not read the offsets as unique.
|
||||
- **A retired fixture keeps its shipment day at its peril.**
|
||||
`rescueStrandedPaidForDay` sweeps every unlinked booking whose
|
||||
`payment_status` is PAID and whose `scheduled_date` falls on the day being
|
||||
filled, and re-places it on the fresh train. A previous run's paid bookings
|
||||
therefore climb back aboard — 18 stowaway wagons on a 28-wagon day, until
|
||||
`releaseUnpaidHolds` / `resetCorridorDay` started nulling `scheduled_date`.
|
||||
filled, and re-places it on the fresh train. Within a run this still bites
|
||||
across files on one shard; *between* runs it no longer can — `it:test`
|
||||
re-clones every shard database from the template first.
|
||||
- **Only a FULL train rests at DONE.** An under-filled day CONCLUDES and
|
||||
REOPENS (`window_phase` back to OPEN, `booking_cycle_no` 2), so waiting for
|
||||
DONE there waits forever — use `pollCycleConcluded`.
|
||||
- **Wagon stock is finite and shared.** Paid bookings keep their wagons, so
|
||||
`releaseUnpaidHolds()` also frees every earlier `CTR-IT-%` allocation —
|
||||
without it the fifth or sixth file on a warm stack silently gets a short
|
||||
consist.
|
||||
- **Wagon stock is finite and shared** *within a shard*. Paid bookings keep
|
||||
their wagons, so `releaseUnpaidHolds()` also frees every earlier `CTR-IT-%`
|
||||
allocation — without it the fifth or sixth file on one shard silently gets a
|
||||
short consist.
|
||||
- **`poll()` samples every 250ms, but keeps the caller's deadline.** Callers pass
|
||||
`attempts × intervalMs`; that product is the timeout, and only the sampling
|
||||
rate changed. Override with `IT_POLL_INTERVAL_MS`.
|
||||
- **The tick cadence is env-driven.** `BOOKING_WINDOW_TICK_CRON` is `*/1` here
|
||||
and `*/10` in production. Every window phase, allocation and expiry the suite
|
||||
waits on lands on that tick. A spec that proves sensitive to it should pin its
|
||||
own value rather than the whole suite reverting.
|
||||
- **One tenant per booking.** `seedTenantContracts` mints a company per booking
|
||||
because a company may hold only one unpaid reservation at a time; staff book
|
||||
and pay on their behalf, which is also the real Path B flow.
|
||||
|
||||
182
integration/scripts/gen-shards.mjs
Normal file
182
integration/scripts/gen-shards.mjs
Normal file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Generate `integration/.it-shards.yaml` — one payment API + one gateway mock
|
||||
* per shard. Compose has no loops, so the per-shard services are written out.
|
||||
*
|
||||
* Every shard is the real topology in miniature; nothing crosses between them:
|
||||
*
|
||||
* gateway-mock-it-{i} its own `modes`/`calls`/`orders` — the mock keeps those
|
||||
* process-global, and 20 of 25 specs call
|
||||
* POST /__control/reset in beforeAll, so a shared mock
|
||||
* would have each starting spec wipe every running spec's
|
||||
* live orders (which fails as a wrong *assertion*: an
|
||||
* unknown order still gets a correctly-signed webhook,
|
||||
* just with the mock's default amount).
|
||||
* payment-api-it-{i} its own DB_NAME (the shard's database, `edr_payment`
|
||||
* schema inside it), its own broker vhost, and its own
|
||||
* FREIGHT_API_BASE_URL — the inbound CBE-bill query has
|
||||
* to land on THIS shard's in-process app, and a single
|
||||
* container could only ever point at one of them.
|
||||
*
|
||||
* Written by it.mjs, gitignored. Do not edit by hand.
|
||||
*/
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
/**
|
||||
* All shards share one build and one image tag, so the payment API image is
|
||||
* built once and reused rather than N times.
|
||||
*/
|
||||
const PAYMENT_IMAGE = "edr-payment-api-it:local";
|
||||
|
||||
export function renderShards({
|
||||
shards,
|
||||
apiPortBase,
|
||||
paymentPortBase,
|
||||
gatewayPortBase,
|
||||
dbPrefix,
|
||||
telebirrKey,
|
||||
}) {
|
||||
const services = [];
|
||||
|
||||
for (let i = 0; i < shards; i++) {
|
||||
const gw = `gateway-mock-it-${i}`;
|
||||
const pay = `payment-api-it-${i}`;
|
||||
const db = `${dbPrefix}${i}`;
|
||||
const vhost = `payment_s${i}`;
|
||||
// The freight app for this shard runs on the HOST, inside the vitest worker.
|
||||
const freightBase = `http://host.docker.internal:${apiPortBase + i}/api`;
|
||||
|
||||
services.push(`
|
||||
${gw}:
|
||||
image: node:20-alpine
|
||||
volumes:
|
||||
- ./integration/gateway-mock:/app:ro
|
||||
working_dir: /app
|
||||
environment:
|
||||
PORT: "4600"
|
||||
# Same secrets the payment API gets — so webhooks the mock signs pass the
|
||||
# API's REAL signature verification instead of bypassing it.
|
||||
CBE_SECRET_KEY: it-cbe-secret
|
||||
CBE_MERCHANT_ID: it-cbe-merchant
|
||||
PAYMENT_API_URL: http://${pay}:3003
|
||||
command: ["node", "server.js"]
|
||||
ports:
|
||||
- "${gatewayPortBase + i}:4600"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
${pay}:
|
||||
image: ${PAYMENT_IMAGE}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: apps/edr-payment-api/Dockerfile
|
||||
secrets:
|
||||
- npmrc
|
||||
depends_on:
|
||||
postgres-freight-e2e:
|
||||
condition: service_healthy
|
||||
rabbitmq-it:
|
||||
condition: service_healthy
|
||||
${gw}:
|
||||
condition: service_healthy
|
||||
# The freight app is on the host now, not in this network.
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
environment:
|
||||
PORT: "3003"
|
||||
NODE_ENV: test
|
||||
# Payment tables live in their own schema of THIS SHARD's database;
|
||||
# main.ts ensurePaymentSchema() creates it, migrationsRun does the rest.
|
||||
# Same database as freight, so every raw edr_payment.* query in the specs
|
||||
# reads through the one connection.
|
||||
DB_HOST: postgres-freight-e2e
|
||||
DB_PORT: "5432"
|
||||
DB_USER: edr_e2e
|
||||
DB_PASSWORD: edr_e2e
|
||||
DB_NAME: ${db}
|
||||
DB_SCHEMA: edr_payment
|
||||
SERVICE_AUTH_TOKEN: e2e-service-token
|
||||
PUBLISHER_TRANSPORT: rabbitmq
|
||||
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/${vhost}
|
||||
PAYMENT_NOTIFY_FREIGHT_URL: ${freightBase}/internal/payments/mark-paid
|
||||
# Fast relay + sweep so retry/reconciliation are observable inside a test
|
||||
# rather than a minute later.
|
||||
OUTBOX_RELAY_INTERVAL_MS: "1000"
|
||||
RECONCILE_STALE_AFTER_MS: "5000"
|
||||
# Every gateway points at this shard's mock. Paths are per-provider prefixes.
|
||||
CBE_BASE_URL: http://${gw}:4600/cbe-birr
|
||||
CBE_MERCHANT_ID: it-cbe-merchant
|
||||
CBE_SECRET_KEY: it-cbe-secret
|
||||
CBE_NOTIFY_URL: http://${pay}:3003/webhooks/cbe-birr
|
||||
CBE_RETURN_URL: http://localhost/return
|
||||
TELEBIRR_BASE_URL: http://${gw}:4600/telebirr
|
||||
TELEBIRR_WEB_BASE_URL: http://${gw}:4600/telebirr/web
|
||||
TELEBIRR_FABRIC_APP_ID: it-fabric
|
||||
TELEBIRR_APP_SECRET: it-secret
|
||||
TELEBIRR_MERCHANT_APP_ID: it-merchant-app
|
||||
TELEBIRR_MERCHANT_CODE: "999999"
|
||||
TELEBIRR_NOTIFY_URL: http://${pay}:3003/webhooks/telebirr
|
||||
# Telebirr PSS-signs every request object — a throwaway key generated per
|
||||
# launch by it.mjs (nothing key-shaped lives in git).
|
||||
TELEBIRR_PRIVATE_KEY: ${JSON.stringify(telebirrKey)}
|
||||
EBIRR_BASE_URL: http://${gw}:4600/ebirr
|
||||
DMONEY_BASE_URL: http://${gw}:4600/dmoney
|
||||
CARD_BASE_URL: http://${gw}:4600/card
|
||||
WAAFI_BASE_URL: http://${gw}:4600/waafi
|
||||
CAC_BASE_URL: http://${gw}:4600/cac
|
||||
CAC_USERNAME: it-cac
|
||||
CAC_PASSWORD: it-cac
|
||||
CAC_APP_KEY: it-cac-key
|
||||
CAC_API_KEY: it-cac-api
|
||||
CAC_COMPANY_SERVICES_ID: "1"
|
||||
# Inbound CBE Unified Bill — we are the biller; bill-query hops back into
|
||||
# the freight API, so this direction runs real code on both sides.
|
||||
CBE_BILL_ENABLED: "true"
|
||||
CBE_BILL_CLIENT_ID: it-cbe-bill
|
||||
CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret
|
||||
CBE_BILL_JWT_SECRET: it-cbe-bill-jwt
|
||||
FREIGHT_API_BASE_URL: ${freightBase}
|
||||
ports:
|
||||
- "${paymentPortBase + i}:3003"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 40s`);
|
||||
}
|
||||
|
||||
return `# GENERATED by integration/scripts/gen-shards.mjs — do not edit.
|
||||
# ${shards} shard(s). Overlaid on docker-compose.e2e.yaml + docker-compose.it.yaml.
|
||||
services:${services.join("\n")}
|
||||
`;
|
||||
}
|
||||
|
||||
export const SHARD_FILE = join(itDir, ".it-shards.yaml");
|
||||
|
||||
export function writeShards(opts) {
|
||||
writeFileSync(SHARD_FILE, renderShards(opts));
|
||||
return SHARD_FILE;
|
||||
}
|
||||
|
||||
export const shardServices = (shards) =>
|
||||
Array.from({ length: shards }, (_, i) => [`gateway-mock-it-${i}`, `payment-api-it-${i}`]).flat();
|
||||
@@ -3,10 +3,13 @@
|
||||
* Freight integration-suite launcher.
|
||||
*
|
||||
* node integration/scripts/it.mjs <up|test|down|logs> [vitest args...]
|
||||
* IT_SHARDS=4 node integration/scripts/it.mjs test
|
||||
* node integration/scripts/it.mjs test --no-reset -- src/payment-happy.it.ts
|
||||
*
|
||||
* Overlays docker-compose.it.yaml on docker-compose.e2e.yaml: same freight
|
||||
* stack, but the payment microservice is real and only the bank gateways are
|
||||
* stubbed. Web/Cypress containers are never started — this suite is HTTP only.
|
||||
* The freight API is NOT containerized here: it boots inside each vitest worker
|
||||
* from `apps/edr-freight-api/dist`, which `test` rebuilds every run. Each worker
|
||||
* is a whole shard — own database, own payment API, own gateway mock, own broker
|
||||
* vhost — so spec files run in parallel without sharing anything mutable.
|
||||
*
|
||||
* Ports are fixed (and distinct from the Cypress e2e defaults) so both stacks
|
||||
* can be up at once; they are separate compose projects.
|
||||
@@ -16,21 +19,24 @@
|
||||
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { SHARD_FILE, shardServices, writeShards } from "./gen-shards.mjs";
|
||||
|
||||
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const repoRoot = resolve(itDir, "..");
|
||||
const composeBase = [
|
||||
"compose",
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.e2e.yaml"),
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.it.yaml"),
|
||||
];
|
||||
|
||||
/** Deliberately offset from the Cypress stack's defaults (3101/5533/9310…). */
|
||||
/**
|
||||
* Deliberately offset from the Cypress stack's defaults (3101/5533/9310…).
|
||||
*
|
||||
* Each per-shard service takes a CONTIGUOUS BLOCK of `IT_SHARDS` ports starting
|
||||
* at its base (freight :3111+i, payment :3131+i, gateway :4600+i), so the bases
|
||||
* must be at least MAX_SHARDS apart. They were originally 3111 and 3113 — two
|
||||
* apart — which silently worked at 1–2 shards and then had shard 2's freight app
|
||||
* try to bind :3113, the port payment-api-it-0 was already published on.
|
||||
*/
|
||||
const PORTS = {
|
||||
E2E_API_PORT: 3111,
|
||||
E2E_DB_PORT: 5543,
|
||||
@@ -39,32 +45,39 @@ const PORTS = {
|
||||
// Unused here (no web containers) but referenced by the base file's build args.
|
||||
E2E_PORTAL_PORT: 5393,
|
||||
E2E_BACKOFFICE_PORT: 5394,
|
||||
IT_PAYMENT_PORT: 3113,
|
||||
IT_PAYMENT_PORT: 3131,
|
||||
IT_GATEWAY_PORT: 4600,
|
||||
IT_RABBIT_PORT: 5772,
|
||||
IT_RABBIT_UI_PORT: 15772,
|
||||
};
|
||||
|
||||
/** Everything the suite needs up — web + cypress are deliberately absent. */
|
||||
const SERVICES = [
|
||||
"postgres-freight-e2e",
|
||||
"minio-e2e",
|
||||
"minio-init-e2e",
|
||||
"freight-migration-e2e",
|
||||
"fayda-mock-e2e",
|
||||
"etrade-mock-e2e",
|
||||
// Still a base-stack dependency of freight-api-e2e (reconcile-before-expire
|
||||
// has its own client); cheap to run alongside the real payment API.
|
||||
"payment-mock-e2e",
|
||||
"gateway-mock-it",
|
||||
"rabbitmq-it",
|
||||
"payment-api-it",
|
||||
"freight-api-e2e",
|
||||
];
|
||||
/** Beyond this the freight block (3111+) would run into the payment block (3131+). */
|
||||
const MAX_SHARDS = 16;
|
||||
|
||||
const RUNNING = SERVICES.filter(
|
||||
(s) => !["minio-init-e2e", "freight-migration-e2e"].includes(s),
|
||||
);
|
||||
/**
|
||||
* How many shards. RAM is the ceiling, not cores: each shard is an in-process
|
||||
* Nest app plus two containers.
|
||||
*/
|
||||
const SHARDS = Math.max(1, Number(process.env.IT_SHARDS ?? 4));
|
||||
if (SHARDS > MAX_SHARDS) {
|
||||
console.error(
|
||||
`\nit: IT_SHARDS=${SHARDS} exceeds ${MAX_SHARDS} — the per-service port blocks would overlap.` +
|
||||
`\n Raise IT_PAYMENT_PORT/IT_GATEWAY_PORT in it.mjs first.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const TEMPLATE_DB = "edr_freight_e2e";
|
||||
const DB_PREFIX = "edr_it_s";
|
||||
|
||||
/** Long-running services shared by every shard. Web + Cypress are absent. */
|
||||
const INFRA = ["postgres-freight-e2e", "minio-e2e", "rabbitmq-it"];
|
||||
|
||||
/**
|
||||
* One-shots: they run, then exit 0. They cannot go in the `up --wait` set —
|
||||
* compose reports an exited container as a failed wait, so a healthy stack
|
||||
* looks broken. Run them with `compose run`, which returns their exit code.
|
||||
*/
|
||||
const ONE_SHOT = ["minio-init-e2e", "freight-migration-e2e"];
|
||||
|
||||
function fail(msg) {
|
||||
console.error(`\nit: ${msg}`);
|
||||
@@ -97,64 +110,271 @@ function fakeFaydaPrivateKeyBase64() {
|
||||
return Buffer.from(JSON.stringify(jwk)).toString("base64");
|
||||
}
|
||||
|
||||
const telebirrKey = process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey();
|
||||
|
||||
/**
|
||||
* The freight app's own environment. This is docker-compose.e2e.yaml's
|
||||
* `freight-api-e2e` block with container hostnames swapped for published host
|
||||
* ports — the app runs on the host now. Per-shard values (DB_NAME, PORT,
|
||||
* PAYMENT_API_URL, PAYMENT_RABBITMQ_URL) are derived per worker in src/app.ts.
|
||||
*/
|
||||
const freightEnv = {
|
||||
NODE_ENV: "test",
|
||||
DB_HOST: "localhost",
|
||||
DB_PORT: String(PORTS.E2E_DB_PORT),
|
||||
DB_USER: "edr_e2e",
|
||||
DB_PASSWORD: "edr_e2e",
|
||||
DB_NAME: TEMPLATE_DB,
|
||||
|
||||
// e2e-only secrets — never reuse outside this stack
|
||||
JWT_SECRET: "e2e-jwt-secret",
|
||||
JWT_ACCESS_TOKEN_SECRET: "e2e-access-secret",
|
||||
JWT_REFRESH_TOKEN_SECRET: "e2e-refresh-secret",
|
||||
JWT_EXPIRES_IN: "1d",
|
||||
JWT_ACCESS_TOKEN_EXPIRES: "1d",
|
||||
JWT_REFRESH_TOKEN_EXPIRES: "7d",
|
||||
// Mandatory: ServiceAuthGuard returns TRUE when this is unset, and
|
||||
// authz.it.ts asserts that the internal surface rejects an unsigned caller.
|
||||
SERVICE_AUTH_TOKEN: "e2e-service-token",
|
||||
|
||||
SEED_EDR_ORG: "true",
|
||||
SUPER_ADMIN_EMAIL: "superadmin@tria.com",
|
||||
SUPER_ADMIN_PHONE: "+251900000000",
|
||||
|
||||
MINIO_ENDPOINT: "localhost",
|
||||
MINIO_PORT: String(PORTS.E2E_MINIO_PORT),
|
||||
MINIO_USE_SSL: "false",
|
||||
MINIO_ACCESS_KEY: "e2e-minio",
|
||||
MINIO_SECRET_KEY: "e2e-minio-secret",
|
||||
MINIO_REGION: "us-east-1",
|
||||
|
||||
// Only gates the SMS/email clients. The payment consumer is wired by
|
||||
// PAYMENT_RABBITMQ_URL alone (payment.module.ts).
|
||||
RABBITMQ_ENABLED: "false",
|
||||
// fayda.config.ts THROWS at load if this is "true" without the full var set,
|
||||
// and the fayda/etrade mocks publish no host ports. No IT spec touches them.
|
||||
FAYDA_ENABLED: "false",
|
||||
// SMS strategy has no kill switch and defaults to a real dev endpoint.
|
||||
OZIKING_SMS_URL: "http://127.0.0.1:9/sms",
|
||||
// app.config.ts otherwise live-scrapes https://ethio.forex on first booking.
|
||||
CBE_EXCHANGE_SCRAPE_URL: "http://127.0.0.1:9/fx",
|
||||
CBE_EXCHANGE_API_URL: "http://127.0.0.1:9/fx",
|
||||
CBE_EXCHANGE_FALLBACK_RATE: "130",
|
||||
FREIGHT_PORTAL_URL: `http://localhost:${PORTS.E2E_PORTAL_PORT}`,
|
||||
|
||||
// Drain tail on every pay window. Production defaults to 5 minutes; a
|
||||
// reservation here lives ~60s, so 5 would push every natural expiry past the
|
||||
// suite's timeouts. One minute keeps the tail real and observable
|
||||
// (src/expired-invoice-late-settle.it.ts asserts both sides of it, and
|
||||
// hardcodes DRAIN_MS = 60_000 to match).
|
||||
FREIGHT_PAYMENT_DRAIN_MINUTES: "1",
|
||||
// The suite's pacing floor: every window phase, allocation and expiry it waits
|
||||
// on lands on this tick. Production stays at */10. Overridable from the
|
||||
// environment so a cadence-sensitive spec (or a bisect) can pin it back.
|
||||
BOOKING_WINDOW_TICK_CRON: process.env.BOOKING_WINDOW_TICK_CRON ?? "*/1 * * * * *",
|
||||
};
|
||||
|
||||
const env = {
|
||||
...process.env,
|
||||
...Object.fromEntries(Object.entries(PORTS).map(([k, v]) => [k, String(v)])),
|
||||
IT_API_URL: `http://localhost:${PORTS.E2E_API_PORT}`,
|
||||
IT_PAYMENT_URL: `http://localhost:${PORTS.IT_PAYMENT_PORT}`,
|
||||
IT_GATEWAY_URL: `http://localhost:${PORTS.IT_GATEWAY_PORT}`,
|
||||
IT_DB_URL: `postgres://edr_e2e:edr_e2e@localhost:${PORTS.E2E_DB_PORT}/edr_freight_e2e`,
|
||||
...freightEnv,
|
||||
IT_SHARDS: String(SHARDS),
|
||||
IT_TEMPLATE_DB: TEMPLATE_DB,
|
||||
IT_DB_PREFIX: DB_PREFIX,
|
||||
IT_API_PORT_BASE: String(PORTS.E2E_API_PORT),
|
||||
IT_PAYMENT_PORT_BASE: String(PORTS.IT_PAYMENT_PORT),
|
||||
IT_GATEWAY_PORT_BASE: String(PORTS.IT_GATEWAY_PORT),
|
||||
FAYDA_PRIVATE_KEY_BASE64:
|
||||
process.env.FAYDA_PRIVATE_KEY_BASE64 ?? fakeFaydaPrivateKeyBase64(),
|
||||
IT_TELEBIRR_PRIVATE_KEY:
|
||||
process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey(),
|
||||
IT_TELEBIRR_PRIVATE_KEY: telebirrKey,
|
||||
};
|
||||
|
||||
function generateShardFile() {
|
||||
writeShards({
|
||||
shards: SHARDS,
|
||||
apiPortBase: PORTS.E2E_API_PORT,
|
||||
paymentPortBase: PORTS.IT_PAYMENT_PORT,
|
||||
gatewayPortBase: PORTS.IT_GATEWAY_PORT,
|
||||
dbPrefix: DB_PREFIX,
|
||||
telebirrKey,
|
||||
});
|
||||
}
|
||||
|
||||
/** The shard file must exist before compose is invoked — it is one of the -f's. */
|
||||
function composeBase() {
|
||||
if (!existsSync(SHARD_FILE)) generateShardFile();
|
||||
return [
|
||||
"compose",
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.e2e.yaml"),
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.it.yaml"),
|
||||
"-f",
|
||||
SHARD_FILE,
|
||||
];
|
||||
}
|
||||
|
||||
function compose(args) {
|
||||
const { status } = spawnSync("docker", [...composeBase, ...args], { stdio: "inherit", env });
|
||||
const { status } = spawnSync("docker", [...composeBase(), ...args], {
|
||||
stdio: "inherit",
|
||||
env,
|
||||
});
|
||||
return status ?? 1;
|
||||
}
|
||||
|
||||
function composeQuiet(args) {
|
||||
return spawnSync("docker", [...composeBase(), ...args], {
|
||||
encoding: "utf8",
|
||||
env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
|
||||
function stackRunning() {
|
||||
try {
|
||||
const out = execFileSync("docker", [...composeBase, "ps", "--services", "--status", "running"], {
|
||||
encoding: "utf8",
|
||||
env,
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
});
|
||||
const out = execFileSync(
|
||||
"docker",
|
||||
[...composeBase(), "ps", "--services", "--status", "running"],
|
||||
{ encoding: "utf8", env, stdio: ["ignore", "pipe", "ignore"] },
|
||||
);
|
||||
const running = new Set(out.split("\n").filter(Boolean));
|
||||
return RUNNING.every((s) => running.has(s));
|
||||
const want = [...INFRA, ...shardServices(SHARDS)];
|
||||
return want.every((s) => running.has(s));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function node(script, args = []) {
|
||||
const { status } = spawnSync("node", [join(itDir, "scripts", script), ...args], {
|
||||
cwd: repoRoot,
|
||||
stdio: "inherit",
|
||||
env,
|
||||
});
|
||||
return status ?? 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* One broker, one vhost per shard — a shared vhost would let one shard's freight
|
||||
* consumer eat another shard's settlement event.
|
||||
*
|
||||
* Retried: the healthcheck now waits for the rabbit *application*, but a cold
|
||||
* boot can still land between "healthy" and "accepting rabbitmqctl", and a
|
||||
* half-provisioned broker fails later as an unexplained missing settlement.
|
||||
*/
|
||||
function createVhosts() {
|
||||
for (let i = 0; i < SHARDS; i++) {
|
||||
const vhost = `payment_s${i}`;
|
||||
let last = "";
|
||||
let ok = false;
|
||||
for (let attempt = 1; attempt <= 10 && !ok; attempt++) {
|
||||
// Idempotent: add_vhost on an existing vhost exits non-zero, which is fine.
|
||||
composeQuiet(["exec", "-T", "rabbitmq-it", "rabbitmqctl", "add_vhost", vhost]);
|
||||
const perm = composeQuiet([
|
||||
"exec", "-T", "rabbitmq-it",
|
||||
"rabbitmqctl", "set_permissions", "-p", vhost, "edr", ".*", ".*", ".*",
|
||||
]);
|
||||
ok = perm.status === 0;
|
||||
last = perm.stderr ?? "";
|
||||
if (!ok) execFileSync("sleep", ["3"]);
|
||||
}
|
||||
if (!ok) fail(`could not grant on rabbit vhost ${vhost} after 10 tries:\n${last}`);
|
||||
}
|
||||
console.log(`it: rabbit vhosts payment_s0…payment_s${SHARDS - 1} ready`);
|
||||
}
|
||||
|
||||
function buildFreight() {
|
||||
console.log("it: building @edr/freight-api (the suite runs dist/, never stale)");
|
||||
const { status } = spawnSync("pnpm", ["--filter", "@edr/freight-api", "run", "build"], {
|
||||
cwd: repoRoot,
|
||||
stdio: "inherit",
|
||||
env,
|
||||
});
|
||||
if (status !== 0) fail("freight API build failed — fix it before running the suite.");
|
||||
}
|
||||
|
||||
function up() {
|
||||
preflight();
|
||||
generateShardFile();
|
||||
console.log(
|
||||
`it: starting stack — freight :${PORTS.E2E_API_PORT} payment :${PORTS.IT_PAYMENT_PORT} ` +
|
||||
`gateway :${PORTS.IT_GATEWAY_PORT} db :${PORTS.E2E_DB_PORT}`,
|
||||
`it: starting ${SHARDS} shard(s) — freight :${PORTS.E2E_API_PORT}..${
|
||||
PORTS.E2E_API_PORT + SHARDS - 1
|
||||
} (in-process) payment :${PORTS.IT_PAYMENT_PORT}.. gateway :${
|
||||
PORTS.IT_GATEWAY_PORT
|
||||
}.. db :${PORTS.E2E_DB_PORT}`,
|
||||
);
|
||||
if (compose(["up", "-d", "--build", "--wait", ...SERVICES]) !== 0) {
|
||||
|
||||
// 1. Shared infrastructure.
|
||||
if (compose(["up", "-d", "--build", "--wait", "--remove-orphans", ...INFRA]) !== 0) {
|
||||
fail(
|
||||
"stack failed to become healthy. Inspect with:\n" +
|
||||
" node integration/scripts/it.mjs logs payment-api-it",
|
||||
"shared services failed to become healthy. Inspect with:\n" +
|
||||
" node integration/scripts/it.mjs logs postgres-freight-e2e",
|
||||
);
|
||||
}
|
||||
createVhosts();
|
||||
|
||||
// 1b. One-shots: the MinIO bucket, then freight's migrations into the
|
||||
// TEMPLATE database every shard is cloned from.
|
||||
for (const svc of ONE_SHOT) {
|
||||
if (compose(["run", "--rm", "--no-deps", "--build", svc]) !== 0) {
|
||||
fail(`${svc} failed — inspect with:\n node integration/scripts/it.mjs logs ${svc}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. The app must exist as dist/ before we can boot it to seed the template.
|
||||
buildFreight();
|
||||
|
||||
// 3. Seed the template (boot seeders + SQL fixtures) and clone it per shard.
|
||||
// Must precede the payment APIs: they connect to the shard databases.
|
||||
if (node("prepare-shards.mjs") !== 0) fail("template seed / shard clone failed.");
|
||||
|
||||
// 4. Per-shard payment API + gateway mock.
|
||||
if (compose(["up", "-d", "--build", "--wait", ...shardServices(SHARDS)]) !== 0) {
|
||||
fail(
|
||||
"shard services failed to become healthy. Inspect with:\n" +
|
||||
" node integration/scripts/it.mjs logs payment-api-it-0",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const [cmd, ...rawExtra] = process.argv.slice(2);
|
||||
/**
|
||||
* Re-clone every shard database from the template so each run is hermetic. The
|
||||
* payment APIs hold connections to those databases, so they come down first —
|
||||
* their `edr_payment` schema is recreated by their own boot migrations.
|
||||
*
|
||||
* This is what retires the warm-stack failure modes the suite used to document:
|
||||
* stowaway paid bookings climbing back aboard, a short consist on the fifth
|
||||
* file, invoice numbers continuing from a previous run.
|
||||
*/
|
||||
function resetShardDbs() {
|
||||
const svc = shardServices(SHARDS);
|
||||
console.log("it: re-cloning shard databases from the template");
|
||||
compose(["stop", ...svc]);
|
||||
if (node("prepare-shards.mjs", ["--clone"]) !== 0) fail("shard database reset failed.");
|
||||
if (compose(["up", "-d", "--wait", ...svc]) !== 0) {
|
||||
fail("shard services failed to become healthy after the reset.");
|
||||
}
|
||||
}
|
||||
|
||||
const [cmd, ...rawArgs] = process.argv.slice(2);
|
||||
const noReset = rawArgs.includes("--no-reset");
|
||||
// `pnpm it:test -- src/foo.it.ts` hands us a literal "--" first. Forwarding it
|
||||
// makes vitest treat everything after it as CLI options and ignore the file
|
||||
// filter — the "one file" run silently becomes the whole suite.
|
||||
const extra = rawExtra[0] === "--" ? rawExtra.slice(1) : rawExtra;
|
||||
const passthrough = rawArgs.filter((a) => a !== "--no-reset");
|
||||
const extra = passthrough[0] === "--" ? passthrough.slice(1) : passthrough;
|
||||
|
||||
switch (cmd) {
|
||||
case "up":
|
||||
up();
|
||||
break;
|
||||
case "test": {
|
||||
if (!stackRunning()) up();
|
||||
if (!stackRunning()) {
|
||||
up();
|
||||
} else {
|
||||
buildFreight();
|
||||
if (!noReset) resetShardDbs();
|
||||
}
|
||||
const { status } = spawnSync(
|
||||
"pnpm",
|
||||
["--filter", "@edr/freight-integration", "run", "test", ...extra],
|
||||
@@ -162,12 +382,14 @@ switch (cmd) {
|
||||
);
|
||||
process.exit(status ?? 1);
|
||||
}
|
||||
case "logs":
|
||||
case "logs": {
|
||||
process.exit(compose(["logs", "--tail", "200", ...extra]));
|
||||
break;
|
||||
case "down":
|
||||
process.exit(compose(["down", "-v", "--remove-orphans"]));
|
||||
break;
|
||||
}
|
||||
case "down": {
|
||||
const status = compose(["down", "-v", "--remove-orphans"]);
|
||||
rmSync(SHARD_FILE, { force: true });
|
||||
process.exit(status);
|
||||
}
|
||||
default:
|
||||
fail(`unknown command "${cmd ?? ""}" — use up | test | logs | down`);
|
||||
}
|
||||
|
||||
134
integration/scripts/prepare-shards.mjs
Normal file
134
integration/scripts/prepare-shards.mjs
Normal file
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the seeded template database, then clone it once per shard.
|
||||
*
|
||||
* node integration/scripts/prepare-shards.mjs # seed template + clone
|
||||
* node integration/scripts/prepare-shards.mjs --clone # clone only (per-run reset)
|
||||
*
|
||||
* Seeding has to happen here, not in vitest's globalSetup, because the order is
|
||||
* load-bearing and half of it is the app itself:
|
||||
*
|
||||
* 1. freight migrations (freight-migration-e2e container, before us)
|
||||
* 2. the app's always-on boot seeders — org, units, positions, permissions
|
||||
* (app.module.ts onApplicationBootstrap)
|
||||
* 3. the SQL fixtures, which declare those as prerequisites
|
||||
* (seed-users.sql: "created by the API's always-on boot seeders")
|
||||
*
|
||||
* Step 2 needs a real Nest boot, and the app now lives inside the vitest workers
|
||||
* — which globalSetup runs in a different process from. So we boot it once here
|
||||
* against the template and every shard is a `CREATE DATABASE … TEMPLATE` copy of
|
||||
* the result: a file copy on the tmpfs postgres, versus re-running migrations
|
||||
* and five seeders per shard.
|
||||
*
|
||||
* Run by it.mjs, which owns the env. No dependencies beyond `pg`.
|
||||
*/
|
||||
import { createRequire } from "node:module";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "pg";
|
||||
|
||||
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const repoRoot = resolve(itDir, "..");
|
||||
const freightDir = join(repoRoot, "apps", "edr-freight-api");
|
||||
|
||||
const TEMPLATE = process.env.IT_TEMPLATE_DB ?? "edr_freight_e2e";
|
||||
const PREFIX = process.env.IT_DB_PREFIX ?? "edr_it_s";
|
||||
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
|
||||
|
||||
const admin = () =>
|
||||
new Client({
|
||||
host: process.env.DB_HOST ?? "localhost",
|
||||
port: Number(process.env.DB_PORT ?? 5543),
|
||||
user: process.env.DB_USER ?? "edr_e2e",
|
||||
password: process.env.DB_PASSWORD ?? "edr_e2e",
|
||||
// Never the template itself: CREATE DATABASE … TEMPLATE refuses while any
|
||||
// session is connected to the source.
|
||||
database: "postgres",
|
||||
});
|
||||
|
||||
const CYPRESS_FIXTURES = join(repoRoot, "e2e", "freight", "cypress", "fixtures");
|
||||
const OWN_FIXTURES = join(itDir, "sql");
|
||||
|
||||
/** Order matters — company needs the users, everything needs the corridor. */
|
||||
const SEEDS = [
|
||||
[CYPRESS_FIXTURES, "seed-users.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-company.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-import-corridor.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-bulk-items.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-g1-train.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-g2-weight.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-government.sql"],
|
||||
[OWN_FIXTURES, "seed-company-b.sql"],
|
||||
[OWN_FIXTURES, "seed-customs-service-type.sql"],
|
||||
];
|
||||
|
||||
async function seedTemplate() {
|
||||
console.log(`it: seeding template ${TEMPLATE}`);
|
||||
|
||||
process.env.DB_NAME = TEMPLATE;
|
||||
// No broker for the template boot: payment.module.ts skips RabbitMQModule
|
||||
// entirely when this is unset, and nothing here publishes.
|
||||
delete process.env.PAYMENT_RABBITMQ_URL;
|
||||
|
||||
const { createFreightApp } = createRequire(join(freightDir, "package.json"))("./dist/main.js");
|
||||
|
||||
// createFreightApp() only builds the graph. `init()` is what fires
|
||||
// onApplicationBootstrap — the seeders we are here for. We never listen.
|
||||
const app = await createFreightApp();
|
||||
try {
|
||||
await app.init();
|
||||
console.log("it: boot seeders done (org, units, positions, permissions)");
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
|
||||
const client = new Client({
|
||||
host: process.env.DB_HOST ?? "localhost",
|
||||
port: Number(process.env.DB_PORT ?? 5543),
|
||||
user: process.env.DB_USER ?? "edr_e2e",
|
||||
password: process.env.DB_PASSWORD ?? "edr_e2e",
|
||||
database: TEMPLATE,
|
||||
});
|
||||
await client.connect();
|
||||
try {
|
||||
for (const [dir, file] of SEEDS) {
|
||||
await client.query(readFileSync(join(dir, file), "utf8"));
|
||||
console.log(`it: seeded ${file}`);
|
||||
}
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function cloneShards() {
|
||||
const pg = admin();
|
||||
await pg.connect();
|
||||
try {
|
||||
// The app's pool and any stray psql hold the template open; without this the
|
||||
// CREATE below fails with "source database is being accessed by other users".
|
||||
await pg.query(
|
||||
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
|
||||
WHERE datname = $1 AND pid <> pg_backend_pid()`,
|
||||
[TEMPLATE],
|
||||
);
|
||||
|
||||
for (let i = 0; i < SHARDS; i++) {
|
||||
const db = `${PREFIX}${i}`;
|
||||
await pg.query(
|
||||
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
|
||||
WHERE datname = $1 AND pid <> pg_backend_pid()`,
|
||||
[db],
|
||||
);
|
||||
await pg.query(`DROP DATABASE IF EXISTS ${db}`);
|
||||
await pg.query(`CREATE DATABASE ${db} TEMPLATE ${TEMPLATE}`);
|
||||
console.log(`it: shard ${i} → ${db}`);
|
||||
}
|
||||
} finally {
|
||||
await pg.end();
|
||||
}
|
||||
}
|
||||
|
||||
const cloneOnly = process.argv.includes("--clone");
|
||||
if (!cloneOnly) await seedTemplate();
|
||||
await cloneShards();
|
||||
133
integration/src/app.ts
Normal file
133
integration/src/app.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* The freight API, booted IN THIS PROCESS — one instance per vitest worker.
|
||||
*
|
||||
* Each worker is a self-contained shard of the whole topology, so spec files can
|
||||
* run in parallel without sharing anything mutable:
|
||||
*
|
||||
* shard i = this in-process freight app on :3111+i
|
||||
* + database edr_it_s{i} (cloned from the seeded template)
|
||||
* + payment-api-it-{i} container on :3113+i, DB_NAME=edr_it_s{i}
|
||||
* + gateway-mock-it-{i} container on :4600+i
|
||||
* + RabbitMQ vhost payment_s{i}
|
||||
*
|
||||
* The payment schema lives INSIDE the shard's own database under the usual
|
||||
* `edr_payment` name, so every raw `edr_payment.*` query in the specs works
|
||||
* unchanged and `paymentDb()` stays an alias for `db()`.
|
||||
*
|
||||
* Why the app is loaded from `dist/` and not from TypeScript: the freight source
|
||||
* is CJS-flavoured — `src/config/database.config.ts` uses `__dirname`,
|
||||
* `require.resolve` and an entity glob, none of which survive vitest's ESM
|
||||
* transform. `createRequire` anchored at the freight app's own package.json
|
||||
* keeps all Nest / typeorm / @tria-plc resolution inside its node_modules, and
|
||||
* `dist` is rebuilt by `it.mjs test` on every run so it can never go stale.
|
||||
*/
|
||||
import { createRequire } from "node:module";
|
||||
import { existsSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const freightDir = join(here, "..", "..", "apps", "edr-freight-api");
|
||||
const distMain = join(freightDir, "dist", "main.js");
|
||||
|
||||
/**
|
||||
* `VITEST_POOL_ID` is 1-based and stable per worker for the life of the run.
|
||||
* Absent outside a worker (global-setup, scripts) — that context gets shard 0.
|
||||
*/
|
||||
export const SHARD = Math.max(0, Number(process.env.VITEST_POOL_ID ?? "1") - 1);
|
||||
|
||||
/**
|
||||
* Each service takes a contiguous block of `IT_SHARDS` ports from its base, so
|
||||
* the bases must stay far apart — see the PORTS comment in scripts/it.mjs.
|
||||
*/
|
||||
const port = (base: string, fallback: number) =>
|
||||
Number(process.env[base] ?? fallback) + SHARD;
|
||||
|
||||
export const API_PORT = port("IT_API_PORT_BASE", 3111);
|
||||
export const PAYMENT_API = `http://localhost:${port("IT_PAYMENT_PORT_BASE", 3131)}`;
|
||||
export const GATEWAY = `http://localhost:${port("IT_GATEWAY_PORT_BASE", 4600)}`;
|
||||
|
||||
export const SHARD_DB = `${process.env.IT_DB_PREFIX ?? "edr_it_s"}${SHARD}`;
|
||||
|
||||
export const DB_URL =
|
||||
`postgres://${process.env.DB_USER ?? "edr_e2e"}:${process.env.DB_PASSWORD ?? "edr_e2e"}` +
|
||||
`@${process.env.DB_HOST ?? "localhost"}:${process.env.DB_PORT ?? "5543"}/${SHARD_DB}`;
|
||||
|
||||
const RABBIT_URL =
|
||||
`amqp://edr:edr_secret@localhost:${process.env.IT_RABBIT_PORT ?? "5772"}` +
|
||||
`/payment_s${SHARD}`;
|
||||
|
||||
/** Prefix every diagnostic — parallel worker output interleaves. */
|
||||
export const tag = (msg: string) => `[shard ${SHARD}] ${msg}`;
|
||||
|
||||
interface FreightMain {
|
||||
createFreightApp: () => Promise<{
|
||||
listen: (port: number) => Promise<unknown>;
|
||||
getHttpServer: () => Server;
|
||||
close: () => Promise<void>;
|
||||
}>;
|
||||
}
|
||||
|
||||
let booted: Promise<Server> | undefined;
|
||||
|
||||
/**
|
||||
* The shard's HTTP server, booted once per worker. supertest takes this
|
||||
* directly in place of a base URL, so every call site in client.ts is a
|
||||
* one-token change.
|
||||
*/
|
||||
export function freightServer(): Promise<Server> {
|
||||
return (booted ??= boot());
|
||||
}
|
||||
|
||||
async function boot(): Promise<Server> {
|
||||
if (!existsSync(distMain)) {
|
||||
throw new Error(
|
||||
tag(
|
||||
`${distMain} is missing — build the API first ` +
|
||||
`(\`pnpm --filter @edr/freight-api run build\`, which \`it.mjs test\` does for you).`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// Must be set BEFORE the require: payment.module.ts reads
|
||||
// PAYMENT_RABBITMQ_URL at module-definition time to decide whether
|
||||
// RabbitMQModule is in the graph at all, and database.config.ts reads DB_NAME
|
||||
// when ConfigModule loads it.
|
||||
process.env.DB_NAME = SHARD_DB;
|
||||
process.env.PORT = String(API_PORT);
|
||||
process.env.PAYMENT_API_URL = PAYMENT_API;
|
||||
process.env.PAYMENT_RABBITMQ_URL = RABBIT_URL;
|
||||
|
||||
const { createFreightApp } = createRequire(join(freightDir, "package.json"))(
|
||||
"./dist/main.js",
|
||||
) as FreightMain;
|
||||
|
||||
// Postgres and the broker are up before vitest starts, but a shard's payment
|
||||
// API may still be finishing its own migrations when the first worker boots.
|
||||
// Retry the whole app rather than a connection: a half-initialised Nest app
|
||||
// cannot be resumed, only closed and rebuilt.
|
||||
let lastErr: unknown;
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
const app = await createFreightApp().catch((err) => {
|
||||
lastErr = err;
|
||||
return undefined;
|
||||
});
|
||||
if (app) {
|
||||
try {
|
||||
// A real port, not an ephemeral one: payment-api-it-{SHARD} calls back
|
||||
// in here for the inbound CBE Unified Bill query (cbe-bill.it.ts).
|
||||
await app.listen(API_PORT);
|
||||
console.log(tag(`freight-api in-process on :${API_PORT} → ${SHARD_DB}`));
|
||||
return app.getHttpServer();
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
await app.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
if (attempt < 3) await new Promise((r) => setTimeout(r, 2000 * attempt));
|
||||
}
|
||||
|
||||
console.error(tag(`freight-api failed to boot on :${API_PORT} (${SHARD_DB})`), lastErr);
|
||||
throw lastErr;
|
||||
}
|
||||
@@ -3,44 +3,75 @@
|
||||
* failures here are the expensive kind: a tenant reading another tenant's
|
||||
* invoice, or an unauthenticated caller marking one paid.
|
||||
*/
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import request from "supertest";
|
||||
import {
|
||||
API,
|
||||
PAYMENT_API,
|
||||
api,
|
||||
closeDb,
|
||||
customerA,
|
||||
customerB,
|
||||
db,
|
||||
freightServer,
|
||||
login,
|
||||
payment,
|
||||
} from "./client";
|
||||
|
||||
/** Company A, from e2e/freight/cypress/fixtures/seed-company.sql. */
|
||||
const TENANT_A_TIN = "0102030405";
|
||||
|
||||
/**
|
||||
* This file used to read "the newest invoice of company A" and `return` early
|
||||
* when there wasn't one — which passed silently on a database no payment spec
|
||||
* had run against yet. Now that every shard starts from a pristine clone, that
|
||||
* would be *every* run. It bills itself instead.
|
||||
*
|
||||
* Straight SQL, not the booking chain: the point of this file is that it is
|
||||
* cheap, and a cross-tenant read is refused on ownership alone — nothing here
|
||||
* cares how the invoice came to exist.
|
||||
*/
|
||||
async function seedTenantAInvoice(): Promise<string> {
|
||||
const rows = await db<{ id: string }>(
|
||||
`INSERT INTO freight.invoices (
|
||||
id, invoice_number, company_id, company_profile_id,
|
||||
subtotal_amount, tax_amount, total_amount, paid_amount, balance_amount,
|
||||
currency, status, source, source_id, type, issued_at, due_at, payments
|
||||
)
|
||||
SELECT gen_random_uuid(), $1, c.id, p.id,
|
||||
1000, 0, 1000, 0, 1000,
|
||||
'ETB', 'ISSUED', 'booking', gen_random_uuid()::text, 'PREPAID',
|
||||
now(), now() + interval '7 days', '[]'::jsonb
|
||||
FROM freight.companies c
|
||||
JOIN freight.company_profiles p ON p.company_id = c.id AND p.deleted_at IS NULL
|
||||
WHERE c.tin = $2 AND c.deleted_at IS NULL
|
||||
LIMIT 1
|
||||
RETURNING id`,
|
||||
[`INV-AUTHZ-${Date.now()}`, TENANT_A_TIN],
|
||||
);
|
||||
const id = rows[0]?.id;
|
||||
if (!id) {
|
||||
throw new Error(
|
||||
`authz: could not bill company ${TENANT_A_TIN} — seed-company.sql missing from this shard?`,
|
||||
);
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
describe("payment authorization boundaries", () => {
|
||||
let invoiceId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
invoiceId = await seedTenantAInvoice();
|
||||
});
|
||||
afterAll(closeDb);
|
||||
|
||||
it("hides one tenant's invoice from the other", async () => {
|
||||
const rows = await db<{ id: string; company_id: string }>(
|
||||
`SELECT i.id, i.company_id FROM freight.invoices i
|
||||
JOIN freight.companies c ON c.id = i.company_id
|
||||
WHERE c.tin = '0102030405' AND i.deleted_at IS NULL
|
||||
ORDER BY i.created_at DESC LIMIT 1`,
|
||||
);
|
||||
if (!rows[0]) return; // nothing billed yet in this run — payment files cover it
|
||||
const res = await api(customerB, "get", `/api/billing/my-invoices/${rows[0].id}`);
|
||||
const res = await api(customerB, "get", `/api/billing/my-invoices/${invoiceId}`);
|
||||
expect([403, 404]).toContain(res.status);
|
||||
});
|
||||
|
||||
it("refuses to let one tenant pay the other's invoice", async () => {
|
||||
const rows = await db<{ id: string }>(
|
||||
`SELECT i.id FROM freight.invoices i
|
||||
JOIN freight.companies c ON c.id = i.company_id
|
||||
WHERE c.tin = '0102030405' AND i.status <> 'PAID' AND i.deleted_at IS NULL
|
||||
ORDER BY i.created_at DESC LIMIT 1`,
|
||||
);
|
||||
if (!rows[0]) return;
|
||||
const res = await api(customerB, "post", `/api/billing/my-invoices/${rows[0].id}/pay`, {
|
||||
const res = await api(customerB, "post", `/api/billing/my-invoices/${invoiceId}/pay`, {
|
||||
method: "CBE_BIRR",
|
||||
platform: "web",
|
||||
});
|
||||
@@ -71,7 +102,9 @@ describe("payment authorization boundaries", () => {
|
||||
amountMinor: 1,
|
||||
currency: "ETB",
|
||||
};
|
||||
const res = await request(API).post("/api/internal/payments/mark-paid").send(body);
|
||||
const res = await request(await freightServer())
|
||||
.post("/api/internal/payments/mark-paid")
|
||||
.send(body);
|
||||
expect([401, 403]).toContain(res.status);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
/**
|
||||
* Plumbing for the integration suite: HTTP against the containerized freight
|
||||
* and payment APIs, SQL against their shared throwaway Postgres, and the
|
||||
* gateway mock's control plane.
|
||||
* Plumbing for the integration suite: HTTP against this worker's IN-PROCESS
|
||||
* freight API and its containerized payment API, SQL against the worker's own
|
||||
* throwaway database, and its own gateway mock's control plane.
|
||||
*
|
||||
* Everything here is shard-scoped — see app.ts for the topology. supertest takes
|
||||
* an `http.Server` exactly where it takes a base URL, so pointing the suite at
|
||||
* the in-process app is a one-token change at each call site.
|
||||
*
|
||||
* This is the Cypress-free port of e2e/freight/cypress/e2e/flows/import-utils.ts —
|
||||
* same request sequences, same SQL, `pg.Pool` instead of `cy.task`.
|
||||
@@ -9,13 +13,11 @@
|
||||
import request from "supertest";
|
||||
import { Pool, type QueryResultRow } from "pg";
|
||||
|
||||
export const API = process.env.IT_API_URL ?? "http://localhost:3111";
|
||||
export const PAYMENT_API = process.env.IT_PAYMENT_URL ?? "http://localhost:3113";
|
||||
export const GATEWAY = process.env.IT_GATEWAY_URL ?? "http://localhost:4600";
|
||||
export const SERVICE_TOKEN = process.env.IT_SERVICE_TOKEN ?? "e2e-service-token";
|
||||
import { DB_URL, GATEWAY, PAYMENT_API, SHARD, freightServer, tag } from "./app";
|
||||
|
||||
const DB_URL =
|
||||
process.env.IT_DB_URL ?? "postgres://edr_e2e:edr_e2e@localhost:5543/edr_freight_e2e";
|
||||
export { GATEWAY, PAYMENT_API, SHARD, freightServer };
|
||||
|
||||
export const SERVICE_TOKEN = process.env.IT_SERVICE_TOKEN ?? "e2e-service-token";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// users (e2e/freight/cypress/fixtures/seed-users.sql + users.json)
|
||||
@@ -45,11 +47,23 @@ export async function db<T extends QueryResultRow = Record<string, unknown>>(
|
||||
return res.rows;
|
||||
}
|
||||
|
||||
export async function closeDb(): Promise<void> {
|
||||
await pool.end();
|
||||
}
|
||||
/**
|
||||
* No-op. The pool is per-WORKER, not per-file: with `isolate: false` the module
|
||||
* registry survives across every spec a worker runs, so the first `afterAll` to
|
||||
* call this would break every later file on that shard. The pool dies with the
|
||||
* worker process.
|
||||
*/
|
||||
export async function closeDb(): Promise<void> {}
|
||||
|
||||
/**
|
||||
* Poll a query until `check` passes. Async settlement here is broker-driven and
|
||||
* tick-driven, so the interval is pure overshoot: at the old 2000ms every wait
|
||||
* in the suite finished up to two seconds after the state it wanted was already
|
||||
* in the database, several hundred times over. The ceiling
|
||||
* (`attempts × intervalMs`) is unchanged.
|
||||
*/
|
||||
const POLL_INTERVAL_MS = Number(process.env.IT_POLL_INTERVAL_MS ?? 250);
|
||||
|
||||
/** Poll a query until `check` passes. Async settlement here is broker-driven. */
|
||||
export async function poll<T extends QueryResultRow = Record<string, unknown>>(
|
||||
label: string,
|
||||
sql: string,
|
||||
@@ -57,14 +71,24 @@ export async function poll<T extends QueryResultRow = Record<string, unknown>>(
|
||||
check: (row: T | undefined) => boolean,
|
||||
{ attempts = 40, intervalMs = 2000 } = {},
|
||||
): Promise<T> {
|
||||
// Callers express patience as attempts × their own interval; keep that
|
||||
// deadline and just sample it finely.
|
||||
const deadlineMs = attempts * intervalMs;
|
||||
const tries = Math.ceil(deadlineMs / POLL_INTERVAL_MS);
|
||||
|
||||
let last: T | undefined;
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
for (let i = 0; i < tries; i++) {
|
||||
last = (await db<T>(sql, params))[0];
|
||||
if (check(last)) return last as T;
|
||||
await sleep(intervalMs);
|
||||
await sleep(POLL_INTERVAL_MS);
|
||||
}
|
||||
throw new Error(
|
||||
`timed out waiting for ${label} after ${attempts} attempts — last row: ${JSON.stringify(last)}`,
|
||||
tag(
|
||||
`timed out waiting for ${label} after ${Math.round(deadlineMs / 1000)}s\n` +
|
||||
` sql: ${sql.replace(/\s+/g, " ").trim()}\n` +
|
||||
` params: ${JSON.stringify(params)}\n` +
|
||||
` last row: ${JSON.stringify(last)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -86,7 +110,7 @@ export async function tokenFor(email: string): Promise<string> {
|
||||
if (cached) return cached;
|
||||
|
||||
const portal = email.endsWith("@gmail.com");
|
||||
const res = await request(API)
|
||||
const res = await request(await freightServer())
|
||||
.post("/api/auth/login")
|
||||
.set("x-client-app", portal ? "portal" : "backoffice")
|
||||
.send({ email, password: portal ? CUSTOMER_PASSWORD : STAFF_PASSWORD });
|
||||
@@ -101,8 +125,11 @@ export async function tokenFor(email: string): Promise<string> {
|
||||
}
|
||||
|
||||
/** Login without caching, so audience-rejection can be asserted. */
|
||||
export function login(email: string, password: string, app: "portal" | "backoffice") {
|
||||
return request(API).post("/api/auth/login").set("x-client-app", app).send({ email, password });
|
||||
export async function login(email: string, password: string, app: "portal" | "backoffice") {
|
||||
return request(await freightServer())
|
||||
.post("/api/auth/login")
|
||||
.set("x-client-app", app)
|
||||
.send({ email, password });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -119,7 +146,9 @@ export async function api(
|
||||
body?: unknown,
|
||||
): Promise<request.Response> {
|
||||
const token = await tokenFor(email);
|
||||
const req = request(API)[method](path).set("Authorization", `Bearer ${token}`);
|
||||
const req = request(await freightServer())
|
||||
[method](path)
|
||||
.set("Authorization", `Bearer ${token}`);
|
||||
return method === "get" || method === "delete" ? req.send() : req.send(body ?? {});
|
||||
}
|
||||
|
||||
@@ -148,7 +177,9 @@ export async function upload(
|
||||
fields: Record<string, string> = {},
|
||||
): Promise<request.Response> {
|
||||
const token = await tokenFor(email);
|
||||
const req = request(API).post(path).set("Authorization", `Bearer ${token}`);
|
||||
const req = request(await freightServer())
|
||||
.post(path)
|
||||
.set("Authorization", `Bearer ${token}`);
|
||||
for (const [k, v] of Object.entries(fields)) req.field(k, v);
|
||||
return req.attach(field, filePath);
|
||||
}
|
||||
|
||||
@@ -1,41 +1,17 @@
|
||||
/**
|
||||
* Runs once before any spec: wait for both APIs, then seed.
|
||||
* Runs once, in its own process, before any spec: confirm every shard's
|
||||
* containerized half is answering, then clear its gateway mock.
|
||||
*
|
||||
* Seeds are the Cypress suite's fixtures, reused verbatim (they are idempotent
|
||||
* `insert … where not exists`), plus one of our own for the second tenant:
|
||||
* seed-users.sql → seed-company.sql (order matters; company needs the users)
|
||||
* seed-import-corridor.sql (yards, locos, wagons, rates, distances)
|
||||
* seed-bulk-items.sql (PER_ITEM break-bulk cargo types)
|
||||
* seed-g1-train.sql (the 53-wagon BUILT container train)
|
||||
* seed-g2-weight.sql (the two 3 500 T weight-bound trains)
|
||||
* seed-government.sql (the kind='government' company)
|
||||
* seed-company-b.sql (user2@gmail.com's company — this suite)
|
||||
* seed-customs-service-type.sql (a service type that bundles customs)
|
||||
* Seeding is NOT here any more. The order is load-bearing —
|
||||
* `seed-users.sql` names its prerequisites as "created by the API's always-on
|
||||
* boot seeders" (iam.organizations / units / positions) — and the freight app
|
||||
* now boots inside the vitest *workers*, which this process cannot reach. So the
|
||||
* template database is seeded once by `scripts/prepare-shards.mjs` (boot the app
|
||||
* → boot seeders → SQL fixtures) and each shard is a clone of it.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { Client } from "pg";
|
||||
|
||||
const API = process.env.IT_API_URL ?? "http://localhost:3111";
|
||||
const PAYMENT_API = process.env.IT_PAYMENT_URL ?? "http://localhost:3113";
|
||||
const GATEWAY = process.env.IT_GATEWAY_URL ?? "http://localhost:4600";
|
||||
const DB_URL =
|
||||
process.env.IT_DB_URL ?? "postgres://edr_e2e:edr_e2e@localhost:5543/edr_freight_e2e";
|
||||
|
||||
const CYPRESS_FIXTURES = join(process.cwd(), "..", "e2e", "freight", "cypress", "fixtures");
|
||||
const OWN_FIXTURES = join(process.cwd(), "sql");
|
||||
|
||||
const SEEDS: Array<[dir: string, file: string]> = [
|
||||
[CYPRESS_FIXTURES, "seed-users.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-company.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-import-corridor.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-bulk-items.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-g1-train.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-g2-weight.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-government.sql"],
|
||||
[OWN_FIXTURES, "seed-company-b.sql"],
|
||||
[OWN_FIXTURES, "seed-customs-service-type.sql"],
|
||||
];
|
||||
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
|
||||
const PAYMENT_BASE = Number(process.env.IT_PAYMENT_PORT_BASE ?? 3131);
|
||||
const GATEWAY_BASE = Number(process.env.IT_GATEWAY_PORT_BASE ?? 4600);
|
||||
|
||||
async function waitFor(label: string, url: string, attempts = 60): Promise<void> {
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
@@ -51,22 +27,20 @@ async function waitFor(label: string, url: string, attempts = 60): Promise<void>
|
||||
}
|
||||
|
||||
export async function setup(): Promise<void> {
|
||||
await Promise.all([
|
||||
waitFor("freight-api", `${API}/api/health`),
|
||||
waitFor("payment-api", `${PAYMENT_API}/health`),
|
||||
waitFor("gateway-mock", `${GATEWAY}/__control/health`),
|
||||
]);
|
||||
const shards = Array.from({ length: SHARDS }, (_, i) => i);
|
||||
|
||||
const client = new Client({ connectionString: DB_URL });
|
||||
await client.connect();
|
||||
try {
|
||||
for (const [dir, file] of SEEDS) {
|
||||
await client.query(readFileSync(join(dir, file), "utf8"));
|
||||
console.log(`it: seeded ${file}`);
|
||||
}
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
await Promise.all(
|
||||
shards.flatMap((i) => [
|
||||
waitFor(`payment-api (shard ${i})`, `http://localhost:${PAYMENT_BASE + i}/health`),
|
||||
waitFor(`gateway-mock (shard ${i})`, `http://localhost:${GATEWAY_BASE + i}/__control/health`),
|
||||
]),
|
||||
);
|
||||
|
||||
await fetch(`${GATEWAY}/__control/reset`, { method: "POST" });
|
||||
await Promise.all(
|
||||
shards.map((i) =>
|
||||
fetch(`http://localhost:${GATEWAY_BASE + i}/__control/reset`, { method: "POST" }),
|
||||
),
|
||||
);
|
||||
|
||||
console.log(`it: ${SHARDS} shard(s) ready`);
|
||||
}
|
||||
|
||||
@@ -1,18 +1,34 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
/**
|
||||
* One worker per shard. A shard owns its whole topology — in-process freight
|
||||
* app, database, payment API, gateway mock, broker vhost — so files can run in
|
||||
* parallel without sharing anything mutable. See src/app.ts.
|
||||
*/
|
||||
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["src/**/*.it.ts"],
|
||||
globalSetup: ["src/global-setup.ts"],
|
||||
// One shared containerized stack and one shared database: files run in
|
||||
// sequence. Concurrency is exercised INSIDE tests (Promise.all), which is
|
||||
// what the guards under test actually see in production.
|
||||
fileParallelism: false,
|
||||
pool: "forks",
|
||||
// Pinned min = max: VITEST_POOL_ID is the shard key, and a worker that vitest
|
||||
// declines to spawn is a shard whose containers are idling.
|
||||
poolOptions: { forks: { minForks: SHARDS, maxForks: SHARDS } },
|
||||
fileParallelism: SHARDS > 1,
|
||||
// The freight app boots ONCE per worker and is memoised in module scope.
|
||||
// With isolation on, every file would pay a fresh Nest boot (60 modules,
|
||||
// TypeORM, 5 seeders) — 25 boots instead of `SHARDS`.
|
||||
isolate: false,
|
||||
testTimeout: 180_000,
|
||||
hookTimeout: 180_000,
|
||||
// Booking/scheduling steps are not idempotent — a retry would assert
|
||||
// against a half-advanced booking.
|
||||
// against a half-advanced booking. (A whole-RUN retry is now safe, because
|
||||
// `it.mjs test` re-clones every shard database first.)
|
||||
retry: 0,
|
||||
// Workers hold a pg pool, three Socket.IO gateways, a RabbitMQ channel and
|
||||
// live cron timers; don't wait on those handles to unwind.
|
||||
teardownTimeout: 20_000,
|
||||
reporters: ["verbose"],
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user