mirror of
https://github.com/Tria-plc/edr-platform.git
synced 2026-08-26 18:42:49 +00:00
test(integration): boot freight API in-process across parallel shards
The suite drove a containerized freight API, so every code change needed an image rebuild before a test could see it, and there was no way to attach a debugger. Files also ran strictly in sequence against one shared database, which is the root of the warm-stack gotchas the README documents: stowaway paid bookings climbing back aboard, a short consist on the fifth file. The freight app now boots inside each vitest worker from dist/, and each worker owns a whole shard of the topology - its own database, payment API, gateway mock and broker vhost - so nothing mutable is shared and files run in parallel. Full suite drops from roughly 20 minutes to 196s at 4 shards. - main.ts exports createFreightApp() so the harness applies the same prefix, pipes, filters and interceptors as production instead of replaying them by hand; self-start is guarded by require.main so the Dockerfile CMD still boots - booking-window tick cadence is env-driven (BOOKING_WINDOW_TICK_CRON), */1 in the suite, */10 unchanged in production - prepare-shards.mjs seeds a template database (boot seeders, then the SQL fixtures that depend on them) and clones it per shard; it.mjs re-clones on every run, so each run is hermetic - gateway mock and payment API are generated per shard: the mock keeps modes and orders process-global and 20 of 25 specs reset it in beforeAll, and the inbound CBE bill query has to reach one specific shard's app - poll() samples every 250ms instead of 2000ms, keeping the caller's deadline - authz.it.ts seeds its own invoice; it previously read another spec's leftover and returned early, which silently passed on a pristine database Known: an unlocked MAX(sequence_no)+1 in train-scheduling.service.ts races under concurrent allocation and leaves a short consist, so 1-3 specs fail intermittently. Pre-existing and reproduces at the production tick cadence.
This commit is contained in:
Nathnael
182
integration/scripts/gen-shards.mjs
Normal file
182
integration/scripts/gen-shards.mjs
Normal file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Generate `integration/.it-shards.yaml` — one payment API + one gateway mock
|
||||
* per shard. Compose has no loops, so the per-shard services are written out.
|
||||
*
|
||||
* Every shard is the real topology in miniature; nothing crosses between them:
|
||||
*
|
||||
* gateway-mock-it-{i} its own `modes`/`calls`/`orders` — the mock keeps those
|
||||
* process-global, and 20 of 25 specs call
|
||||
* POST /__control/reset in beforeAll, so a shared mock
|
||||
* would have each starting spec wipe every running spec's
|
||||
* live orders (which fails as a wrong *assertion*: an
|
||||
* unknown order still gets a correctly-signed webhook,
|
||||
* just with the mock's default amount).
|
||||
* payment-api-it-{i} its own DB_NAME (the shard's database, `edr_payment`
|
||||
* schema inside it), its own broker vhost, and its own
|
||||
* FREIGHT_API_BASE_URL — the inbound CBE-bill query has
|
||||
* to land on THIS shard's in-process app, and a single
|
||||
* container could only ever point at one of them.
|
||||
*
|
||||
* Written by it.mjs, gitignored. Do not edit by hand.
|
||||
*/
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
/**
|
||||
* All shards share one build and one image tag, so the payment API image is
|
||||
* built once and reused rather than N times.
|
||||
*/
|
||||
const PAYMENT_IMAGE = "edr-payment-api-it:local";
|
||||
|
||||
export function renderShards({
|
||||
shards,
|
||||
apiPortBase,
|
||||
paymentPortBase,
|
||||
gatewayPortBase,
|
||||
dbPrefix,
|
||||
telebirrKey,
|
||||
}) {
|
||||
const services = [];
|
||||
|
||||
for (let i = 0; i < shards; i++) {
|
||||
const gw = `gateway-mock-it-${i}`;
|
||||
const pay = `payment-api-it-${i}`;
|
||||
const db = `${dbPrefix}${i}`;
|
||||
const vhost = `payment_s${i}`;
|
||||
// The freight app for this shard runs on the HOST, inside the vitest worker.
|
||||
const freightBase = `http://host.docker.internal:${apiPortBase + i}/api`;
|
||||
|
||||
services.push(`
|
||||
${gw}:
|
||||
image: node:20-alpine
|
||||
volumes:
|
||||
- ./integration/gateway-mock:/app:ro
|
||||
working_dir: /app
|
||||
environment:
|
||||
PORT: "4600"
|
||||
# Same secrets the payment API gets — so webhooks the mock signs pass the
|
||||
# API's REAL signature verification instead of bypassing it.
|
||||
CBE_SECRET_KEY: it-cbe-secret
|
||||
CBE_MERCHANT_ID: it-cbe-merchant
|
||||
PAYMENT_API_URL: http://${pay}:3003
|
||||
command: ["node", "server.js"]
|
||||
ports:
|
||||
- "${gatewayPortBase + i}:4600"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://localhost:4600/__control/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
${pay}:
|
||||
image: ${PAYMENT_IMAGE}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: apps/edr-payment-api/Dockerfile
|
||||
secrets:
|
||||
- npmrc
|
||||
depends_on:
|
||||
postgres-freight-e2e:
|
||||
condition: service_healthy
|
||||
rabbitmq-it:
|
||||
condition: service_healthy
|
||||
${gw}:
|
||||
condition: service_healthy
|
||||
# The freight app is on the host now, not in this network.
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
environment:
|
||||
PORT: "3003"
|
||||
NODE_ENV: test
|
||||
# Payment tables live in their own schema of THIS SHARD's database;
|
||||
# main.ts ensurePaymentSchema() creates it, migrationsRun does the rest.
|
||||
# Same database as freight, so every raw edr_payment.* query in the specs
|
||||
# reads through the one connection.
|
||||
DB_HOST: postgres-freight-e2e
|
||||
DB_PORT: "5432"
|
||||
DB_USER: edr_e2e
|
||||
DB_PASSWORD: edr_e2e
|
||||
DB_NAME: ${db}
|
||||
DB_SCHEMA: edr_payment
|
||||
SERVICE_AUTH_TOKEN: e2e-service-token
|
||||
PUBLISHER_TRANSPORT: rabbitmq
|
||||
PAYMENT_RABBITMQ_URL: amqp://edr:edr_secret@rabbitmq-it:5672/${vhost}
|
||||
PAYMENT_NOTIFY_FREIGHT_URL: ${freightBase}/internal/payments/mark-paid
|
||||
# Fast relay + sweep so retry/reconciliation are observable inside a test
|
||||
# rather than a minute later.
|
||||
OUTBOX_RELAY_INTERVAL_MS: "1000"
|
||||
RECONCILE_STALE_AFTER_MS: "5000"
|
||||
# Every gateway points at this shard's mock. Paths are per-provider prefixes.
|
||||
CBE_BASE_URL: http://${gw}:4600/cbe-birr
|
||||
CBE_MERCHANT_ID: it-cbe-merchant
|
||||
CBE_SECRET_KEY: it-cbe-secret
|
||||
CBE_NOTIFY_URL: http://${pay}:3003/webhooks/cbe-birr
|
||||
CBE_RETURN_URL: http://localhost/return
|
||||
TELEBIRR_BASE_URL: http://${gw}:4600/telebirr
|
||||
TELEBIRR_WEB_BASE_URL: http://${gw}:4600/telebirr/web
|
||||
TELEBIRR_FABRIC_APP_ID: it-fabric
|
||||
TELEBIRR_APP_SECRET: it-secret
|
||||
TELEBIRR_MERCHANT_APP_ID: it-merchant-app
|
||||
TELEBIRR_MERCHANT_CODE: "999999"
|
||||
TELEBIRR_NOTIFY_URL: http://${pay}:3003/webhooks/telebirr
|
||||
# Telebirr PSS-signs every request object — a throwaway key generated per
|
||||
# launch by it.mjs (nothing key-shaped lives in git).
|
||||
TELEBIRR_PRIVATE_KEY: ${JSON.stringify(telebirrKey)}
|
||||
EBIRR_BASE_URL: http://${gw}:4600/ebirr
|
||||
DMONEY_BASE_URL: http://${gw}:4600/dmoney
|
||||
CARD_BASE_URL: http://${gw}:4600/card
|
||||
WAAFI_BASE_URL: http://${gw}:4600/waafi
|
||||
CAC_BASE_URL: http://${gw}:4600/cac
|
||||
CAC_USERNAME: it-cac
|
||||
CAC_PASSWORD: it-cac
|
||||
CAC_APP_KEY: it-cac-key
|
||||
CAC_API_KEY: it-cac-api
|
||||
CAC_COMPANY_SERVICES_ID: "1"
|
||||
# Inbound CBE Unified Bill — we are the biller; bill-query hops back into
|
||||
# the freight API, so this direction runs real code on both sides.
|
||||
CBE_BILL_ENABLED: "true"
|
||||
CBE_BILL_CLIENT_ID: it-cbe-bill
|
||||
CBE_BILL_CLIENT_SECRET: it-cbe-bill-secret
|
||||
CBE_BILL_JWT_SECRET: it-cbe-bill-jwt
|
||||
FREIGHT_API_BASE_URL: ${freightBase}
|
||||
ports:
|
||||
- "${paymentPortBase + i}:3003"
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://localhost:3003/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 40s`);
|
||||
}
|
||||
|
||||
return `# GENERATED by integration/scripts/gen-shards.mjs — do not edit.
|
||||
# ${shards} shard(s). Overlaid on docker-compose.e2e.yaml + docker-compose.it.yaml.
|
||||
services:${services.join("\n")}
|
||||
`;
|
||||
}
|
||||
|
||||
export const SHARD_FILE = join(itDir, ".it-shards.yaml");
|
||||
|
||||
export function writeShards(opts) {
|
||||
writeFileSync(SHARD_FILE, renderShards(opts));
|
||||
return SHARD_FILE;
|
||||
}
|
||||
|
||||
export const shardServices = (shards) =>
|
||||
Array.from({ length: shards }, (_, i) => [`gateway-mock-it-${i}`, `payment-api-it-${i}`]).flat();
|
||||
@@ -3,10 +3,13 @@
|
||||
* Freight integration-suite launcher.
|
||||
*
|
||||
* node integration/scripts/it.mjs <up|test|down|logs> [vitest args...]
|
||||
* IT_SHARDS=4 node integration/scripts/it.mjs test
|
||||
* node integration/scripts/it.mjs test --no-reset -- src/payment-happy.it.ts
|
||||
*
|
||||
* Overlays docker-compose.it.yaml on docker-compose.e2e.yaml: same freight
|
||||
* stack, but the payment microservice is real and only the bank gateways are
|
||||
* stubbed. Web/Cypress containers are never started — this suite is HTTP only.
|
||||
* The freight API is NOT containerized here: it boots inside each vitest worker
|
||||
* from `apps/edr-freight-api/dist`, which `test` rebuilds every run. Each worker
|
||||
* is a whole shard — own database, own payment API, own gateway mock, own broker
|
||||
* vhost — so spec files run in parallel without sharing anything mutable.
|
||||
*
|
||||
* Ports are fixed (and distinct from the Cypress e2e defaults) so both stacks
|
||||
* can be up at once; they are separate compose projects.
|
||||
@@ -16,21 +19,24 @@
|
||||
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { SHARD_FILE, shardServices, writeShards } from "./gen-shards.mjs";
|
||||
|
||||
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const repoRoot = resolve(itDir, "..");
|
||||
const composeBase = [
|
||||
"compose",
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.e2e.yaml"),
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.it.yaml"),
|
||||
];
|
||||
|
||||
/** Deliberately offset from the Cypress stack's defaults (3101/5533/9310…). */
|
||||
/**
|
||||
* Deliberately offset from the Cypress stack's defaults (3101/5533/9310…).
|
||||
*
|
||||
* Each per-shard service takes a CONTIGUOUS BLOCK of `IT_SHARDS` ports starting
|
||||
* at its base (freight :3111+i, payment :3131+i, gateway :4600+i), so the bases
|
||||
* must be at least MAX_SHARDS apart. They were originally 3111 and 3113 — two
|
||||
* apart — which silently worked at 1–2 shards and then had shard 2's freight app
|
||||
* try to bind :3113, the port payment-api-it-0 was already published on.
|
||||
*/
|
||||
const PORTS = {
|
||||
E2E_API_PORT: 3111,
|
||||
E2E_DB_PORT: 5543,
|
||||
@@ -39,32 +45,39 @@ const PORTS = {
|
||||
// Unused here (no web containers) but referenced by the base file's build args.
|
||||
E2E_PORTAL_PORT: 5393,
|
||||
E2E_BACKOFFICE_PORT: 5394,
|
||||
IT_PAYMENT_PORT: 3113,
|
||||
IT_PAYMENT_PORT: 3131,
|
||||
IT_GATEWAY_PORT: 4600,
|
||||
IT_RABBIT_PORT: 5772,
|
||||
IT_RABBIT_UI_PORT: 15772,
|
||||
};
|
||||
|
||||
/** Everything the suite needs up — web + cypress are deliberately absent. */
|
||||
const SERVICES = [
|
||||
"postgres-freight-e2e",
|
||||
"minio-e2e",
|
||||
"minio-init-e2e",
|
||||
"freight-migration-e2e",
|
||||
"fayda-mock-e2e",
|
||||
"etrade-mock-e2e",
|
||||
// Still a base-stack dependency of freight-api-e2e (reconcile-before-expire
|
||||
// has its own client); cheap to run alongside the real payment API.
|
||||
"payment-mock-e2e",
|
||||
"gateway-mock-it",
|
||||
"rabbitmq-it",
|
||||
"payment-api-it",
|
||||
"freight-api-e2e",
|
||||
];
|
||||
/** Beyond this the freight block (3111+) would run into the payment block (3131+). */
|
||||
const MAX_SHARDS = 16;
|
||||
|
||||
const RUNNING = SERVICES.filter(
|
||||
(s) => !["minio-init-e2e", "freight-migration-e2e"].includes(s),
|
||||
);
|
||||
/**
|
||||
* How many shards. RAM is the ceiling, not cores: each shard is an in-process
|
||||
* Nest app plus two containers.
|
||||
*/
|
||||
const SHARDS = Math.max(1, Number(process.env.IT_SHARDS ?? 4));
|
||||
if (SHARDS > MAX_SHARDS) {
|
||||
console.error(
|
||||
`\nit: IT_SHARDS=${SHARDS} exceeds ${MAX_SHARDS} — the per-service port blocks would overlap.` +
|
||||
`\n Raise IT_PAYMENT_PORT/IT_GATEWAY_PORT in it.mjs first.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const TEMPLATE_DB = "edr_freight_e2e";
|
||||
const DB_PREFIX = "edr_it_s";
|
||||
|
||||
/** Long-running services shared by every shard. Web + Cypress are absent. */
|
||||
const INFRA = ["postgres-freight-e2e", "minio-e2e", "rabbitmq-it"];
|
||||
|
||||
/**
|
||||
* One-shots: they run, then exit 0. They cannot go in the `up --wait` set —
|
||||
* compose reports an exited container as a failed wait, so a healthy stack
|
||||
* looks broken. Run them with `compose run`, which returns their exit code.
|
||||
*/
|
||||
const ONE_SHOT = ["minio-init-e2e", "freight-migration-e2e"];
|
||||
|
||||
function fail(msg) {
|
||||
console.error(`\nit: ${msg}`);
|
||||
@@ -97,64 +110,271 @@ function fakeFaydaPrivateKeyBase64() {
|
||||
return Buffer.from(JSON.stringify(jwk)).toString("base64");
|
||||
}
|
||||
|
||||
const telebirrKey = process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey();
|
||||
|
||||
/**
|
||||
* The freight app's own environment. This is docker-compose.e2e.yaml's
|
||||
* `freight-api-e2e` block with container hostnames swapped for published host
|
||||
* ports — the app runs on the host now. Per-shard values (DB_NAME, PORT,
|
||||
* PAYMENT_API_URL, PAYMENT_RABBITMQ_URL) are derived per worker in src/app.ts.
|
||||
*/
|
||||
const freightEnv = {
|
||||
NODE_ENV: "test",
|
||||
DB_HOST: "localhost",
|
||||
DB_PORT: String(PORTS.E2E_DB_PORT),
|
||||
DB_USER: "edr_e2e",
|
||||
DB_PASSWORD: "edr_e2e",
|
||||
DB_NAME: TEMPLATE_DB,
|
||||
|
||||
// e2e-only secrets — never reuse outside this stack
|
||||
JWT_SECRET: "e2e-jwt-secret",
|
||||
JWT_ACCESS_TOKEN_SECRET: "e2e-access-secret",
|
||||
JWT_REFRESH_TOKEN_SECRET: "e2e-refresh-secret",
|
||||
JWT_EXPIRES_IN: "1d",
|
||||
JWT_ACCESS_TOKEN_EXPIRES: "1d",
|
||||
JWT_REFRESH_TOKEN_EXPIRES: "7d",
|
||||
// Mandatory: ServiceAuthGuard returns TRUE when this is unset, and
|
||||
// authz.it.ts asserts that the internal surface rejects an unsigned caller.
|
||||
SERVICE_AUTH_TOKEN: "e2e-service-token",
|
||||
|
||||
SEED_EDR_ORG: "true",
|
||||
SUPER_ADMIN_EMAIL: "superadmin@tria.com",
|
||||
SUPER_ADMIN_PHONE: "+251900000000",
|
||||
|
||||
MINIO_ENDPOINT: "localhost",
|
||||
MINIO_PORT: String(PORTS.E2E_MINIO_PORT),
|
||||
MINIO_USE_SSL: "false",
|
||||
MINIO_ACCESS_KEY: "e2e-minio",
|
||||
MINIO_SECRET_KEY: "e2e-minio-secret",
|
||||
MINIO_REGION: "us-east-1",
|
||||
|
||||
// Only gates the SMS/email clients. The payment consumer is wired by
|
||||
// PAYMENT_RABBITMQ_URL alone (payment.module.ts).
|
||||
RABBITMQ_ENABLED: "false",
|
||||
// fayda.config.ts THROWS at load if this is "true" without the full var set,
|
||||
// and the fayda/etrade mocks publish no host ports. No IT spec touches them.
|
||||
FAYDA_ENABLED: "false",
|
||||
// SMS strategy has no kill switch and defaults to a real dev endpoint.
|
||||
OZIKING_SMS_URL: "http://127.0.0.1:9/sms",
|
||||
// app.config.ts otherwise live-scrapes https://ethio.forex on first booking.
|
||||
CBE_EXCHANGE_SCRAPE_URL: "http://127.0.0.1:9/fx",
|
||||
CBE_EXCHANGE_API_URL: "http://127.0.0.1:9/fx",
|
||||
CBE_EXCHANGE_FALLBACK_RATE: "130",
|
||||
FREIGHT_PORTAL_URL: `http://localhost:${PORTS.E2E_PORTAL_PORT}`,
|
||||
|
||||
// Drain tail on every pay window. Production defaults to 5 minutes; a
|
||||
// reservation here lives ~60s, so 5 would push every natural expiry past the
|
||||
// suite's timeouts. One minute keeps the tail real and observable
|
||||
// (src/expired-invoice-late-settle.it.ts asserts both sides of it, and
|
||||
// hardcodes DRAIN_MS = 60_000 to match).
|
||||
FREIGHT_PAYMENT_DRAIN_MINUTES: "1",
|
||||
// The suite's pacing floor: every window phase, allocation and expiry it waits
|
||||
// on lands on this tick. Production stays at */10. Overridable from the
|
||||
// environment so a cadence-sensitive spec (or a bisect) can pin it back.
|
||||
BOOKING_WINDOW_TICK_CRON: process.env.BOOKING_WINDOW_TICK_CRON ?? "*/1 * * * * *",
|
||||
};
|
||||
|
||||
const env = {
|
||||
...process.env,
|
||||
...Object.fromEntries(Object.entries(PORTS).map(([k, v]) => [k, String(v)])),
|
||||
IT_API_URL: `http://localhost:${PORTS.E2E_API_PORT}`,
|
||||
IT_PAYMENT_URL: `http://localhost:${PORTS.IT_PAYMENT_PORT}`,
|
||||
IT_GATEWAY_URL: `http://localhost:${PORTS.IT_GATEWAY_PORT}`,
|
||||
IT_DB_URL: `postgres://edr_e2e:edr_e2e@localhost:${PORTS.E2E_DB_PORT}/edr_freight_e2e`,
|
||||
...freightEnv,
|
||||
IT_SHARDS: String(SHARDS),
|
||||
IT_TEMPLATE_DB: TEMPLATE_DB,
|
||||
IT_DB_PREFIX: DB_PREFIX,
|
||||
IT_API_PORT_BASE: String(PORTS.E2E_API_PORT),
|
||||
IT_PAYMENT_PORT_BASE: String(PORTS.IT_PAYMENT_PORT),
|
||||
IT_GATEWAY_PORT_BASE: String(PORTS.IT_GATEWAY_PORT),
|
||||
FAYDA_PRIVATE_KEY_BASE64:
|
||||
process.env.FAYDA_PRIVATE_KEY_BASE64 ?? fakeFaydaPrivateKeyBase64(),
|
||||
IT_TELEBIRR_PRIVATE_KEY:
|
||||
process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey(),
|
||||
IT_TELEBIRR_PRIVATE_KEY: telebirrKey,
|
||||
};
|
||||
|
||||
function generateShardFile() {
|
||||
writeShards({
|
||||
shards: SHARDS,
|
||||
apiPortBase: PORTS.E2E_API_PORT,
|
||||
paymentPortBase: PORTS.IT_PAYMENT_PORT,
|
||||
gatewayPortBase: PORTS.IT_GATEWAY_PORT,
|
||||
dbPrefix: DB_PREFIX,
|
||||
telebirrKey,
|
||||
});
|
||||
}
|
||||
|
||||
/** The shard file must exist before compose is invoked — it is one of the -f's. */
|
||||
function composeBase() {
|
||||
if (!existsSync(SHARD_FILE)) generateShardFile();
|
||||
return [
|
||||
"compose",
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.e2e.yaml"),
|
||||
"-f",
|
||||
join(repoRoot, "docker-compose.it.yaml"),
|
||||
"-f",
|
||||
SHARD_FILE,
|
||||
];
|
||||
}
|
||||
|
||||
function compose(args) {
|
||||
const { status } = spawnSync("docker", [...composeBase, ...args], { stdio: "inherit", env });
|
||||
const { status } = spawnSync("docker", [...composeBase(), ...args], {
|
||||
stdio: "inherit",
|
||||
env,
|
||||
});
|
||||
return status ?? 1;
|
||||
}
|
||||
|
||||
function composeQuiet(args) {
|
||||
return spawnSync("docker", [...composeBase(), ...args], {
|
||||
encoding: "utf8",
|
||||
env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
|
||||
function stackRunning() {
|
||||
try {
|
||||
const out = execFileSync("docker", [...composeBase, "ps", "--services", "--status", "running"], {
|
||||
encoding: "utf8",
|
||||
env,
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
});
|
||||
const out = execFileSync(
|
||||
"docker",
|
||||
[...composeBase(), "ps", "--services", "--status", "running"],
|
||||
{ encoding: "utf8", env, stdio: ["ignore", "pipe", "ignore"] },
|
||||
);
|
||||
const running = new Set(out.split("\n").filter(Boolean));
|
||||
return RUNNING.every((s) => running.has(s));
|
||||
const want = [...INFRA, ...shardServices(SHARDS)];
|
||||
return want.every((s) => running.has(s));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function node(script, args = []) {
|
||||
const { status } = spawnSync("node", [join(itDir, "scripts", script), ...args], {
|
||||
cwd: repoRoot,
|
||||
stdio: "inherit",
|
||||
env,
|
||||
});
|
||||
return status ?? 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* One broker, one vhost per shard — a shared vhost would let one shard's freight
|
||||
* consumer eat another shard's settlement event.
|
||||
*
|
||||
* Retried: the healthcheck now waits for the rabbit *application*, but a cold
|
||||
* boot can still land between "healthy" and "accepting rabbitmqctl", and a
|
||||
* half-provisioned broker fails later as an unexplained missing settlement.
|
||||
*/
|
||||
function createVhosts() {
|
||||
for (let i = 0; i < SHARDS; i++) {
|
||||
const vhost = `payment_s${i}`;
|
||||
let last = "";
|
||||
let ok = false;
|
||||
for (let attempt = 1; attempt <= 10 && !ok; attempt++) {
|
||||
// Idempotent: add_vhost on an existing vhost exits non-zero, which is fine.
|
||||
composeQuiet(["exec", "-T", "rabbitmq-it", "rabbitmqctl", "add_vhost", vhost]);
|
||||
const perm = composeQuiet([
|
||||
"exec", "-T", "rabbitmq-it",
|
||||
"rabbitmqctl", "set_permissions", "-p", vhost, "edr", ".*", ".*", ".*",
|
||||
]);
|
||||
ok = perm.status === 0;
|
||||
last = perm.stderr ?? "";
|
||||
if (!ok) execFileSync("sleep", ["3"]);
|
||||
}
|
||||
if (!ok) fail(`could not grant on rabbit vhost ${vhost} after 10 tries:\n${last}`);
|
||||
}
|
||||
console.log(`it: rabbit vhosts payment_s0…payment_s${SHARDS - 1} ready`);
|
||||
}
|
||||
|
||||
function buildFreight() {
|
||||
console.log("it: building @edr/freight-api (the suite runs dist/, never stale)");
|
||||
const { status } = spawnSync("pnpm", ["--filter", "@edr/freight-api", "run", "build"], {
|
||||
cwd: repoRoot,
|
||||
stdio: "inherit",
|
||||
env,
|
||||
});
|
||||
if (status !== 0) fail("freight API build failed — fix it before running the suite.");
|
||||
}
|
||||
|
||||
function up() {
|
||||
preflight();
|
||||
generateShardFile();
|
||||
console.log(
|
||||
`it: starting stack — freight :${PORTS.E2E_API_PORT} payment :${PORTS.IT_PAYMENT_PORT} ` +
|
||||
`gateway :${PORTS.IT_GATEWAY_PORT} db :${PORTS.E2E_DB_PORT}`,
|
||||
`it: starting ${SHARDS} shard(s) — freight :${PORTS.E2E_API_PORT}..${
|
||||
PORTS.E2E_API_PORT + SHARDS - 1
|
||||
} (in-process) payment :${PORTS.IT_PAYMENT_PORT}.. gateway :${
|
||||
PORTS.IT_GATEWAY_PORT
|
||||
}.. db :${PORTS.E2E_DB_PORT}`,
|
||||
);
|
||||
if (compose(["up", "-d", "--build", "--wait", ...SERVICES]) !== 0) {
|
||||
|
||||
// 1. Shared infrastructure.
|
||||
if (compose(["up", "-d", "--build", "--wait", "--remove-orphans", ...INFRA]) !== 0) {
|
||||
fail(
|
||||
"stack failed to become healthy. Inspect with:\n" +
|
||||
" node integration/scripts/it.mjs logs payment-api-it",
|
||||
"shared services failed to become healthy. Inspect with:\n" +
|
||||
" node integration/scripts/it.mjs logs postgres-freight-e2e",
|
||||
);
|
||||
}
|
||||
createVhosts();
|
||||
|
||||
// 1b. One-shots: the MinIO bucket, then freight's migrations into the
|
||||
// TEMPLATE database every shard is cloned from.
|
||||
for (const svc of ONE_SHOT) {
|
||||
if (compose(["run", "--rm", "--no-deps", "--build", svc]) !== 0) {
|
||||
fail(`${svc} failed — inspect with:\n node integration/scripts/it.mjs logs ${svc}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. The app must exist as dist/ before we can boot it to seed the template.
|
||||
buildFreight();
|
||||
|
||||
// 3. Seed the template (boot seeders + SQL fixtures) and clone it per shard.
|
||||
// Must precede the payment APIs: they connect to the shard databases.
|
||||
if (node("prepare-shards.mjs") !== 0) fail("template seed / shard clone failed.");
|
||||
|
||||
// 4. Per-shard payment API + gateway mock.
|
||||
if (compose(["up", "-d", "--build", "--wait", ...shardServices(SHARDS)]) !== 0) {
|
||||
fail(
|
||||
"shard services failed to become healthy. Inspect with:\n" +
|
||||
" node integration/scripts/it.mjs logs payment-api-it-0",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const [cmd, ...rawExtra] = process.argv.slice(2);
|
||||
/**
|
||||
* Re-clone every shard database from the template so each run is hermetic. The
|
||||
* payment APIs hold connections to those databases, so they come down first —
|
||||
* their `edr_payment` schema is recreated by their own boot migrations.
|
||||
*
|
||||
* This is what retires the warm-stack failure modes the suite used to document:
|
||||
* stowaway paid bookings climbing back aboard, a short consist on the fifth
|
||||
* file, invoice numbers continuing from a previous run.
|
||||
*/
|
||||
function resetShardDbs() {
|
||||
const svc = shardServices(SHARDS);
|
||||
console.log("it: re-cloning shard databases from the template");
|
||||
compose(["stop", ...svc]);
|
||||
if (node("prepare-shards.mjs", ["--clone"]) !== 0) fail("shard database reset failed.");
|
||||
if (compose(["up", "-d", "--wait", ...svc]) !== 0) {
|
||||
fail("shard services failed to become healthy after the reset.");
|
||||
}
|
||||
}
|
||||
|
||||
const [cmd, ...rawArgs] = process.argv.slice(2);
|
||||
const noReset = rawArgs.includes("--no-reset");
|
||||
// `pnpm it:test -- src/foo.it.ts` hands us a literal "--" first. Forwarding it
|
||||
// makes vitest treat everything after it as CLI options and ignore the file
|
||||
// filter — the "one file" run silently becomes the whole suite.
|
||||
const extra = rawExtra[0] === "--" ? rawExtra.slice(1) : rawExtra;
|
||||
const passthrough = rawArgs.filter((a) => a !== "--no-reset");
|
||||
const extra = passthrough[0] === "--" ? passthrough.slice(1) : passthrough;
|
||||
|
||||
switch (cmd) {
|
||||
case "up":
|
||||
up();
|
||||
break;
|
||||
case "test": {
|
||||
if (!stackRunning()) up();
|
||||
if (!stackRunning()) {
|
||||
up();
|
||||
} else {
|
||||
buildFreight();
|
||||
if (!noReset) resetShardDbs();
|
||||
}
|
||||
const { status } = spawnSync(
|
||||
"pnpm",
|
||||
["--filter", "@edr/freight-integration", "run", "test", ...extra],
|
||||
@@ -162,12 +382,14 @@ switch (cmd) {
|
||||
);
|
||||
process.exit(status ?? 1);
|
||||
}
|
||||
case "logs":
|
||||
case "logs": {
|
||||
process.exit(compose(["logs", "--tail", "200", ...extra]));
|
||||
break;
|
||||
case "down":
|
||||
process.exit(compose(["down", "-v", "--remove-orphans"]));
|
||||
break;
|
||||
}
|
||||
case "down": {
|
||||
const status = compose(["down", "-v", "--remove-orphans"]);
|
||||
rmSync(SHARD_FILE, { force: true });
|
||||
process.exit(status);
|
||||
}
|
||||
default:
|
||||
fail(`unknown command "${cmd ?? ""}" — use up | test | logs | down`);
|
||||
}
|
||||
|
||||
134
integration/scripts/prepare-shards.mjs
Normal file
134
integration/scripts/prepare-shards.mjs
Normal file
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the seeded template database, then clone it once per shard.
|
||||
*
|
||||
* node integration/scripts/prepare-shards.mjs # seed template + clone
|
||||
* node integration/scripts/prepare-shards.mjs --clone # clone only (per-run reset)
|
||||
*
|
||||
* Seeding has to happen here, not in vitest's globalSetup, because the order is
|
||||
* load-bearing and half of it is the app itself:
|
||||
*
|
||||
* 1. freight migrations (freight-migration-e2e container, before us)
|
||||
* 2. the app's always-on boot seeders — org, units, positions, permissions
|
||||
* (app.module.ts onApplicationBootstrap)
|
||||
* 3. the SQL fixtures, which declare those as prerequisites
|
||||
* (seed-users.sql: "created by the API's always-on boot seeders")
|
||||
*
|
||||
* Step 2 needs a real Nest boot, and the app now lives inside the vitest workers
|
||||
* — which globalSetup runs in a different process from. So we boot it once here
|
||||
* against the template and every shard is a `CREATE DATABASE … TEMPLATE` copy of
|
||||
* the result: a file copy on the tmpfs postgres, versus re-running migrations
|
||||
* and five seeders per shard.
|
||||
*
|
||||
* Run by it.mjs, which owns the env. No dependencies beyond `pg`.
|
||||
*/
|
||||
import { createRequire } from "node:module";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "pg";
|
||||
|
||||
const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const repoRoot = resolve(itDir, "..");
|
||||
const freightDir = join(repoRoot, "apps", "edr-freight-api");
|
||||
|
||||
const TEMPLATE = process.env.IT_TEMPLATE_DB ?? "edr_freight_e2e";
|
||||
const PREFIX = process.env.IT_DB_PREFIX ?? "edr_it_s";
|
||||
const SHARDS = Number(process.env.IT_SHARDS ?? 1);
|
||||
|
||||
const admin = () =>
|
||||
new Client({
|
||||
host: process.env.DB_HOST ?? "localhost",
|
||||
port: Number(process.env.DB_PORT ?? 5543),
|
||||
user: process.env.DB_USER ?? "edr_e2e",
|
||||
password: process.env.DB_PASSWORD ?? "edr_e2e",
|
||||
// Never the template itself: CREATE DATABASE … TEMPLATE refuses while any
|
||||
// session is connected to the source.
|
||||
database: "postgres",
|
||||
});
|
||||
|
||||
const CYPRESS_FIXTURES = join(repoRoot, "e2e", "freight", "cypress", "fixtures");
|
||||
const OWN_FIXTURES = join(itDir, "sql");
|
||||
|
||||
/** Order matters — company needs the users, everything needs the corridor. */
|
||||
const SEEDS = [
|
||||
[CYPRESS_FIXTURES, "seed-users.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-company.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-import-corridor.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-bulk-items.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-g1-train.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-g2-weight.sql"],
|
||||
[CYPRESS_FIXTURES, "seed-government.sql"],
|
||||
[OWN_FIXTURES, "seed-company-b.sql"],
|
||||
[OWN_FIXTURES, "seed-customs-service-type.sql"],
|
||||
];
|
||||
|
||||
async function seedTemplate() {
|
||||
console.log(`it: seeding template ${TEMPLATE}`);
|
||||
|
||||
process.env.DB_NAME = TEMPLATE;
|
||||
// No broker for the template boot: payment.module.ts skips RabbitMQModule
|
||||
// entirely when this is unset, and nothing here publishes.
|
||||
delete process.env.PAYMENT_RABBITMQ_URL;
|
||||
|
||||
const { createFreightApp } = createRequire(join(freightDir, "package.json"))("./dist/main.js");
|
||||
|
||||
// createFreightApp() only builds the graph. `init()` is what fires
|
||||
// onApplicationBootstrap — the seeders we are here for. We never listen.
|
||||
const app = await createFreightApp();
|
||||
try {
|
||||
await app.init();
|
||||
console.log("it: boot seeders done (org, units, positions, permissions)");
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
|
||||
const client = new Client({
|
||||
host: process.env.DB_HOST ?? "localhost",
|
||||
port: Number(process.env.DB_PORT ?? 5543),
|
||||
user: process.env.DB_USER ?? "edr_e2e",
|
||||
password: process.env.DB_PASSWORD ?? "edr_e2e",
|
||||
database: TEMPLATE,
|
||||
});
|
||||
await client.connect();
|
||||
try {
|
||||
for (const [dir, file] of SEEDS) {
|
||||
await client.query(readFileSync(join(dir, file), "utf8"));
|
||||
console.log(`it: seeded ${file}`);
|
||||
}
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function cloneShards() {
|
||||
const pg = admin();
|
||||
await pg.connect();
|
||||
try {
|
||||
// The app's pool and any stray psql hold the template open; without this the
|
||||
// CREATE below fails with "source database is being accessed by other users".
|
||||
await pg.query(
|
||||
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
|
||||
WHERE datname = $1 AND pid <> pg_backend_pid()`,
|
||||
[TEMPLATE],
|
||||
);
|
||||
|
||||
for (let i = 0; i < SHARDS; i++) {
|
||||
const db = `${PREFIX}${i}`;
|
||||
await pg.query(
|
||||
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
|
||||
WHERE datname = $1 AND pid <> pg_backend_pid()`,
|
||||
[db],
|
||||
);
|
||||
await pg.query(`DROP DATABASE IF EXISTS ${db}`);
|
||||
await pg.query(`CREATE DATABASE ${db} TEMPLATE ${TEMPLATE}`);
|
||||
console.log(`it: shard ${i} → ${db}`);
|
||||
}
|
||||
} finally {
|
||||
await pg.end();
|
||||
}
|
||||
}
|
||||
|
||||
const cloneOnly = process.argv.includes("--clone");
|
||||
if (!cloneOnly) await seedTemplate();
|
||||
await cloneShards();
|
||||
Reference in New Issue
Block a user