#!/usr/bin/env node /** * Freight integration-suite launcher. * * node integration/scripts/it.mjs [vitest args...] * IT_SHARDS=4 node integration/scripts/it.mjs test * node integration/scripts/it.mjs test --no-reset -- src/payment-happy.it.ts * * The freight API is NOT containerized here: it boots inside each vitest worker * from `apps/edr-freight-api/dist`, which `test` rebuilds every run. Each worker * is a whole shard — own database, own payment API, own gateway mock, own broker * vhost — so spec files run in parallel without sharing anything mutable. * * Ports are fixed (and distinct from the Cypress e2e defaults) so both stacks * can be up at once; they are separate compose projects. * * No dependencies — plain Node spawning `docker compose` and `pnpm`. */ import { execFileSync, spawnSync } from "node:child_process"; import { generateKeyPairSync } from "node:crypto"; import { existsSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { SHARD_FILE, shardServices, writeShards } from "./gen-shards.mjs"; const itDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const repoRoot = resolve(itDir, ".."); /** * Deliberately offset from the Cypress stack's defaults (3101/5533/9310…). * * Each per-shard service takes a CONTIGUOUS BLOCK of `IT_SHARDS` ports starting * at its base (freight :3111+i, payment :3131+i, gateway :4600+i), so the bases * must be at least MAX_SHARDS apart. They were originally 3111 and 3113 — two * apart — which silently worked at 1–2 shards and then had shard 2's freight app * try to bind :3113, the port payment-api-it-0 was already published on. */ const PORTS = { E2E_API_PORT: 3111, E2E_DB_PORT: 5543, E2E_MINIO_PORT: 9320, E2E_MINIO_CONSOLE_PORT: 9321, // Unused here (no web containers) but referenced by the base file's build args. E2E_PORTAL_PORT: 5393, E2E_BACKOFFICE_PORT: 5394, IT_PAYMENT_PORT: 3131, IT_GATEWAY_PORT: 4600, IT_RABBIT_PORT: 5772, IT_RABBIT_UI_PORT: 15772, }; /** Beyond this the freight block (3111+) would run into the payment block (3131+). */ const MAX_SHARDS = 16; /** * How many shards. RAM is the ceiling, not cores: each shard is an in-process * Nest app plus two containers. */ const SHARDS = Math.max(1, Number(process.env.IT_SHARDS ?? 4)); if (SHARDS > MAX_SHARDS) { console.error( `\nit: IT_SHARDS=${SHARDS} exceeds ${MAX_SHARDS} — the per-service port blocks would overlap.` + `\n Raise IT_PAYMENT_PORT/IT_GATEWAY_PORT in it.mjs first.`, ); process.exit(1); } const TEMPLATE_DB = "edr_freight_e2e"; const DB_PREFIX = "edr_it_s"; /** Long-running services shared by every shard. Web + Cypress are absent. */ const INFRA = ["postgres-freight-e2e", "minio-e2e", "rabbitmq-it"]; /** * One-shots: they run, then exit 0. They cannot go in the `up --wait` set — * compose reports an exited container as a failed wait, so a healthy stack * looks broken. Run them with `compose run`, which returns their exit code. */ const ONE_SHOT = ["minio-init-e2e", "freight-migration-e2e"]; function fail(msg) { console.error(`\nit: ${msg}`); process.exit(1); } function preflight() { try { execFileSync("docker", ["info"], { stdio: "ignore" }); } catch { fail("docker is not running (or not installed) — start Docker and retry."); } if (!existsSync(join(repoRoot, ".npmrc"))) { fail(".npmrc missing at repo root — image builds need GitHub Packages auth for @tria-plc."); } } /** Throwaway RSA PEM — Telebirr PSS-signs every request object; the mock never * verifies it, but the provider refuses to build a request without a real key. */ function fakeTelebirrPrivateKey() { const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); return privateKey.export({ type: "pkcs8", format: "pem" }).toString(); } /** Throwaway RSA JWK for FAYDA_PRIVATE_KEY_BASE64 (see e2e.mjs — same reason). */ function fakeFaydaPrivateKeyBase64() { const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); const jwk = privateKey.export({ format: "jwk" }); Object.assign(jwk, { kty: "RSA", use: "sig", alg: "RS256", kid: "it-fayda-mock" }); return Buffer.from(JSON.stringify(jwk)).toString("base64"); } const telebirrKey = process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey(); /** * The freight app's own environment. This is docker-compose.e2e.yaml's * `freight-api-e2e` block with container hostnames swapped for published host * ports — the app runs on the host now. Per-shard values (DB_NAME, PORT, * PAYMENT_API_URL, PAYMENT_RABBITMQ_URL) are derived per worker in src/app.ts. */ const freightEnv = { NODE_ENV: "test", DB_HOST: "localhost", DB_PORT: String(PORTS.E2E_DB_PORT), DB_USER: "edr_e2e", DB_PASSWORD: "edr_e2e", DB_NAME: TEMPLATE_DB, // e2e-only secrets — never reuse outside this stack JWT_SECRET: "e2e-jwt-secret", JWT_ACCESS_TOKEN_SECRET: "e2e-access-secret", JWT_REFRESH_TOKEN_SECRET: "e2e-refresh-secret", JWT_EXPIRES_IN: "1d", JWT_ACCESS_TOKEN_EXPIRES: "1d", JWT_REFRESH_TOKEN_EXPIRES: "7d", // Mandatory: ServiceAuthGuard returns TRUE when this is unset, and // authz.it.ts asserts that the internal surface rejects an unsigned caller. SERVICE_AUTH_TOKEN: "e2e-service-token", SEED_EDR_ORG: "true", SUPER_ADMIN_EMAIL: "superadmin@tria.com", SUPER_ADMIN_PHONE: "+251900000000", MINIO_ENDPOINT: "localhost", MINIO_PORT: String(PORTS.E2E_MINIO_PORT), MINIO_USE_SSL: "false", MINIO_ACCESS_KEY: "e2e-minio", MINIO_SECRET_KEY: "e2e-minio-secret", MINIO_REGION: "us-east-1", // Only gates the SMS/email clients. The payment consumer is wired by // PAYMENT_RABBITMQ_URL alone (payment.module.ts). RABBITMQ_ENABLED: "false", // fayda.config.ts THROWS at load if this is "true" without the full var set, // and the fayda/etrade mocks publish no host ports. No IT spec touches them. FAYDA_ENABLED: "false", // SMS strategy has no kill switch and defaults to a real dev endpoint. OZIKING_SMS_URL: "http://127.0.0.1:9/sms", // app.config.ts otherwise live-scrapes https://ethio.forex on first booking. CBE_EXCHANGE_SCRAPE_URL: "http://127.0.0.1:9/fx", CBE_EXCHANGE_API_URL: "http://127.0.0.1:9/fx", CBE_EXCHANGE_FALLBACK_RATE: "130", FREIGHT_PORTAL_URL: `http://localhost:${PORTS.E2E_PORTAL_PORT}`, // Drain tail on every pay window. Production defaults to 5 minutes; a // reservation here lives ~60s, so 5 would push every natural expiry past the // suite's timeouts. One minute keeps the tail real and observable // (src/expired-invoice-late-settle.it.ts asserts both sides of it, and // hardcodes DRAIN_MS = 60_000 to match). FREIGHT_PAYMENT_DRAIN_MINUTES: "1", // The suite's pacing floor: every window phase, allocation and expiry it waits // on lands on this tick. Production stays at */10. Overridable from the // environment so a cadence-sensitive spec (or a bisect) can pin it back. BOOKING_WINDOW_TICK_CRON: process.env.BOOKING_WINDOW_TICK_CRON ?? "*/1 * * * * *", }; const env = { ...process.env, ...Object.fromEntries(Object.entries(PORTS).map(([k, v]) => [k, String(v)])), ...freightEnv, IT_SHARDS: String(SHARDS), IT_TEMPLATE_DB: TEMPLATE_DB, IT_DB_PREFIX: DB_PREFIX, IT_API_PORT_BASE: String(PORTS.E2E_API_PORT), IT_PAYMENT_PORT_BASE: String(PORTS.IT_PAYMENT_PORT), IT_GATEWAY_PORT_BASE: String(PORTS.IT_GATEWAY_PORT), FAYDA_PRIVATE_KEY_BASE64: process.env.FAYDA_PRIVATE_KEY_BASE64 ?? fakeFaydaPrivateKeyBase64(), IT_TELEBIRR_PRIVATE_KEY: telebirrKey, }; function generateShardFile() { writeShards({ shards: SHARDS, apiPortBase: PORTS.E2E_API_PORT, paymentPortBase: PORTS.IT_PAYMENT_PORT, gatewayPortBase: PORTS.IT_GATEWAY_PORT, dbPrefix: DB_PREFIX, telebirrKey, }); } /** The shard file must exist before compose is invoked — it is one of the -f's. */ function composeBase() { if (!existsSync(SHARD_FILE)) generateShardFile(); return [ "compose", "-f", join(repoRoot, "docker-compose.e2e.yaml"), "-f", join(repoRoot, "docker-compose.it.yaml"), "-f", SHARD_FILE, ]; } function compose(args) { const { status } = spawnSync("docker", [...composeBase(), ...args], { stdio: "inherit", env, }); return status ?? 1; } function composeQuiet(args) { return spawnSync("docker", [...composeBase(), ...args], { encoding: "utf8", env, stdio: ["ignore", "pipe", "pipe"], }); } function stackRunning() { try { const out = execFileSync( "docker", [...composeBase(), "ps", "--services", "--status", "running"], { encoding: "utf8", env, stdio: ["ignore", "pipe", "ignore"] }, ); const running = new Set(out.split("\n").filter(Boolean)); const want = [...INFRA, ...shardServices(SHARDS)]; return want.every((s) => running.has(s)); } catch { return false; } } function node(script, args = []) { const { status } = spawnSync("node", [join(itDir, "scripts", script), ...args], { cwd: repoRoot, stdio: "inherit", env, }); return status ?? 1; } /** * One broker, one vhost per shard — a shared vhost would let one shard's freight * consumer eat another shard's settlement event. * * Retried: the healthcheck now waits for the rabbit *application*, but a cold * boot can still land between "healthy" and "accepting rabbitmqctl", and a * half-provisioned broker fails later as an unexplained missing settlement. */ function createVhosts() { for (let i = 0; i < SHARDS; i++) { const vhost = `payment_s${i}`; let last = ""; let ok = false; for (let attempt = 1; attempt <= 10 && !ok; attempt++) { // Idempotent: add_vhost on an existing vhost exits non-zero, which is fine. composeQuiet(["exec", "-T", "rabbitmq-it", "rabbitmqctl", "add_vhost", vhost]); const perm = composeQuiet([ "exec", "-T", "rabbitmq-it", "rabbitmqctl", "set_permissions", "-p", vhost, "edr", ".*", ".*", ".*", ]); ok = perm.status === 0; last = perm.stderr ?? ""; if (!ok) execFileSync("sleep", ["3"]); } if (!ok) fail(`could not grant on rabbit vhost ${vhost} after 10 tries:\n${last}`); } console.log(`it: rabbit vhosts payment_s0…payment_s${SHARDS - 1} ready`); } function buildFreight() { console.log("it: building @edr/freight-api (the suite runs dist/, never stale)"); const { status } = spawnSync("pnpm", ["--filter", "@edr/freight-api", "run", "build"], { cwd: repoRoot, stdio: "inherit", env, }); if (status !== 0) fail("freight API build failed — fix it before running the suite."); } function up() { preflight(); generateShardFile(); console.log( `it: starting ${SHARDS} shard(s) — freight :${PORTS.E2E_API_PORT}..${ PORTS.E2E_API_PORT + SHARDS - 1 } (in-process) payment :${PORTS.IT_PAYMENT_PORT}.. gateway :${ PORTS.IT_GATEWAY_PORT }.. db :${PORTS.E2E_DB_PORT}`, ); // 1. Shared infrastructure. if (compose(["up", "-d", "--build", "--wait", "--remove-orphans", ...INFRA]) !== 0) { fail( "shared services failed to become healthy. Inspect with:\n" + " node integration/scripts/it.mjs logs postgres-freight-e2e", ); } createVhosts(); // 1b. One-shots: the MinIO bucket, then freight's migrations into the // TEMPLATE database every shard is cloned from. for (const svc of ONE_SHOT) { if (compose(["run", "--rm", "--no-deps", "--build", svc]) !== 0) { fail(`${svc} failed — inspect with:\n node integration/scripts/it.mjs logs ${svc}`); } } // 2. The app must exist as dist/ before we can boot it to seed the template. buildFreight(); // 3. Seed the template (boot seeders + SQL fixtures) and clone it per shard. // Must precede the payment APIs: they connect to the shard databases. if (node("prepare-shards.mjs") !== 0) fail("template seed / shard clone failed."); // 4. Per-shard payment API + gateway mock. if (compose(["up", "-d", "--build", "--wait", ...shardServices(SHARDS)]) !== 0) { fail( "shard services failed to become healthy. Inspect with:\n" + " node integration/scripts/it.mjs logs payment-api-it-0", ); } } /** * Re-clone every shard database from the template so each run is hermetic. The * payment APIs hold connections to those databases, so they come down first — * their `edr_payment` schema is recreated by their own boot migrations. * * This is what retires the warm-stack failure modes the suite used to document: * stowaway paid bookings climbing back aboard, a short consist on the fifth * file, invoice numbers continuing from a previous run. */ function resetShardDbs() { const svc = shardServices(SHARDS); console.log("it: re-cloning shard databases from the template"); compose(["stop", ...svc]); if (node("prepare-shards.mjs", ["--clone"]) !== 0) fail("shard database reset failed."); if (compose(["up", "-d", "--wait", ...svc]) !== 0) { fail("shard services failed to become healthy after the reset."); } } const [cmd, ...rawArgs] = process.argv.slice(2); const noReset = rawArgs.includes("--no-reset"); // `pnpm it:test -- src/foo.it.ts` hands us a literal "--" first. Forwarding it // makes vitest treat everything after it as CLI options and ignore the file // filter — the "one file" run silently becomes the whole suite. const passthrough = rawArgs.filter((a) => a !== "--no-reset"); const extra = passthrough[0] === "--" ? passthrough.slice(1) : passthrough; switch (cmd) { case "up": up(); break; case "test": { if (!stackRunning()) { up(); } else { buildFreight(); if (!noReset) resetShardDbs(); } const { status } = spawnSync( "pnpm", ["--filter", "@edr/freight-integration", "run", "test", ...extra], { cwd: repoRoot, stdio: "inherit", env }, ); process.exit(status ?? 1); } case "logs": { process.exit(compose(["logs", "--tail", "200", ...extra])); } case "down": { const status = compose(["down", "-v", "--remove-orphans"]); rmSync(SHARD_FILE, { force: true }); process.exit(status); } default: fail(`unknown command "${cmd ?? ""}" — use up | test | logs | down`); }