edr-platform/integration/scripts/it.mjs

#!/usr/bin/env node
/**
 * Freight integration-suite launcher.
 *
 *   node integration/scripts/it.mjs <up|test|down|logs> [vitest args...]
 *   IT_SHARDS=4 node integration/scripts/it.mjs test
 *   node integration/scripts/it.mjs test --no-reset -- src/payment-happy.it.ts
 *
 * The freight API is NOT containerized here: it boots inside each vitest worker
 * from `apps/edr-freight-api/dist`, which `test` rebuilds every run. Each worker
 * is a whole shard — own database, own payment API, own gateway mock, own broker
 * vhost — so spec files run in parallel without sharing anything mutable.
 *
 * Ports are fixed (and distinct from the Cypress e2e defaults) so both stacks
 * can be up at once; they are separate compose projects.
 *
 * No dependencies — plain Node spawning `docker compose` and `pnpm`.
 */

import { execFileSync, spawnSync } from "node:child_process";
import { generateKeyPairSync } from "node:crypto";
import { existsSync, rmSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";

import { SHARD_FILE, shardServices, writeShards } from "./gen-shards.mjs";

const itDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const repoRoot = resolve(itDir, "..");

/**
 * Deliberately offset from the Cypress stack's defaults (3101/5533/9310…).
 *
 * Each per-shard service takes a CONTIGUOUS BLOCK of `IT_SHARDS` ports starting
 * at its base (freight :3111+i, payment :3131+i, gateway :4600+i), so the bases
 * must be at least MAX_SHARDS apart. They were originally 3111 and 3113 — two
 * apart — which silently worked at 1–2 shards and then had shard 2's freight app
 * try to bind :3113, the port payment-api-it-0 was already published on.
 */
const PORTS = {
  E2E_API_PORT: 3111,
  E2E_DB_PORT: 5543,
  E2E_MINIO_PORT: 9320,
  E2E_MINIO_CONSOLE_PORT: 9321,
  // Unused here (no web containers) but referenced by the base file's build args.
  E2E_PORTAL_PORT: 5393,
  E2E_BACKOFFICE_PORT: 5394,
  IT_PAYMENT_PORT: 3131,
  IT_GATEWAY_PORT: 4600,
  IT_RABBIT_PORT: 5772,
  IT_RABBIT_UI_PORT: 15772,
};

/** Beyond this the freight block (3111+) would run into the payment block (3131+). */
const MAX_SHARDS = 16;

/**
 * How many shards. RAM is the ceiling, not cores: each shard is an in-process
 * Nest app plus two containers.
 */
const SHARDS = Math.max(1, Number(process.env.IT_SHARDS ?? 4));
if (SHARDS > MAX_SHARDS) {
  console.error(
    `\nit: IT_SHARDS=${SHARDS} exceeds ${MAX_SHARDS} — the per-service port blocks would overlap.` +
      `\n    Raise IT_PAYMENT_PORT/IT_GATEWAY_PORT in it.mjs first.`,
  );
  process.exit(1);
}
const TEMPLATE_DB = "edr_freight_e2e";
const DB_PREFIX = "edr_it_s";

/** Long-running services shared by every shard. Web + Cypress are absent. */
const INFRA = ["postgres-freight-e2e", "minio-e2e", "rabbitmq-it"];

/**
 * One-shots: they run, then exit 0. They cannot go in the `up --wait` set —
 * compose reports an exited container as a failed wait, so a healthy stack
 * looks broken. Run them with `compose run`, which returns their exit code.
 */
const ONE_SHOT = ["minio-init-e2e", "freight-migration-e2e"];

function fail(msg) {
  console.error(`\nit: ${msg}`);
  process.exit(1);
}

function preflight() {
  try {
    execFileSync("docker", ["info"], { stdio: "ignore" });
  } catch {
    fail("docker is not running (or not installed) — start Docker and retry.");
  }
  if (!existsSync(join(repoRoot, ".npmrc"))) {
    fail(".npmrc missing at repo root — image builds need GitHub Packages auth for @tria-plc.");
  }
}

/** Throwaway RSA PEM — Telebirr PSS-signs every request object; the mock never
 *  verifies it, but the provider refuses to build a request without a real key. */
function fakeTelebirrPrivateKey() {
  const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
  return privateKey.export({ type: "pkcs8", format: "pem" }).toString();
}

/** Throwaway RSA JWK for FAYDA_PRIVATE_KEY_BASE64 (see e2e.mjs — same reason). */
function fakeFaydaPrivateKeyBase64() {
  const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
  const jwk = privateKey.export({ format: "jwk" });
  Object.assign(jwk, { kty: "RSA", use: "sig", alg: "RS256", kid: "it-fayda-mock" });
  return Buffer.from(JSON.stringify(jwk)).toString("base64");
}

const telebirrKey = process.env.IT_TELEBIRR_PRIVATE_KEY ?? fakeTelebirrPrivateKey();

/**
 * The freight app's own environment. This is docker-compose.e2e.yaml's
 * `freight-api-e2e` block with container hostnames swapped for published host
 * ports — the app runs on the host now. Per-shard values (DB_NAME, PORT,
 * PAYMENT_API_URL, PAYMENT_RABBITMQ_URL) are derived per worker in src/app.ts.
 */
const freightEnv = {
  NODE_ENV: "test",
  DB_HOST: "localhost",
  DB_PORT: String(PORTS.E2E_DB_PORT),
  DB_USER: "edr_e2e",
  DB_PASSWORD: "edr_e2e",
  DB_NAME: TEMPLATE_DB,

  // e2e-only secrets — never reuse outside this stack
  JWT_SECRET: "e2e-jwt-secret",
  JWT_ACCESS_TOKEN_SECRET: "e2e-access-secret",
  JWT_REFRESH_TOKEN_SECRET: "e2e-refresh-secret",
  JWT_EXPIRES_IN: "1d",
  JWT_ACCESS_TOKEN_EXPIRES: "1d",
  JWT_REFRESH_TOKEN_EXPIRES: "7d",
  // Mandatory: ServiceAuthGuard returns TRUE when this is unset, and
  // authz.it.ts asserts that the internal surface rejects an unsigned caller.
  SERVICE_AUTH_TOKEN: "e2e-service-token",

  SEED_EDR_ORG: "true",
  SUPER_ADMIN_EMAIL: "superadmin@tria.com",
  SUPER_ADMIN_PHONE: "+251900000000",

  MINIO_ENDPOINT: "localhost",
  MINIO_PORT: String(PORTS.E2E_MINIO_PORT),
  MINIO_USE_SSL: "false",
  MINIO_ACCESS_KEY: "e2e-minio",
  MINIO_SECRET_KEY: "e2e-minio-secret",
  MINIO_REGION: "us-east-1",

  // Only gates the SMS/email clients. The payment consumer is wired by
  // PAYMENT_RABBITMQ_URL alone (payment.module.ts).
  RABBITMQ_ENABLED: "false",
  // fayda.config.ts THROWS at load if this is "true" without the full var set,
  // and the fayda/etrade mocks publish no host ports. No IT spec touches them.
  FAYDA_ENABLED: "false",
  // SMS strategy has no kill switch and defaults to a real dev endpoint.
  OZIKING_SMS_URL: "http://127.0.0.1:9/sms",
  // app.config.ts otherwise live-scrapes https://ethio.forex on first booking.
  CBE_EXCHANGE_SCRAPE_URL: "http://127.0.0.1:9/fx",
  CBE_EXCHANGE_API_URL: "http://127.0.0.1:9/fx",
  CBE_EXCHANGE_FALLBACK_RATE: "130",
  FREIGHT_PORTAL_URL: `http://localhost:${PORTS.E2E_PORTAL_PORT}`,

  // Drain tail on every pay window. Production defaults to 5 minutes; a
  // reservation here lives ~60s, so 5 would push every natural expiry past the
  // suite's timeouts. One minute keeps the tail real and observable
  // (src/expired-invoice-late-settle.it.ts asserts both sides of it, and
  // hardcodes DRAIN_MS = 60_000 to match).
  FREIGHT_PAYMENT_DRAIN_MINUTES: "1",
  // The suite's pacing floor: every window phase, allocation and expiry it waits
  // on lands on this tick. Production stays at */10. Overridable from the
  // environment so a cadence-sensitive spec (or a bisect) can pin it back.
  BOOKING_WINDOW_TICK_CRON: process.env.BOOKING_WINDOW_TICK_CRON ?? "*/1 * * * * *",
};

const env = {
  ...process.env,
  ...Object.fromEntries(Object.entries(PORTS).map(([k, v]) => [k, String(v)])),
  ...freightEnv,
  IT_SHARDS: String(SHARDS),
  IT_TEMPLATE_DB: TEMPLATE_DB,
  IT_DB_PREFIX: DB_PREFIX,
  IT_API_PORT_BASE: String(PORTS.E2E_API_PORT),
  IT_PAYMENT_PORT_BASE: String(PORTS.IT_PAYMENT_PORT),
  IT_GATEWAY_PORT_BASE: String(PORTS.IT_GATEWAY_PORT),
  FAYDA_PRIVATE_KEY_BASE64:
    process.env.FAYDA_PRIVATE_KEY_BASE64 ?? fakeFaydaPrivateKeyBase64(),
  IT_TELEBIRR_PRIVATE_KEY: telebirrKey,
};

function generateShardFile() {
  writeShards({
    shards: SHARDS,
    apiPortBase: PORTS.E2E_API_PORT,
    paymentPortBase: PORTS.IT_PAYMENT_PORT,
    gatewayPortBase: PORTS.IT_GATEWAY_PORT,
    dbPrefix: DB_PREFIX,
    telebirrKey,
  });
}

/** The shard file must exist before compose is invoked — it is one of the -f's. */
function composeBase() {
  if (!existsSync(SHARD_FILE)) generateShardFile();
  return [
    "compose",
    "-f",
    join(repoRoot, "docker-compose.e2e.yaml"),
    "-f",
    join(repoRoot, "docker-compose.it.yaml"),
    "-f",
    SHARD_FILE,
  ];
}

function compose(args) {
  const { status } = spawnSync("docker", [...composeBase(), ...args], {
    stdio: "inherit",
    env,
  });
  return status ?? 1;
}

function composeQuiet(args) {
  return spawnSync("docker", [...composeBase(), ...args], {
    encoding: "utf8",
    env,
    stdio: ["ignore", "pipe", "pipe"],
  });
}

function stackRunning() {
  try {
    const out = execFileSync(
      "docker",
      [...composeBase(), "ps", "--services", "--status", "running"],
      { encoding: "utf8", env, stdio: ["ignore", "pipe", "ignore"] },
    );
    const running = new Set(out.split("\n").filter(Boolean));
    const want = [...INFRA, ...shardServices(SHARDS)];
    return want.every((s) => running.has(s));
  } catch {
    return false;
  }
}

function node(script, args = []) {
  const { status } = spawnSync("node", [join(itDir, "scripts", script), ...args], {
    cwd: repoRoot,
    stdio: "inherit",
    env,
  });
  return status ?? 1;
}

/**
 * One broker, one vhost per shard — a shared vhost would let one shard's freight
 * consumer eat another shard's settlement event.
 *
 * Retried: the healthcheck now waits for the rabbit *application*, but a cold
 * boot can still land between "healthy" and "accepting rabbitmqctl", and a
 * half-provisioned broker fails later as an unexplained missing settlement.
 */
function createVhosts() {
  for (let i = 0; i < SHARDS; i++) {
    const vhost = `payment_s${i}`;
    let last = "";
    let ok = false;
    for (let attempt = 1; attempt <= 10 && !ok; attempt++) {
      // Idempotent: add_vhost on an existing vhost exits non-zero, which is fine.
      composeQuiet(["exec", "-T", "rabbitmq-it", "rabbitmqctl", "add_vhost", vhost]);
      const perm = composeQuiet([
        "exec", "-T", "rabbitmq-it",
        "rabbitmqctl", "set_permissions", "-p", vhost, "edr", ".*", ".*", ".*",
      ]);
      ok = perm.status === 0;
      last = perm.stderr ?? "";
      if (!ok) execFileSync("sleep", ["3"]);
    }
    if (!ok) fail(`could not grant on rabbit vhost ${vhost} after 10 tries:\n${last}`);
  }
  console.log(`it: rabbit vhosts payment_s0…payment_s${SHARDS - 1} ready`);
}

function buildFreight() {
  console.log("it: building @edr/freight-api (the suite runs dist/, never stale)");
  const { status } = spawnSync("pnpm", ["--filter", "@edr/freight-api", "run", "build"], {
    cwd: repoRoot,
    stdio: "inherit",
    env,
  });
  if (status !== 0) fail("freight API build failed — fix it before running the suite.");
}

function up() {
  preflight();
  generateShardFile();
  console.log(
    `it: starting ${SHARDS} shard(s) — freight :${PORTS.E2E_API_PORT}..${
      PORTS.E2E_API_PORT + SHARDS - 1
    } (in-process)  payment :${PORTS.IT_PAYMENT_PORT}..  gateway :${
      PORTS.IT_GATEWAY_PORT
    }..  db :${PORTS.E2E_DB_PORT}`,
  );

  // 1. Shared infrastructure.
  if (compose(["up", "-d", "--build", "--wait", "--remove-orphans", ...INFRA]) !== 0) {
    fail(
      "shared services failed to become healthy. Inspect with:\n" +
        "  node integration/scripts/it.mjs logs postgres-freight-e2e",
    );
  }
  createVhosts();

  // 1b. One-shots: the MinIO bucket, then freight's migrations into the
  //     TEMPLATE database every shard is cloned from.
  for (const svc of ONE_SHOT) {
    if (compose(["run", "--rm", "--no-deps", "--build", svc]) !== 0) {
      fail(`${svc} failed — inspect with:\n  node integration/scripts/it.mjs logs ${svc}`);
    }
  }

  // 2. The app must exist as dist/ before we can boot it to seed the template.
  buildFreight();

  // 3. Seed the template (boot seeders + SQL fixtures) and clone it per shard.
  //    Must precede the payment APIs: they connect to the shard databases.
  if (node("prepare-shards.mjs") !== 0) fail("template seed / shard clone failed.");

  // 4. Per-shard payment API + gateway mock.
  if (compose(["up", "-d", "--build", "--wait", ...shardServices(SHARDS)]) !== 0) {
    fail(
      "shard services failed to become healthy. Inspect with:\n" +
        "  node integration/scripts/it.mjs logs payment-api-it-0",
    );
  }
}

/**
 * Re-clone every shard database from the template so each run is hermetic. The
 * payment APIs hold connections to those databases, so they come down first —
 * their `edr_payment` schema is recreated by their own boot migrations.
 *
 * This is what retires the warm-stack failure modes the suite used to document:
 * stowaway paid bookings climbing back aboard, a short consist on the fifth
 * file, invoice numbers continuing from a previous run.
 */
function resetShardDbs() {
  const svc = shardServices(SHARDS);
  console.log("it: re-cloning shard databases from the template");
  compose(["stop", ...svc]);
  if (node("prepare-shards.mjs", ["--clone"]) !== 0) fail("shard database reset failed.");
  if (compose(["up", "-d", "--wait", ...svc]) !== 0) {
    fail("shard services failed to become healthy after the reset.");
  }
}

const [cmd, ...rawArgs] = process.argv.slice(2);
const noReset = rawArgs.includes("--no-reset");
// `pnpm it:test -- src/foo.it.ts` hands us a literal "--" first. Forwarding it
// makes vitest treat everything after it as CLI options and ignore the file
// filter — the "one file" run silently becomes the whole suite.
const passthrough = rawArgs.filter((a) => a !== "--no-reset");
const extra = passthrough[0] === "--" ? passthrough.slice(1) : passthrough;

switch (cmd) {
  case "up":
    up();
    break;
  case "test": {
    if (!stackRunning()) {
      up();
    } else {
      buildFreight();
      if (!noReset) resetShardDbs();
    }
    const { status } = spawnSync(
      "pnpm",
      ["--filter", "@edr/freight-integration", "run", "test", ...extra],
      { cwd: repoRoot, stdio: "inherit", env },
    );
    process.exit(status ?? 1);
  }
  case "logs": {
    process.exit(compose(["logs", "--tail", "200", ...extra]));
  }
  case "down": {
    const status = compose(["down", "-v", "--remove-orphans"]);
    rmSync(SHARD_FILE, { force: true });
    process.exit(status);
  }
  default:
    fail(`unknown command "${cmd ?? ""}" — use up | test | logs | down`);
}