Enhance internal payment handling and e2e testing setup

apps/edr-freight-api/src/modules/payment/internal-payment.controller.ts

@@ -16,6 +16,7 @@ import {
   BillQueryRequestDto,
   BillQueryResponseDto,
 } from "./internal-payment.dto";
+import { Public } from "@edr/api-common";
 import { PaymentService } from "./payment.service";
 import { BillingService } from "../billing/billing.service";
 import { ServiceAuthGuard } from "../../common/guards/service-auth.guard";
@@ -28,6 +29,10 @@ import { ServiceAuthGuard } from "../../common/guards/service-auth.guard";
  * this HTTP endpoint remains as a transport-agnostic fallback.
  */
 @ApiTags("Internal Payments")
+// Service-to-service, not user-to-service: exempt from the global JwtGuard
+// (there is no end-user JWT on a relay call) and authenticated instead by the
+// shared service token that ServiceAuthGuard checks.
+@Public()
 @UseGuards(ServiceAuthGuard)
 @Controller("internal/payments")
 export class InternalPaymentController {

docker-compose.e2e.yaml

@@ -324,6 +324,8 @@ services:
       CYPRESS_BASE_URL: http://localhost:${E2E_BACKOFFICE_PORT:-5383}
       CYPRESS_API_URL: http://localhost:${E2E_API_PORT:-3101}
       CYPRESS_PORTAL_URL: http://localhost:${E2E_PORTAL_PORT:-5373}
+      # Must match freight-api-e2e's SERVICE_AUTH_TOKEN above.
+      CYPRESS_SERVICE_AUTH_TOKEN: e2e-service-token
     volumes:
       - .:/repo
 

e2e/freight/cypress.config.ts

@@ -42,6 +42,8 @@ export default defineConfig({
       defaultPassword: process.env.CYPRESS_DEFAULT_PASSWORD ?? "password@tria",
       // Demo portal users: hardcoded in DemoUsersSeeder.
       demoPassword: "12345678",
+      // Shared secret for /api/internal/* — SERVICE_AUTH_TOKEN in docker-compose.e2e.yaml.
+      serviceAuthToken: process.env.CYPRESS_SERVICE_AUTH_TOKEN ?? "e2e-service-token",
     },
     setupNodeEvents(on) {
       const dbUrl =

e2e/freight/cypress/e2e/flows/export_one_time.cy.ts

@@ -450,6 +450,11 @@ describe("export one-time journeys: container + bulk on one train", { retries: 0
       cy.task("db:query", {
         sql: `UPDATE freight.train_schedules
               SET window_opens_at = LEAST(window_opens_at, now()),
+                  -- The e2e rules run a 1.002-minute window duration, so the
+                  -- CREATE-time close for a departing-today schedule is already
+                  -- in the past — hold the close out or the next 10s tick slams
+                  -- the window shut mid-flow.
+                  window_closes_at = GREATEST(window_closes_at, now() + interval '45 minutes'),
                   window_phase = 'OPEN',
                   booking_window_status = 'OPEN'
               WHERE id = $1 AND booking_window_status <> 'FULL'`,

e2e/freight/cypress/e2e/flows/g1-utils.ts

@@ -37,6 +37,7 @@ import {
   db,
   dbSchedule,
   forceWindowOpen,
+  holdPayWindows,
   opsStaff,
   ORIGIN,
   pollDb,
@@ -190,8 +191,35 @@ export function closeWindowAndRunBatch(departure: Date) {
 
   cy.loginBackoffice(opsStaff);
   withSchedule(departure, (s) => cy.visit(`/dashboard/operations/batch-board/${s.id}`));
-  cy.contains("Doc review", { timeout: 120000 }).should("exist");
+  // The e2e rules run a 1-MINUTE doc review, and the login + visit above can
-  cy.contains("button", "Doc review complete — run batch", { timeout: 120000 }).click();
+  // outlive it — the tick then runs the batch itself and the button never
+  // renders. Click the button while the phase is still DOC_REVIEW; once the
+  // engine has advanced on its own there is nothing left to click, and the
+  // poll below asserts the batch ran either way.
+  withSchedule(departure, (s) => {
+    const tryRunBatch = (attempt: number): void => {
+      db<{ p: string }>(
+        `SELECT window_phase AS p FROM freight.train_schedules WHERE id = $1`,
+        [s.id],
+      ).then(({ rows }) => {
+        if (rows[0].p !== "DOC_REVIEW") return; // tick already ran the batch
+        cy.get("body").then(($body) => {
+          const button = $body.find(
+            'button:contains("Doc review complete — run batch")',
+          );
+          if (button.length > 0) {
+            cy.wrap(button.first()).click({ force: true });
+            return;
+          }
+          expect(attempt, "batch board rendered its doc-review action").to.be.lessThan(
+            20,
+          );
+          cy.wait(3000, { log: false }).then(() => tryRunBatch(attempt + 1));
+        });
+      });
+    };
+    tryRunBatch(0);
+  });
 
   withSchedule(departure, (s) =>
     pollDb<ScheduleRow>(
@@ -199,10 +227,16 @@ export function closeWindowAndRunBatch(departure: Date) {
       `SELECT window_phase FROM freight.train_schedules WHERE id = $1`,
       [s.id],
       // DONE when the batch reserved nobody — itself a scenario outcome.
-      (row) => !!row && ["PAYMENT", "DONE"].includes(row.window_phase as string),
+      // PRE_WINDOW/OPEN when an under-filled day concluded and re-opened for
+      // its next cycle (window duration is 1 minute in e2e).
+      (row) =>
+        !!row &&
+        ["PAYMENT", "DONE", "PRE_WINDOW", "OPEN"].includes(row.window_phase as string),
       20,
     ),
   );
+  // The batch stamped 1-minute pay windows; hold them while the spec pays.
+  holdPayWindows();
 }
 
 /**
@@ -224,6 +258,15 @@ export function expectBoard(
   cy.loginBackoffice(opsStaff);
   withSchedule(departure, (s) => cy.visit(`/dashboard/operations/batch-board/${s.id}`));
   cy.contains(/Priority Tracking/, { timeout: 120000 }).click();
+  // While a window is OPEN the tab defaults to the Forecast view (an
+  // under-filled day re-opens for its next cycle — g1_s3's core scenario) and
+  // the live lanes are hidden behind the "Live state" toggle. On settled
+  // boards the toggle is not rendered at all, so only click it when present.
+  cy.contains(/Priority ranking|Live state/, { timeout: 120000 })
+    .invoke("text")
+    .then((text) => {
+      if (text.includes("Live state")) cy.contains("Live state").click();
+    });
   cy.contains("Priority ranking", { timeout: 120000 }).should("be.visible");
 
   if (opts.inBatch !== undefined) {

e2e/freight/cypress/e2e/flows/g1_s3_underfill_day_stays_open.cy.ts

@@ -91,7 +91,17 @@ describe("G1·S3: an under-filled train keeps its day open", { retries: 0 }, ()
     configureAndOpenSchedule({ departure: DEPARTURE });
   });
 
-  it("A and B book through the API; C books 24×20FT through the portal", () => {
+  // The API bookings and the portal booking are SEPARATE tests on purpose.
+  // cy.loginPortal's cross-origin visit makes Cypress reload the runner,
+  // re-evaluate the bundle (re-running `before()` and regenerating the
+  // module-scope stamp) and restart the CURRENT test from the top. With A and
+  // B in the same test as the portal visit they were booked twice — once per
+  // pass, under two stamps, even on a freshly wiped DB — and the orphaned
+  // first pair expired at payment end, corrupting the board counts and the
+  // free-wagon arithmetic. In their own test the completed API step is never
+  // re-entered; the restart only repeats the login. Same structure as g1_s2,
+  // which is why that spec never double-booked.
+  it("A and B book through the API", () => {
     let isoSeed = 8600;
     (["A", "B"] as const).forEach((suffix) => {
       const shape = SHAPES[suffix];
@@ -105,8 +115,12 @@ describe("G1·S3: an under-filled train keeps its day open", { retries: 0 }, ()
       });
       isoSeed += shape.twenty + shape.forty;
     });
+  });
 
+  it("C books 24×20FT through the portal shipment form", () => {
     cy.loginPortal(customer);
+    // dbContractId picks the NEWEST *-C contract, so the duplicate seeded by
+    // the reload's before() pass is inert.
     dbContractId("C").then((contractId) => {
       bookContainersVisually({
         contractId,

e2e/freight/cypress/e2e/flows/import-utils.ts

@@ -640,8 +640,47 @@ export function expectClearanceOnBookingInvoice(suffix: string) {
   );
 }
 
+/**
+ * Push every still-unpaid fixture reservation's pay deadline out 10 minutes.
+ *
+ * The e2e rules run a 1-MINUTE payment window (seed-import-corridor.sql), and
+ * a spec's pay loop — login, settle, poll, per booking — always outlives it:
+ * without this the 10s tick expires the holds the spec is queued up to pay.
+ * Called right after the batch reserves (completeDocReview,
+ * closeWindowAndRunBatch), after an export FCFS accept, and again before each
+ * payment. Blanket over CTR-IMP-% on purpose: specs run one at a time, and the
+ * first payment must rescue its yet-unpaid siblings, whichever schedule they
+ * reserved onto.
+ *
+ * Two invariants preserved:
+ *  - export parity ("the pay window never outlives the window close"): while a
+ *    booking's window is still open, the extension clamps to window_closes_at;
+ *  - expiry scenarios: specs that TEST expiry pull deadlines back into the
+ *    past afterwards (forceReservationExpiry / forceOfferLapse), and
+ *    endPaymentPhase now expires its schedule's unpaid holds itself — so the
+ *    extension never masks an expiry.
+ */
+export function holdPayWindows() {
+  db(
+    `UPDATE freight.bookings b
+     SET payment_deadline = LEAST(
+           now() + interval '10 minutes',
+           COALESCE(
+             (SELECT ts.window_closes_at FROM freight.train_schedules ts
+              WHERE ts.id = b.train_schedule_id
+                AND ts.window_closes_at > now()),
+             now() + interval '10 minutes'))
+     FROM freight.contracts ct
+     WHERE ct.id = b.contract_id AND ct.reference LIKE 'CTR-IMP-%'
+       AND b.deleted_at IS NULL
+       AND b.status IN ('SELECTED_FOR_BATCH', 'AWAITING_PAYMENT')
+       AND b.payment_deadline IS NOT NULL`,
+  );
+}
+
 /** Staff force-pay; polls PAID + SCHEDULED. */
 export function markPaid(suffix: string) {
+  holdPayWindows();
   withBooking(suffix, (b) => {
     apiPost(opsStaff, `/api/train-scheduling/bookings/${b.id}/mark-paid`)
       .its("status")
@@ -668,6 +707,7 @@ export function markPaid(suffix: string) {
  * path that applies a pending split offer (staff mark-paid skips it).
  */
 export function settleViaGateway(suffix: string) {
+  holdPayWindows();
   withBooking(suffix, (b) => {
     db<{ intent_id: string; currency: string; total: string }>(
       `WITH inv AS (
@@ -698,6 +738,9 @@ export function settleViaGateway(suffix: string) {
       cy.request({
         method: "POST",
         url: `${apiUrl()}/api/internal/payments/mark-paid`,
+        headers: {
+          "x-service-token": Cypress.env("serviceAuthToken") as string,
+        },
         body: {
           version: 1,
           eventId: crypto.randomUUID(),
@@ -917,6 +960,25 @@ export function resetCorridorDay(departure: Date, destCode = DEST, originCode =
        AND b.scheduled_date = $1::date`,
     [eatDayStr(departure)],
   );
+  // Finally, soft-delete every remaining unpinned fixture booking on the day.
+  // Reset runs before the current run books anything, so all of them are
+  // prior-run debris — and merely leaving them unpinned is not enough:
+  //  - EXPIRED ones render in the board's "Expired" lane (it lists by DAY),
+  //    so `expired: 0` could never pass against a warm DB;
+  //  - PAID ones sit in the day pool, and when an under-filled day re-opens
+  //    for its next cycle the engine's batch fill re-links them to the LIVE
+  //    schedule mid-run — observed as 15 ghosts re-pinned within one second,
+  //    inflating "In the batch (N)" past what the spec created.
+  db(
+    `UPDATE freight.bookings b
+     SET deleted_at = now()
+     FROM freight.contracts ct
+     WHERE ct.id = b.contract_id AND ct.reference LIKE 'CTR-IMP-%'
+       AND b.deleted_at IS NULL
+       AND b.train_schedule_id IS NULL
+       AND b.scheduled_date = $1::date`,
+    [eatDayStr(departure)],
+  );
 }
 
 /** Create the reversed 6-stop corridor (ET → DJ = EXPORT) if missing. */
@@ -959,6 +1021,10 @@ export function acceptExport(suffix: string) {
       .should("be.oneOf", [200, 201]);
   });
   pollBookingStatus(suffix, ["SELECTED_FOR_BATCH", "AWAITING_PAYMENT"], 10);
+  // The accept stamped a 1-minute pay window (export_payment_window_minutes);
+  // a spec accepting several bookings would lose the first before the last is
+  // even accepted. Still clamped to the window close — see holdPayWindows.
+  holdPayWindows();
 }
 
 export interface ScheduleRow {
@@ -1228,18 +1294,54 @@ export function closeBookingWindow(scheduleId: string) {
  * (Lands on DONE instead when the batch reserved nobody.)
  */
 export function completeDocReview(scheduleId: string) {
-  apiPost(opsStaff, `/api/train-scheduling/schedules/${scheduleId}/doc-review-complete`)
+  apiPost(
-    .its("status")
+    opsStaff,
-    .should("be.oneOf", [200, 201]);
+    `/api/train-scheduling/schedules/${scheduleId}/doc-review-complete`,
+    undefined,
+    false,
+  ).then((res) => {
+    if (res.status >= 400) {
+      // The e2e rules run a 1-MINUTE doc review: the tick may have run the
+      // batch on its own while the spec was still logging in or asserting.
+      // That is the engine doing the right thing on schedule — but a 4xx with
+      // the phase still stuck in DOC_REVIEW is a real failure.
+      db<{ p: string }>(
+        `SELECT window_phase AS p FROM freight.train_schedules WHERE id = $1`,
+        [scheduleId],
+      ).then(({ rows }) => {
+        expect(
+          rows[0]?.p,
+          `doc-review-complete ${res.status} — engine advanced on its own`,
+        ).to.be.oneOf(["PAYMENT", "DONE", "PRE_WINDOW", "OPEN"]);
+      });
+    }
+  });
   pollSchedulePhase(
     scheduleId,
-    ["PAYMENT", "DONE", "PRE_WINDOW"],
+    // OPEN: with a 1-minute window duration an under-filled day can already
+    // have re-opened for its next cycle by the first poll read.
+    ["PAYMENT", "DONE", "PRE_WINDOW", "OPEN"],
     `schedule ${scheduleId} payment phase`,
   );
+  holdPayWindows();
 }
 
 /** End the payment phase now — the tick settles (allocate paid / expire unpaid). */
 export function endPaymentPhase(scheduleId: string) {
+  // holdPayWindows pushed the unpaid holds' own deadlines out so a pay loop
+  // could outlive the 1-minute window; ending the phase means those holds must
+  // now expire, so pull them back first — the settle only expires reservations
+  // whose OWN deadline has passed, and holds the cycle open for the rest.
+  db(
+    `UPDATE freight.bookings b
+     SET payment_deadline = now() - interval '1 second'
+     FROM freight.contracts ct
+     WHERE ct.id = b.contract_id AND ct.reference LIKE 'CTR-IMP-%'
+       AND b.deleted_at IS NULL
+       AND b.train_schedule_id = $1
+       AND b.status IN ('SELECTED_FOR_BATCH', 'AWAITING_PAYMENT')`,
+    [scheduleId],
+  );
   db(
     `UPDATE freight.train_schedules
      SET payment_phase_ends_at = now() - interval '1 second'

e2e/freight/cypress/e2e/flows/segment_weight.cy.ts

@@ -428,6 +428,11 @@ describe(
           cy.task("db:query", {
             sql: `UPDATE freight.train_schedules
                   SET window_opens_at = LEAST(window_opens_at, now()),
+                      -- The e2e rules run a 1.002-minute window duration, so the
+                      -- CREATE-time close for a departing-today schedule is already
+                      -- in the past — hold the close out or the next 10s tick slams
+                      -- the window shut mid-flow.
+                      window_closes_at = GREATEST(window_closes_at, now() + interval '45 minutes'),
                       window_phase = 'OPEN',
                       booking_window_status = 'OPEN'
                   WHERE id = $1 AND booking_window_status <> 'FULL'`,

e2e/freight/cypress/fixtures/seed-import-corridor.sql

@@ -450,14 +450,20 @@ WHERE NOT EXISTS (
 );
 
 -- ---------------------------------------------------------------------------
--- e2e window durations: 1 minute instead of the 30/60 production defaults.
+-- e2e window durations — the dev-environment settings, verbatim:
+--   window duration 0.0167 h (1.002 min), doc review 1 min, payment 1 min
+--   (import AND export).
 --
--- Most specs never wait these out — closeWindowAndRunBatch clicks "Doc review
+-- Specs still arrange the timestamps they need (forceWindowOpen holds a
--- complete" and endPaymentPhase pulls the deadline into the past — so this is
+-- window open for 45 min; endPaymentPhase ends the pay phase early), but
--- a safety net for the paths that DO let a phase elapse on its own, not the
+-- every ENGINE-stamped deadline now comes from these 1-minute rules: the
--- main speed lever. That one is the 10s @Cron tick in booking-window.service.
+-- batch's pay windows, the doc-review auto-advance, and re-opened cycles all
+-- elapse in about a minute on their own via the 10s @Cron tick in
+-- booking-window.service. holdPayWindows (import-utils.ts) is what keeps a
+-- spec's queued-up payments from expiring under the 1-minute pay window.
 -- ---------------------------------------------------------------------------
 UPDATE freight.train_scheduling_global_rules
-   SET doc_review_minutes = 1,
+   SET window_duration_hours = 0.0167,
+       doc_review_minutes = 1,
        payment_window_minutes = 1,
        export_payment_window_minutes = 1;

e2e/freight/scripts/e2e.mjs

@@ -16,7 +16,7 @@
  */
 
 import { execFileSync, spawnSync } from "node:child_process";
-import { generateKeyPairSync } from "node:crypto";
+import { createHash, generateKeyPairSync } from "node:crypto";
 import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createServer } from "node:net";
 import { dirname, join, resolve } from "node:path";
@@ -25,7 +25,17 @@ import { fileURLToPath } from "node:url";
 const e2eDir = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const repoRoot = resolve(e2eDir, "..", "..");
 const stateFile = join(e2eDir, ".e2e-ports.json");
-const composeBase = ["compose", "-f", join(repoRoot, "docker-compose.e2e.yaml")];
+// Per-checkout compose project: parallel checkouts on one docker daemon
+// otherwise share the yaml's fixed `name:` and recreate/kill each other's
+// containers mid-run.
+const projectName = `edr-freight-e2e-${createHash("sha1").update(repoRoot).digest("hex").slice(0, 6)}`;
+const composeBase = [
+  "compose",
+  "-p",
+  projectName,
+  "-f",
+  join(repoRoot, "docker-compose.e2e.yaml"),
+];
 
 const DEFAULT_PORTS = {
   E2E_API_PORT: 3101,