From 7c566075828fc9a62efbf7f78603b66f3a9fd76a Mon Sep 17 00:00:00 2001 From: Nathnael Date: Wed, 19 Aug 2026 10:11:48 +0000 Subject: [PATCH 01/66] fix(filters): honor a date filter's own operator, not always "between" DateBody always initialized its op state to DEFAULT_OP.date ("between"), ignoring a def's `operators` restriction. A single-operator exact-date filter (e.g. `operators: ["before"]`) opened on the range UI with no way off it, since OperatorSelect hides itself when there's only one choice. Default to the def's first allowed operator instead. --- .../backoffice/src/components/filters/bodies/DateBody.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/edr-freight-web/backoffice/src/components/filters/bodies/DateBody.tsx b/apps/edr-freight-web/backoffice/src/components/filters/bodies/DateBody.tsx index b2032170d..32970abef 100644 --- a/apps/edr-freight-web/backoffice/src/components/filters/bodies/DateBody.tsx +++ b/apps/edr-freight-web/backoffice/src/components/filters/bodies/DateBody.tsx @@ -15,7 +15,11 @@ import type { FilterBodyProps } from "./TextBody"; // i18n.language !== "en" branch here when this body is first wired into a // record-management page (Phase 4 of the filter-bar rollout). export function DateBody({ def, value, onChange, onClose }: FilterBodyProps) { - const [op, setOp] = useState(value?.op ?? DEFAULT_OP.date); + // DEFAULT_OP.date is always "between" — a def restricted to a single + // non-default operator (e.g. `operators: ["before"]` for an exact-date + // filter) would otherwise open on the range UI with no way to switch off + // it, since OperatorSelect hides itself when there's only one choice. + const [op, setOp] = useState(value?.op ?? def.operators?.[0] ?? DEFAULT_OP.date); // Mantine 9's date inputs speak `YYYY-MM-DD` strings, not Date objects. const [from, setFrom] = useState(value?.v[0]?.slice(0, 10) ?? null); const [to, setTo] = useState(value?.v[1]?.slice(0, 10) ?? null); From 6687def4afc36c36e6c9483e7af982efb297d0f5 Mon Sep 17 00:00:00 2001 From: Nathnael Date: Wed, 19 Aug 2026 10:12:03 +0000 Subject: [PATCH 02/66] feat(reports): use the shared FilterBar instead of ReportFilters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swaps ReportView's bespoke ReportFilters for the same FilterBar/useFilters combo BookingRequestsPage uses — pills, saved-view-ready state, URL sync. toFilterDefs() maps the report catalog's own filter vocabulary (daterange/date/select/multiselect/text) onto FilterDef, matched against every report definition's backend param handling. The search box only shows for reports that actually implement `search` server-side, so it's not a dead control on the rest. ReportFilters.tsx is now dead, removed. --- .../src/components/reports/ReportFilters.tsx | 110 -------------- .../src/components/reports/ReportView.tsx | 139 ++++++++++++++---- 2 files changed, 110 insertions(+), 139 deletions(-) delete mode 100644 apps/edr-freight-web/backoffice/src/components/reports/ReportFilters.tsx diff --git a/apps/edr-freight-web/backoffice/src/components/reports/ReportFilters.tsx b/apps/edr-freight-web/backoffice/src/components/reports/ReportFilters.tsx deleted file mode 100644 index e44e97ec8..000000000 --- a/apps/edr-freight-web/backoffice/src/components/reports/ReportFilters.tsx +++ /dev/null @@ -1,110 +0,0 @@ -import { Group, MultiSelect, Select, TextInput } from "@mantine/core"; -import { DateInput, DatePickerInput } from "@mantine/dates"; -import { Search } from "lucide-react"; - -import { getDateRangePresets } from "@/components/common/dateRangePresets"; -import type { ReportFilterDef } from "@/types/reports"; - -export interface ReportFilterValues { - [param: string]: string | undefined; -} - -interface ReportFiltersProps { - filters: ReportFilterDef[]; - values: ReportFilterValues; - onChange: (values: ReportFilterValues) => void; -} - -const toDate = (value: string | undefined): Date | null => (value ? new Date(value) : null); -const fromDate = (value: string | null): string | undefined => value ?? undefined; - -/** Renders one widget per report-declared filter and reports raw param values back up. */ -export function ReportFilters({ filters, values, onChange }: ReportFiltersProps) { - if (!filters.length) return null; - - const set = (patch: ReportFilterValues) => onChange({ ...values, ...patch }); - - return ( - - {filters.map((filter) => { - switch (filter.type) { - case "daterange": - return ( - - set({ [`${filter.key}From`]: fromDate(from), [`${filter.key}To`]: fromDate(to) }) - } - presets={getDateRangePresets()} - radius="md" - size="sm" - clearable - w={230} - /> - ); - case "date": - return ( - set({ [filter.key]: fromDate(d) })} - radius="md" - size="sm" - clearable - w={150} - /> - ); - case "select": - return ( - + setDraft((d) => ({ ...d, [field.name]: e.target.value })) + } + /> + + {field.unit} + + +

+ {invalid(field) + ? field.integer + ? "Must be a whole number above zero" + : "Must be above zero" + : field.hint} +

+ + ))} + + + ))} + + {!canEdit && ( +

+ You can view these standards but not change them. +

+ )} + + ); +} diff --git a/apps/edr-freight-web/backoffice/src/services/operationsStandards.service.ts b/apps/edr-freight-web/backoffice/src/services/operationsStandards.service.ts new file mode 100644 index 000000000..f9086570a --- /dev/null +++ b/apps/edr-freight-web/backoffice/src/services/operationsStandards.service.ts @@ -0,0 +1,51 @@ +import { api as client } from "../auth/http"; +import { unwrap } from "@/utils/endpoint"; +import { URL_CONSTANTS } from "@/constants/URLS"; +import type { ApiResponse } from "@/types/apiResponse"; + +const BASE = URL_CONSTANTS.OPERATIONS_STANDARDS.BASE; + +/** + * The railway's operating standards — the numbers the operations reports + * measure actual performance against. One row, edited here. + */ +export interface OperationsStandards { + id: string; + stationStandardHoursEthiopia: number; + stationStandardHoursDjibouti: number; + cycleStandardHoursContainer: number; + cycleStandardHoursBulkDmp: number; + cycleStandardHoursBulkNagad: number; + cycleStandardHoursBulkBcc: number; + defaultLegStandardHours: number; + delayToleranceMinutes: number; + chargedTonsFull20ft: number; + chargedTonsFull40ft: number; + chargedTonsEmpty20ft: number; + chargedTonsEmpty40ft: number; + chargedTonsPerWagonGeneral: number; + chargedTonsPerWagonPerishable: number; + defaultFullTrainsetWagons: number; + updatedAt?: string; +} + +export type OperationsStandardsPatch = Partial< + Omit +>; + +export const operationsStandardsService = { + get: async (): Promise => { + const response = await client.get>(BASE); + return unwrap(response.data); + }, + + update: async ( + patch: OperationsStandardsPatch, + ): Promise => { + const response = await client.patch>( + BASE, + patch, + ); + return unwrap(response.data); + }, +}; diff --git a/apps/edr-freight-web/backoffice/src/services/ruleEngine/ruleEngine.service.ts b/apps/edr-freight-web/backoffice/src/services/ruleEngine/ruleEngine.service.ts index 48f944c21..fc7bce8f0 100644 --- a/apps/edr-freight-web/backoffice/src/services/ruleEngine/ruleEngine.service.ts +++ b/apps/edr-freight-web/backoffice/src/services/ruleEngine/ruleEngine.service.ts @@ -96,6 +96,7 @@ const RESOURCE_BASE: Record = { "weight-limit-rules": URL_CONSTANTS.RULE_ENGINE.WEIGHT_LIMIT_RULES, yards: URL_CONSTANTS.RULE_ENGINE.YARDS, "yard-distances": URL_CONSTANTS.RULE_ENGINE.YARD_DISTANCES, + "operations-targets": URL_CONSTANTS.RULE_ENGINE.OPERATIONS_TARGETS, "shipping-lines": URL_CONSTANTS.RULE_ENGINE.SHIPPING_LINES, rates: URL_CONSTANTS.RULE_ENGINE.RATES, "approval-rules": URL_CONSTANTS.RULE_ENGINE.APPROVAL_RULES, diff --git a/apps/edr-freight-web/backoffice/src/theme/freight-brand.ts b/apps/edr-freight-web/backoffice/src/theme/freight-brand.ts index 6c1efd3f3..4117f503c 100644 --- a/apps/edr-freight-web/backoffice/src/theme/freight-brand.ts +++ b/apps/edr-freight-web/backoffice/src/theme/freight-brand.ts @@ -86,7 +86,7 @@ export const freightMantineTheme = createTheme({ black: "#10202F", fontFamily: - '"Inter", ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif', + '"Space Grotesk", ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif', defaultRadius: "md", @@ -123,7 +123,7 @@ export const freightMantineTheme = createTheme({ }, headings: { - fontFamily: '"Inter", var(--mantine-font-family)', + fontFamily: '"Space Grotesk", var(--mantine-font-family)', fontWeight: "700", sizes: { h1: { fontSize: "36px", lineHeight: "1.1", fontWeight: "800" }, diff --git a/apps/edr-freight-web/backoffice/src/types/rule-engine/index.ts b/apps/edr-freight-web/backoffice/src/types/rule-engine/index.ts index 5d886043a..9f530d95b 100644 --- a/apps/edr-freight-web/backoffice/src/types/rule-engine/index.ts +++ b/apps/edr-freight-web/backoffice/src/types/rule-engine/index.ts @@ -11,7 +11,8 @@ export type RuleEngineResourceSlug = | "shipping-lines" | "rates" | "approval-rules" - | "transit-agents"; + | "transit-agents" + | "operations-targets"; /** * Mirrors the API's shared `PaginationMeta` (@edr/types). The `has*` flags are From fc55f48371ee8f5cbbaeb58907b9bd3f557ecea2 Mon Sep 17 00:00:00 2001 From: Hagernesh Date: Wed, 19 Aug 2026 18:21:10 +0000 Subject: [PATCH 07/66] feat(eims): implement POST /v1/bulkRegister MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New endpoints: POST invoices/eims/bulk-register { invoiceIds: [...] } — trigger POST eims/webhook/bulk-register — MoR's callback Fundamentally different shape from single register: bulkRegister answers only {conversationId, status:202} immediately: MoR processes the array asynchronously and pushes the real per-invoice results (a mix of accepted/rejected in one array, per the collection's own examples) to a webhook configured out of band. So this ships as two halves that don't share a call stack — EimsBulkRegistrationService. registerBulk() reserves a contiguous block of counters (durable reservation, same doctrine as single register, extended to N items) and submits; handleBulkCallback(), invoked by the new EimsWebhookController whenever MoR gets around to it, settles. New EimsSystemState.inFlightConversationId is the bulk equivalent of inFlightInvoiceId — a whole batch outstanding, not one invoice — and the two markers block each other since they share the same counter sequence. The conversation id isn't known until MoR's 202 arrives, so reservation stamps a locally-generated placeholder first (same commit-before-the-network-call reasoning as single register), then swaps it for MoR's real id right after — the only value the callback can actually use to find the batch again. Only the first invoice in a bulk batch chains via PreviousIrn — every other item gets an empty string, matching the collection's own two-invoice example exactly (MoR doesn't expect a batch to chain to IRNs that don't exist yet at submission time). Webhook has no auth (MoR has no JWT to send) — the conversation id embedded in the payload is what stands between this and a forged callback: an item only ever touches an invoice actually holding that exact id, and an unknown id is logged and ignored, never applied. Migration 3580000000000: eims_system_state.in_flight_conversation_id, invoices.eims_bulk_conversation_id (tags which batch an invoice was submitted in, so a stuck batch — webhook never arrived — can be found and reconciled by conversation id). Applied to dev DB and recorded in freight.migrations directly (idempotent IF NOT EXISTS DDL). Not live-testable from this sandbox (no route to MoR's real gateway). Signing the whole array as one envelope, the way single /v1/register was confirmed live to need despite the collection's raw example showing no envelope, is the reasonable extension of that confirmed behavior, not a blind guess — but it has not itself been exercised against the real gateway. Left for the first live bulk attempt to confirm, same as every other MoR-facing assumption this integration has made. --- .../3580000000000-EimsBulkRegistration.ts | 35 ++ .../billing/entities/invoice.entity.ts | 8 + .../dto/bulk-register-eims-invoice.dto.ts | 11 + .../eims-bulk-registration.service.spec.ts | 359 ++++++++++++ .../eims/eims-bulk-registration.service.ts | 518 ++++++++++++++++++ .../modules/eims/eims-invoice.controller.ts | 15 + .../modules/eims/eims-registration.types.ts | 57 ++ .../modules/eims/eims-webhook.controller.ts | 25 + .../src/modules/eims/eims.module.ts | 6 +- .../eims/entities/eims-system-state.entity.ts | 7 + 10 files changed, 1040 insertions(+), 1 deletion(-) create mode 100644 apps/edr-freight-api/src/migrations/3580000000000-EimsBulkRegistration.ts create mode 100644 apps/edr-freight-api/src/modules/eims/dto/bulk-register-eims-invoice.dto.ts create mode 100644 apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.spec.ts create mode 100644 apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.ts create mode 100644 apps/edr-freight-api/src/modules/eims/eims-webhook.controller.ts diff --git a/apps/edr-freight-api/src/migrations/3580000000000-EimsBulkRegistration.ts b/apps/edr-freight-api/src/migrations/3580000000000-EimsBulkRegistration.ts new file mode 100644 index 000000000..140a7ad6a --- /dev/null +++ b/apps/edr-freight-api/src/migrations/3580000000000-EimsBulkRegistration.ts @@ -0,0 +1,35 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +/** + * Columns for `POST /v1/bulkRegister` — see `EimsBulkRegistrationService`. + * + * `eims_system_state.in_flight_conversation_id` is the bulk equivalent of `in_flight_invoice_id`: + * a whole batch, not one invoice, is what's outstanding while MoR processes it asynchronously. + * `invoices.eims_bulk_conversation_id` tags which batch an invoice was submitted in, so a stuck + * batch (webhook never arrived) can be found and reconciled by conversation id. + */ +export class EimsBulkRegistration3580000000000 implements MigrationInterface { + name = "EimsBulkRegistration3580000000000"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE freight.eims_system_state + ADD COLUMN IF NOT EXISTS in_flight_conversation_id text + `); + await queryRunner.query(` + ALTER TABLE freight.invoices + ADD COLUMN IF NOT EXISTS eims_bulk_conversation_id text + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE freight.eims_system_state + DROP COLUMN IF EXISTS in_flight_conversation_id + `); + await queryRunner.query(` + ALTER TABLE freight.invoices + DROP COLUMN IF EXISTS eims_bulk_conversation_id + `); + } +} diff --git a/apps/edr-freight-api/src/modules/billing/entities/invoice.entity.ts b/apps/edr-freight-api/src/modules/billing/entities/invoice.entity.ts index 43b6e3271..66bf0f452 100644 --- a/apps/edr-freight-api/src/modules/billing/entities/invoice.entity.ts +++ b/apps/edr-freight-api/src/modules/billing/entities/invoice.entity.ts @@ -203,4 +203,12 @@ export class Invoice extends BaseEntity { @ManyToOne(() => Invoice) @JoinColumn({ name: "related_invoice_id" }) relatedInvoice?: Invoice | null; + + /** + * Which `POST /v1/bulkRegister` batch this invoice was submitted in, if any — MoR's own + * conversation id, not one we generate. Lets a stuck batch (webhook never arrived) be found and + * reconciled. Null for every invoice filed through single `/v1/register`. + */ + @Column({ name: "eims_bulk_conversation_id", type: "text", nullable: true }) + eimsBulkConversationId?: string | null; } diff --git a/apps/edr-freight-api/src/modules/eims/dto/bulk-register-eims-invoice.dto.ts b/apps/edr-freight-api/src/modules/eims/dto/bulk-register-eims-invoice.dto.ts new file mode 100644 index 000000000..2509275e9 --- /dev/null +++ b/apps/edr-freight-api/src/modules/eims/dto/bulk-register-eims-invoice.dto.ts @@ -0,0 +1,11 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { ArrayMinSize, IsArray, IsUUID } from "class-validator"; + +/** `POST invoices/eims/bulk-register` body — see `EimsBulkRegistrationService.registerBulk`. */ +export class BulkRegisterEimsInvoiceDto { + @ApiProperty({ type: [String], description: "Invoice IDs to register with MoR EIMS in one batch." }) + @IsArray() + @ArrayMinSize(1) + @IsUUID("4", { each: true }) + invoiceIds!: string[]; +} diff --git a/apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.spec.ts b/apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.spec.ts new file mode 100644 index 000000000..459ee8f71 --- /dev/null +++ b/apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.spec.ts @@ -0,0 +1,359 @@ +import { BadRequestException, ConflictException } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { DataSource } from "typeorm"; + +import { Invoice } from "../billing/entities/invoice.entity"; +import { NotificationsService } from "../notifications/notifications.service"; +import { eimsConfig, eimsInvoiceConfig } from "./eims-test-fixtures"; +import { EimsAuthService } from "./eims-auth.service"; +import { EimsBulkRegistrationService } from "./eims-bulk-registration.service"; +import { EimsClientService } from "./eims-client.service"; +import { EimsSellerCacheService } from "./eims-seller-cache.service"; +import { EimsSystemState } from "./entities/eims-system-state.entity"; +import { EimsApiException } from "./eims.errors"; +import { EimsInvoiceStatus } from "./eims-registration.types"; +import { buildEimsSeller } from "./eims-invoice-context"; + +const SYSTEM_NUMBER = "B0360154BA"; +const INVOICE_A = "11111111-1111-4111-8111-111111111111"; +const INVOICE_B = "22222222-2222-4222-8222-222222222222"; +const CONVERSATION_ID = "2345678901-1735900502800-c04f8dd6-e6e2-4198-b871-c6e504fc14f5"; + +const invoiceRow = (over: Partial = {}): Invoice => + ({ + id: INVOICE_A, + invoiceNumber: "INV-20260807-00001", + currency: "ETB", + companyId: "company-1", + issuedAt: new Date(2026, 7, 7, 9, 5, 3), + totalAmount: "10000.00", + eimsStatus: EimsInvoiceStatus.NotSubmitted, + eimsIrn: null, + eimsDocumentType: "INV", + eimsBulkConversationId: null, + company: { + name: "ABC Trading PLC", + tin: "0999930000", + vatNumber: "123475885858", + phone: "0912345678", + region: "13", + zone: "SHA", + woreda: "574", + kebele: "03", + houseNo: "NEW", + country: "Ethiopia", + }, + ...over, + }) as unknown as Invoice; + +const LINES = (id: string) => [ + { + invoiceId: id, + chargeType: "RAIL_FREIGHT", + description: "Addis to Djibouti", + quantity: "1.00", + unitRate: "10000.00", + amount: "10000.00", + }, +]; + +/** In-memory stand-in covering the query/manager surface this service actually calls. */ +class FakeDb { + invoices = new Map(); + state: EimsSystemState; + companyContact: { phone: string | null; email: string | null } | null = null; + + constructor(invoices: Invoice[], state: Partial = {}) { + for (const inv of invoices) this.invoices.set(inv.id, inv); + this.state = { + id: "state-1", + systemNumber: SYSTEM_NUMBER, + nextInvoiceCounter: 1, + nextDocumentNumber: 1, + previousIrn: null, + inFlightInvoiceId: null, + inFlightCounter: null, + inFlightDocumentNumber: null, + inFlightConversationId: null, + blockedReason: null, + ...state, + } as EimsSystemState; + } + + private matches(entity: Invoice | EimsSystemState, where: Record): boolean { + return Object.entries(where).every(([key, value]) => (entity as never)[key] === value); + } + + private queryBuilder(entityCtor: unknown) { + let where: Record = {}; + const builder = { + setLock: () => builder, + where: (_clause: string, params: Record) => { + where = { ...where, ...this.normalizeParams(params) }; + return builder; + }, + andWhere: (_clause: string, params: Record) => { + where = { ...where, ...this.normalizeParams(params) }; + return builder; + }, + getOne: async () => this.find(entityCtor, where)[0] ?? null, + getMany: async () => this.find(entityCtor, where), + }; + return builder; + } + + private normalizeParams(params: Record): Record { + // Test-only mapping from the SQL param names used in the service's own queries to entity fields. + const map: Record = { + invoiceId: "id", + systemNumber: "systemNumber", + id: "eimsBulkConversationId", + }; + const out: Record = {}; + for (const [k, v] of Object.entries(params)) out[map[k] ?? k] = v; + return out; + } + + private find(entityCtor: unknown, where: Record): Array { + const isState = entityCtor === EimsSystemState; + const pool: Array = isState ? [this.state] : [...this.invoices.values()]; + return pool.filter((e) => this.matches(e, where)); + } + + private manager = { + createQueryBuilder: (entityCtor: unknown) => this.queryBuilder(entityCtor), + query: async () => [], + findOne: async (entityCtor: unknown, options: { where: Record }) => + this.find(entityCtor, options.where)[0] ?? null, + update: async (entityCtor: unknown, idOrWhere: string | Record, patch: Record) => { + const targets = + typeof idOrWhere === "string" + ? this.find(entityCtor, { id: idOrWhere }) + : this.find(entityCtor, idOrWhere); + for (const t of targets) Object.assign(t, patch); + return { affected: targets.length }; + }, + getRepository: (entityCtor: unknown) => ({ + findOne: async (options: { where: { id: string } }) => this.find(entityCtor, { id: options.where.id })[0] ?? null, + }), + }; + + asDataSource(): DataSource { + return { + manager: this.manager, + // Routed by SQL text: the lines lookup and sendCompanyChannels' contact lookup share this + // one entry point in the real DataSource. + query: async (sql: string) => { + if (sql.includes("invoice_lines")) { + return [...this.invoices.keys()].flatMap((id) => LINES(id)); + } + return this.companyContact ? [this.companyContact] : []; + }, + transaction: async (body: (m: unknown) => Promise) => body(this.manager), + getRepository: () => ({ + find: async (options: { where: { id: { value: string[] } } }) => { + const ids = options.where.id.value ?? []; + return ids.map((id: string) => this.invoices.get(id)).filter(Boolean); + }, + createQueryBuilder: (alias: string) => { + void alias; + return this.queryBuilder(Invoice); + }, + count: async (options: { where: Record }) => this.find(Invoice, options.where).length, + }), + } as unknown as DataSource; + } +} + +const build = (db: FakeDb, postSigned: jest.Mock, directSend: jest.Mock = jest.fn().mockResolvedValue(undefined)) => + new EimsBulkRegistrationService( + db.asDataSource(), + { get: () => eimsConfig({ invoice: eimsInvoiceConfig() }) } as unknown as ConfigService, + { postSigned } as unknown as EimsClientService, + { getSessionContext: async () => ({ systemNumber: SYSTEM_NUMBER, systemType: "SYS" }) } as unknown as EimsAuthService, + { directSend } as unknown as NotificationsService, + { getSellerDetails: (c: unknown) => buildEimsSeller(c as never) } as unknown as EimsSellerCacheService, + ); + +const accepted = (conversationId = CONVERSATION_ID) => ({ conversationId, status: 202 }); + +describe("EimsBulkRegistrationService.registerBulk", () => { + it("reserves sequential counters, sends one signed array, and claims MoR's real conversation id", async () => { + const db = new FakeDb( + [invoiceRow(), invoiceRow({ id: INVOICE_B, invoiceNumber: "INV-20260807-00002" })], + { nextInvoiceCounter: 5, nextDocumentNumber: 5, previousIrn: "prev-irn" }, + ); + const postSigned = jest.fn().mockResolvedValue(accepted()); + + const result = await build(db, postSigned).registerBulk([INVOICE_A, INVOICE_B]); + + expect(result).toEqual({ conversationId: CONVERSATION_ID, accepted: [INVOICE_A, INVOICE_B], alreadyRegistered: [] }); + const [, request] = postSigned.mock.calls[0]; + expect(request).toHaveLength(2); + expect(request[0].SourceSystem.InvoiceCounter).toBe(5); + expect(request[0].DocumentDetails.DocumentNumber).toBe("5"); + expect(request[0].ReferenceDetails.PreviousIrn).toBe("prev-irn"); + expect(request[1].SourceSystem.InvoiceCounter).toBe(6); + // Only the first item in a bulk batch chains — the rest have no IRN to reference yet. + expect(request[1].ReferenceDetails.PreviousIrn).toBe(""); + + expect(db.invoices.get(INVOICE_A)).toMatchObject({ eimsStatus: EimsInvoiceStatus.Submitting, eimsBulkConversationId: CONVERSATION_ID }); + expect(db.invoices.get(INVOICE_B)).toMatchObject({ eimsStatus: EimsInvoiceStatus.Submitting, eimsBulkConversationId: CONVERSATION_ID }); + expect(db.state.nextInvoiceCounter).toBe(7); + expect(db.state.inFlightConversationId).toBe(CONVERSATION_ID); + }); + + it("skips an already-registered invoice, without consuming a counter for it", async () => { + const db = new FakeDb([ + invoiceRow({ eimsIrn: "already-irn", eimsStatus: EimsInvoiceStatus.Registered }), + invoiceRow({ id: INVOICE_B, invoiceNumber: "INV-20260807-00002" }), + ]); + const postSigned = jest.fn().mockResolvedValue(accepted()); + + const result = await build(db, postSigned).registerBulk([INVOICE_A, INVOICE_B]); + + expect(result.alreadyRegistered).toEqual([INVOICE_A]); + expect(result.accepted).toEqual([INVOICE_B]); + const [, request] = postSigned.mock.calls[0]; + expect(request).toHaveLength(1); + }); + + it("refuses the whole batch — no reservation, no HTTP call — when a DEB note has no registered original", async () => { + const db = new FakeDb([ + invoiceRow({ eimsDocumentType: "DEB", relatedInvoice: { eimsIrn: null, invoiceNumber: "INV-orig" } as never }), + ]); + const postSigned = jest.fn(); + + await expect(build(db, postSigned).registerBulk([INVOICE_A])).rejects.toBeInstanceOf(BadRequestException); + expect(postSigned).not.toHaveBeenCalled(); + expect(db.state.inFlightConversationId).toBeNull(); + }); + + it("refuses when a single-invoice submission is already in flight", async () => { + const db = new FakeDb([invoiceRow()], { inFlightInvoiceId: "some-other-invoice" }); + await expect(build(db, jest.fn()).registerBulk([INVOICE_A])).rejects.toBeInstanceOf(ConflictException); + }); + + it("refuses when another bulk batch is already in flight", async () => { + const db = new FakeDb([invoiceRow()], { inFlightConversationId: "other-conversation" }); + await expect(build(db, jest.fn()).registerBulk([INVOICE_A])).rejects.toBeInstanceOf(ConflictException); + }); + + it("a deterministic rejection rolls back the whole block and clears the in-flight marker", async () => { + const db = new FakeDb( + [invoiceRow(), invoiceRow({ id: INVOICE_B, invoiceNumber: "INV-20260807-00002" })], + { nextInvoiceCounter: 5, nextDocumentNumber: 5 }, + ); + const postSigned = jest.fn().mockRejectedValue(new EimsApiException("SCHEMA_VALIDATION", "bad", 400)); + + await expect(build(db, postSigned).registerBulk([INVOICE_A, INVOICE_B])).rejects.toBeInstanceOf(EimsApiException); + + expect(db.state.nextInvoiceCounter).toBe(5); + expect(db.state.nextDocumentNumber).toBe(5); + expect(db.state.inFlightConversationId).toBeNull(); + expect(db.invoices.get(INVOICE_A)?.eimsStatus).toBe(EimsInvoiceStatus.Failed); + expect(db.invoices.get(INVOICE_A)?.eimsBulkConversationId).toBeNull(); + }); + + it("an ambiguous failure blocks the system number and leaves counters consumed", async () => { + const db = new FakeDb([invoiceRow()], { nextInvoiceCounter: 5, nextDocumentNumber: 5 }); + const postSigned = jest.fn().mockRejectedValue(new EimsApiException("TIMEOUT", "timed out")); + + await expect(build(db, postSigned).registerBulk([INVOICE_A])).rejects.toBeInstanceOf(EimsApiException); + + expect(db.state.nextInvoiceCounter).toBe(6); + expect(db.state.blockedReason).toMatch(/never acknowledged/); + expect(db.invoices.get(INVOICE_A)?.eimsStatus).toBe(EimsInvoiceStatus.Unknown); + }); + + it("refuses an empty invoice list", async () => { + const db = new FakeDb([invoiceRow()]); + await expect(build(db, jest.fn()).registerBulk([])).rejects.toBeInstanceOf(BadRequestException); + }); +}); + +describe("EimsBulkRegistrationService.handleBulkCallback", () => { + const submittingRow = (over: Partial) => + invoiceRow({ + eimsStatus: EimsInvoiceStatus.Submitting, + eimsBulkConversationId: CONVERSATION_ID, + ...over, + }); + + it("settles a mixed success/error callback, advancing previousIrn to the last accepted item", async () => { + const db = new FakeDb( + [ + submittingRow({ eimsInvoiceCounter: 5, eimsDocumentNumber: "5" }), + submittingRow({ id: INVOICE_B, invoiceNumber: "INV-20260807-00002", eimsInvoiceCounter: 6, eimsDocumentNumber: "6" }), + ], + { inFlightConversationId: CONVERSATION_ID }, + ); + + const results = await build(db, jest.fn()).handleBulkCallback([ + { irn: "irn-a", status: "A", documentNumber: "5" }, + { ruleError: [{ portion: "DocumentDetails", errorMessage: ["bad date"] }], status: "ERROR", docNo: "6" }, + { conversionId: CONVERSATION_ID }, + ]); + + expect(db.invoices.get(INVOICE_A)).toMatchObject({ eimsStatus: EimsInvoiceStatus.Registered, eimsIrn: "irn-a" }); + expect(db.invoices.get(INVOICE_B)).toMatchObject({ eimsStatus: EimsInvoiceStatus.Failed }); + expect(db.state.previousIrn).toBe("irn-a"); + expect(db.state.inFlightConversationId).toBeNull(); + expect(results).toEqual( + expect.arrayContaining([ + expect.objectContaining({ invoiceId: INVOICE_A, success: true, irn: "irn-a" }), + expect.objectContaining({ invoiceId: INVOICE_B, success: false }), + ]), + ); + }); + + it("ignores a callback for an unknown or already-settled conversation", async () => { + const db = new FakeDb([invoiceRow()]); + const results = await build(db, jest.fn()).handleBulkCallback([ + { irn: "irn-x", status: "A", documentNumber: "1" }, + { conversionId: "no-such-conversation" }, + ]); + expect(results).toEqual([]); + }); + + it("does not clear the in-flight marker while another invoice in the batch is still submitting", async () => { + const db = new FakeDb( + [ + submittingRow({ eimsInvoiceCounter: 5, eimsDocumentNumber: "5" }), + submittingRow({ id: INVOICE_B, invoiceNumber: "INV-20260807-00002", eimsInvoiceCounter: 6, eimsDocumentNumber: "6" }), + ], + { inFlightConversationId: CONVERSATION_ID }, + ); + + // Callback only reports on one of the two invoices in this batch. + await build(db, jest.fn()).handleBulkCallback([ + { irn: "irn-a", status: "A", documentNumber: "5" }, + { conversionId: CONVERSATION_ID }, + ]); + + expect(db.state.inFlightConversationId).toBe(CONVERSATION_ID); + }); + + it("reports the current state without re-settling an invoice that already resolved", async () => { + const db = new FakeDb( + [ + invoiceRow({ + eimsStatus: EimsInvoiceStatus.Registered, + eimsIrn: "irn-a", + eimsDocumentNumber: "1", + eimsBulkConversationId: CONVERSATION_ID, + }), + ], + { inFlightConversationId: CONVERSATION_ID }, + ); + + const results = await build(db, jest.fn()).handleBulkCallback([ + { irn: "irn-a", status: "A", documentNumber: "1" }, + { conversionId: CONVERSATION_ID }, + ]); + + expect(results).toEqual([ + expect.objectContaining({ invoiceId: INVOICE_A, success: true, message: expect.stringContaining("Already settled") }), + ]); + }); +}); diff --git a/apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.ts b/apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.ts new file mode 100644 index 000000000..4638d3555 --- /dev/null +++ b/apps/edr-freight-api/src/modules/eims/eims-bulk-registration.service.ts @@ -0,0 +1,518 @@ +import { randomUUID } from "node:crypto"; +import { BadRequestException, ConflictException, Injectable, Logger, NotFoundException } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { InjectDataSource } from "@nestjs/typeorm"; +import { DataSource, EntityManager, In } from "typeorm"; +import type { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity.js"; + +import { EimsConfig } from "../../config/eims.config"; +import { Invoice } from "../billing/entities/invoice.entity"; +import { EimsDocumentType, EimsMapperLine, toEimsInvoice } from "../billing/eims-invoice.mapper"; +import { sendCompanyChannels } from "../notifications/notify-company.util"; +import { NotificationsService } from "../notifications/notifications.service"; +import { EimsAuthService } from "./eims-auth.service"; +import { EimsClientService } from "./eims-client.service"; +import { EimsApiException, EimsConfigException } from "./eims.errors"; +import { EimsSellerCacheService } from "./eims-seller-cache.service"; +import { EimsSystemState } from "./entities/eims-system-state.entity"; +import { assertEimsInvoiceConfig, buildEimsContext } from "./eims-invoice-context"; +import { + EimsBulkCallbackItem, + EimsBulkRegisterAcceptedResponse, + EimsBulkRegisterItemResult, + EimsBulkRegisterRequest, + EimsInvoiceError, + EimsInvoiceStatus, +} from "./eims-registration.types"; + +const DETERMINISTIC_KINDS = new Set(["SCHEMA_VALIDATION", "RULE_VALIDATION", "AUTH", "FORBIDDEN"]); + +interface BulkReservation { + stateId: string; + invoice: Invoice & { lines: EimsMapperLine[] }; + documentType: EimsDocumentType; + relatedDocument: string | null; + invoiceCounter: number; + documentNumber: string; + previousIrn: string; +} + +/** + * Registers many invoices with MoR EIMS in one call — `POST /v1/bulkRegister`. + * + * Fundamentally different shape from `EimsInvoiceRegistrationService.registerInvoiceWithEims`: + * that endpoint answers synchronously (an IRN or a rejection, in the HTTP response itself). Bulk + * does not — it returns only `{conversationId, status:202}` immediately, and the real per-invoice + * results (a mix of accepted/rejected in one array, per the collection's own examples) arrive later + * as a POST to a webhook MoR was configured with out of band. That means this service has two + * halves that don't share a call stack: `registerBulk` reserves and submits; `handleBulkCallback` + * — invoked by `EimsWebhookController`, whenever MoR gets around to it — settles. + * + * Reservation follows the same durable-reservation doctrine as the single-invoice service (counters + * consumed and the holder recorded, committed, before the HTTP call leaves the process), extended + * to a contiguous block of N counters instead of one. The "something is in flight" marker is + * `EimsSystemState.inFlightConversationId`, not `inFlightInvoiceId` — a whole batch is outstanding, + * not one invoice — and the two markers block each other: a single registration cannot start while + * a bulk batch is pending, and vice versa, because they share the same counter sequence. + * + * The conversation id is not known until MoR's 202 response arrives, so reservation stamps a + * locally-generated placeholder token first (same "commit the reservation before the network call" + * reasoning as the single flow), then swaps it for MoR's real conversation id right after — the only + * value the webhook callback can actually use to find this batch again. + * + * Not live-testable from this sandbox (no route to MoR's real gateway) — signing the whole array as + * one envelope, the way single `/v1/register` was confirmed live to need despite the collection's + * raw example showing no envelope, is the reasonable extension of that confirmed behavior, not a + * blind guess, but it has not itself been exercised against the real gateway. + */ +@Injectable() +export class EimsBulkRegistrationService { + private readonly logger = new Logger(EimsBulkRegistrationService.name); + + constructor( + @InjectDataSource() private readonly dataSource: DataSource, + private readonly config: ConfigService, + private readonly client: EimsClientService, + private readonly auth: EimsAuthService, + private readonly notifications: NotificationsService, + private readonly sellerCache: EimsSellerCacheService, + ) {} + + private get cfg(): EimsConfig { + return this.config.get("eims")!; + } + + /** + * Reserve counters for every eligible invoice and submit them as one `/v1/bulkRegister` call. + * An invoice that already has an IRN is silently skipped (idempotent, matching single register); + * everything else must pass the same DEB/CRE precondition single register checks, or the whole + * call is refused before anything is reserved. + */ + async registerBulk( + invoiceIds: string[], + ): Promise<{ conversationId: string | null; accepted: string[]; alreadyRegistered: string[] }> { + const cfg = this.cfg; + assertEimsInvoiceConfig(cfg); + + const ids = [...new Set(invoiceIds)]; + if (ids.length === 0) { + throw new BadRequestException({ code: "EIMS_BULK_EMPTY", message: "No invoice ids given" }); + } + + const invoices = await this.loadInvoicesForMapping(ids); + const alreadyRegistered = invoices.filter((inv) => inv.eimsIrn).map((inv) => inv.id); + const pending = invoices.filter((inv) => !inv.eimsIrn); + + // Same DEB/CRE precondition as single register, checked for every pending invoice before any + // counter is touched: a bad member must fail the whole batch, not surface mid-submission. + const prepared = pending.map((invoice) => { + const documentType = (invoice.eimsDocumentType as EimsDocumentType | undefined) ?? "INV"; + let relatedDocument: string | null = null; + if (documentType !== "INV") { + if (!invoice.relatedInvoice) { + throw new BadRequestException({ + code: "EIMS_RELATED_INVOICE_REQUIRED", + message: `Invoice ${invoice.invoiceNumber} is a ${documentType} but has no related invoice set.`, + }); + } + if (!invoice.relatedInvoice.eimsIrn) { + throw new BadRequestException({ + code: "EIMS_RELATED_INVOICE_NOT_REGISTERED", + message: `Invoice ${invoice.invoiceNumber} is a ${documentType} against invoice ${invoice.relatedInvoice.invoiceNumber}, which was never registered with EIMS — nothing to reference.`, + }); + } + relatedDocument = invoice.relatedInvoice.eimsIrn; + } + return { invoice, documentType, relatedDocument }; + }); + + if (prepared.length === 0) { + return { conversationId: null, accepted: [], alreadyRegistered }; + } + + const session = await this.auth.getSessionContext(); + const placeholder = `local:${randomUUID()}`; + const reservations = await this.reserveBulk(prepared, session.systemNumber, placeholder); + + let conversationId: string; + try { + const requests: EimsBulkRegisterRequest = reservations.map((r) => + toEimsInvoice( + r.invoice, + this.sellerCache.getSellerDetails(cfg), + buildEimsContext(cfg, { + documentNumber: r.documentNumber, + invoiceCounter: r.invoiceCounter, + previousIrn: r.previousIrn, + session, + documentType: r.documentType, + reason: r.invoice.eimsReason, + relatedDocument: r.relatedDocument, + }), + ), + ); + const response = await this.client.postSigned( + "/v1/bulkRegister", + requests, + ); + if (!response?.conversationId) { + throw new EimsApiException( + "SCHEMA_VALIDATION", + "EIMS bulkRegister returned no conversationId", + response?.status, + ); + } + conversationId = response.conversationId; + } catch (err) { + await this.settleBulkFailure(reservations, err); + throw err; + } + + await this.claimConversationId(placeholder, conversationId); + this.logger.log( + `Bulk-registered ${reservations.length} invoice(s) with EIMS (conversation ${conversationId}), awaiting callback`, + ); + return { conversationId, accepted: reservations.map((r) => r.invoice.id), alreadyRegistered }; + } + + /** + * Settle a batch's callback, whenever MoR gets around to sending it. Called by + * `EimsWebhookController` with the raw parsed array body — no auth on that route (MoR calls it, + * not a logged-in user), so the only thing standing between this and a forged callback is the + * conversation id itself: an item is only ever applied to an invoice actually holding that exact + * id, and an unknown id is logged and ignored rather than touching anything. + */ + async handleBulkCallback(items: EimsBulkCallbackItem[]): Promise { + const settlements = items.filter( + (item): item is Exclude => + "irn" in item || "ruleError" in item, + ); + + const conversationId = this.markerFrom(items); + const invoices = await this.dataSource + .getRepository(Invoice) + .createQueryBuilder("invoice") + .where("invoice.eims_bulk_conversation_id = :id", { id: conversationId }) + .getMany(); + + if (invoices.length === 0) { + this.logger.warn( + `EIMS bulk callback for an unknown or already-settled conversation — ignored (${settlements.length} item(s))`, + ); + return []; + } + + const byDocumentNumber = new Map(invoices.map((inv) => [inv.eimsDocumentNumber, inv])); + // Process in invoiceCounter order so `previousIrn` ends up as the last-accepted item's IRN — + // the same "advance the chain" semantics as single register's settleSuccess. + const ordered = [...settlements].sort((a, b) => { + const invA = byDocumentNumber.get("documentNumber" in a ? a.documentNumber : a.docNo); + const invB = byDocumentNumber.get("documentNumber" in b ? b.documentNumber : b.docNo); + return (invA?.eimsInvoiceCounter ?? 0) - (invB?.eimsInvoiceCounter ?? 0); + }); + + const results: EimsBulkRegisterItemResult[] = []; + for (const item of ordered) { + const docNumber = "documentNumber" in item ? item.documentNumber : item.docNo; + const invoice = byDocumentNumber.get(docNumber); + if (!invoice) { + this.logger.warn(`EIMS bulk callback item for unknown document number ${docNumber} — ignored`); + continue; + } + if (invoice.eimsStatus !== EimsInvoiceStatus.Submitting) { + // Already settled — a duplicate callback delivery. Report the current state, touch nothing. + results.push({ + invoiceId: invoice.id, + invoiceNumber: invoice.invoiceNumber, + success: invoice.eimsStatus === EimsInvoiceStatus.Registered, + message: `Already settled (${invoice.eimsStatus})`, + irn: invoice.eimsIrn ?? undefined, + }); + continue; + } + + if ("irn" in item) { + await this.settleBulkItemSuccess(invoice, item.irn, conversationId, item.signedQR); + results.push({ + invoiceId: invoice.id, + invoiceNumber: invoice.invoiceNumber, + success: true, + message: `Registered with EIMS (IRN ${item.irn})`, + irn: item.irn, + }); + } else { + const message = item.ruleError.flatMap((e) => e.errorMessage).join("; ") || "EIMS bulk rule validation error"; + await this.settleBulkItemFailure(invoice, message); + results.push({ invoiceId: invoice.id, invoiceNumber: invoice.invoiceNumber, success: false, message }); + } + } + + // Clear the batch's in-flight marker only once nothing submitted under this conversation is + // still waiting — a partial/incremental callback (not expected per the collection's docs, but + // not ruled out either) must not prematurely unblock the system number. + const stillPending = await this.dataSource + .getRepository(Invoice) + .count({ where: { eimsBulkConversationId: conversationId, eimsStatus: EimsInvoiceStatus.Submitting } }); + if (stillPending === 0) { + await this.dataSource.manager.update( + EimsSystemState, + { inFlightConversationId: conversationId }, + { inFlightConversationId: null }, + ); + this.logger.log(`EIMS bulk conversation ${conversationId} fully settled (${results.length} item(s))`); + } + + return results; + } + + // ── transactions ───────────────────────────────────────────────────────────────────────────── + + /** TX1. Reserve a contiguous block of N counters, one per invoice, in the given order. */ + private async reserveBulk( + prepared: Array<{ invoice: Invoice & { lines: EimsMapperLine[] }; documentType: EimsDocumentType; relatedDocument: string | null }>, + systemNumber: string, + placeholder: string, + ): Promise { + return this.dataSource.transaction(async (manager) => { + const state = await this.lockSystemState(manager, systemNumber); + + if (state.blockedReason) { + throw new ConflictException({ + code: "EIMS_SYSTEM_BLOCKED", + message: `EIMS registration is blocked for system ${systemNumber}: ${state.blockedReason}. Resolve the affected invoice before registering anything else.`, + }); + } + if (state.inFlightInvoiceId) { + throw new ConflictException({ + code: "EIMS_SUBMISSION_IN_FLIGHT", + message: `A submission for invoice ${state.inFlightInvoiceId} is already in flight on system ${systemNumber}. Wait for it to settle, or resolve it if the process was interrupted.`, + }); + } + if (state.inFlightConversationId) { + throw new ConflictException({ + code: "EIMS_BULK_IN_FLIGHT", + message: `A bulk submission (conversation ${state.inFlightConversationId}) is already in flight on system ${systemNumber}. Wait for its callback, or resolve it if the process was interrupted.`, + }); + } + + let counter = Number(state.nextInvoiceCounter); + let docNumber = Number(state.nextDocumentNumber); + let previousIrn = state.previousIrn ?? ""; + const reservations: BulkReservation[] = []; + + // Locked in the caller's given order — stable, avoids two concurrent bulk calls deadlocking + // on the opposite lock order. + for (const { invoice, documentType, relatedDocument } of prepared) { + const locked = await this.lockInvoice(manager, invoice.id); + const thisCounter = counter++; + const thisDocNumber = String(docNumber++); + const thisPreviousIrn = reservations.length === 0 ? previousIrn : ""; + + await manager.update(Invoice, invoice.id, { + eimsStatus: EimsInvoiceStatus.Submitting, + eimsInvoiceCounter: thisCounter, + eimsDocumentNumber: thisDocNumber, + eimsSubmittedAt: new Date(), + eimsLastError: null, + eimsBulkConversationId: placeholder, + } as QueryDeepPartialEntity); + + reservations.push({ + stateId: state.id, + invoice: Object.assign(locked, { lines: invoice.lines }), + documentType, + relatedDocument, + invoiceCounter: thisCounter, + documentNumber: thisDocNumber, + previousIrn: thisPreviousIrn, + }); + } + + await manager.update(EimsSystemState, state.id, { + nextInvoiceCounter: counter, + nextDocumentNumber: docNumber, + inFlightConversationId: placeholder, + }); + + return reservations; + }); + } + + /** Swap the local placeholder for MoR's real conversation id, on both the state row and every invoice. */ + private async claimConversationId(placeholder: string, conversationId: string): Promise { + await this.dataSource.transaction(async (manager) => { + await manager.update(EimsSystemState, { inFlightConversationId: placeholder }, { inFlightConversationId: conversationId }); + await manager.update(Invoice, { eimsBulkConversationId: placeholder }, { eimsBulkConversationId: conversationId }); + }); + } + + /** + * TX2b for the whole batch — the same determinism doctrine as single register's settleFailure, + * applied once since `/v1/bulkRegister` either accepts the whole array (202) or fails as one HTTP + * call; there is no per-item answer yet at this point, only after the callback. + */ + private async settleBulkFailure(reservations: BulkReservation[], err: unknown): Promise { + const api = err instanceof EimsApiException ? err : null; + const deterministic = api ? DETERMINISTIC_KINDS.has(api.kind) : true; + const status = deterministic ? EimsInvoiceStatus.Failed : EimsInvoiceStatus.Unknown; + const localKind = err instanceof EimsConfigException ? "CONFIG" : "LOCAL"; + const lastError: EimsInvoiceError = { + kind: api?.kind ?? localKind, + message: (err as Error)?.message ?? "unknown error", + httpStatus: api?.httpStatus, + details: api?.details, + at: new Date().toISOString(), + }; + const first = reservations[0]; + + await this.dataSource.transaction(async (manager) => { + for (const r of reservations) { + await manager.update(Invoice, r.invoice.id, { + eimsStatus: status, + eimsLastError: lastError, + ...(deterministic ? { eimsBulkConversationId: null } : {}), + } as QueryDeepPartialEntity); + } + await manager.update( + EimsSystemState, + first.stateId, + deterministic + ? { + // The whole block returns: MoR never counted a refused batch against either sequence. + nextInvoiceCounter: first.invoiceCounter, + nextDocumentNumber: Number(first.documentNumber), + inFlightConversationId: null, + } + : { + blockedReason: + `A bulk submission of ${reservations.length} invoice(s) (starting counter ${first.invoiceCounter}) ` + + `was sent but never acknowledged (${lastError.kind}). No further document can be filed until it is resolved.`, + }, + ); + }); + + this.logger.error(`EIMS bulk submission ${status}: ${lastError.message}`); + } + + /** One callback item accepted. */ + private async settleBulkItemSuccess( + invoice: Invoice, + irn: string, + conversationId: string, + signedQR?: string, + ): Promise { + await this.dataSource.transaction(async (manager) => { + await this.lockInvoice(manager, invoice.id); + await manager.update(Invoice, invoice.id, { + eimsStatus: EimsInvoiceStatus.Registered, + eimsIrn: irn, + eimsSignedQr: signedQR ?? null, + eimsLastError: null, + }); + // Looked up by conversation id, not system number — this batch's state row is whichever one + // is holding this conversation, which is exactly what `inFlightConversationId` already tracks. + await manager.update(EimsSystemState, { inFlightConversationId: conversationId }, { previousIrn: irn }); + }); + this.logger.log(`Invoice ${invoice.invoiceNumber} registered with EIMS via bulk (IRN ${irn})`); + + if (invoice.companyId) { + try { + await sendCompanyChannels( + this.dataSource, + this.notifications, + invoice.companyId, + `Invoice ${invoice.invoiceNumber} has been registered with MoR EIMS. Reference (IRN): ${irn}`, + ); + } catch (err) { + this.logger.warn(`EIMS buyer notification failed for invoice ${invoice.id}: ${(err as Error).message}`); + } + } + } + + /** + * One callback item rejected. Unlike single register's settleFailure, the counter/document + * number are not returned — MoR's own bulk processing already advanced the whole array's + * allocation regardless of this item's individual outcome, so there is nothing local to roll back. + */ + private async settleBulkItemFailure(invoice: Invoice, message: string): Promise { + const lastError: EimsInvoiceError = { kind: "RULE_VALIDATION", message, at: new Date().toISOString() }; + await this.dataSource.manager.update(Invoice, invoice.id, { + eimsStatus: EimsInvoiceStatus.Failed, + eimsLastError: lastError, + } as QueryDeepPartialEntity); + this.logger.error(`Invoice ${invoice.invoiceNumber} EIMS bulk registration FAILED: ${message}`); + } + + // ── internals ──────────────────────────────────────────────────────────────────────────────── + + private markerFrom(items: EimsBulkCallbackItem[]): string { + const marker = items.find((i) => "conversationId" in i || "conversionId" in i) as + | { conversationId?: string; conversionId?: string } + | undefined; + return marker?.conversationId ?? marker?.conversionId ?? ""; + } + + + private async lockInvoice(manager: EntityManager, invoiceId: string): Promise { + const invoice = await manager + .createQueryBuilder(Invoice, "invoice") + .setLock("pessimistic_write") + .where("invoice.id = :invoiceId", { invoiceId }) + .getOne(); + if (!invoice) throw new NotFoundException(`Invoice ${invoiceId} not found`); + return invoice; + } + + private async lockSystemState(manager: EntityManager, systemNumber: string): Promise { + const select = () => + manager + .createQueryBuilder(EimsSystemState, "state") + .setLock("pessimistic_write") + .where("state.system_number = :systemNumber", { systemNumber }) + .getOne(); + + const existing = await select(); + if (existing) return existing; + + await manager.query( + `INSERT INTO freight.eims_system_state (system_number) VALUES ($1) ON CONFLICT (system_number) DO NOTHING`, + [systemNumber], + ); + const created = await select(); + if (!created) throw new Error(`Could not initialise EIMS system state for ${systemNumber}`); + return created; + } + + private async loadInvoicesForMapping(invoiceIds: string[]): Promise> { + const invoices = await this.dataSource.getRepository(Invoice).find({ + where: { id: In(invoiceIds) }, + relations: { company: true, companyProfile: true, relatedInvoice: true }, + }); + const found = new Set(invoices.map((inv) => inv.id)); + const missing = invoiceIds.filter((id) => !found.has(id)); + if (missing.length > 0) { + throw new NotFoundException(`Invoice(s) not found: ${missing.join(", ")}`); + } + + const lines: Array = await this.dataSource.query( + `SELECT invoice_id AS "invoiceId", charge_type AS "chargeType", description, quantity, + unit_rate AS "unitRate", amount, currency, metadata + FROM freight.invoice_lines + WHERE invoice_id = ANY($1) AND deleted_at IS NULL + ORDER BY created_at ASC`, + [invoiceIds], + ); + const linesByInvoice = new Map(); + for (const line of lines) { + const { invoiceId, ...rest } = line; + if (!linesByInvoice.has(invoiceId)) linesByInvoice.set(invoiceId, []); + linesByInvoice.get(invoiceId)!.push(rest); + } + + // Preserve the caller's given order — reservation and result ordering both depend on it. + return invoiceIds.map((id) => { + const invoice = invoices.find((inv) => inv.id === id)!; + return Object.assign(invoice, { lines: linesByInvoice.get(id) ?? [] }); + }); + } +} diff --git a/apps/edr-freight-api/src/modules/eims/eims-invoice.controller.ts b/apps/edr-freight-api/src/modules/eims/eims-invoice.controller.ts index 11eb214dc..db28eeafe 100644 --- a/apps/edr-freight-api/src/modules/eims/eims-invoice.controller.ts +++ b/apps/edr-freight-api/src/modules/eims/eims-invoice.controller.ts @@ -6,10 +6,12 @@ import { BookingStaff } from "../../common/booking-guards"; import { FREIGHT_PERMS } from "../../seed/freight-permissions.registry"; import { sendPdf } from "../billing/billing.controller"; import { BulkCancelEimsRegistrationDto } from "./dto/bulk-cancel-eims-registration.dto"; +import { BulkRegisterEimsInvoiceDto } from "./dto/bulk-register-eims-invoice.dto"; import { CancelEimsRegistrationDto } from "./dto/cancel-eims-registration.dto"; import { RegisterSalesReceiptDto } from "./dto/register-sales-receipt.dto"; import { RegisterWithholdingReceiptDto } from "./dto/register-withholding-receipt.dto"; import { ResolveEimsRegistrationDto } from "./dto/resolve-eims-registration.dto"; +import { EimsBulkRegistrationService } from "./eims-bulk-registration.service"; import { EimsCancellationService } from "./eims-cancellation.service"; import { EimsInvoiceRegistrationService } from "./eims-invoice-registration.service"; import { EimsReceiptService } from "./eims-receipt.service"; @@ -39,6 +41,7 @@ import { EimsReceiptService } from "./eims-receipt.service"; export class EimsInvoiceController { constructor( private readonly registration: EimsInvoiceRegistrationService, + private readonly bulkRegistration: EimsBulkRegistrationService, private readonly cancellation: EimsCancellationService, private readonly receipts: EimsReceiptService, ) {} @@ -53,6 +56,18 @@ export class EimsInvoiceController { return this.registration.registerInvoiceWithEims(id); } + @Post("eims/bulk-register") + @BookingStaff(FREIGHT_PERMS.invoices.eimsRegister) + @ApiOperation({ + summary: + "Submit multiple invoices to MoR EIMS in one call. Asynchronous — this only confirms MoR " + + "accepted the batch (a conversation id), not the per-invoice outcome. Real results (IRN or " + + "rejection per invoice) arrive later via MoR's own callback; poll GET :id/eims/status.", + }) + bulkRegister(@Body() dto: BulkRegisterEimsInvoiceDto) { + return this.bulkRegistration.registerBulk(dto.invoiceIds); + } + @Post(":id/eims/verify") @BookingStaff(FREIGHT_PERMS.invoices.eimsRegister) @ApiOperation({ summary: "Verify the invoice's stored IRN against EIMS" }) diff --git a/apps/edr-freight-api/src/modules/eims/eims-registration.types.ts b/apps/edr-freight-api/src/modules/eims/eims-registration.types.ts index 60941825c..7cc58c0a9 100644 --- a/apps/edr-freight-api/src/modules/eims/eims-registration.types.ts +++ b/apps/edr-freight-api/src/modules/eims/eims-registration.types.ts @@ -1,3 +1,4 @@ +import { EimsInvoiceRequest } from "../billing/eims-invoice.mapper"; import { EimsErrorResponse } from "./eims.types"; /** @@ -129,6 +130,62 @@ export interface EimsBulkCancelItemResult { message: string; } +/** + * `POST /v1/bulkRegister` — same array-of-full-documents shape as single register (`EimsInvoiceRequest` + * from `eims-invoice.mapper.ts`), one element per invoice, sent as one signed envelope. + */ +export type EimsBulkRegisterRequest = EimsInvoiceRequest[]; + +/** + * Immediate response to `bulkRegister` — unlike single register, this is not the result, just an + * acknowledgement. The real per-invoice outcomes arrive later via `EimsBulkCallbackItem`s pushed to + * a webhook MoR was configured with out of band (see `EimsBulkRegistrationService`). + */ +export interface EimsBulkRegisterAcceptedResponse { + conversationId: string; + status: number; +} + +/** A settled item in the async callback — `irn` present means MoR accepted this document. */ +export interface EimsBulkCallbackSuccessItem { + irn: string; + status: string; + documentNumber: string; + signedQR?: string; + signedInvoice?: string; +} + +/** A rejected item in the async callback — `docNo` echoes what we submitted as `DocumentNumber`. */ +export interface EimsBulkCallbackErrorItem { + ruleError: Array<{ portion: string; errorMessage: string[] }>; + status: string; + docNo: string; +} + +/** + * The callback array's last element, per the collection's own examples — never a settlement result, + * just the batch id echoed back. Spelled two different ways across the collection's own docs + * ("conversationId" on the initial 202, "conversionId" in the callback examples); accept both. + */ +export interface EimsBulkCallbackMarker { + conversationId?: string; + conversionId?: string; +} + +export type EimsBulkCallbackItem = + | EimsBulkCallbackSuccessItem + | EimsBulkCallbackErrorItem + | EimsBulkCallbackMarker; + +/** One invoice's outcome once a bulk batch's callback has been processed. */ +export interface EimsBulkRegisterItemResult { + invoiceId: string; + invoiceNumber: string; + success: boolean; + message: string; + irn?: string; +} + /** Persisted failure detail. Carries the gateway's own error fields only — never our envelope. */ export interface EimsInvoiceError { kind: string; diff --git a/apps/edr-freight-api/src/modules/eims/eims-webhook.controller.ts b/apps/edr-freight-api/src/modules/eims/eims-webhook.controller.ts new file mode 100644 index 000000000..e2b3ab49e --- /dev/null +++ b/apps/edr-freight-api/src/modules/eims/eims-webhook.controller.ts @@ -0,0 +1,25 @@ +import { Body, Controller, HttpCode, Post } from "@nestjs/common"; +import { ApiExcludeController } from "@nestjs/swagger"; +import { Public } from "@edr/api-common"; +import { EimsBulkCallbackItem } from "./eims-registration.types"; +import { EimsBulkRegistrationService } from "./eims-bulk-registration.service"; + +/** + * MoR's own callback for `POST /v1/bulkRegister`, not a route a person calls. Public — MoR has no + * JWT to send — so the conversation id embedded in the payload is the only thing standing between + * this and a forged callback: `EimsBulkRegistrationService.handleBulkCallback` only ever touches + * invoices actually holding that exact id, and an unrecognised one is logged and ignored. See the + * "Callback Mechanism" section of the collection's own docs for the payload shape. + */ +@ApiExcludeController() +@Controller("eims/webhook") +export class EimsWebhookController { + constructor(private readonly bulk: EimsBulkRegistrationService) {} + + @Public() + @Post("bulk-register") + @HttpCode(200) + bulkRegisterCallback(@Body() items: EimsBulkCallbackItem[]) { + return this.bulk.handleBulkCallback(items); + } +} diff --git a/apps/edr-freight-api/src/modules/eims/eims.module.ts b/apps/edr-freight-api/src/modules/eims/eims.module.ts index 5d55a7ee8..739bf7089 100644 --- a/apps/edr-freight-api/src/modules/eims/eims.module.ts +++ b/apps/edr-freight-api/src/modules/eims/eims.module.ts @@ -9,6 +9,7 @@ import { NotificationInboxModule } from "../notification-inbox/notification-inbo import { NotificationsModule } from "../notifications/notifications.module"; import { EimsAuthService } from "./eims-auth.service"; import { EimsAutoSubmitService } from "./eims-auto-submit.service"; +import { EimsBulkRegistrationService } from "./eims-bulk-registration.service"; import { EimsCancellationService } from "./eims-cancellation.service"; import { EimsClientService } from "./eims-client.service"; import { EimsCredentialsProvider } from "./eims-credentials.provider"; @@ -17,6 +18,7 @@ import { EimsInvoiceRegistrationService } from "./eims-invoice-registration.serv import { EimsReceiptService } from "./eims-receipt.service"; import { EimsSellerCacheService } from "./eims-seller-cache.service"; import { EimsSignerService } from "./eims-signer.service"; +import { EimsWebhookController } from "./eims-webhook.controller"; import { EimsReceipt } from "./entities/eims-receipt.entity"; import { EimsSystemState } from "./entities/eims-system-state.entity"; @@ -40,13 +42,14 @@ import { EimsSystemState } from "./entities/eims-system-state.entity"; // so this stays a plain one-directional import, not a new cycle. CompaniesModule, ], - controllers: [EimsInvoiceController], + controllers: [EimsInvoiceController, EimsWebhookController], providers: [ EimsCredentialsProvider, EimsSignerService, EimsAuthService, EimsClientService, EimsInvoiceRegistrationService, + EimsBulkRegistrationService, EimsAutoSubmitService, EimsCancellationService, EimsReceiptService, @@ -56,6 +59,7 @@ import { EimsSystemState } from "./entities/eims-system-state.entity"; EimsAuthService, EimsClientService, EimsInvoiceRegistrationService, + EimsBulkRegistrationService, EimsCancellationService, EimsReceiptService, ], diff --git a/apps/edr-freight-api/src/modules/eims/entities/eims-system-state.entity.ts b/apps/edr-freight-api/src/modules/eims/entities/eims-system-state.entity.ts index 328aa8d0f..67b5d74d4 100644 --- a/apps/edr-freight-api/src/modules/eims/entities/eims-system-state.entity.ts +++ b/apps/edr-freight-api/src/modules/eims/entities/eims-system-state.entity.ts @@ -54,4 +54,11 @@ export class EimsSystemState extends BaseEntity { */ @Column({ name: "blocked_reason", type: "text", nullable: true }) blockedReason?: string | null; + + /** + * Bulk equivalent of `in_flight_invoice_id` — a whole batch, not one invoice, is outstanding + * while MoR processes `POST /v1/bulkRegister` asynchronously. See `EimsBulkRegistrationService`. + */ + @Column({ name: "in_flight_conversation_id", type: "text", nullable: true }) + inFlightConversationId?: string | null; } From f93b3cd9932837ff70d60733844fa2ccc7788b61 Mon Sep 17 00:00:00 2001 From: Stephanos A Date: Thu, 20 Aug 2026 07:46:15 +0300 Subject: [PATCH 08/66] feat: add seat conflict assertion for route bookings --- .../src/modules/bookings/bookings.service.ts | 68 +++++++++++++++ .../modules/bookings/guest-booking.service.ts | 1 + .../src/modules/seats/seats.service.spec.ts | 86 +++++++++++++++++++ .../src/modules/seats/seats.service.ts | 59 +++++++++++++ 4 files changed, 214 insertions(+) diff --git a/apps/edr-passenger-api/src/modules/bookings/bookings.service.ts b/apps/edr-passenger-api/src/modules/bookings/bookings.service.ts index 7f0a6eee5..6330c8cc6 100644 --- a/apps/edr-passenger-api/src/modules/bookings/bookings.service.ts +++ b/apps/edr-passenger-api/src/modules/bookings/bookings.service.ts @@ -13,6 +13,7 @@ import { FareEngineService } from '../fare-engine/fare-engine.service'; import { PaymentsService } from '../payments/payments.service'; import { MAX_PAYMENT_HOURS } from '../../common/utils/payment-deadline.utils'; import { Currency, PassengerCategory, IdDocumentType } from '@prisma/client'; +import { JourneyDirection } from '../seats/seats.dto'; import { resolveCurrencyFromNationality } from '../fare-engine/fare-engine.dto'; import { DeleteOperationException } from '../../common/exceptions/delete-operation.exception'; import { AuditService } from '../../common/audit.service'; @@ -856,6 +857,14 @@ export class BookingsService { const destStop = schedule.stopTimes.find(s => s.stationId === dto.destinationStationId); if (!originStop || !destStop) throw new NotFoundException('Origin or destination not found'); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.scheduleId, + seatIds: requestedSeatIds, + originStationId: dto.originStationId, + destinationStationId: dto.destinationStationId, + journeyDirection: JourneyDirection.ONE_WAY, + }); + const [passengersData, iamContact] = await Promise.all([ this.processPassengers(dto.passengers as any[]), this.resolveIamContact(dto.passengerId), @@ -1052,6 +1061,21 @@ export class BookingsService { throw new NotFoundException('Origin or destination stops not found'); } + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.scheduleId, + seatIds: holdObSeatIds, + originStationId: dto.originStationId, + destinationStationId: dto.destinationStationId, + journeyDirection: JourneyDirection.OUTBOUND, + }); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.returnScheduleId, + seatIds: holdRetSeatIds, + originStationId: dto.returnOriginStationId, + destinationStationId: dto.returnDestinationStationId, + journeyDirection: JourneyDirection.RETURN, + }); + const [passengersData, iamContact] = await Promise.all([ this.processRoundTripPassengers(dto.passengers as any[]), this.resolveIamContact(dto.passengerId), @@ -1295,6 +1319,21 @@ export class BookingsService { if (!leg1OriginStop || !leg1DestStop) throw new NotFoundException('Leg-1 origin or transit station not found on schedule'); if (!leg2OriginStop || !leg2DestStop) throw new NotFoundException('Transit or leg-2 destination station not found on leg-2 schedule'); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.scheduleId, + seatIds: leg1SeatIds, + originStationId: dto.originStationId, + destinationStationId: dto.transitStationId, + journeyDirection: JourneyDirection.ONE_WAY, + }); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.leg2ScheduleId, + seatIds: leg2SeatIds, + originStationId: dto.transitStationId, + destinationStationId: dto.leg2DestinationStationId, + journeyDirection: JourneyDirection.ONE_WAY, + }); + const [passengersData, iamContact] = await Promise.all([ this.processPassengers(dto.passengers as any[]), this.resolveIamContact(dto.passengerId), @@ -1490,6 +1529,35 @@ export class BookingsService { if (!retL1Origin || !retL1Dest) throw new NotFoundException('Return leg-1: origin or transit station not found'); if (!retL2Origin || !retL2Dest) throw new NotFoundException('Return leg-2: transit or destination not found'); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.scheduleId, + seatIds: (dto.passengers as any[]).map(p => p.seatId), + originStationId: dto.originStationId, + destinationStationId: dto.transitStationId, + journeyDirection: JourneyDirection.OUTBOUND, + }); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.leg2ScheduleId, + seatIds: (dto.passengers as any[]).map(p => p.leg2SeatId ?? p.seatId), + originStationId: dto.transitStationId, + destinationStationId: dto.leg2DestinationStationId, + journeyDirection: JourneyDirection.OUTBOUND, + }); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.returnScheduleId, + seatIds: (dto.passengers as any[]).map(p => p.returnSeatId), + originStationId: dto.returnOriginStationId, + destinationStationId: dto.returnTransitStationId, + journeyDirection: JourneyDirection.RETURN, + }); + await this.seatsService.assertNoRouteSeatConflict({ + scheduleId: dto.returnLeg2ScheduleId, + seatIds: (dto.passengers as any[]).map(p => p.returnLeg2SeatId ?? p.returnSeatId), + originStationId: dto.returnTransitStationId, + destinationStationId: dto.returnLeg2DestinationStationId, + journeyDirection: JourneyDirection.RETURN, + }); + const [passengersData, iamContact] = await Promise.all([ this.processRoundTripPassengers(dto.passengers as any[]), this.resolveIamContact(dto.passengerId), diff --git a/apps/edr-passenger-api/src/modules/bookings/guest-booking.service.ts b/apps/edr-passenger-api/src/modules/bookings/guest-booking.service.ts index c8061184f..4b3a9a53d 100644 --- a/apps/edr-passenger-api/src/modules/bookings/guest-booking.service.ts +++ b/apps/edr-passenger-api/src/modules/bookings/guest-booking.service.ts @@ -13,6 +13,7 @@ import { PaymentsService } from '../payments/payments.service'; import { SmsClientService } from '../notifications/sms-client.service'; import { CreateGuestBookingDto, SavedPassengerProfileDto, IssueReservationBookingDto, ReservationBookingKind } from './guest-booking.dto'; import { Currency, PassengerCategory, IdDocumentType, PaymentMethodType, PaymentIntentStatus } from '@prisma/client'; +import { JourneyDirection } from '../seats/seats.dto'; import { resolveCheckinCutoff } from '../../common/utils/checkin-cutoff.utils'; import { computePaymentDeadline } from '../../common/utils/payment-deadline.utils'; import { randomUUID } from 'crypto'; diff --git a/apps/edr-passenger-api/src/modules/seats/seats.service.spec.ts b/apps/edr-passenger-api/src/modules/seats/seats.service.spec.ts index 2c4215dc8..e80966d6a 100644 --- a/apps/edr-passenger-api/src/modules/seats/seats.service.spec.ts +++ b/apps/edr-passenger-api/src/modules/seats/seats.service.spec.ts @@ -2,16 +2,47 @@ import { Test, TestingModule } from '@nestjs/testing'; import { SeatsService } from './seats.service'; import { PrismaService } from '../../common/prisma.service'; import { ConflictException } from '@nestjs/common'; +import { SegmentsService } from '../segments/segments.service'; +import { SystemConfigService } from '../system-config/system-config.service'; +import { AuditService } from '../../common/audit.service'; +import { SmsClientService } from '../notifications/sms-client.service'; +import { JourneyDirection } from './seats.dto'; describe('SeatsService - Auto Assign', () => { let service: SeatsService; let prisma: PrismaService; + let segmentsService: SegmentsService; const mockPrisma = { seat: { findMany: jest.fn(), updateMany: jest.fn(), }, + tripStopTime: { + findMany: jest.fn(), + }, + trainSchedule: { + findUnique: jest.fn(), + }, + seatBlock: { + findMany: jest.fn(), + }, + }; + + const mockSegmentsService = { + getSeatAvailabilityMap: jest.fn(), + }; + + const mockSystemConfigService = { + getNumber: jest.fn().mockResolvedValue(30), + }; + + const mockAuditService = { + log: jest.fn(), + }; + + const mockSmsClientService = { + send: jest.fn(), }; beforeEach(async () => { @@ -19,12 +50,67 @@ describe('SeatsService - Auto Assign', () => { providers: [ SeatsService, { provide: PrismaService, useValue: mockPrisma }, + { provide: SegmentsService, useValue: mockSegmentsService }, + { provide: SystemConfigService, useValue: mockSystemConfigService }, + { provide: AuditService, useValue: mockAuditService }, + { provide: SmsClientService, useValue: mockSmsClientService }, ], }).compile(); service = module.get(SeatsService); prisma = module.get(PrismaService); + segmentsService = module.get(SegmentsService); jest.clearAllMocks(); + + mockPrisma.trainSchedule.findUnique.mockResolvedValue({ + originStationId: 'origin-station', + destinationStationId: 'destination-station', + }); + mockPrisma.tripStopTime.findMany.mockResolvedValue([ + { stationId: 'origin-station', sequence: 0 }, + { stationId: 'destination-station', sequence: 1 }, + ]); + mockPrisma.seatBlock.findMany.mockResolvedValue([]); + mockSegmentsService.getSeatAvailabilityMap.mockResolvedValue(new Map()); + }); + + describe('assertNoRouteSeatConflict', () => { + it('rejects overlapping origin-destination seat reuse on the same schedule', async () => { + mockPrisma.tripStopTime.findMany.mockResolvedValue([ + { stationId: 'seb', sequence: 1 }, + { stationId: 'ada', sequence: 2 }, + { stationId: 'dd', sequence: 3 }, + ]); + mockSegmentsService.getSeatAvailabilityMap.mockResolvedValue(new Map([['seat-1', 'BOOKED']])); + mockPrisma.seat.findMany.mockResolvedValue([ + { id: 'seat-1', seatNumber: '12', coach: { number: 'C1' } }, + ]); + + await expect(service.assertNoRouteSeatConflict({ + scheduleId: 'sched-1', + seatIds: ['seat-1'], + originStationId: 'seb', + destinationStationId: 'dd', + journeyDirection: JourneyDirection.ONE_WAY, + })).rejects.toThrow(ConflictException); + }); + + it('allows the same seat number to be reused on non-overlapping legs', async () => { + mockPrisma.tripStopTime.findMany.mockResolvedValue([ + { stationId: 'seb', sequence: 1 }, + { stationId: 'ada', sequence: 2 }, + { stationId: 'dd', sequence: 3 }, + ]); + mockSegmentsService.getSeatAvailabilityMap.mockResolvedValue(new Map()); + + await expect(service.assertNoRouteSeatConflict({ + scheduleId: 'sched-1', + seatIds: ['seat-1'], + originStationId: 'seb', + destinationStationId: 'ada', + journeyDirection: JourneyDirection.ONE_WAY, + })).resolves.toBeUndefined(); + }); }); describe('autoAssignSeats', () => { diff --git a/apps/edr-passenger-api/src/modules/seats/seats.service.ts b/apps/edr-passenger-api/src/modules/seats/seats.service.ts index 383f7d96c..83dc6aea4 100644 --- a/apps/edr-passenger-api/src/modules/seats/seats.service.ts +++ b/apps/edr-passenger-api/src/modules/seats/seats.service.ts @@ -209,6 +209,57 @@ export class SeatsService { // (availabilityByClass) use — so the seatmap and search results can never disagree // about seat availability again. Previously this method carried its own // separately-written copy of the same hold/JourneySegment-overlap logic. + async assertNoRouteSeatConflict(args: { + scheduleId: string; + seatIds: string[]; + originStationId?: string; + destinationStationId?: string; + journeyDirection?: JourneyDirection; + }): Promise { + const { scheduleId, seatIds, originStationId, destinationStationId, journeyDirection = JourneyDirection.ONE_WAY } = args; + if (!seatIds.length) return; + + const stopTimes = await this.prisma.tripStopTime.findMany({ + where: { scheduleId }, + select: { stationId: true, sequence: true }, + }); + + let reqFrom = -Infinity; + let reqTo = Infinity; + if (originStationId && destinationStationId) { + const seqOf = (stationId: string) => stopTimes.find(s => s.stationId === stationId)?.sequence; + const resolvedFrom = seqOf(originStationId); + const resolvedTo = seqOf(destinationStationId); + if (resolvedFrom !== undefined && resolvedTo !== undefined) { + reqFrom = resolvedFrom; + reqTo = resolvedTo; + } + } + + const availability = await this.segmentsService.getSeatAvailabilityMap( + scheduleId, + seatIds, + stopTimes, + reqFrom, + reqTo, + journeyDirection, + ); + + if (availability.size === 0) return; + + const seats = await this.prisma.seat.findMany({ + where: { id: { in: seatIds } }, + select: { id: true, seatNumber: true }, + }); + const byId = new Map(seats.map(seat => [seat.id, seat.seatNumber])); + const conflicts = seatIds.filter(id => availability.has(id)); + + if (conflicts.length > 0) { + const labels = conflicts.map(id => byId.get(id) ?? id).join(', '); + throw new ConflictException(`Seat(s) ${labels} are already assigned for this route on this schedule`); + } + } + async resolveEffectiveStatuses( scheduleId: string, seatIds: string[], @@ -424,6 +475,14 @@ export class SeatsService { throw new BadRequestException('Origin must come before destination'); const currentDirection = dto.journeyDirection || JourneyDirection.ONE_WAY; + await this.assertNoRouteSeatConflict({ + scheduleId: dto.scheduleId, + seatIds, + originStationId: dto.originStationId, + destinationStationId: dto.destinationStationId, + journeyDirection: currentDirection, + }); + const activeHolds = await tx.seatHold.findMany({ where: { scheduleId: dto.scheduleId, expiresAt: { gt: new Date() } }, select: { seatIds: true, createdBy: true }, From fc2b5ee0e3d8b394704633e84192262d237898a4 Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 05:16:15 +0000 Subject: [PATCH 09/66] refactor(reports): export through the shared tabular writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the writer extraction whose other half landed in fb21ad154. reports.controller now builds a TabularDoc and calls TabularExportService, so report-export.service.ts and report-export-request.util.ts are dead and removed — HEAD was carrying both copies with the controller still on the old one. Reports gain CSV for free, and the PDF path now passes buildTabularFallbackPdf as its fallback: previously it passed none, so a box without Chromium silently returned PdfRenderService's ~900-character generic text dump instead of a table. Adds a spec covering the CSV writer's quoting of embedded commas and double quotes — the reason this uses ExcelJS's csv writer rather than a hand-rolled join. --- .../exports/tabular-export.service.spec.ts | 65 ++++++++++ .../report-export-request.util.spec.ts | 62 ---------- .../reports/report-export-request.util.ts | 29 ----- .../modules/reports/report-export.service.ts | 117 ------------------ .../src/modules/reports/reports.controller.ts | 40 +++--- .../src/modules/reports/reports.module.ts | 9 +- 6 files changed, 95 insertions(+), 227 deletions(-) create mode 100644 apps/edr-freight-api/src/modules/exports/tabular-export.service.spec.ts delete mode 100644 apps/edr-freight-api/src/modules/reports/report-export-request.util.spec.ts delete mode 100644 apps/edr-freight-api/src/modules/reports/report-export-request.util.ts delete mode 100644 apps/edr-freight-api/src/modules/reports/report-export.service.ts diff --git a/apps/edr-freight-api/src/modules/exports/tabular-export.service.spec.ts b/apps/edr-freight-api/src/modules/exports/tabular-export.service.spec.ts new file mode 100644 index 000000000..7f262b48e --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/tabular-export.service.spec.ts @@ -0,0 +1,65 @@ +import { PdfRenderService } from '../billing/documents/pdf-render.service'; +import { TabularDoc, TabularExportService } from './tabular-export.service'; + +/** The PDF path is puppeteer-backed; these specs only cover the sheet writers. */ +const service = new TabularExportService(null as unknown as PdfRenderService); + +const doc: TabularDoc = { + title: 'Bookings', + description: 'every booking', + label: 'test', + columns: [ + { key: 'ref', label: 'Reference', type: 'string' }, + { key: 'customer', label: 'Customer', type: 'string' }, + { key: 'amount', label: 'Amount', type: 'money' }, + { key: 'gov', label: 'Government', type: 'boolean' }, + ], + rows: [ + { ref: 'BK-1', customer: 'Acme, Inc.', amount: 1234.5, gov: true }, + { ref: 'BK-2', customer: 'Quote "Q" Ltd', amount: null, gov: false }, + ], + kpis: [{ label: 'Bookings', value: 2 }], +}; + +describe('TabularExportService.toCsv', () => { + it('quotes a value containing the delimiter — the reason we do not hand-roll join(",")', async () => { + const csv = (await service.toCsv(doc)).toString('utf8'); + expect(csv).toContain('"Acme, Inc."'); + }); + + it('escapes embedded double quotes by doubling them', async () => { + const csv = (await service.toCsv(doc)).toString('utf8'); + expect(csv).toContain('"Quote ""Q"" Ltd"'); + }); + + it('starts at the header row — no KPI preamble, so the file parses as a plain table', async () => { + const csv = (await service.toCsv(doc)).toString('utf8'); + expect(csv.split('\n')[0]).toBe('Reference,Customer,Amount,Government'); + expect(csv).not.toContain('Bookings: 2'); + }); + + it('emits one line per row plus the header', async () => { + const csv = (await service.toCsv(doc)).toString('utf8'); + expect(csv.trim().split('\n').filter(Boolean)).toHaveLength(3); + }); + + it('only the selected columns are written, in the order given', async () => { + const csv = ( + await service.toCsv({ ...doc, columns: [doc.columns[2], doc.columns[0]] }) + ).toString('utf8'); + expect(csv.split('\n')[0]).toBe('Amount,Reference'); + }); +}); + +describe('TabularExportService.toXlsx', () => { + it('writes a real xlsx (a zip, so it starts with the PK magic bytes)', async () => { + const buffer = await service.toXlsx(doc); + expect(buffer.subarray(0, 2).toString('utf8')).toBe('PK'); + expect(buffer.length).toBeGreaterThan(1000); + }); + + it('a title longer than Excel\'s 31-char sheet-name limit does not throw', async () => { + const longTitle = 'A'.repeat(60); + await expect(service.toXlsx({ ...doc, title: longTitle })).resolves.toBeInstanceOf(Buffer); + }); +}); diff --git a/apps/edr-freight-api/src/modules/reports/report-export-request.util.spec.ts b/apps/edr-freight-api/src/modules/reports/report-export-request.util.spec.ts deleted file mode 100644 index dc6fa6230..000000000 --- a/apps/edr-freight-api/src/modules/reports/report-export-request.util.spec.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { PDF_ROW_CAP, XLSX_ROW_CAP } from './report-export.service'; -import { resolveExportCap, resolveExportColumns, resolveExportFormat } from './report-export-request.util'; -import { ReportColumn } from './report.types'; - -describe('resolveExportFormat', () => { - it('only \'pdf\' exports as pdf', () => { - expect(resolveExportFormat('pdf')).toBe('pdf'); - }); - - it.each([undefined, 'xlsx', 'csv', ''])('%p falls back to xlsx', (raw) => { - expect(resolveExportFormat(raw)).toBe('xlsx'); - }); -}); - -describe('resolveExportCap', () => { - it('missing limit uses the full format cap', () => { - expect(resolveExportCap('xlsx', undefined)).toBe(XLSX_ROW_CAP); - expect(resolveExportCap('pdf', undefined)).toBe(PDF_ROW_CAP); - }); - - it('a limit under the cap is used as-is', () => { - expect(resolveExportCap('pdf', '100')).toBe(100); - }); - - it('a limit over the cap is clamped down', () => { - expect(resolveExportCap('pdf', String(PDF_ROW_CAP + 1000))).toBe(PDF_ROW_CAP); - expect(resolveExportCap('xlsx', String(XLSX_ROW_CAP + 1))).toBe(XLSX_ROW_CAP); - }); - - it.each(['0', '-5', 'not-a-number', ''])('non-positive/invalid limit %p falls back to the cap', (raw) => { - expect(resolveExportCap('xlsx', raw)).toBe(XLSX_ROW_CAP); - }); -}); - -describe('resolveExportColumns', () => { - const columns: ReportColumn[] = [ - { key: 'a', label: 'A', type: 'string' }, - { key: 'b', label: 'B', type: 'number' }, - { key: 'c', label: 'C', type: 'money' }, - ]; - const def = { columns }; - - it('missing fields returns every column', () => { - expect(resolveExportColumns(def, undefined)).toEqual(columns); - }); - - it('empty fields string returns every column', () => { - expect(resolveExportColumns(def, '')).toEqual(columns); - }); - - it('a known subset filters to just those columns, in the report\'s own order', () => { - expect(resolveExportColumns(def, 'c,a')).toEqual([columns[0], columns[2]]); - }); - - it('unknown keys are dropped, not passed through', () => { - expect(resolveExportColumns(def, 'a,ghost')).toEqual([columns[0]]); - }); - - it('all-unknown keys falls back to every column instead of a blank sheet', () => { - expect(resolveExportColumns(def, 'ghost,also-ghost')).toEqual(columns); - }); -}); diff --git a/apps/edr-freight-api/src/modules/reports/report-export-request.util.ts b/apps/edr-freight-api/src/modules/reports/report-export-request.util.ts deleted file mode 100644 index 18f2fa322..000000000 --- a/apps/edr-freight-api/src/modules/reports/report-export-request.util.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { PDF_ROW_CAP, XLSX_ROW_CAP } from './report-export.service'; -import { ReportColumn, ReportDefinition } from './report.types'; - -export type ExportFormat = 'xlsx' | 'pdf'; - -/** Anything but the literal string 'pdf' exports as xlsx. */ -export function resolveExportFormat(raw: string | undefined): ExportFormat { - return raw === 'pdf' ? 'pdf' : 'xlsx'; -} - -/** Caller's requested row limit, clamped to the format's hard cap. A - * missing/non-positive/non-numeric limit means "as many as the format allows". */ -export function resolveExportCap(format: ExportFormat, rawLimit: string | undefined): number { - const formatCap = format === 'pdf' ? PDF_ROW_CAP : XLSX_ROW_CAP; - const requested = Number(rawLimit); - return requested > 0 ? Math.min(requested, formatCap) : formatCap; -} - -/** Caller's requested column subset, whitelisted against the report's own - * columns. Missing, empty, or all-unknown `rawFields` falls back to every - * column rather than shipping a blank sheet. */ -export function resolveExportColumns( - def: Pick, - rawFields: string | undefined, -): ReportColumn[] { - const requested = rawFields?.split(',').filter(Boolean); - const filtered = requested?.length ? def.columns.filter((c) => requested.includes(c.key)) : def.columns; - return filtered.length ? filtered : def.columns; -} diff --git a/apps/edr-freight-api/src/modules/reports/report-export.service.ts b/apps/edr-freight-api/src/modules/reports/report-export.service.ts deleted file mode 100644 index f0919c9c7..000000000 --- a/apps/edr-freight-api/src/modules/reports/report-export.service.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import ExcelJS from 'exceljs'; - -import { PdfRenderService } from '../billing/documents/pdf-render.service'; -import { ReportColumn, ReportDefinition, ReportKpi } from './report.types'; - -// ponytail: in-memory Workbook, cap below. Switch to ExcelJS's streaming -// WorkbookWriter if a report ever needs to outgrow XLSX_ROW_CAP. -export const XLSX_ROW_CAP = 50_000; -// ponytail: HTML→PDF render cost grows with row count; larger exports must -// use XLSX instead. -export const PDF_ROW_CAP = 5_000; - -const NUMBER_FORMAT: Partial> = { - money: '#,##0.00', - tons: '#,##0.0', - percent: '0"%"', - number: '#,##0', -}; - -function formatCell(value: unknown, type: ReportColumn['type']): string { - if (value === null || value === undefined) return ''; - if (type === 'money' || type === 'number') { - return Number(value).toLocaleString('en-US', { maximumFractionDigits: 2 }); - } - if (type === 'tons') return `${Number(value).toLocaleString('en-US')} t`; - if (type === 'percent') return `${value}%`; - return String(value); -} - -@Injectable() -export class ReportExportService { - constructor(private readonly pdfRender: PdfRenderService) {} - - async toXlsx( - def: ReportDefinition, - rows: Record[], - kpis: ReportKpi[], - columns: ReportColumn[] = def.columns, - ): Promise { - const workbook = new ExcelJS.Workbook(); - const sheet = workbook.addWorksheet(def.title.slice(0, 31)); - - if (kpis.length) { - sheet.addRow(kpis.map((k) => `${k.label}: ${k.value.toLocaleString()}${k.unit ? ` ${k.unit}` : ''}`)); - sheet.addRow([]); - } - - const headerRow = sheet.addRow(columns.map((c) => c.label)); - headerRow.font = { bold: true }; - - for (const row of rows) { - sheet.addRow(columns.map((c) => row[c.key] ?? null)); - } - - columns.forEach((col, i) => { - const format = NUMBER_FORMAT[col.type]; - const excelCol = sheet.getColumn(i + 1); - excelCol.width = Math.max(col.label.length + 2, 12); - if (format) excelCol.numFmt = format; - }); - - const buffer = await workbook.xlsx.writeBuffer(); - return Buffer.from(buffer); - } - - async toPdf( - def: ReportDefinition, - rows: Record[], - kpis: ReportKpi[], - columns: ReportColumn[] = def.columns, - ): Promise { - const html = this.buildHtml(def, rows, kpis, columns); - return this.pdfRender.htmlToPdfBuffer(html, { label: `report:${def.key}`, landscape: true }); - } - - private buildHtml( - def: ReportDefinition, - rows: Record[], - kpis: ReportKpi[], - columns: ReportColumn[], - ): string { - const esc = (v: unknown) => - String(v ?? '').replace(/&/g, '&').replace(//g, '>'); - - const kpiHtml = kpis.length - ? `
${kpis - .map( - (k) => - `
${esc(k.label)}
${k.value.toLocaleString()}${k.unit ? ` ${esc(k.unit)}` : ''}
`, - ) - .join('')}
` - : ''; - - const head = columns.map((c) => `${esc(c.label)}`).join(''); - const body = rows - .map( - (row) => - `${columns.map((c) => `${esc(formatCell(row[c.key], c.type))}`).join('')}`, - ) - .join(''); - - return ` -

${esc(def.title)}

-

${esc(def.description)}

- ${kpiHtml} - ${head}${body}
- `; - } -} diff --git a/apps/edr-freight-api/src/modules/reports/reports.controller.ts b/apps/edr-freight-api/src/modules/reports/reports.controller.ts index 107e11097..33a4214c8 100644 --- a/apps/edr-freight-api/src/modules/reports/reports.controller.ts +++ b/apps/edr-freight-api/src/modules/reports/reports.controller.ts @@ -10,8 +10,13 @@ import { BookingStaff } from '../../common/booking-guards'; import { assertFreightPermission, hasFreightPermission } from '../../common/freight-permission.util'; import { FREIGHT_PERMS, reportPermissionKey } from '../../seed/freight-permissions.registry'; import { UserTradeAccessService } from '../user-trade-access/user-trade-access.service'; -import { ReportExportService } from './report-export.service'; -import { resolveExportCap, resolveExportColumns, resolveExportFormat } from './report-export-request.util'; +import { + EXPORT_MIME, + pickByKey, + resolveExportCap, + resolveExportFormat, +} from '../exports/export-request.util'; +import { TabularExportService } from '../exports/tabular-export.service'; import { RawReportQuery, ReportRunnerService } from './report-runner.service'; import { REPORTS, getReport } from './report.registry'; import { ReportCatalogEntry, ReportDefinition, ReportFilterOption } from './report.types'; @@ -59,7 +64,7 @@ async function resolveFilterOptions( export class ReportsController { constructor( private readonly runner: ReportRunnerService, - private readonly exportService: ReportExportService, + private readonly exportService: TabularExportService, private readonly userTradeAccessService: UserTradeAccessService, @InjectDataSource() private readonly dataSource: DataSource, ) {} @@ -86,7 +91,7 @@ export class ReportsController { } @Get(':key/export') - @ApiOperation({ summary: 'Export a report to xlsx or pdf' }) + @ApiOperation({ summary: 'Export a report to xlsx, csv or pdf' }) async export( @Param('key') key: string, @Query() query: RawReportQuery & { format?: string; fields?: string; limit?: string }, @@ -97,22 +102,27 @@ export class ReportsController { const directions = await this.userTradeAccessService.resolveAllowedDirections(user); const format = resolveExportFormat(query.format); const cap = resolveExportCap(format, query.limit); - const exportColumns = resolveExportColumns(def, query.fields); + const exportColumns = pickByKey(def.columns, query.fields); const { items, kpis } = await this.runner.runAll(def, query, directions, cap); + const doc = { + title: def.title, + description: def.description, + label: `report:${def.key}`, + columns: exportColumns, + rows: items, + kpis, + }; const buffer = format === 'pdf' - ? await this.exportService.toPdf(def, items, kpis, exportColumns) - : await this.exportService.toXlsx(def, items, kpis, exportColumns); + ? await this.exportService.toPdf(doc) + : format === 'csv' + ? await this.exportService.toCsv(doc) + : await this.exportService.toXlsx(doc); - const filename = `${def.key}.${format === 'pdf' ? 'pdf' : 'xlsx'}`; - res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); - res.setHeader( - 'Content-Type', - format === 'pdf' - ? 'application/pdf' - : 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - ); + const mime = EXPORT_MIME[format]; + res.setHeader('Content-Disposition', `attachment; filename="${def.key}.${mime.ext}"`); + res.setHeader('Content-Type', mime.type); res.send(buffer); } diff --git a/apps/edr-freight-api/src/modules/reports/reports.module.ts b/apps/edr-freight-api/src/modules/reports/reports.module.ts index 2f98e9e04..e60f16362 100644 --- a/apps/edr-freight-api/src/modules/reports/reports.module.ts +++ b/apps/edr-freight-api/src/modules/reports/reports.module.ts @@ -1,14 +1,15 @@ import { Module } from '@nestjs/common'; -import { DocumentsModule } from '../billing/documents/documents.module'; +import { ExportsModule } from '../exports/exports.module'; import { UserTradeAccessModule } from '../user-trade-access/user-trade-access.module'; -import { ReportExportService } from './report-export.service'; import { ReportRunnerService } from './report-runner.service'; import { ReportsController } from './reports.controller'; @Module({ - imports: [UserTradeAccessModule, DocumentsModule], + // ExportsModule provides the shared tabular writer (xlsx/csv/pdf) and pulls + // DocumentsModule in for the PDF renderer. + imports: [UserTradeAccessModule, ExportsModule], controllers: [ReportsController], - providers: [ReportRunnerService, ReportExportService], + providers: [ReportRunnerService], }) export class ReportsModule {} From ce90be5c887014449763080ed15b40a348e226af Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 05:28:51 +0000 Subject: [PATCH 10/66] fix(reports): stop the 'first N rows' option failing on large exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The export path used one number for two different things: the format's hard row cap, and the caller's explicit 'give me the first N rows'. Because resolveExportCap() returned min(requested, formatCap) and runAll() then threw when the result reached it, picking 'Records: First 100' in the export dialog 400'd on any report with more than 100 rows — the user asked to be truncated and got an error instead. Splits them: formatRowCap() is the hard, non-caller-controllable ceiling that still throws when exceeded (a silently short file hides missing rows), while resolveRowLimit() is the deliberate truncation and is honoured by slicing. Verified against a 223-row dataset: limit=5 now returns 5 rows, and no limit returns all 223. --- .../exports/export-request.util.spec.ts | 41 ++++++++++++------- .../modules/exports/export-request.util.ts | 29 ++++++++++--- .../modules/reports/report-runner.service.ts | 32 ++++++++++----- .../src/modules/reports/reports.controller.ts | 9 ++-- 4 files changed, 78 insertions(+), 33 deletions(-) diff --git a/apps/edr-freight-api/src/modules/exports/export-request.util.spec.ts b/apps/edr-freight-api/src/modules/exports/export-request.util.spec.ts index 1a22be1d5..1b473b12a 100644 --- a/apps/edr-freight-api/src/modules/exports/export-request.util.spec.ts +++ b/apps/edr-freight-api/src/modules/exports/export-request.util.spec.ts @@ -1,4 +1,10 @@ -import { EXPORT_MIME, pickByKey, resolveExportCap, resolveExportFormat } from './export-request.util'; +import { + EXPORT_MIME, + formatRowCap, + pickByKey, + resolveExportFormat, + resolveRowLimit, +} from './export-request.util'; import { CSV_ROW_CAP, PDF_ROW_CAP, XLSX_ROW_CAP } from './tabular-export.service'; describe('resolveExportFormat', () => { @@ -15,25 +21,32 @@ describe('resolveExportFormat', () => { }); }); -describe('resolveExportCap', () => { - it('missing limit uses the full format cap', () => { - expect(resolveExportCap('xlsx', undefined)).toBe(XLSX_ROW_CAP); - expect(resolveExportCap('csv', undefined)).toBe(CSV_ROW_CAP); - expect(resolveExportCap('pdf', undefined)).toBe(PDF_ROW_CAP); +describe('formatRowCap', () => { + it('is the format\'s hard ceiling and is not caller-controllable', () => { + expect(formatRowCap('xlsx')).toBe(XLSX_ROW_CAP); + expect(formatRowCap('csv')).toBe(CSV_ROW_CAP); + expect(formatRowCap('pdf')).toBe(PDF_ROW_CAP); + }); +}); + +describe('resolveRowLimit', () => { + it('no limit means "everything, up to the cap"', () => { + expect(resolveRowLimit('xlsx', undefined)).toBeUndefined(); }); - it('a limit under the cap is used as-is', () => { - expect(resolveExportCap('pdf', '100')).toBe(100); + it('an explicit limit is the caller asking to be truncated — kept as-is', () => { + // Distinct from the cap: 100 here must yield 100 rows, not a 400, even + // when the unfiltered result is far larger. + expect(resolveRowLimit('pdf', '100')).toBe(100); }); - it('a limit over the cap is clamped down', () => { - expect(resolveExportCap('pdf', String(PDF_ROW_CAP + 1000))).toBe(PDF_ROW_CAP); - expect(resolveExportCap('xlsx', String(XLSX_ROW_CAP + 1))).toBe(XLSX_ROW_CAP); - expect(resolveExportCap('csv', String(CSV_ROW_CAP + 1))).toBe(CSV_ROW_CAP); + it('an explicit limit over the format cap is clamped down', () => { + expect(resolveRowLimit('pdf', String(PDF_ROW_CAP + 1000))).toBe(PDF_ROW_CAP); + expect(resolveRowLimit('csv', String(CSV_ROW_CAP + 1))).toBe(CSV_ROW_CAP); }); - it.each(['0', '-5', 'not-a-number', ''])('non-positive/invalid limit %p falls back to the cap', (raw) => { - expect(resolveExportCap('xlsx', raw)).toBe(XLSX_ROW_CAP); + it.each(['0', '-5', 'not-a-number', ''])('non-positive/invalid limit %p means no limit', (raw) => { + expect(resolveRowLimit('xlsx', raw)).toBeUndefined(); }); }); diff --git a/apps/edr-freight-api/src/modules/exports/export-request.util.ts b/apps/edr-freight-api/src/modules/exports/export-request.util.ts index d71db912e..39f1f2e8c 100644 --- a/apps/edr-freight-api/src/modules/exports/export-request.util.ts +++ b/apps/edr-freight-api/src/modules/exports/export-request.util.ts @@ -9,13 +9,30 @@ export function resolveExportFormat(raw: string | undefined): ExportFormat { return 'xlsx'; } -/** Caller's requested row limit, clamped to the format's hard cap. A - * missing/non-positive/non-numeric limit means "as many as the format allows". */ -export function resolveExportCap(format: ExportFormat, rawLimit: string | undefined): number { - const formatCap = - format === 'pdf' ? PDF_ROW_CAP : format === 'csv' ? CSV_ROW_CAP : XLSX_ROW_CAP; +/** + * The format's hard ceiling. Not caller-controllable: exceeding it is an error, + * because a silently short file is worse than a clear failure. + */ +export function formatRowCap(format: ExportFormat): number { + return format === 'pdf' ? PDF_ROW_CAP : format === 'csv' ? CSV_ROW_CAP : XLSX_ROW_CAP; +} + +/** + * The caller's deliberate "just the first N rows", clamped to the format cap. + * `undefined` means "everything, up to the cap". + * + * This is a DIFFERENT thing from the cap and must not share a number with it. + * Conflating them (as this code did originally) makes the dialog's + * "Records: First 100" option fail outright on any export with more than 100 + * rows — the user explicitly asked to be truncated, so truncating is the + * correct answer, not a 400. + */ +export function resolveRowLimit( + format: ExportFormat, + rawLimit: string | undefined, +): number | undefined { const requested = Number(rawLimit); - return requested > 0 ? Math.min(requested, formatCap) : formatCap; + return requested > 0 ? Math.min(requested, formatRowCap(format)) : undefined; } /** Content type + file extension per format, for the download response headers. */ diff --git a/apps/edr-freight-api/src/modules/reports/report-runner.service.ts b/apps/edr-freight-api/src/modules/reports/report-runner.service.ts index 8ba2c5da2..d38a5ba8a 100644 --- a/apps/edr-freight-api/src/modules/reports/report-runner.service.ts +++ b/apps/edr-freight-api/src/modules/reports/report-runner.service.ts @@ -122,12 +122,19 @@ export class ReportRunnerService { }; } - /** Same query, no paging — used by the export path. */ + /** + * Same query, no paging — used by the export path. + * + * `limit` is the caller's deliberate "first N" (the dialog's "Records: First + * 100"), honoured by truncating. `cap` is the format's hard ceiling, which + * throws instead. These used to be one number, which made "First 100" fail + * outright on any report with more than 100 rows. + */ async runAll( def: ReportDefinition, raw: RawReportQuery, directions: string[] | null, - limit: number, + { cap, limit }: { cap: number; limit?: number }, ): Promise<{ columns: typeof def.columns; items: Record[]; kpis: ReportRunResult['kpis'] }> { const params = coerceParams(def, raw); const ctx = { ds: this.ds, params, directions }; @@ -136,14 +143,19 @@ export class ReportRunnerService { // export is supposed to match what the user is looking at. const sort = resolveSort(def, raw.sortBy, raw.sortOrder); if (sort) qb.orderBy(sort.expr, sort.dir); - // limit + 1: fetching exactly `limit` cannot distinguish "there are exactly - // limit rows" from "there are more" — which is why the old `>= limit` check - // rejected a legitimate export of exactly the cap. - const items = await qb.limit(limit + 1).getRawMany(); - if (items.length > limit) { - throw new BadRequestException( - `Export exceeds the ${limit}-row cap for this format. Narrow the filters.`, - ); + + const ceiling = limit ?? cap; + // ceiling + 1: fetching exactly `ceiling` cannot distinguish "there are + // exactly that many rows" from "there are more" — which is why the old + // `>= limit` check rejected a legitimate export of exactly the cap. + const items = await qb.limit(ceiling + 1).getRawMany(); + if (items.length > ceiling) { + if (limit === undefined) { + throw new BadRequestException( + `Export exceeds the ${cap}-row cap for this format. Narrow the filters.`, + ); + } + items.length = limit; } const kpis = def.summary ? await def.summary(ctx) : []; return { columns: def.columns, items, kpis }; diff --git a/apps/edr-freight-api/src/modules/reports/reports.controller.ts b/apps/edr-freight-api/src/modules/reports/reports.controller.ts index 33a4214c8..774c992fb 100644 --- a/apps/edr-freight-api/src/modules/reports/reports.controller.ts +++ b/apps/edr-freight-api/src/modules/reports/reports.controller.ts @@ -12,9 +12,10 @@ import { FREIGHT_PERMS, reportPermissionKey } from '../../seed/freight-permissio import { UserTradeAccessService } from '../user-trade-access/user-trade-access.service'; import { EXPORT_MIME, + formatRowCap, pickByKey, - resolveExportCap, resolveExportFormat, + resolveRowLimit, } from '../exports/export-request.util'; import { TabularExportService } from '../exports/tabular-export.service'; import { RawReportQuery, ReportRunnerService } from './report-runner.service'; @@ -101,10 +102,12 @@ export class ReportsController { const def = this.resolve(key, user); const directions = await this.userTradeAccessService.resolveAllowedDirections(user); const format = resolveExportFormat(query.format); - const cap = resolveExportCap(format, query.limit); const exportColumns = pickByKey(def.columns, query.fields); - const { items, kpis } = await this.runner.runAll(def, query, directions, cap); + const { items, kpis } = await this.runner.runAll(def, query, directions, { + cap: formatRowCap(format), + limit: resolveRowLimit(format, query.limit), + }); const doc = { title: def.title, description: def.description, From 62f7b91315ead96a0691bce61f17713fc0a90f1e Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 05:29:04 +0000 Subject: [PATCH 11/66] feat(exports): dataset-driven table export, starting with bookings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a parallel export system the reports module can also draw on. A dataset describes a table's exportable fields — including related-entity detail the list page never shows — and the engine assembles a query from whichever fields the caller picked. GET /exports catalog (metadata only; select/requires never ship) GET /exports/:key/count exact row count + per-format caps GET /exports/:key/download csv | xlsx | pdf Two invariants carry the design: - Every lazy join is a LEFT join, and ExportJoin has no 'kind' field to make anything else expressible. An inner join added because a checkbox was ticked would change the rowset, so two exports of the same filters would disagree on their row count. - Because of that, the count cannot depend on field selection, so /count runs base + alwaysJoin only and is exact rather than an estimate. Verified: count and the delivered file both report 223 rows. One-to-many relations (a booking's containers) aggregate in a correlated subquery rather than joining, so a row can never multiply. Export rides each dataset's existing view permission — no new permission keys and no seeder change. Sensitive columns are simply never declared as fields: raw gateway payloads, signature blobs, error dumps, raw jsonb snapshots, internal user UUIDs and review notes are all absent by construction. bookings ships 77 fields across 10 groups. scripts/validate-export-datasets.ts EXPLAINs every dataset's widest query, its count query, and each field on its own against the real database — the per-field pass is what catches a field referencing a join it forgot to declare, which otherwise only fails when that one field is picked alone. --- apps/edr-freight-api/src/app.module.ts | 2 + .../exports/datasets/bookings.dataset.ts | 223 ++++++++++++++++++ .../src/modules/exports/export-filter.util.ts | 81 +++++++ .../exports/export-query.builder.spec.ts | 92 ++++++++ .../modules/exports/export-query.builder.ts | 101 ++++++++ .../modules/exports/export-runner.service.ts | 69 ++++++ .../src/modules/exports/export.registry.ts | 15 ++ .../src/modules/exports/export.types.ts | 135 +++++++++++ .../src/modules/exports/exports.controller.ts | 156 ++++++++++++ .../src/modules/exports/exports.module.ts | 18 +- .../src/scripts/validate-export-datasets.ts | 83 +++++++ 11 files changed, 969 insertions(+), 6 deletions(-) create mode 100644 apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-filter.util.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-query.builder.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-runner.service.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export.registry.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export.types.ts create mode 100644 apps/edr-freight-api/src/modules/exports/exports.controller.ts create mode 100644 apps/edr-freight-api/src/scripts/validate-export-datasets.ts diff --git a/apps/edr-freight-api/src/app.module.ts b/apps/edr-freight-api/src/app.module.ts index b148c7eea..34825d13c 100644 --- a/apps/edr-freight-api/src/app.module.ts +++ b/apps/edr-freight-api/src/app.module.ts @@ -100,6 +100,7 @@ import { RoutesModule } from "./modules/routes/routes.module"; import { WarehousesModule } from "./modules/warehouses/warehouses.module"; import { OverviewModule } from "./modules/overview/overview.module"; import { ReportsModule } from "./modules/reports/reports.module"; +import { ExportsModule } from "./modules/exports/exports.module"; import { UserTradeAccessModule } from "./modules/user-trade-access/user-trade-access.module"; import { VehiclesModule } from "./modules/vehicles/vehicles.module"; import { DriversModule } from "./modules/drivers/drivers.module"; @@ -234,6 +235,7 @@ if (!process.env.APPLICATION_NAME) { WarehousesModule, OverviewModule, ReportsModule, + ExportsModule, UserTradeAccessModule, VehiclesModule, DriversModule, diff --git a/apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts b/apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts new file mode 100644 index 000000000..1e2cc9098 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts @@ -0,0 +1,223 @@ +import { FREIGHT_PERMS } from '../../../seed/freight-permissions.registry'; +import { Booking } from '../../bookings/entities/booking.entity'; +import { Company } from '../../companies/entities/company.entity'; +import { CompanyProfile } from '../../companies/entities/company-profile.entity'; +import { Contract } from '../../contracts/entities/contract.entity'; +import { CargoType } from '../../rule-engine/entities/cargo-type.entity'; +import { ServiceType } from '../../rule-engine/entities/service-type.entity'; +import { ShippingLine } from '../../rule-engine/entities/shipping-line.entity'; +import { Yard } from '../../rule-engine/entities/yard.entity'; +import { ShippingLineCompany } from '../../shipping-lines/entities/shipping-line-company.entity'; +import { Train } from '../../trains/entities/train.entity'; +import { applyDirectionScope } from '../../user-trade-access/trade-scope.util'; +import { ExportDataset } from '../export.types'; + +/** + * Domain semantics shared with `reports/definitions/bookings-list.report.ts`. + * Kept identical on purpose — for PER_ITEM bulk bookings `cargo_total_weight_vgm` + * holds an item COUNT, not tonnage, and `adjusted_total_amount` silently + * overrides `total_amount`. Getting either wrong misreports money or weight. + */ +const TONS = 'COALESCE(b.bulk_total_weight_tons, b.cargo_total_weight_vgm)'; +const REVENUE = 'COALESCE(b.adjusted_total_amount, b.total_amount)'; + +const STATUS_OPTIONS = [ + 'DRAFT', 'SUBMITTED', 'UNDER_REVIEW', 'APPROVED', 'REJECTED', + 'CANCELLED', 'EXPIRED', 'SCHEDULED', 'LOADED', 'IN_TRANSIT', + 'ARRIVED', 'DELIVERED', 'COMPLETED', +].map((v) => ({ value: v, label: v.replace(/_/g, ' ') })); + +export const bookingsDataset: ExportDataset = { + key: 'bookings', + title: 'Bookings', + description: 'Every booking, with customer, route, cargo, contract and payment detail', + group: 'Commercial', + permission: FREIGHT_PERMS.bookings.view, + base: { entity: Booking, alias: 'b' }, + + // Every join is a LEFT join (see ExportJoin) — ticking a field must never + // change which rows come back. + joins: [ + { alias: 'c', entity: Company, on: 'c.id = b.company_id' }, + { alias: 'cp', entity: CompanyProfile, on: 'cp.id = b.company_profile_id' }, + { alias: 'slc', entity: ShippingLineCompany, on: 'slc.id = b.shipping_line_company_id' }, + { alias: 'o', entity: Yard, on: 'o.id = b.origin_yard_id' }, + { alias: 'd', entity: Yard, on: 'd.id = b.destination_yard_id' }, + { alias: 'cty', entity: CargoType, on: 'cty.id = b.cargo_type_id' }, + { alias: 'st', entity: ServiceType, on: 'st.id = b.service_type_id' }, + { alias: 'sl', entity: ShippingLine, on: 'sl.id = b.shipping_line_id' }, + { alias: 'ct', entity: Contract, on: 'ct.id = b.contract_id' }, + { alias: 't', entity: Train, on: 't.id = b.train_id' }, + // Transitive: the contract's own customer, reachable only once `ct` is in. + { alias: 'ctc', entity: Company, on: 'ctc.id = ct.company_id', requires: ['ct'] }, + ], + // `search` matches the customer name, so `c` is always present — which is + // also why the count query joins it. + alwaysJoin: ['c'], + + groups: [ + { id: 'booking', label: 'Booking' }, + { id: 'customer', label: 'Customer' }, + { id: 'route', label: 'Route' }, + { id: 'cargo', label: 'Cargo' }, + { id: 'payment', label: 'Payment' }, + { id: 'scheduling', label: 'Scheduling' }, + { id: 'contract', label: 'Contract' }, + { id: 'firstMile', label: 'First mile' }, + { id: 'lastMile', label: 'Last mile' }, + { id: 'clearance', label: 'Clearance' }, + ], + + fields: [ + // ---- Booking ------------------------------------------------------- + { key: 'reference', label: 'Reference', type: 'string', group: 'booking', default: true, select: 'b.reference', sortExpr: 'b.reference' }, + { key: 'status', label: 'Status', type: 'string', group: 'booking', default: true, select: 'b.status', sortExpr: 'b.status' }, + { key: 'bookingType', label: 'Booking type', type: 'string', group: 'booking', select: 'b.booking_type' }, + { key: 'contractKind', label: 'Contract kind', type: 'string', group: 'booking', select: 'b.contract_kind' }, + { key: 'createdAt', label: 'Created', type: 'datetime', group: 'booking', default: true, select: `to_char(b.created_at, 'YYYY-MM-DD HH24:MI')`, sortExpr: 'b.created_at' }, + { key: 'updatedAt', label: 'Updated', type: 'datetime', group: 'booking', select: `to_char(b.updated_at, 'YYYY-MM-DD HH24:MI')`, sortExpr: 'b.updated_at' }, + { key: 'expiresAt', label: 'Expires', type: 'date', group: 'booking', select: `to_char(b.expires_at, 'YYYY-MM-DD')` }, + { key: 'createdByRole', label: 'Created by role', type: 'string', group: 'booking', select: 'b.created_by_role' }, + { key: 'isSplit', label: 'Split booking', type: 'boolean', group: 'booking', select: 'b.is_split' }, + { key: 'priorityScore', label: 'Priority score', type: 'number', group: 'booking', select: 'b.priority_score', sortExpr: 'b.priority_score' }, + { key: 'versionNumber', label: 'Version', type: 'number', group: 'booking', select: 'b.version_number' }, + { key: 'pnrCode', label: 'PNR code', type: 'string', group: 'booking', select: 'b.pnr_code' }, + + // ---- Customer (the "more than the UI shows" payload) ---------------- + { key: 'customer', label: 'Customer', type: 'string', group: 'customer', default: true, requires: ['c'], select: 'c.name', sortExpr: 'c.name' }, + { key: 'customerType', label: 'Customer type', type: 'string', group: 'customer', requires: ['c'], select: 'c.type' }, + { key: 'customerKind', label: 'Customer kind', type: 'string', group: 'customer', requires: ['c'], select: 'c.kind' }, + { key: 'customerStatus', label: 'Customer status', type: 'string', group: 'customer', requires: ['c'], select: 'c.status' }, + { key: 'customerTin', label: 'Customer TIN', type: 'string', group: 'customer', requires: ['c'], select: 'c.tin' }, + { key: 'customerVat', label: 'Customer VAT no.', type: 'string', group: 'customer', requires: ['c'], select: 'c.vat_number' }, + { key: 'customerPhone', label: 'Customer phone', type: 'string', group: 'customer', requires: ['c'], select: 'c.phone' }, + { key: 'customerEmail', label: 'Customer email', type: 'string', group: 'customer', requires: ['c'], select: 'c.email' }, + { key: 'customerContact', label: 'Contact person', type: 'string', group: 'customer', requires: ['c'], select: 'c.contact_person_name' }, + { key: 'customerContactPhone', label: 'Contact phone', type: 'string', group: 'customer', requires: ['c'], select: 'c.contact_person_phone' }, + { key: 'customerAddress', label: 'Customer address', type: 'string', group: 'customer', requires: ['c'], select: 'c.address' }, + { key: 'customerCountry', label: 'Customer country', type: 'string', group: 'customer', requires: ['c'], select: 'c.country' }, + { key: 'customerRegion', label: 'Customer region', type: 'string', group: 'customer', requires: ['c'], select: 'c.region' }, + { key: 'customerProfileRef', label: 'Profile reference', type: 'string', group: 'customer', requires: ['cp'], select: 'cp.reference' }, + { key: 'customerProfileType', label: 'Profile type', type: 'string', group: 'customer', requires: ['cp'], select: 'cp.type' }, + { key: 'isGovernment', label: 'Government', type: 'boolean', group: 'customer', select: 'b.is_government' }, + { key: 'governmentInstitution', label: 'Government institution', type: 'string', group: 'customer', select: 'b.government_institution' }, + { key: 'shippingLineCompany', label: 'Shipping line company', type: 'string', group: 'customer', requires: ['slc'], select: 'slc.name' }, + + // ---- Route ---------------------------------------------------------- + { key: 'origin', label: 'Origin', type: 'string', group: 'route', default: true, requires: ['o'], select: 'o.label' }, + { key: 'originCode', label: 'Origin code', type: 'string', group: 'route', requires: ['o'], select: 'o.code' }, + { key: 'destination', label: 'Destination', type: 'string', group: 'route', default: true, requires: ['d'], select: 'd.label' }, + { key: 'destinationCode', label: 'Destination code', type: 'string', group: 'route', requires: ['d'], select: 'd.code' }, + { key: 'tradeDirection', label: 'Direction', type: 'string', group: 'route', default: true, select: 'b.trade_direction', sortExpr: 'b.trade_direction' }, + { key: 'serviceType', label: 'Service type', type: 'string', group: 'route', requires: ['st'], select: 'st.service_name' }, + + // ---- Cargo ----------------------------------------------------------- + { key: 'cargo', label: 'Cargo', type: 'string', group: 'cargo', default: true, requires: ['cty'], select: 'COALESCE(cty.cargo_type_name, b.cargo_free_text)' }, + { key: 'freightType', label: 'Freight type', type: 'string', group: 'cargo', default: true, select: 'b.freight_type' }, + { key: 'tons', label: 'Tonnage', type: 'tons', group: 'cargo', default: true, select: `ROUND(${TONS})::float8`, sortExpr: TONS }, + { key: 'containerWeightVgm', label: 'Container VGM', type: 'number', group: 'cargo', select: 'b.cargo_total_weight_vgm' }, + { key: 'bulkWeightTons', label: 'Bulk weight (t)', type: 'tons', group: 'cargo', select: 'b.bulk_total_weight_tons' }, + { key: 'isHazardous', label: 'Hazardous', type: 'boolean', group: 'cargo', select: 'b.is_hazardous' }, + { key: 'isReefer', label: 'Reefer', type: 'boolean', group: 'cargo', select: 'b.is_reefer' }, + { key: 'shippingLine', label: 'Shipping line', type: 'string', group: 'cargo', requires: ['sl'], select: 'sl.label' }, + { + // One-to-many, so it aggregates in a correlated subquery rather than a + // join — a join here would multiply rows and break the count contract. + key: 'containerNumbers', label: 'Container numbers', type: 'string', group: 'cargo', + select: `(SELECT string_agg(bc.container_number, ' | ' ORDER BY bc.container_number) + FROM freight.booking_container bc + WHERE bc.booking_id = b.id AND bc.deleted_at IS NULL)`, + }, + + // ---- Payment ---------------------------------------------------------- + { key: 'amount', label: 'Amount', type: 'money', group: 'payment', default: true, select: `ROUND(${REVENUE}, 2)::float8`, sortExpr: REVENUE }, + { key: 'totalAmount', label: 'Total amount (pre-adjustment)', type: 'money', group: 'payment', select: 'b.total_amount::float8' }, + { key: 'adjustedTotalAmount', label: 'Adjusted total', type: 'money', group: 'payment', select: 'b.adjusted_total_amount::float8' }, + { key: 'adjustmentReason', label: 'Adjustment reason', type: 'string', group: 'payment', select: 'b.adjustment_reason' }, + { key: 'paymentStatus', label: 'Payment status', type: 'string', group: 'payment', default: true, select: 'b.payment_status', sortExpr: 'b.payment_status' }, + { key: 'paymentCurrency', label: 'Currency', type: 'string', group: 'payment', select: 'b.payment_currency' }, + { key: 'paymentDeadline', label: 'Payment deadline', type: 'datetime', group: 'payment', select: `to_char(b.payment_deadline, 'YYYY-MM-DD HH24:MI')` }, + + // ---- Scheduling -------------------------------------------------------- + { key: 'scheduledDate', label: 'Scheduled date', type: 'date', group: 'scheduling', default: true, select: `to_char(b.scheduled_date, 'YYYY-MM-DD')`, sortExpr: 'b.scheduled_date' }, + { key: 'schedulingStatus', label: 'Scheduling status', type: 'string', group: 'scheduling', select: 'b.scheduling_status' }, + { key: 'wagonsRequired', label: 'Wagons required', type: 'number', group: 'scheduling', select: 'b.wagons_required' }, + { key: 'trainCode', label: 'Train', type: 'string', group: 'scheduling', requires: ['t'], select: 't.code' }, + { key: 'loadedAt', label: 'Loaded at', type: 'datetime', group: 'scheduling', select: `to_char(b.loaded_at, 'YYYY-MM-DD HH24:MI')` }, + { key: 'arrivedAt', label: 'Arrived at', type: 'datetime', group: 'scheduling', select: `to_char(b.arrived_at, 'YYYY-MM-DD HH24:MI')` }, + + // ---- Contract ---------------------------------------------------------- + { key: 'contractReference', label: 'Contract reference', type: 'string', group: 'contract', requires: ['ct'], select: 'ct.reference' }, + { key: 'contractStatus', label: 'Contract status', type: 'string', group: 'contract', requires: ['ct'], select: 'ct.status' }, + { key: 'contractCustomer', label: 'Contract customer', type: 'string', group: 'contract', requires: ['ctc'], select: 'ctc.name' }, + { key: 'contractType', label: 'Contract type', type: 'string', group: 'contract', select: 'b.contract_type' }, + { key: 'contractValidFrom', label: 'Contract valid from', type: 'date', group: 'contract', select: `to_char(b.contract_valid_from, 'YYYY-MM-DD')` }, + { key: 'contractValidUntil', label: 'Contract valid until', type: 'date', group: 'contract', select: `to_char(b.contract_valid_until, 'YYYY-MM-DD')` }, + { key: 'fullyExecutedAt', label: 'Fully executed at', type: 'datetime', group: 'contract', select: `to_char(b.fully_executed_at, 'YYYY-MM-DD HH24:MI')` }, + + // ---- First / last mile --------------------------------------------------- + { key: 'firstMileAddress', label: 'Pickup address', type: 'string', group: 'firstMile', select: 'b.first_mile_pickup_address' }, + { key: 'lastMileAddress', label: 'Delivery address', type: 'string', group: 'lastMile', select: 'b.last_mile_delivery_address' }, + { key: 'customerTruckPlate', label: 'Customer truck plate', type: 'string', group: 'lastMile', select: 'b.customer_truck_plate_number' }, + { key: 'customerTruckDriver', label: 'Customer truck driver', type: 'string', group: 'lastMile', select: 'b.customer_truck_driver_name' }, + { key: 'exportHandoverMode', label: 'Handover mode', type: 'string', group: 'lastMile', select: 'b.export_handover_mode' }, + + // ---- Clearance ------------------------------------------------------------- + { key: 'customsClearingEnabled', label: 'Customs clearing', type: 'boolean', group: 'clearance', select: 'b.customs_clearing_enabled' }, + { key: 'customsClearingAgent', label: 'Clearing agent', type: 'string', group: 'clearance', select: 'b.customs_clearing_agent' }, + { key: 'clearancePhase', label: 'Clearance phase', type: 'string', group: 'clearance', select: 'b.clearance_current_phase' }, + { key: 'dutyRequired', label: 'Duty required', type: 'boolean', group: 'clearance', select: 'b.duty_required' }, + { key: 'vesselArrivalDate', label: 'Vessel arrival', type: 'date', group: 'clearance', select: `to_char(b.vessel_arrival_date, 'YYYY-MM-DD')` }, + { key: 'doCollectedDate', label: 'DO collected', type: 'date', group: 'clearance', select: `to_char(b.do_collected_date, 'YYYY-MM-DD')` }, + { key: 'doubleHandling', label: 'Double handling', type: 'boolean', group: 'clearance', select: 'b.double_handling' }, + ], + + filters: [ + { key: 'created', label: 'Created', type: 'daterange' }, + { key: 'statuses', label: 'Status', type: 'multiselect', options: STATUS_OPTIONS }, + { + key: 'tradeDirection', label: 'Direction', type: 'select', + options: ['IMPORT', 'EXPORT', 'DOMESTIC'].map((v) => ({ value: v, label: v })), + }, + { + key: 'freightType', label: 'Freight type', type: 'select', + options: ['CONTAINER', 'BULK'].map((v) => ({ value: v, label: v })), + }, + { key: 'paymentStatus', label: 'Payment status', type: 'select', options: [ + { value: 'PENDING', label: 'Pending' }, + { value: 'PNR_GENERATED', label: 'PNR generated' }, + { value: 'VERIFICATION_IN_PROGRESS', label: 'Verification in progress' }, + { value: 'PAID', label: 'Paid' }, + { value: 'FAILED', label: 'Failed' }, + ] }, + { key: 'companyId', label: 'Customer', type: 'text' }, + { key: 'search', label: 'Search reference or customer', type: 'text' }, + ], + + defaultSort: { key: 'createdAt', dir: 'DESC' }, + + scope(ctx, qb) { + const { params, directions } = ctx; + // andWhere, not where: `where()` resets any condition already on the + // builder, so scope() would silently drop anything a caller added first. + qb.andWhere('b.deleted_at IS NULL'); + + if (params.createdFrom) qb.andWhere('b.created_at >= :createdFrom', { createdFrom: params.createdFrom }); + if (params.createdTo) qb.andWhere('b.created_at < :createdTo', { createdTo: params.createdTo }); + + const statuses = params.statuses as string[] | null; + if (statuses?.length) qb.andWhere('b.status IN (:...statuses)', { statuses }); + + if (params.tradeDirection) qb.andWhere('b.trade_direction = :tradeDirection', { tradeDirection: params.tradeDirection }); + if (params.freightType) qb.andWhere('b.freight_type = :freightType', { freightType: params.freightType }); + if (params.paymentStatus) qb.andWhere('b.payment_status = :paymentStatus', { paymentStatus: params.paymentStatus }); + if (params.companyId) qb.andWhere('b.company_id = :companyId', { companyId: params.companyId }); + if (params.search) { + qb.andWhere('(b.reference ILIKE :search OR c.name ILIKE :search)', { search: `%${params.search as string}%` }); + } + + // Trade-direction ACL. Without this the export returns rows the user's own + // list page would not show them. + applyDirectionScope(qb, 'b.trade_direction', directions); + }, +}; diff --git a/apps/edr-freight-api/src/modules/exports/export-filter.util.ts b/apps/edr-freight-api/src/modules/exports/export-filter.util.ts new file mode 100644 index 000000000..c302d9d3b --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-filter.util.ts @@ -0,0 +1,81 @@ +import { DataSource } from 'typeorm'; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export type ExportFilterType = 'daterange' | 'date' | 'select' | 'multiselect' | 'text'; + +export interface ExportFilterOption { + value: string; + label: string; +} + +export interface ExportFilterDef { + key: string; + label: string; + type: ExportFilterType; + /** Static choices. Mutually exclusive with `optionsQuery`. */ + options?: ExportFilterOption[]; + /** Reference-data choices resolved from the DB and cached for the process. */ + optionsQuery?: (ds: DataSource) => Promise; +} + +/** Raw query-string bag. Per-registry filter keys, so `forbidNonWhitelisted` can't police it. */ +export type RawFilterQuery = Record; + +/** + * Coerce raw query strings into typed filter params per a filter declaration + * list. Unknown keys are dropped rather than rejected. + * + * Shared by the report runner and the export runner so the `daterange` + * handling in particular cannot drift between them: `To` is pushed forward a + * day because callers mean an INCLUSIVE end date while the SQL bound is + * exclusive (`created_at < :dateTo`). + */ +export function coerceFilterParams( + filters: ExportFilterDef[], + raw: RawFilterQuery, +): Record { + const params: Record = {}; + for (const filter of filters) { + if (filter.type === 'daterange') { + const from = raw[`${filter.key}From`]; + const to = raw[`${filter.key}To`]; + params[`${filter.key}From`] = from ? new Date(from).toISOString() : null; + params[`${filter.key}To`] = to + ? new Date(new Date(to).getTime() + DAY_MS).toISOString() + : null; + } else if (filter.type === 'multiselect') { + const csv = raw[filter.key]; + const items = csv?.split(',').map((s) => s.trim()).filter(Boolean) ?? []; + params[filter.key] = items.length ? items : null; + } else { + params[filter.key] = raw[filter.key]?.trim() || null; + } + } + return params; +} + +/** + * Process-lifetime cache for `optionsQuery` results — small, rarely-changing + * reference lists (23 stations, 18 cargo types) hit on every catalog load. + * + * ponytail: keyed by filter key alone, so two registries sharing a filter key + * share one option list. Key by `${registry}:${filterKey}` if that ever bites. + */ +const optionsCache = new Map(); + +export async function resolveFilterOptions( + filters: ExportFilterDef[], + ds: DataSource, +): Promise { + return Promise.all( + filters.map(async (filter) => { + if (!filter.optionsQuery) return filter; + const cached = optionsCache.get(filter.key); + if (cached) return { ...filter, options: cached, optionsQuery: undefined }; + const options = await filter.optionsQuery(ds); + optionsCache.set(filter.key, options); + return { ...filter, options, optionsQuery: undefined }; + }), + ); +} diff --git a/apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts b/apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts new file mode 100644 index 000000000..af6dd923d --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts @@ -0,0 +1,92 @@ +import { resolveJoins } from './export-query.builder'; +import { ExportDataset, ExportField } from './export.types'; + +const field = (key: string, requires?: string[]): ExportField => ({ + key, + label: key, + type: 'string', + group: 'g', + select: `x.${key}`, + requires, +}); + +/** Entities are never dereferenced by resolveJoins — only the alias graph matters. */ +const entity = {} as ExportDataset['joins'][number]['entity']; + +const dataset = ( + joins: ExportDataset['joins'], + alwaysJoin?: string[], +): ExportDataset => + ({ + key: 'test', + joins, + alwaysJoin, + fields: [], + }) as unknown as ExportDataset; + +describe('resolveJoins', () => { + it('pulls in only the joins the selected fields ask for', () => { + const ds = dataset([ + { alias: 'a', entity, on: 'a.id = b.a_id' }, + { alias: 'z', entity, on: 'z.id = b.z_id' }, + ]); + expect(resolveJoins(ds, [field('one', ['a'])]).map((j) => j.alias)).toEqual(['a']); + }); + + it('selecting nothing still applies alwaysJoin — the count query relies on this', () => { + const ds = dataset( + [ + { alias: 'a', entity, on: 'a.id = b.a_id' }, + { alias: 'z', entity, on: 'z.id = b.z_id' }, + ], + ['a'], + ); + expect(resolveJoins(ds, []).map((j) => j.alias)).toEqual(['a']); + }); + + it('resolves a transitive dependency, dependency first', () => { + const ds = dataset([ + { alias: 'ct', entity, on: 'ct.id = b.contract_id' }, + { alias: 'ctc', entity, on: 'ctc.id = ct.company_id', requires: ['ct'] }, + ]); + expect(resolveJoins(ds, [field('x', ['ctc'])]).map((j) => j.alias)).toEqual(['ct', 'ctc']); + }); + + it('resolves a multi-hop chain in order', () => { + const ds = dataset([ + { alias: 'a', entity, on: 'a.id = b.a_id' }, + { alias: 'bb', entity, on: 'bb.id = a.b_id', requires: ['a'] }, + { alias: 'cc', entity, on: 'cc.id = bb.c_id', requires: ['bb'] }, + ]); + expect(resolveJoins(ds, [field('x', ['cc'])]).map((j) => j.alias)).toEqual(['a', 'bb', 'cc']); + }); + + it('emits a shared join once, not per field that needs it', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]); + const joins = resolveJoins(ds, [field('one', ['a']), field('two', ['a'])]); + expect(joins.map((j) => j.alias)).toEqual(['a']); + }); + + it('does not duplicate a join already pulled in by alwaysJoin', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }], ['a']); + expect(resolveJoins(ds, [field('one', ['a'])]).map((j) => j.alias)).toEqual(['a']); + }); + + it('throws on a cycle rather than looping forever', () => { + const ds = dataset([ + { alias: 'a', entity, on: 'a.id = bb.a_id', requires: ['bb'] }, + { alias: 'bb', entity, on: 'bb.id = a.b_id', requires: ['a'] }, + ]); + expect(() => resolveJoins(ds, [field('x', ['a'])])).toThrow(/join cycle/); + }); + + it('throws on an undeclared alias — a typo must fail loudly, not silently 42P01', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]); + expect(() => resolveJoins(ds, [field('x', ['ghost'])])).toThrow(/unknown join alias "ghost"/); + }); + + it('a field with no requires pulls in no joins at all', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]); + expect(resolveJoins(ds, [field('plain')])).toEqual([]); + }); +}); diff --git a/apps/edr-freight-api/src/modules/exports/export-query.builder.ts b/apps/edr-freight-api/src/modules/exports/export-query.builder.ts new file mode 100644 index 000000000..6f4c091d0 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-query.builder.ts @@ -0,0 +1,101 @@ +import { ObjectLiteral, SelectQueryBuilder } from 'typeorm'; + +import { ExportContext, ExportDataset, ExportField, ExportJoin } from './export.types'; + +/** + * Sort expression fallback: the SELECT alias TypeORM emitted, quoted. TypeORM + * double-quotes `addSelect` aliases (preserving case), so ordering by the bare + * key lets Postgres fold it to lowercase and 42703 on any camelCase alias. + */ +export const aliasSortExpr = (key: string): string => `"${key.replace(/"/g, '""')}"`; + +/** + * Selected fields -> the joins they need, transitively, dependencies first. + * DFS post-order over `requires`, memoized. Deterministic: `alwaysJoin` first, + * then fields in the dataset's own declaration order. + */ +export function resolveJoins(dataset: ExportDataset, fields: ExportField[]): ExportJoin[] { + const byAlias = new Map(dataset.joins.map((j) => [j.alias, j])); + const out: ExportJoin[] = []; + const done = new Set(); + const onStack = new Set(); + + const visit = (alias: string): void => { + if (done.has(alias)) return; + if (onStack.has(alias)) { + throw new Error(`export "${dataset.key}": join cycle at alias "${alias}"`); + } + const join = byAlias.get(alias); + if (!join) { + throw new Error(`export "${dataset.key}": unknown join alias "${alias}"`); + } + onStack.add(alias); + for (const dep of join.requires ?? []) visit(dep); + onStack.delete(alias); + done.add(alias); + out.push(join); + }; + + for (const alias of dataset.alwaysJoin ?? []) visit(alias); + for (const field of fields) for (const alias of field.requires ?? []) visit(alias); + return out; +} + +/** The download query: base + only the joins the selected fields need. */ +export function buildExportQuery( + dataset: ExportDataset, + fields: ExportField[], + ctx: ExportContext, +): SelectQueryBuilder { + const qb = ctx.ds.createQueryBuilder().from(dataset.base.entity, dataset.base.alias); + for (const join of resolveJoins(dataset, fields)) { + qb.leftJoin(join.entity, join.alias, join.on); + } + for (const field of fields) qb.addSelect(field.select, field.key); + dataset.scope(ctx, qb); + return qb; +} + +/** + * The count query: same base, same `scope()`, same WHERE — but no field joins + * and no selects. Exact rather than an estimate, because every lazy join is a + * left join to a to-one side and so cannot change the row count. + */ +export function buildExportCountQuery( + dataset: ExportDataset, + ctx: ExportContext, +): SelectQueryBuilder { + const qb = ctx.ds + .createQueryBuilder() + .select('COUNT(*)::int', 'total') + .from(dataset.base.entity, dataset.base.alias); + for (const join of resolveJoins(dataset, [])) { + qb.leftJoin(join.entity, join.alias, join.on); + } + dataset.scope(ctx, qb); + return qb; +} + +/** + * Resolve a requested sort against the SELECTED fields. Restricting to selected + * fields means a sort can never pull in a join the projection didn't already + * need — which is what keeps the count query's join set correct. + */ +export function resolveExportSort( + dataset: ExportDataset, + fields: ExportField[], + sortBy?: string, + sortOrder?: string, +): { expr: string; dir: 'ASC' | 'DESC' } | null { + const dir = sortOrder?.toUpperCase() === 'DESC' ? 'DESC' : 'ASC'; + const requested = sortBy && fields.find((f) => f.key === sortBy && f.sortExpr); + if (requested) return { expr: requested.sortExpr ?? aliasSortExpr(requested.key), dir }; + + if (!dataset.defaultSort) return null; + const fallback = fields.find((f) => f.key === dataset.defaultSort!.key); + if (!fallback) return null; + return { + expr: fallback.sortExpr ?? aliasSortExpr(fallback.key), + dir: dataset.defaultSort.dir, + }; +} diff --git a/apps/edr-freight-api/src/modules/exports/export-runner.service.ts b/apps/edr-freight-api/src/modules/exports/export-runner.service.ts new file mode 100644 index 000000000..816aeb38d --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-runner.service.ts @@ -0,0 +1,69 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; +import { DataSource } from 'typeorm'; + +import { coerceFilterParams, RawFilterQuery } from './export-filter.util'; +import { + buildExportCountQuery, + buildExportQuery, + resolveExportSort, +} from './export-query.builder'; +import { ExportDataset, ExportField } from './export.types'; + +@Injectable() +export class ExportRunnerService { + constructor(@InjectDataSource() private readonly ds: DataSource) {} + + private context(dataset: ExportDataset, raw: RawFilterQuery, directions: string[] | null) { + return { ds: this.ds, params: coerceFilterParams(dataset.filters, raw), directions }; + } + + /** + * Exact row count for the current filters. Exact rather than estimated + * because lazy joins are all left joins to to-one sides, so the count cannot + * depend on which fields the caller picked. + */ + async count( + dataset: ExportDataset, + raw: RawFilterQuery, + directions: string[] | null, + ): Promise { + const qb = buildExportCountQuery(dataset, this.context(dataset, raw, directions)); + const row = await qb.getRawOne<{ total: number }>(); + return Number(row?.total ?? 0); + } + + /** + * Matching rows. + * + * `limit` is the caller's deliberate "first N" truncation — honoured + * silently, because they asked for it. `cap` is the format's hard ceiling — + * exceeding it throws, because a silently short file is worse than a clear + * error: nothing downstream reveals that rows are missing. + */ + async run( + dataset: ExportDataset, + fields: ExportField[], + raw: RawFilterQuery, + directions: string[] | null, + { cap, limit }: { cap: number; limit?: number }, + ): Promise[]> { + const ctx = this.context(dataset, raw, directions); + const qb = buildExportQuery(dataset, fields, ctx); + + const sort = resolveExportSort(dataset, fields, raw.sortBy, raw.sortOrder); + if (sort) qb.orderBy(sort.expr, sort.dir); + + const ceiling = limit ?? cap; + // ceiling + 1: fetching exactly `ceiling` cannot distinguish "there are + // exactly that many rows" from "there are more". + const items = await qb.limit(ceiling + 1).getRawMany(); + if (items.length <= ceiling) return items; + + // Asked to be truncated -> truncate. Hit the hard cap -> say so. + if (limit !== undefined) return items.slice(0, limit); + throw new BadRequestException( + `This export has more than ${cap.toLocaleString()} rows, the limit for this format. Narrow the filters, or export a smaller number of rows.`, + ); + } +} diff --git a/apps/edr-freight-api/src/modules/exports/export.registry.ts b/apps/edr-freight-api/src/modules/exports/export.registry.ts new file mode 100644 index 000000000..dc7a3c968 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export.registry.ts @@ -0,0 +1,15 @@ +import { bookingsDataset } from './datasets/bookings.dataset'; +import { ExportDataset } from './export.types'; + +/** + * Every exportable dataset. + * + * Adding one = a new file under `datasets/` + an entry here. No frontend edit, + * no route, no permission seed — the dialog is driven entirely by the catalog + * this registry serves, and a dataset reuses its module's existing `view` key. + */ +export const DATASETS: ExportDataset[] = [bookingsDataset]; + +const BY_KEY = new Map(DATASETS.map((d) => [d.key, d])); + +export const getDataset = (key: string): ExportDataset | undefined => BY_KEY.get(key); diff --git a/apps/edr-freight-api/src/modules/exports/export.types.ts b/apps/edr-freight-api/src/modules/exports/export.types.ts new file mode 100644 index 000000000..db12f0211 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export.types.ts @@ -0,0 +1,135 @@ +import { + DataSource, + EntityTarget, + ObjectLiteral, + ObjectType, + SelectQueryBuilder, +} from 'typeorm'; + +import { ExportFilterDef } from './export-filter.util'; +import { ExportFieldType } from './tabular-export.service'; + +export type { ExportFieldType }; + +/** + * A lazily-applied relation. + * + * There is deliberately no `kind: 'inner' | 'left'` here — every join is + * emitted as a LEFT JOIN, and the type makes anything else unrepresentable. + * An inner join added only because someone ticked a checkbox would silently + * change the rowset (ticking "Customer TIN" would drop every booking with a + * null company_id), so two exports of the same filters would disagree on their + * row count. Anything that genuinely must narrow rows belongs in `scope()`, + * where it is unconditional and visible. + * + * The payoff: because a left join to a to-one side can neither add nor remove + * rows, the row count is independent of which fields are selected — which is + * what lets the count endpoint be exact rather than an estimate. + */ +export interface ExportJoin { + /** Alias used by field `select` expressions and by `requires`. */ + alias: string; + /** Entity class. Narrower than `EntityTarget` to match TypeORM's join overload. */ + entity: ObjectType; + /** ON condition; may reference the base alias and any alias in `requires`. */ + on: string; + /** Other join aliases this join's ON clause depends on. Resolved transitively. */ + requires?: string[]; +} + +/** + * One exportable column. + * + * `select` must yield exactly ONE value per base row. To surface a one-to-many + * relation (a company's profiles, a booking's containers), aggregate inside a + * correlated subquery — `(SELECT string_agg(...) FROM ... WHERE ... = base.id)` + * — rather than adding a join, which would multiply rows and break the count. + * + * Sensitive columns are simply never declared: raw gateway payloads + * (payments.raw_initiation, client_action), signature/crypto blobs + * (invoices.eims_signed_qr), internal error dumps (eims_last_error), raw jsonb + * snapshots (pricing_breakdown, document_snapshot, financial_terms, + * attributes, business_license_files), bare internal user UUIDs, and internal + * review/rejection notes. Fields are opt-in, so omission is the whole + * enforcement mechanism. + */ +export interface ExportField { + /** Response key, sheet header id, and the picker's checkbox id. */ + key: string; + label: string; + type: ExportFieldType; + /** Scalar SQL projected as `key`. */ + select: string; + /** Join aliases `select` references. Omit for base-table-only fields. */ + requires?: string[]; + /** Picker group id; must exist in the dataset's `groups`. */ + group: string; + /** Pre-ticked when the dialog opens with no preset. */ + default?: boolean; + /** ORDER BY expression. Presence makes the field sortable. */ + sortExpr?: string; +} + +export interface ExportGroup { + id: string; + label: string; +} + +export interface ExportContext { + ds: DataSource; + /** Filter values, already coerced by `coerceFilterParams`. */ + params: Record; + /** Trade-scope directions. `null` = unrestricted, `[]` = show nothing. */ + directions: string[] | null; +} + +export interface ExportDataset { + key: string; + title: string; + description: string; + group: 'Commercial' | 'Operations' | 'Finance' | 'Fleet'; + /** + * Permission to export this dataset. Reuses the module's existing `view` + * key — if you may see these rows on their list page, you may export them. + * The export never returns a row the list endpoint would not. + */ + permission: string; + base: { entity: EntityTarget; alias: string }; + joins: ExportJoin[]; + /** + * Aliases applied unconditionally because `scope()` references them. This is + * the only reason a join is eager, and the count query applies exactly these. + */ + alwaysJoin?: string[]; + groups: ExportGroup[]; + fields: ExportField[]; + filters: ExportFilterDef[]; + /** Must name a field whose `sortExpr` references only the base alias. */ + defaultSort?: { key: string; dir: 'ASC' | 'DESC' }; + /** + * Base WHERE (soft-delete guard), filter application, and the trade-direction + * ACL. Runs identically for the count and download queries, so the row count + * the dialog shows is exactly what lands in the file. + * + * A dataset whose table carries a trade direction MUST apply it here, or the + * export leaks rows the user cannot see on the list page. + */ + scope(ctx: ExportContext, qb: SelectQueryBuilder): void; +} + +/** + * What `GET /exports` serves. `select` / `requires` / `sortExpr` are raw SQL + * and a map of the schema — they never leave the server. + */ +export interface ExportCatalogEntry { + key: string; + title: string; + description: string; + group: ExportDataset['group']; + groups: ExportGroup[]; + fields: Pick[]; + filters: ExportFilterDef[]; + formats: ('csv' | 'xlsx' | 'pdf')[]; + caps: { csv: number; xlsx: number; pdf: number }; + defaultSort?: { key: string; dir: 'ASC' | 'DESC' }; +} diff --git a/apps/edr-freight-api/src/modules/exports/exports.controller.ts b/apps/edr-freight-api/src/modules/exports/exports.controller.ts new file mode 100644 index 000000000..89ca7419a --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/exports.controller.ts @@ -0,0 +1,156 @@ +import { Controller, Get, NotFoundException, Param, Query, Res, UseGuards } from '@nestjs/common'; +import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { InjectDataSource } from '@nestjs/typeorm'; +import { DataSource } from 'typeorm'; +import { CurrentUser } from '@edr/api-common'; +import { JwtGuard } from '@tria-plc/api-common/modules/auth/services/jwt.guard'; +import type { Response } from 'express'; +import type { TCurrentUser } from '@tria-plc/api-common/modules/auth/types/current-user.type'; + +import { assertFreightPermission, hasFreightPermission } from '../../common/freight-permission.util'; +import { UserTradeAccessService } from '../user-trade-access/user-trade-access.service'; +import { resolveFilterOptions } from './export-filter.util'; +import { + EXPORT_MIME, + formatRowCap, + pickByKey, + resolveExportFormat, + resolveRowLimit, +} from './export-request.util'; +import { ExportRunnerService } from './export-runner.service'; +import { DATASETS, getDataset } from './export.registry'; +import { ExportCatalogEntry, ExportDataset, ExportField } from './export.types'; +import { CSV_ROW_CAP, PDF_ROW_CAP, TabularExportService, XLSX_ROW_CAP } from './tabular-export.service'; + +/** Raw query bag — filter keys are per-dataset, so DTO whitelisting can't police it. */ +type RawExportQuery = Record; + +const CAPS = { csv: CSV_ROW_CAP, xlsx: XLSX_ROW_CAP, pdf: PDF_ROW_CAP }; + +/** + * Metadata only. `select` / `requires` / `sortExpr` are raw SQL and a map of + * the schema — they never leave the server. + */ +const toCatalogEntry = (dataset: ExportDataset): ExportCatalogEntry => ({ + key: dataset.key, + title: dataset.title, + description: dataset.description, + group: dataset.group, + groups: dataset.groups, + fields: dataset.fields.map(({ key, label, type, group, default: isDefault }) => ({ + key, + label, + type, + group, + default: isDefault, + })), + filters: dataset.filters, + formats: ['csv', 'xlsx', 'pdf'], + caps: CAPS, + defaultSort: dataset.defaultSort, +}); + +/** + * Generic table export. One dataset per major table, each describing far more + * fields than its list page shows — including related-entity detail. + */ +@ApiTags('Exports') +@ApiBearerAuth() +@Controller('exports') +@UseGuards(JwtGuard) +export class ExportsController { + constructor( + private readonly runner: ExportRunnerService, + private readonly writer: TabularExportService, + private readonly userTradeAccessService: UserTradeAccessService, + @InjectDataSource() private readonly dataSource: DataSource, + ) {} + + @Get() + @ApiOperation({ summary: 'List datasets the caller has permission to export' }) + async catalog(@CurrentUser() user: TCurrentUser): Promise { + const allowed = DATASETS.filter((d) => hasFreightPermission(user, d.permission)); + return Promise.all( + allowed.map(async (d) => ({ + ...toCatalogEntry(d), + filters: await resolveFilterOptions(d.filters, this.dataSource), + })), + ); + } + + @Get(':key/count') + @ApiOperation({ summary: 'Exact row count for the given filters, plus the per-format caps' }) + async count( + @Param('key') key: string, + @Query() query: RawExportQuery, + @CurrentUser() user: TCurrentUser, + ): Promise<{ total: number; caps: typeof CAPS }> { + const dataset = this.resolve(key, user); + const directions = await this.userTradeAccessService.resolveAllowedDirections(user); + const total = await this.runner.count(dataset, query, directions); + return { total, caps: CAPS }; + } + + @Get(':key/download') + @ApiOperation({ summary: 'Export a dataset to csv, xlsx or pdf' }) + async download( + @Param('key') key: string, + @Query() query: RawExportQuery & { format?: string; fields?: string; limit?: string }, + @CurrentUser() user: TCurrentUser, + @Res() res: Response, + ): Promise { + const dataset = this.resolve(key, user); + const directions = await this.userTradeAccessService.resolveAllowedDirections(user); + const format = resolveExportFormat(query.format); + const fields = this.resolveFields(dataset, query.fields); + + const rows = await this.runner.run(dataset, fields, query, directions, { + cap: formatRowCap(format), + limit: resolveRowLimit(format, query.limit), + }); + const doc = { + title: dataset.title, + description: dataset.description, + label: `export:${dataset.key}`, + columns: fields.map(({ key: k, label, type }) => ({ key: k, label, type })), + rows, + }; + const buffer = + format === 'pdf' + ? await this.writer.toPdf(doc) + : format === 'csv' + ? await this.writer.toCsv(doc) + : await this.writer.toXlsx(doc); + + const mime = EXPORT_MIME[format]; + const stamp = new Date().toISOString().slice(0, 10); + res.setHeader('Content-Disposition', `attachment; filename="${dataset.key}-${stamp}.${mime.ext}"`); + res.setHeader('Content-Type', mime.type); + res.send(buffer); + } + + /** + * Requested fields, whitelisted against the dataset. No `fields=` means the + * DEFAULT set, not everything — a booking export has ~70 fields and dumping + * all of them on an unparameterised call is nobody's intent. + */ + private resolveFields(dataset: ExportDataset, raw: string | undefined): ExportField[] { + if (raw?.trim()) { + const picked = pickByKey(dataset.fields, raw); + // pickByKey falls back to everything when nothing matched; for a dataset + // the safer read of "all keys unknown" is still the default set. + if (picked.length !== dataset.fields.length) return picked; + } + const defaults = dataset.fields.filter((f) => f.default); + return defaults.length ? defaults : dataset.fields; + } + + private resolve(key: string, user: TCurrentUser): ExportDataset { + const dataset = getDataset(key); + if (!dataset) throw new NotFoundException(`Unknown export dataset: ${key}`); + // Export rides the dataset's own list-page view permission: if you may see + // these rows, you may export them. + assertFreightPermission(user, dataset.permission); + return dataset; + } +} diff --git a/apps/edr-freight-api/src/modules/exports/exports.module.ts b/apps/edr-freight-api/src/modules/exports/exports.module.ts index 6f56826bf..e3c64bd4c 100644 --- a/apps/edr-freight-api/src/modules/exports/exports.module.ts +++ b/apps/edr-freight-api/src/modules/exports/exports.module.ts @@ -1,17 +1,23 @@ import { Module } from '@nestjs/common'; import { DocumentsModule } from '../billing/documents/documents.module'; +import { UserTradeAccessModule } from '../user-trade-access/user-trade-access.module'; +import { ExportRunnerService } from './export-runner.service'; +import { ExportsController } from './exports.controller'; import { TabularExportService } from './tabular-export.service'; /** - * Export infrastructure. Currently just the shared tabular writer (xlsx / csv / - * pdf) that both the reports module and — once the dataset registry lands — the - * generic table exports write through. No domain dependencies, so any module can - * import it. + * Generic table export: a dataset registry describing far more fields than each + * list page shows (related-entity detail included), plus the shared tabular + * writer (csv / xlsx / pdf) the reports module also writes through. + * + * `TabularExportService` is exported so ReportsModule can reuse it without + * pulling in the dataset machinery. */ @Module({ - imports: [DocumentsModule], - providers: [TabularExportService], + imports: [DocumentsModule, UserTradeAccessModule], + controllers: [ExportsController], + providers: [TabularExportService, ExportRunnerService], exports: [TabularExportService], }) export class ExportsModule {} diff --git a/apps/edr-freight-api/src/scripts/validate-export-datasets.ts b/apps/edr-freight-api/src/scripts/validate-export-datasets.ts new file mode 100644 index 000000000..5ee036d3d --- /dev/null +++ b/apps/edr-freight-api/src/scripts/validate-export-datasets.ts @@ -0,0 +1,83 @@ +/** + * EXPLAIN-validates every export dataset against the real database. + * + * CLAUDE.md hard rule: raw SQL must be validated against a real DB before it + * ships. Every dataset is hand-written SQL expressions over wide tables where + * column drift is documented history, so a typo is a runtime 500 no type-check + * can catch. This builds each dataset's WIDEST query (all fields selected, so + * every join and every subquery is exercised) plus its count query, and runs + * both through EXPLAIN. + * + * npx ts-node -r tsconfig-paths/register src/scripts/validate-export-datasets.ts + */ +import 'dotenv/config'; + +import AppDataSource from '../data-source'; +import { buildExportCountQuery, buildExportQuery } from '../modules/exports/export-query.builder'; +import { DATASETS } from '../modules/exports/export.registry'; + +async function main(): Promise { + await AppDataSource.initialize(); + let failed = 0; + + for (const dataset of DATASETS) { + const ctx = { ds: AppDataSource, params: {}, directions: null }; + + const cases: [string, () => { sql: string; params: unknown[] }][] = [ + [ + `${dataset.key} (all ${dataset.fields.length} fields)`, + () => { + const qb = buildExportQuery(dataset, dataset.fields, ctx); + return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] }; + }, + ], + [ + `${dataset.key} (count)`, + () => { + const qb = buildExportCountQuery(dataset, ctx); + return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] }; + }, + ], + ]; + + // Each field ALONE. The all-fields query above cannot catch a field that + // references an alias it forgot to declare in `requires` — some other + // field's `requires` pulls that join in, so it only 42P01s when that one + // checkbox is ticked on its own. This is the check that finds it. + for (const field of dataset.fields) { + cases.push([ + `${dataset.key}.${field.key}`, + () => { + const qb = buildExportQuery(dataset, [field], ctx); + return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] }; + }, + ]); + } + + let fieldFailures = 0; + for (const [label, build] of cases) { + const isPerField = label.startsWith(`${dataset.key}.`); + try { + const { sql } = build(); + // Parameters are all optional filters and unset here, so the generated + // SQL carries no placeholders — EXPLAIN it directly. + await AppDataSource.query(`EXPLAIN ${sql}`); + if (!isPerField) console.log(` ok ${label}`); + } catch (error) { + failed += 1; + if (isPerField) fieldFailures += 1; + console.error(` FAIL ${label}`); + console.error(` ${(error as Error).message.split('\n')[0]}`); + } + } + if (!fieldFailures) { + console.log(` ok ${dataset.key} (each of ${dataset.fields.length} fields alone)`); + } + } + + await AppDataSource.destroy(); + console.log(failed ? `\n${failed} query/queries failed.` : '\nAll export dataset SQL validated.'); + process.exit(failed ? 1 : 0); +} + +void main(); From 42b9f30057d3a94bff7549db62b2ab161e992e81 Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 06:44:29 +0000 Subject: [PATCH 12/66] feat(export-ui): field-picker export dialog, mounted on bookings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Stripe-style export dialog over the /exports catalog: searchable field picker grouped by related entity, format choice, row scope, saved presets, and a live row count. The picker is what makes 77 fields usable. Groups auto-expand only when they already hold a selection, so the dialog opens showing the default columns and their groups rather than a wall of checkboxes; searching force-expands so a match can't hide inside a collapsed group. Group headers carry a tri-state checkbox and an n/total badge. The row count comes from /exports/:key/count with the page's own filters, so the button reads 'Export 223 rows' before anything is downloaded, and turns into a cap warning with a one-click 'export the first N' escape when the result is too large for the chosen format. ExportButton takes plain params rather than a UseFilters instance — four of the pages that need this haven't migrated to FilterBar yet, and coupling to the hook would have blocked them. Pagination keys are stripped in one place instead of at every call site. It renders nothing when the catalog omits the dataset, so the catalog's permission filtering IS the UI gate. Presets reuse useSavedViews unchanged by encoding the preset as a query string; a preset naming a field the catalog no longer offers is dropped on load rather than 400ing the download. Download errors go through extractDownloadErrorMessage, without which the server's row-cap message degrades to 'Request failed with status code 400'. --- .../src/components/export/ExportButton.tsx | 86 ++++ .../src/components/export/ExportDialog.tsx | 420 ++++++++++++++++++ .../backoffice/src/constants/URLS.ts | 6 + .../pages/bookings/BookingRequestsPage.tsx | 5 +- .../backoffice/src/services/api.ts | 18 + .../src/services/exports.service.ts | 42 ++ .../backoffice/src/types/exports.ts | 54 +++ 7 files changed, 630 insertions(+), 1 deletion(-) create mode 100644 apps/edr-freight-web/backoffice/src/components/export/ExportButton.tsx create mode 100644 apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx create mode 100644 apps/edr-freight-web/backoffice/src/services/exports.service.ts create mode 100644 apps/edr-freight-web/backoffice/src/types/exports.ts diff --git a/apps/edr-freight-web/backoffice/src/components/export/ExportButton.tsx b/apps/edr-freight-web/backoffice/src/components/export/ExportButton.tsx new file mode 100644 index 000000000..b79017cdb --- /dev/null +++ b/apps/edr-freight-web/backoffice/src/components/export/ExportButton.tsx @@ -0,0 +1,86 @@ +import { useMemo, useState } from "react"; +import { Button, Tooltip } from "@mantine/core"; +import { useQuery } from "@tanstack/react-query"; +import { Download } from "lucide-react"; + +import { api } from "@/services/api"; +import type { ExportParams } from "@/types/exports"; + +import { ExportDialog } from "./ExportDialog"; + +/** + * Pagination is a screen concern, never an export one — stripped here, once, + * rather than at each of the pages that mount this. + */ +const PAGINATION_KEYS = ["page", "pageSize", "skip", "take"]; + +export interface ExportButtonProps { + /** Catalog dataset key, e.g. "bookings". */ + datasetKey: string; + /** + * The page's current filters — `useFilters().params` verbatim, or a + * non-migrated page's hand-built filter object. Deliberately not typed as + * `UseFilters`: four of the pages that need this haven't migrated yet. + */ + params?: Record; + label?: string; + size?: "xs" | "sm"; +} + +/** + * Opens the export dialog for one dataset. Renders nothing when the caller + * lacks permission for that dataset — the catalog only returns what they may + * export, so an absent entry IS the permission check. + */ +export function ExportButton({ + datasetKey, + params, + label = "Export", + size = "xs", +}: ExportButtonProps) { + const [opened, setOpened] = useState(false); + const { data: catalog, isLoading } = useQuery( + api.exports.catalog.queryOptions({ staleTime: 5 * 60_000 }), + ); + + const dataset = catalog?.find((d) => d.key === datasetKey); + + const exportParams = useMemo(() => { + const out: ExportParams = {}; + for (const [key, value] of Object.entries(params ?? {})) { + if (PAGINATION_KEYS.includes(key)) continue; + if (value === undefined || value === null || value === "") continue; + out[key] = value as string | number; + } + return out; + }, [params]); + + if (isLoading || !dataset) return null; + + return ( + <> + + + + + {opened && ( + setOpened(false)} + dataset={dataset} + params={exportParams} + /> + )} + + ); +} + +export default ExportButton; diff --git a/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx b/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx new file mode 100644 index 000000000..195d05f08 --- /dev/null +++ b/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx @@ -0,0 +1,420 @@ +import { useMemo, useState } from "react"; +import { + Accordion, + Alert, + Anchor, + Badge, + Button, + Checkbox, + Chip, + Divider, + Group, + Loader, + Modal, + Popover, + Radio, + ScrollArea, + Select, + SimpleGrid, + Stack, + Text, + TextInput, +} from "@mantine/core"; +import { useQuery } from "@tanstack/react-query"; +import { Download, FileSpreadsheet, FileText, Search, Table, TriangleAlert, X } from "lucide-react"; + +import { extractDownloadErrorMessage } from "@/components/warehouses/options"; +import { saveBlob } from "@/components/warehouses/pdf"; +import { useSavedViews } from "@/components/filters"; +import { useToast } from "@/hooks/use-toast"; +import { api } from "@/services/api"; +import { exportsService } from "@/services/exports.service"; +import type { + ExportDatasetEntry, + ExportFormat, + ExportParams, +} from "@/types/exports"; + +const FORMAT_META: Record = { + csv: { label: "CSV", Icon: Table, hint: "Best for many columns" }, + xlsx: { label: "Excel", Icon: FileSpreadsheet, hint: "Typed number columns" }, + pdf: { label: "PDF", Icon: FileText, hint: "Few columns only" }, +}; + +const ROW_SCOPES = [ + { value: "all", label: "All matching filters" }, + { value: "100", label: "First 100" }, + { value: "1000", label: "First 1,000" }, + { value: "5000", label: "First 5,000" }, +]; + +/** Beyond this a PDF's columns are too narrow to read; we warn, the server allows it. */ +const PDF_FIELD_WARN = 12; + +export interface ExportDialogProps { + opened: boolean; + onClose: () => void; + dataset: ExportDatasetEntry; + /** The page's current filters. Pagination keys are stripped by ExportButton. */ + params: ExportParams; +} + +export function ExportDialog({ opened, onClose, dataset, params }: ExportDialogProps) { + const { toast } = useToast(); + const defaultKeys = useMemo( + () => dataset.fields.filter((f) => f.default).map((f) => f.key), + [dataset.fields], + ); + + const [selected, setSelected] = useState(defaultKeys); + const [format, setFormat] = useState("csv"); + const [scope, setScope] = useState("all"); + const [search, setSearch] = useState(""); + const [exporting, setExporting] = useState(false); + const [presetName, setPresetName] = useState(""); + const [savePresetOpen, setSavePresetOpen] = useState(false); + + // A preset is stored as a query string so the existing saved-views hook can + // hold it unchanged — see useExportPresets note below. + const presets = useSavedViews(`export:${dataset.key}`); + + const { data: countData, isLoading: countLoading } = useQuery({ + ...api.exports.count.queryOptions({ input: { key: dataset.key, params } }), + enabled: opened, + staleTime: 30_000, + }); + + const total = countData?.total; + const cap = dataset.caps[format]; + const limit = scope === "all" ? undefined : Number(scope); + const rowsToExport = total === undefined ? undefined : Math.min(total, limit ?? total); + const overCap = total !== undefined && limit === undefined && total > cap; + + const selectedSet = useMemo(() => new Set(selected), [selected]); + const fieldKeys = useMemo(() => new Set(dataset.fields.map((f) => f.key)), [dataset.fields]); + + const visibleByGroup = useMemo(() => { + const q = search.trim().toLowerCase(); + const out = new Map(); + for (const group of dataset.groups) { + const fields = dataset.fields.filter( + (f) => f.group === group.id && (!q || f.label.toLowerCase().includes(q)), + ); + if (fields.length) out.set(group.id, fields); + } + return out; + }, [dataset.fields, dataset.groups, search]); + + // Searching force-expands so matches aren't hidden inside collapsed groups. + // Otherwise open only groups that already have something selected, which is + // what keeps 77 fields tractable on open. + const openGroups = search.trim() + ? [...visibleByGroup.keys()] + : dataset.groups + .filter((g) => dataset.fields.some((f) => f.group === g.id && selectedSet.has(f.key))) + .map((g) => g.id); + + const toggleField = (key: string) => + setSelected((prev) => (prev.includes(key) ? prev.filter((k) => k !== key) : [...prev, key])); + + const toggleGroup = (groupId: string) => { + const keys = dataset.fields.filter((f) => f.group === groupId).map((f) => f.key); + const allOn = keys.every((k) => selectedSet.has(k)); + setSelected((prev) => + allOn ? prev.filter((k) => !keys.includes(k)) : [...new Set([...prev, ...keys])], + ); + }; + + const applyPreset = (query: string) => { + const p = new URLSearchParams(query); + // Drop any key the catalog no longer offers — a stale preset must not 400 + // the download by asking for a field that has since been removed. + const keys = (p.get("fields") ?? "").split(",").filter((k) => fieldKeys.has(k)); + if (keys.length) setSelected(keys); + const f = p.get("format") as ExportFormat | null; + if (f && dataset.formats.includes(f)) setFormat(f); + }; + + const savePreset = () => { + const name = presetName.trim(); + if (!name) return; + presets.save( + new URLSearchParams({ name, format, fields: selected.join(",") }).toString(), + ); + setPresetName(""); + setSavePresetOpen(false); + }; + + const handleDownload = async () => { + setExporting(true); + try { + const blob = await exportsService.download(dataset.key, format, selected, { + ...params, + ...(limit ? { limit } : {}), + }); + saveBlob(blob, `${dataset.key}-${new Date().toISOString().slice(0, 10)}.${format}`); + onClose(); + } catch (error) { + // Blob error bodies need the async decoder, or the server's row-cap + // message degrades to "Request failed with status code 400". + toast({ + variant: "destructive", + title: "Export failed", + description: await extractDownloadErrorMessage(error), + }); + } finally { + setExporting(false); + } + }; + + return ( + + + {/* Presets */} + + setSelected(defaultKeys)}> + Default columns + + setSelected(dataset.fields.map((f) => f.key))} + > + All columns + + {presets.views.map((view) => { + const name = new URLSearchParams(view.query).get("name") ?? "Preset"; + return ( + applyPreset(view.query)} + > + + {name} + { + e.stopPropagation(); + presets.remove(view.id); + }} + /> + + + ); + })} + + + + + + + setPresetName(e.currentTarget.value)} + onKeyDown={(e) => e.key === "Enter" && savePreset()} + autoFocus + /> + + + + + + + + + {/* Pick the data on the left, configure the file on the right. Stacks + on a phone, where neither column has room to sit beside the other. */} +
+ {/* Fields */} +
+ + } + value={search} + onChange={(e) => setSearch(e.currentTarget.value)} + /> + + + {selected.length} of {dataset.fields.length} fields selected + + + + + + + {dataset.groups.map((group) => { + const fields = visibleByGroup.get(group.id); + if (!fields) return null; + const groupKeys = dataset.fields + .filter((f) => f.group === group.id) + .map((f) => f.key); + const on = groupKeys.filter((k) => selectedSet.has(k)).length; + return ( + + + + 0 && on < groupKeys.length} + onClick={(e) => { + e.stopPropagation(); + toggleGroup(group.id); + }} + onChange={() => undefined} + /> + + {group.label} + + + {on}/{groupKeys.length} + + + + + + {fields.map((field) => ( + toggleField(field.key)} + /> + ))} + + + + ); + })} + + + +
+ + {/* Options */} +
+ +
+ + Format + + setFormat(v as ExportFormat)}> + + {dataset.formats.map((f) => { + const { label, Icon } = FORMAT_META[f]; + return ( + + + + + {label} + + + + ); + })} + + +
+ + + } /> From 6e95c5b8b8b02958994cfb8ad83997365ba02e07 Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 08:20:00 +0000 Subject: [PATCH 15/66] fix(backoffice): map overview layouts to the org's real position keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overview layout table matched invented keys (`edr_operations_officer`, `edr_marketing`, …) that only ever existed as IAM roles. The positions actually configured under the unit use their own keys — `edr_freight_app/opn`, `ethiopian_gl`, `edr_freight_app/finance` — so most staff fell through to the executive fallback regardless of desk. Map every position key in the current org tree, roots and sub-positions alike, and keep the legacy role-form keys so accounts that model the desks as roles still resolve. Finance was previously unmapped entirely. Also drop a stray console.log from resolveOverviewLayout. Safety (`edr_freight_app/sf_146`) stays unmapped — no such layout exists yet. Co-Authored-By: Claude Opus 5 (1M context) --- .../overview/role-dashboards.config.ts | 87 +++++++++++++++---- .../src/pages/dashboard/OverviewPage.tsx | 36 +++++--- 2 files changed, 94 insertions(+), 29 deletions(-) diff --git a/apps/edr-freight-web/backoffice/src/components/overview/role-dashboards.config.ts b/apps/edr-freight-web/backoffice/src/components/overview/role-dashboards.config.ts index 509f17a3c..b683a3fbe 100644 --- a/apps/edr-freight-web/backoffice/src/components/overview/role-dashboards.config.ts +++ b/apps/edr-freight-web/backoffice/src/components/overview/role-dashboards.config.ts @@ -4,44 +4,93 @@ import { getPositionKeys } from "@/lib/permissions"; /** One overview composition. Every backoffice user lands on exactly one of these. */ export type OverviewLayoutKey = | "executive" - | "operations" + | "operation" | "occ" - | "marketing" + | "marketer" | "finance" | "clearance"; export const OVERVIEW_LAYOUT_LABEL: Record = { executive: "Executive dashboard", - operations: "Operations dashboard", + operation: "Operations dashboard", occ: "Control centre dashboard", - marketing: "Marketing dashboard", + marketer: "Marketing dashboard", finance: "Finance dashboard", clearance: "Clearance & logistics dashboard", }; /** - * Role/position key → layout, in match priority order: a user holding several + * Position/role key → layout, in match priority order: a user holding several * of these keys gets the first match, so the specific operational view wins - * over the broad executive one. Position keys are matched too because the IAM - * payload models the GL desks as positions (`ethiopian_gl`) on some accounts - * and as roles (`edr_gl_ethiopia`) on others — see `getPositionKeys`. + * over the broad executive one. Roles are matched alongside positions because + * the IAM payload models the GL desks as positions (`ethiopian_gl`) on some + * accounts and as roles (`edr_gl_ethiopia`) on others — see `getPositionKeys`. + * + * The `edr_freight_app/…` keys are the org's real position keys (root desks and + * their sub-positions) as configured under Unit → Departments. They are typed + * by hand in the Add/Edit Department form, so a new sub-position appears here + * only once someone adds it — unmapped keys fall through to `executive`. */ const ROLE_LAYOUTS: Array<[key: string, layout: OverviewLayoutKey]> = [ - ["edr_operations_officer", "operations"], - ["truck_machinery_chief", "operations"], - ["edr_line_staff", "occ"], - ["edr_gl_ethiopia", "clearance"], - ["edr_gl_djibouti", "clearance"], + // ── Clearance & logistics: both GL desks, root and sub-positions ────────── ["ethiopian_gl", "clearance"], + ["edr_freight_app/gl_003", "clearance"], // Ethiopian GL Chief + ["edr_freight_app/off_001", "clearance"], // Ethiopian GL Director + ["edr_freight_app/off_0056", "clearance"], // Ethiopian GL Officer ["djibouti_gl", "clearance"], - ["edr_marketing", "marketing"], - ["edr_finance", "finance"], - ["edr_director", "executive"], - ["edr_ceo", "executive"], - ["edr_org_manager", "executive"], + ["edr_freight_app/dj_gl_001", "clearance"], // Djibouti GL Director + ["edr_freight_app/dj_gl_002", "clearance"], // Djibouti GL Chief + ["edr_freight_app/dj_gl_003", "clearance"], // Djibouti GL Officer + ["edr_gl_ethiopia", "clearance"], // legacy role form + ["edr_gl_djibouti", "clearance"], // legacy role form + + // ── Control centre ─────────────────────────────────────────────────────── + ["edr_freight_app/occ_001", "occ"], // OCC + ["edr_freight_app/occ_005", "occ"], // OCC Director + ["edr_line_staff", "occ"], // legacy role form + + // ── Operations: operations desk, track & machinery, rolling stock ───────── + ["edr_freight_app/opn", "operation"], // Operation + ["edr_freight_app/opcf", "operation"], // Operation Chief + ["edr_freight_app/opdr", "operation"], // Operation Director + ["edr_freight_app/opco", "operation"], // Operation Officer + ["edr_freight_app/opp_005", "operation"], // Operation Dispatcher + ["edr_freight_app/opp_0067", "operation"], // Gelan Operation Director + ["edr_freight_app/track_001", "operation"], // Track And Machinery + ["edr_freight_app/ttk_001", "operation"], // Track Director + ["edr_freight_app/tto_001", "operation"], // Track Operator + ["edr_freight_app/rool_001", "operation"], // Rolling Stock + ["edr_freight_app/rl_003", "operation"], // Rolling Stock Director + ["edr_freight_app/rl_009", "operation"], // Rolling Stock Team Lead + ["edr_freight_app/rl_0090", "operation"], // Rolling Stock Dispatcher + ["operation", "operation"], + ["operations_chief", "operation"], + ["dispatcher", "operation"], + ["truck_machinery_chief", "operation"], + ["edr_operations_officer", "operation"], // legacy role form + + // ── Marketing ──────────────────────────────────────────────────────────── + ["edr_freight_app/edr_test_org_0022", "marketer"], // Commercial Marketing + ["edr_freight_app/edr_test_org_00567", "marketer"], // Marketing Director + ["edr_freight_app/edr_test_org_0054", "marketer"], // Marketing Chief + ["edr_freight_app/edr_test_org_0013", "marketer"], // Marketing Officer + ["marketer", "marketer"], + ["edr_marketing", "marketer"], // legacy role form + + // ── Finance ────────────────────────────────────────────────────────────── + ["edr_freight_app/finance", "finance"], + ["edr_finance", "finance"], // legacy role form + + // ── Executive: org-wide desks with no operational queue of their own ────── + ["ceo", "executive"], + ["director", "executive"], + ["chief", "executive"], + ["edr_ceo", "executive"], // legacy role form + ["edr_director", "executive"], // legacy role form + ["edr_org_manager", "executive"], // legacy role form ]; -/** Unmapped roles (superadmin, IAM admins, new roles) keep the executive layout. */ +/** Unmapped keys (superadmin, IAM admins, Safety, new positions) keep the executive layout. */ export function resolveOverviewLayout( user: AuthUser | null | undefined, ): OverviewLayoutKey { diff --git a/apps/edr-freight-web/backoffice/src/pages/dashboard/OverviewPage.tsx b/apps/edr-freight-web/backoffice/src/pages/dashboard/OverviewPage.tsx index 0d1f5a6e7..eadd2f1c6 100644 --- a/apps/edr-freight-web/backoffice/src/pages/dashboard/OverviewPage.tsx +++ b/apps/edr-freight-web/backoffice/src/pages/dashboard/OverviewPage.tsx @@ -24,14 +24,21 @@ import { useOverview } from "@/hooks/useOverview"; import type { OverviewRange } from "@/types/overview"; import "@/components/overview/summary/overview-summary.css"; -const RANGE_LABEL: Record = { "7d": "7d", "30d": "30d", "90d": "90d" }; +const RANGE_LABEL: Record = { + "7d": "7d", + "30d": "30d", + "90d": "90d", +}; /** Which composition each role sees below the hero. */ -const LAYOUTS: Record ReactElement> = { +const LAYOUTS: Record< + OverviewLayoutKey, + (props: RoleOverviewProps) => ReactElement +> = { executive: ExecutiveOverview, - operations: OperationsOverview, + operation: OperationsOverview, occ: OccOverview, - marketing: MarketingOverview, + marketer: MarketingOverview, finance: FinanceOverview, clearance: ClearanceOverview, }; @@ -51,15 +58,17 @@ const OverviewPage = () => { const [range, setRange] = useState("30d"); const queryClient = useQueryClient(); const { user } = useAuth(); - const { data, isLoading, isError, error, refetch, isFetching } = useOverview(range); + const { data, isLoading, isError, error, refetch, isFetching } = + useOverview(range); // Hero, range control and headline KPIs are role-neutral; everything below // them is chosen by role key. const layoutKey = resolveOverviewLayout(user); - const RoleLayout = LAYOUTS[layoutKey]; + const RoleLayout = layoutKey ? LAYOUTS[layoutKey] : null; const accessDenied = - (error as { response?: { status?: number } } | null)?.response?.status === 403; + (error as { response?: { status?: number } } | null)?.response?.status === + 403; const handleRefresh = () => { void refetch(); @@ -79,7 +88,9 @@ const OverviewPage = () => { label={OVERVIEW_LAYOUT_LABEL[layoutKey]} /> {data ? ( -
+
{ > Check your connection and try again. - @@ -117,7 +133,7 @@ const OverviewPage = () => { - ) : data ? ( + ) : data && RoleLayout ? ( ) : null} From 87ec7cec07c205f68e952e081637c7d9b7493b1c Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 08:25:47 +0000 Subject: [PATCH 16/66] fix(export-ui): let field groups be expanded and collapsed by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accordion's open state was derived from the selection on every render, which made it fully controlled with nothing driving it. Clicking a group that had no fields selected opened it for one render and the recomputed value immediately shut it again, so such a group could only be opened by selecting something inside it — and conversely a group with a selection could not be collapsed at all. Open state is now real state with an onChange, seeded from the fields marked default rather than the live selection, so clearing every field doesn't close the groups underneath the user. Search still force-opens every group holding a match, but only as a display override — the manual state survives and returns when the search clears. --- .../src/components/export/ExportDialog.tsx | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx b/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx index 195d05f08..6c7af128c 100644 --- a/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx +++ b/apps/edr-freight-web/backoffice/src/components/export/ExportDialog.tsx @@ -105,14 +105,22 @@ export function ExportDialog({ opened, onClose, dataset, params }: ExportDialogP return out; }, [dataset.fields, dataset.groups, search]); - // Searching force-expands so matches aren't hidden inside collapsed groups. - // Otherwise open only groups that already have something selected, which is - // what keeps 77 fields tractable on open. - const openGroups = search.trim() - ? [...visibleByGroup.keys()] - : dataset.groups - .filter((g) => dataset.fields.some((f) => f.group === g.id && selectedSet.has(f.key))) - .map((g) => g.id); + // Which groups are expanded. Real state, NOT derived from the selection: + // deriving it made the accordion fully controlled with no way to change it, + // so clicking a group that had nothing selected re-collapsed on the next + // render and the group could only be opened by selecting a field in it. + // Seeded from `default` (not the live selection) so clearing every field + // doesn't slam the open groups shut underneath the user. + const [expanded, setExpanded] = useState(() => + dataset.groups + .filter((g) => dataset.fields.some((f) => f.group === g.id && f.default)) + .map((g) => g.id), + ); + + // Searching force-opens every group holding a match, so a hit can't hide + // inside a collapsed section. It only overrides what is displayed — the + // user's own expand state is untouched and returns when the search clears. + const openGroups = search.trim() ? [...visibleByGroup.keys()] : expanded; const toggleField = (key: string) => setSelected((prev) => (prev.includes(key) ? prev.filter((k) => k !== key) : [...prev, key])); @@ -264,7 +272,13 @@ export function ExportDialog({ opened, onClose, dataset, params }: ExportDialogP - + {dataset.groups.map((group) => { const fields = visibleByGroup.get(group.id); if (!fields) return null; From d7752f438629c9449851a9507f0606ffa56c276c Mon Sep 17 00:00:00 2001 From: Marshal Date: Thu, 20 Aug 2026 08:36:02 +0000 Subject: [PATCH 17/66] fix(portal): show declaration, T1 and Djibouti clearance documents to the customer --- .../src/modules/audit/audit-endpoints.ts | 1 + .../booking-lifecycle-notifier.service.ts | 11 ++ .../bookings/booking-transition.service.ts | 53 ++++++++ .../modules/bookings/bookings.controller.ts | 19 +++ .../entities/booking-review-note.entity.ts | 6 + .../contracts/booking-clearance.service.ts | 17 +++ .../detail/AdditionalDocsRequestCard.tsx | 113 ++++++++++++++++++ .../bookings/DocumentClearanceDetailPage.tsx | 9 ++ .../src/services/bookings.service.ts | 5 + .../bookings/clearance/ClearanceFlow.tsx | 83 +++++++++++++ .../bookings/clearance/useClearanceFlow.ts | 13 ++ packages/types/src/freight/index.ts | 11 ++ 12 files changed, 341 insertions(+) create mode 100644 apps/edr-freight-web/backoffice/src/components/bookings/detail/AdditionalDocsRequestCard.tsx diff --git a/apps/edr-freight-api/src/modules/audit/audit-endpoints.ts b/apps/edr-freight-api/src/modules/audit/audit-endpoints.ts index deda35e1c..019d99920 100644 --- a/apps/edr-freight-api/src/modules/audit/audit-endpoints.ts +++ b/apps/edr-freight-api/src/modules/audit/audit-endpoints.ts @@ -47,6 +47,7 @@ export const AUDIT_ENDPOINTS: Readonly> = { "POST /api/bookings/:id/clearance/proceed": ["Customer requests operation with a schedule day", "POST", "Booking"], "POST /api/bookings/:id/clearance/release-order": ["Upload Booking Release Order", "POST", "Booking"], "POST /api/bookings/:id/clearance/review": ["GL reviews a clearance document (Approve | Query)", "POST", "Booking"], + "POST /api/bookings/:id/clearance/doc-requests": ["GL asks the customer for additional clearance documents", "POST", "Booking"], "POST /api/bookings/:id/clearance/charges/port-document": ["GL Djibouti uploads the port-charges document", "POST", "Booking"], "PATCH /api/bookings/:id/clearance/charges/:chargeId/bill": ["GL Ethiopia sets or revises a clearance charge's amount + currency", "PATCH", "Booking"], "POST /api/bookings/:id/clearance/charges/:chargeId/send": ["GL Ethiopia issues the clearance charge invoice to the customer", "POST", "Booking"], diff --git a/apps/edr-freight-api/src/modules/bookings/booking-lifecycle-notifier.service.ts b/apps/edr-freight-api/src/modules/bookings/booking-lifecycle-notifier.service.ts index 78c07bd22..f457e782a 100644 --- a/apps/edr-freight-api/src/modules/bookings/booking-lifecycle-notifier.service.ts +++ b/apps/edr-freight-api/src/modules/bookings/booking-lifecycle-notifier.service.ts @@ -202,6 +202,17 @@ export class BookingLifecycleNotifierService { } /** A clearance document was queried and needs the customer to re-upload. */ + /** GL asked the customer for additional clearance document(s). */ + additionalDocsRequested(b: Booking, note: string): void { + const msg = + `Additional document(s) requested on booking ${b.reference}: ` + + `${note} Please upload them from the portal.`; + void this.notifyContact(b, msg, 'ADDITIONAL DOCUMENTS REQUESTED'); + this.inApp(b, 'Additional documents requested', msg, { + type: NotificationType.DOCUMENT_ACTION, + }); + } + documentQueried(b: Booking, fileKey: string, note: string): void { const msg = `A clearance document on booking ${b.reference} needs attention: "${fileKey}". ` + diff --git a/apps/edr-freight-api/src/modules/bookings/booking-transition.service.ts b/apps/edr-freight-api/src/modules/bookings/booking-transition.service.ts index 805b2216f..6d4d9073b 100644 --- a/apps/edr-freight-api/src/modules/bookings/booking-transition.service.ts +++ b/apps/edr-freight-api/src/modules/bookings/booking-transition.service.ts @@ -643,6 +643,12 @@ export class BookingTransitionService { }>; allApproved: boolean; documentsOpen: boolean; + docRequests: Array<{ + id: string; + note: string; + byName: string | null; + at: string; + }>; phase?: string | null; milestones?: unknown[]; nextAction?: unknown; @@ -674,9 +680,14 @@ export class BookingTransitionService { bookingId, "CHANGES_REQUESTED", ); + const docRequestNotes = await this.bookingsRepository.findReviewNotes( + bookingId, + "ADDITIONAL_DOC_REQUEST", + ); const reviewerNames = await this.bookingsRepository.resolveStaffNames([ ...reviews.map((r) => r.reviewedByStaffId), ...queryNotes.map((n) => n.authorId), + ...docRequestNotes.map((n) => n.authorId), ]); const documents: Awaited< @@ -763,9 +774,51 @@ export class BookingTransitionService { documents, allApproved, documentsOpen: clearanceDocumentsOpen(booking), + docRequests: docRequestNotes.map((n) => ({ + id: n.id, + note: n.note, + byName: n.authorId ? (reviewerNames.get(n.authorId) ?? null) : null, + at: n.createdAt.toISOString(), + })), }; } + /** + * GL asks the customer for additional clearance document(s). Stored as a + * review-note thread shown on both the GL clearance page and the customer's + * portal; the customer answers with an ad-hoc upload. Allowed for as long as + * documents are open (until the shipment is paid). + */ + async requestAdditionalDocuments( + bookingId: string, + note: string, + staffId: string, + ): Promise { + const booking = await this.bookingsService.findById(bookingId); + if (!clearanceDocumentsOpen(booking)) { + throw new ConflictException( + `Clearance documents are closed for this booking (status "${booking.status}").`, + ); + } + if (!note?.trim()) { + throw new BadRequestException("Describe the document(s) you need."); + } + await this.bookingsRepository.createReviewNote( + bookingId, + note.trim(), + "ADDITIONAL_DOC_REQUEST", + staffId, + ); + await this.clearanceEvents.record({ + bookingId, + action: "ADDITIONAL_DOCS_REQUESTED", + label: "Requested additional document(s) from the customer", + actorId: staffId, + metadata: { note: note.trim() }, + }); + this.notifier.additionalDocsRequested(booking, note.trim()); + } + /** * True when every REQUIRED field of the booking's customer-input clearance set * has an APPROVED review row. The 100% gate before clearance can be finalized. diff --git a/apps/edr-freight-api/src/modules/bookings/bookings.controller.ts b/apps/edr-freight-api/src/modules/bookings/bookings.controller.ts index b1be94016..05d312384 100644 --- a/apps/edr-freight-api/src/modules/bookings/bookings.controller.ts +++ b/apps/edr-freight-api/src/modules/bookings/bookings.controller.ts @@ -1083,6 +1083,25 @@ export class BookingsController { return this.transitionService.enrichBookingResponse(booking); } + @Post(":id/clearance/doc-requests") + @BookingStaff(FREIGHT_PERMS.contracts.clearanceEtActions) + @ApiOperation({ + summary: + "GL asks the customer for additional clearance document(s) — shown on the portal with author and time", + }) + async requestAdditionalDocuments( + @Param("id", ParseUUIDPipe) id: string, + @Body("note") note: string, + @CurrentUser() user: AuthUserPayload, + ) { + await this.transitionService.requestAdditionalDocuments( + id, + note, + resolveAuthUserId(user), + ); + return { success: true }; + } + @Get(":id/clearance/history") @BookingStaff([ FREIGHT_PERMS.contracts.clearanceEtActions, diff --git a/apps/edr-freight-api/src/modules/bookings/entities/booking-review-note.entity.ts b/apps/edr-freight-api/src/modules/bookings/entities/booking-review-note.entity.ts index 969098196..8bac3ea2b 100644 --- a/apps/edr-freight-api/src/modules/bookings/entities/booking-review-note.entity.ts +++ b/apps/edr-freight-api/src/modules/bookings/entities/booking-review-note.entity.ts @@ -11,6 +11,12 @@ export const REVIEW_NOTE_TYPES = [ * (price/files). One row per round — the draft/change-request loop can repeat. */ 'DRAFT_DECL_CHANGE_REQUEST', + /** + * GL asked the customer for additional clearance document(s). Shown as a + * thread on both the GL clearance page and the customer's portal — the + * customer answers by uploading an ad-hoc document. + */ + 'ADDITIONAL_DOC_REQUEST', ] as const; export type ReviewNoteType = (typeof REVIEW_NOTE_TYPES)[number]; diff --git a/apps/edr-freight-api/src/modules/contracts/booking-clearance.service.ts b/apps/edr-freight-api/src/modules/contracts/booking-clearance.service.ts index d9fe1825f..d393c011d 100644 --- a/apps/edr-freight-api/src/modules/contracts/booking-clearance.service.ts +++ b/apps/edr-freight-api/src/modules/contracts/booking-clearance.service.ts @@ -66,6 +66,12 @@ export interface BookingClearanceView { }>; allApproved: boolean; documentsOpen: boolean; + docRequests: Array<{ + id: string; + note: string; + byName: string | null; + at: string; + }>; phase?: string | null; milestones?: Array<{ id: string; @@ -206,9 +212,14 @@ export class BookingClearanceService { bookingId, 'CHANGES_REQUESTED', ); + const docRequestNotes = await this.bookingsRepository.findReviewNotes( + bookingId, + 'ADDITIONAL_DOC_REQUEST', + ); const reviewerNames = await this.bookingsRepository.resolveStaffNames([ ...reviews.map((r) => r.reviewedByStaffId), ...queryNotes.map((n) => n.authorId), + ...docRequestNotes.map((n) => n.authorId), ]); const documents: BookingClearanceView['documents'] = []; @@ -357,6 +368,12 @@ export class BookingClearanceService { documents, allApproved, documentsOpen: clearanceDocumentsOpen(booking), + docRequests: docRequestNotes.map((n) => ({ + id: n.id, + note: n.note, + byName: n.authorId ? (reviewerNames.get(n.authorId) ?? null) : null, + at: n.createdAt.toISOString(), + })), phase, milestones: milestones.map((m) => ({ id: m.id, diff --git a/apps/edr-freight-web/backoffice/src/components/bookings/detail/AdditionalDocsRequestCard.tsx b/apps/edr-freight-web/backoffice/src/components/bookings/detail/AdditionalDocsRequestCard.tsx new file mode 100644 index 000000000..fd9abe744 --- /dev/null +++ b/apps/edr-freight-web/backoffice/src/components/bookings/detail/AdditionalDocsRequestCard.tsx @@ -0,0 +1,113 @@ +import { useState } from "react"; +import { useMutation } from "@tanstack/react-query"; +import { + Box, + Button, + Group, + Paper, + Stack, + Text, + Textarea, +} from "@mantine/core"; +import { MessageSquarePlus, Send } from "lucide-react"; +import toast from "react-hot-toast"; +import type { Freight } from "@edr/types"; + +import { SectionCard } from "./SectionCard"; +import { bookingsService } from "@/services/bookings.service"; +import { formatDateTime } from "@/lib/format"; +import { extractErrorMessage } from "@/utils/errorExtractor"; + +export interface AdditionalDocsRequestCardProps { + bookingId: string; + /** Past requests, newest first. */ + requests: Freight.ClearanceDocRequest[]; + /** False once the shipment is paid — documents (and requests) are closed. */ + canRequest: boolean; + onSent?: () => void; +} + +/** + * GL asks the customer for additional clearance document(s) in plain words. + * The message, its author and its time show on the customer's portal beside + * the upload box, so the customer knows exactly what to send and who asked. + */ +export function AdditionalDocsRequestCard({ + bookingId, + requests, + canRequest, + onSent, +}: AdditionalDocsRequestCardProps) { + const [note, setNote] = useState(""); + + const send = useMutation({ + mutationFn: () => bookingsService.requestAdditionalDocuments(bookingId, note), + onSuccess: () => { + toast.success("Request sent to the customer"); + setNote(""); + onSent?.(); + }, + onError: (e) => + toast.error(extractErrorMessage(e, "Could not send the request")), + }); + + return ( + + + {canRequest ? ( + +