From 62f7b91315ead96a0691bce61f17713fc0a90f1e Mon Sep 17 00:00:00 2001 From: Nathnael Date: Thu, 20 Aug 2026 05:29:04 +0000 Subject: [PATCH] feat(exports): dataset-driven table export, starting with bookings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a parallel export system the reports module can also draw on. A dataset describes a table's exportable fields — including related-entity detail the list page never shows — and the engine assembles a query from whichever fields the caller picked. GET /exports catalog (metadata only; select/requires never ship) GET /exports/:key/count exact row count + per-format caps GET /exports/:key/download csv | xlsx | pdf Two invariants carry the design: - Every lazy join is a LEFT join, and ExportJoin has no 'kind' field to make anything else expressible. An inner join added because a checkbox was ticked would change the rowset, so two exports of the same filters would disagree on their row count. - Because of that, the count cannot depend on field selection, so /count runs base + alwaysJoin only and is exact rather than an estimate. Verified: count and the delivered file both report 223 rows. One-to-many relations (a booking's containers) aggregate in a correlated subquery rather than joining, so a row can never multiply. Export rides each dataset's existing view permission — no new permission keys and no seeder change. Sensitive columns are simply never declared as fields: raw gateway payloads, signature blobs, error dumps, raw jsonb snapshots, internal user UUIDs and review notes are all absent by construction. bookings ships 77 fields across 10 groups. scripts/validate-export-datasets.ts EXPLAINs every dataset's widest query, its count query, and each field on its own against the real database — the per-field pass is what catches a field referencing a join it forgot to declare, which otherwise only fails when that one field is picked alone. --- apps/edr-freight-api/src/app.module.ts | 2 + .../exports/datasets/bookings.dataset.ts | 223 ++++++++++++++++++ .../src/modules/exports/export-filter.util.ts | 81 +++++++ .../exports/export-query.builder.spec.ts | 92 ++++++++ .../modules/exports/export-query.builder.ts | 101 ++++++++ .../modules/exports/export-runner.service.ts | 69 ++++++ .../src/modules/exports/export.registry.ts | 15 ++ .../src/modules/exports/export.types.ts | 135 +++++++++++ .../src/modules/exports/exports.controller.ts | 156 ++++++++++++ .../src/modules/exports/exports.module.ts | 18 +- .../src/scripts/validate-export-datasets.ts | 83 +++++++ 11 files changed, 969 insertions(+), 6 deletions(-) create mode 100644 apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-filter.util.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-query.builder.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export-runner.service.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export.registry.ts create mode 100644 apps/edr-freight-api/src/modules/exports/export.types.ts create mode 100644 apps/edr-freight-api/src/modules/exports/exports.controller.ts create mode 100644 apps/edr-freight-api/src/scripts/validate-export-datasets.ts diff --git a/apps/edr-freight-api/src/app.module.ts b/apps/edr-freight-api/src/app.module.ts index b148c7eea..34825d13c 100644 --- a/apps/edr-freight-api/src/app.module.ts +++ b/apps/edr-freight-api/src/app.module.ts @@ -100,6 +100,7 @@ import { RoutesModule } from "./modules/routes/routes.module"; import { WarehousesModule } from "./modules/warehouses/warehouses.module"; import { OverviewModule } from "./modules/overview/overview.module"; import { ReportsModule } from "./modules/reports/reports.module"; +import { ExportsModule } from "./modules/exports/exports.module"; import { UserTradeAccessModule } from "./modules/user-trade-access/user-trade-access.module"; import { VehiclesModule } from "./modules/vehicles/vehicles.module"; import { DriversModule } from "./modules/drivers/drivers.module"; @@ -234,6 +235,7 @@ if (!process.env.APPLICATION_NAME) { WarehousesModule, OverviewModule, ReportsModule, + ExportsModule, UserTradeAccessModule, VehiclesModule, DriversModule, diff --git a/apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts b/apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts new file mode 100644 index 000000000..1e2cc9098 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/datasets/bookings.dataset.ts @@ -0,0 +1,223 @@ +import { FREIGHT_PERMS } from '../../../seed/freight-permissions.registry'; +import { Booking } from '../../bookings/entities/booking.entity'; +import { Company } from '../../companies/entities/company.entity'; +import { CompanyProfile } from '../../companies/entities/company-profile.entity'; +import { Contract } from '../../contracts/entities/contract.entity'; +import { CargoType } from '../../rule-engine/entities/cargo-type.entity'; +import { ServiceType } from '../../rule-engine/entities/service-type.entity'; +import { ShippingLine } from '../../rule-engine/entities/shipping-line.entity'; +import { Yard } from '../../rule-engine/entities/yard.entity'; +import { ShippingLineCompany } from '../../shipping-lines/entities/shipping-line-company.entity'; +import { Train } from '../../trains/entities/train.entity'; +import { applyDirectionScope } from '../../user-trade-access/trade-scope.util'; +import { ExportDataset } from '../export.types'; + +/** + * Domain semantics shared with `reports/definitions/bookings-list.report.ts`. + * Kept identical on purpose — for PER_ITEM bulk bookings `cargo_total_weight_vgm` + * holds an item COUNT, not tonnage, and `adjusted_total_amount` silently + * overrides `total_amount`. Getting either wrong misreports money or weight. + */ +const TONS = 'COALESCE(b.bulk_total_weight_tons, b.cargo_total_weight_vgm)'; +const REVENUE = 'COALESCE(b.adjusted_total_amount, b.total_amount)'; + +const STATUS_OPTIONS = [ + 'DRAFT', 'SUBMITTED', 'UNDER_REVIEW', 'APPROVED', 'REJECTED', + 'CANCELLED', 'EXPIRED', 'SCHEDULED', 'LOADED', 'IN_TRANSIT', + 'ARRIVED', 'DELIVERED', 'COMPLETED', +].map((v) => ({ value: v, label: v.replace(/_/g, ' ') })); + +export const bookingsDataset: ExportDataset = { + key: 'bookings', + title: 'Bookings', + description: 'Every booking, with customer, route, cargo, contract and payment detail', + group: 'Commercial', + permission: FREIGHT_PERMS.bookings.view, + base: { entity: Booking, alias: 'b' }, + + // Every join is a LEFT join (see ExportJoin) — ticking a field must never + // change which rows come back. + joins: [ + { alias: 'c', entity: Company, on: 'c.id = b.company_id' }, + { alias: 'cp', entity: CompanyProfile, on: 'cp.id = b.company_profile_id' }, + { alias: 'slc', entity: ShippingLineCompany, on: 'slc.id = b.shipping_line_company_id' }, + { alias: 'o', entity: Yard, on: 'o.id = b.origin_yard_id' }, + { alias: 'd', entity: Yard, on: 'd.id = b.destination_yard_id' }, + { alias: 'cty', entity: CargoType, on: 'cty.id = b.cargo_type_id' }, + { alias: 'st', entity: ServiceType, on: 'st.id = b.service_type_id' }, + { alias: 'sl', entity: ShippingLine, on: 'sl.id = b.shipping_line_id' }, + { alias: 'ct', entity: Contract, on: 'ct.id = b.contract_id' }, + { alias: 't', entity: Train, on: 't.id = b.train_id' }, + // Transitive: the contract's own customer, reachable only once `ct` is in. + { alias: 'ctc', entity: Company, on: 'ctc.id = ct.company_id', requires: ['ct'] }, + ], + // `search` matches the customer name, so `c` is always present — which is + // also why the count query joins it. + alwaysJoin: ['c'], + + groups: [ + { id: 'booking', label: 'Booking' }, + { id: 'customer', label: 'Customer' }, + { id: 'route', label: 'Route' }, + { id: 'cargo', label: 'Cargo' }, + { id: 'payment', label: 'Payment' }, + { id: 'scheduling', label: 'Scheduling' }, + { id: 'contract', label: 'Contract' }, + { id: 'firstMile', label: 'First mile' }, + { id: 'lastMile', label: 'Last mile' }, + { id: 'clearance', label: 'Clearance' }, + ], + + fields: [ + // ---- Booking ------------------------------------------------------- + { key: 'reference', label: 'Reference', type: 'string', group: 'booking', default: true, select: 'b.reference', sortExpr: 'b.reference' }, + { key: 'status', label: 'Status', type: 'string', group: 'booking', default: true, select: 'b.status', sortExpr: 'b.status' }, + { key: 'bookingType', label: 'Booking type', type: 'string', group: 'booking', select: 'b.booking_type' }, + { key: 'contractKind', label: 'Contract kind', type: 'string', group: 'booking', select: 'b.contract_kind' }, + { key: 'createdAt', label: 'Created', type: 'datetime', group: 'booking', default: true, select: `to_char(b.created_at, 'YYYY-MM-DD HH24:MI')`, sortExpr: 'b.created_at' }, + { key: 'updatedAt', label: 'Updated', type: 'datetime', group: 'booking', select: `to_char(b.updated_at, 'YYYY-MM-DD HH24:MI')`, sortExpr: 'b.updated_at' }, + { key: 'expiresAt', label: 'Expires', type: 'date', group: 'booking', select: `to_char(b.expires_at, 'YYYY-MM-DD')` }, + { key: 'createdByRole', label: 'Created by role', type: 'string', group: 'booking', select: 'b.created_by_role' }, + { key: 'isSplit', label: 'Split booking', type: 'boolean', group: 'booking', select: 'b.is_split' }, + { key: 'priorityScore', label: 'Priority score', type: 'number', group: 'booking', select: 'b.priority_score', sortExpr: 'b.priority_score' }, + { key: 'versionNumber', label: 'Version', type: 'number', group: 'booking', select: 'b.version_number' }, + { key: 'pnrCode', label: 'PNR code', type: 'string', group: 'booking', select: 'b.pnr_code' }, + + // ---- Customer (the "more than the UI shows" payload) ---------------- + { key: 'customer', label: 'Customer', type: 'string', group: 'customer', default: true, requires: ['c'], select: 'c.name', sortExpr: 'c.name' }, + { key: 'customerType', label: 'Customer type', type: 'string', group: 'customer', requires: ['c'], select: 'c.type' }, + { key: 'customerKind', label: 'Customer kind', type: 'string', group: 'customer', requires: ['c'], select: 'c.kind' }, + { key: 'customerStatus', label: 'Customer status', type: 'string', group: 'customer', requires: ['c'], select: 'c.status' }, + { key: 'customerTin', label: 'Customer TIN', type: 'string', group: 'customer', requires: ['c'], select: 'c.tin' }, + { key: 'customerVat', label: 'Customer VAT no.', type: 'string', group: 'customer', requires: ['c'], select: 'c.vat_number' }, + { key: 'customerPhone', label: 'Customer phone', type: 'string', group: 'customer', requires: ['c'], select: 'c.phone' }, + { key: 'customerEmail', label: 'Customer email', type: 'string', group: 'customer', requires: ['c'], select: 'c.email' }, + { key: 'customerContact', label: 'Contact person', type: 'string', group: 'customer', requires: ['c'], select: 'c.contact_person_name' }, + { key: 'customerContactPhone', label: 'Contact phone', type: 'string', group: 'customer', requires: ['c'], select: 'c.contact_person_phone' }, + { key: 'customerAddress', label: 'Customer address', type: 'string', group: 'customer', requires: ['c'], select: 'c.address' }, + { key: 'customerCountry', label: 'Customer country', type: 'string', group: 'customer', requires: ['c'], select: 'c.country' }, + { key: 'customerRegion', label: 'Customer region', type: 'string', group: 'customer', requires: ['c'], select: 'c.region' }, + { key: 'customerProfileRef', label: 'Profile reference', type: 'string', group: 'customer', requires: ['cp'], select: 'cp.reference' }, + { key: 'customerProfileType', label: 'Profile type', type: 'string', group: 'customer', requires: ['cp'], select: 'cp.type' }, + { key: 'isGovernment', label: 'Government', type: 'boolean', group: 'customer', select: 'b.is_government' }, + { key: 'governmentInstitution', label: 'Government institution', type: 'string', group: 'customer', select: 'b.government_institution' }, + { key: 'shippingLineCompany', label: 'Shipping line company', type: 'string', group: 'customer', requires: ['slc'], select: 'slc.name' }, + + // ---- Route ---------------------------------------------------------- + { key: 'origin', label: 'Origin', type: 'string', group: 'route', default: true, requires: ['o'], select: 'o.label' }, + { key: 'originCode', label: 'Origin code', type: 'string', group: 'route', requires: ['o'], select: 'o.code' }, + { key: 'destination', label: 'Destination', type: 'string', group: 'route', default: true, requires: ['d'], select: 'd.label' }, + { key: 'destinationCode', label: 'Destination code', type: 'string', group: 'route', requires: ['d'], select: 'd.code' }, + { key: 'tradeDirection', label: 'Direction', type: 'string', group: 'route', default: true, select: 'b.trade_direction', sortExpr: 'b.trade_direction' }, + { key: 'serviceType', label: 'Service type', type: 'string', group: 'route', requires: ['st'], select: 'st.service_name' }, + + // ---- Cargo ----------------------------------------------------------- + { key: 'cargo', label: 'Cargo', type: 'string', group: 'cargo', default: true, requires: ['cty'], select: 'COALESCE(cty.cargo_type_name, b.cargo_free_text)' }, + { key: 'freightType', label: 'Freight type', type: 'string', group: 'cargo', default: true, select: 'b.freight_type' }, + { key: 'tons', label: 'Tonnage', type: 'tons', group: 'cargo', default: true, select: `ROUND(${TONS})::float8`, sortExpr: TONS }, + { key: 'containerWeightVgm', label: 'Container VGM', type: 'number', group: 'cargo', select: 'b.cargo_total_weight_vgm' }, + { key: 'bulkWeightTons', label: 'Bulk weight (t)', type: 'tons', group: 'cargo', select: 'b.bulk_total_weight_tons' }, + { key: 'isHazardous', label: 'Hazardous', type: 'boolean', group: 'cargo', select: 'b.is_hazardous' }, + { key: 'isReefer', label: 'Reefer', type: 'boolean', group: 'cargo', select: 'b.is_reefer' }, + { key: 'shippingLine', label: 'Shipping line', type: 'string', group: 'cargo', requires: ['sl'], select: 'sl.label' }, + { + // One-to-many, so it aggregates in a correlated subquery rather than a + // join — a join here would multiply rows and break the count contract. + key: 'containerNumbers', label: 'Container numbers', type: 'string', group: 'cargo', + select: `(SELECT string_agg(bc.container_number, ' | ' ORDER BY bc.container_number) + FROM freight.booking_container bc + WHERE bc.booking_id = b.id AND bc.deleted_at IS NULL)`, + }, + + // ---- Payment ---------------------------------------------------------- + { key: 'amount', label: 'Amount', type: 'money', group: 'payment', default: true, select: `ROUND(${REVENUE}, 2)::float8`, sortExpr: REVENUE }, + { key: 'totalAmount', label: 'Total amount (pre-adjustment)', type: 'money', group: 'payment', select: 'b.total_amount::float8' }, + { key: 'adjustedTotalAmount', label: 'Adjusted total', type: 'money', group: 'payment', select: 'b.adjusted_total_amount::float8' }, + { key: 'adjustmentReason', label: 'Adjustment reason', type: 'string', group: 'payment', select: 'b.adjustment_reason' }, + { key: 'paymentStatus', label: 'Payment status', type: 'string', group: 'payment', default: true, select: 'b.payment_status', sortExpr: 'b.payment_status' }, + { key: 'paymentCurrency', label: 'Currency', type: 'string', group: 'payment', select: 'b.payment_currency' }, + { key: 'paymentDeadline', label: 'Payment deadline', type: 'datetime', group: 'payment', select: `to_char(b.payment_deadline, 'YYYY-MM-DD HH24:MI')` }, + + // ---- Scheduling -------------------------------------------------------- + { key: 'scheduledDate', label: 'Scheduled date', type: 'date', group: 'scheduling', default: true, select: `to_char(b.scheduled_date, 'YYYY-MM-DD')`, sortExpr: 'b.scheduled_date' }, + { key: 'schedulingStatus', label: 'Scheduling status', type: 'string', group: 'scheduling', select: 'b.scheduling_status' }, + { key: 'wagonsRequired', label: 'Wagons required', type: 'number', group: 'scheduling', select: 'b.wagons_required' }, + { key: 'trainCode', label: 'Train', type: 'string', group: 'scheduling', requires: ['t'], select: 't.code' }, + { key: 'loadedAt', label: 'Loaded at', type: 'datetime', group: 'scheduling', select: `to_char(b.loaded_at, 'YYYY-MM-DD HH24:MI')` }, + { key: 'arrivedAt', label: 'Arrived at', type: 'datetime', group: 'scheduling', select: `to_char(b.arrived_at, 'YYYY-MM-DD HH24:MI')` }, + + // ---- Contract ---------------------------------------------------------- + { key: 'contractReference', label: 'Contract reference', type: 'string', group: 'contract', requires: ['ct'], select: 'ct.reference' }, + { key: 'contractStatus', label: 'Contract status', type: 'string', group: 'contract', requires: ['ct'], select: 'ct.status' }, + { key: 'contractCustomer', label: 'Contract customer', type: 'string', group: 'contract', requires: ['ctc'], select: 'ctc.name' }, + { key: 'contractType', label: 'Contract type', type: 'string', group: 'contract', select: 'b.contract_type' }, + { key: 'contractValidFrom', label: 'Contract valid from', type: 'date', group: 'contract', select: `to_char(b.contract_valid_from, 'YYYY-MM-DD')` }, + { key: 'contractValidUntil', label: 'Contract valid until', type: 'date', group: 'contract', select: `to_char(b.contract_valid_until, 'YYYY-MM-DD')` }, + { key: 'fullyExecutedAt', label: 'Fully executed at', type: 'datetime', group: 'contract', select: `to_char(b.fully_executed_at, 'YYYY-MM-DD HH24:MI')` }, + + // ---- First / last mile --------------------------------------------------- + { key: 'firstMileAddress', label: 'Pickup address', type: 'string', group: 'firstMile', select: 'b.first_mile_pickup_address' }, + { key: 'lastMileAddress', label: 'Delivery address', type: 'string', group: 'lastMile', select: 'b.last_mile_delivery_address' }, + { key: 'customerTruckPlate', label: 'Customer truck plate', type: 'string', group: 'lastMile', select: 'b.customer_truck_plate_number' }, + { key: 'customerTruckDriver', label: 'Customer truck driver', type: 'string', group: 'lastMile', select: 'b.customer_truck_driver_name' }, + { key: 'exportHandoverMode', label: 'Handover mode', type: 'string', group: 'lastMile', select: 'b.export_handover_mode' }, + + // ---- Clearance ------------------------------------------------------------- + { key: 'customsClearingEnabled', label: 'Customs clearing', type: 'boolean', group: 'clearance', select: 'b.customs_clearing_enabled' }, + { key: 'customsClearingAgent', label: 'Clearing agent', type: 'string', group: 'clearance', select: 'b.customs_clearing_agent' }, + { key: 'clearancePhase', label: 'Clearance phase', type: 'string', group: 'clearance', select: 'b.clearance_current_phase' }, + { key: 'dutyRequired', label: 'Duty required', type: 'boolean', group: 'clearance', select: 'b.duty_required' }, + { key: 'vesselArrivalDate', label: 'Vessel arrival', type: 'date', group: 'clearance', select: `to_char(b.vessel_arrival_date, 'YYYY-MM-DD')` }, + { key: 'doCollectedDate', label: 'DO collected', type: 'date', group: 'clearance', select: `to_char(b.do_collected_date, 'YYYY-MM-DD')` }, + { key: 'doubleHandling', label: 'Double handling', type: 'boolean', group: 'clearance', select: 'b.double_handling' }, + ], + + filters: [ + { key: 'created', label: 'Created', type: 'daterange' }, + { key: 'statuses', label: 'Status', type: 'multiselect', options: STATUS_OPTIONS }, + { + key: 'tradeDirection', label: 'Direction', type: 'select', + options: ['IMPORT', 'EXPORT', 'DOMESTIC'].map((v) => ({ value: v, label: v })), + }, + { + key: 'freightType', label: 'Freight type', type: 'select', + options: ['CONTAINER', 'BULK'].map((v) => ({ value: v, label: v })), + }, + { key: 'paymentStatus', label: 'Payment status', type: 'select', options: [ + { value: 'PENDING', label: 'Pending' }, + { value: 'PNR_GENERATED', label: 'PNR generated' }, + { value: 'VERIFICATION_IN_PROGRESS', label: 'Verification in progress' }, + { value: 'PAID', label: 'Paid' }, + { value: 'FAILED', label: 'Failed' }, + ] }, + { key: 'companyId', label: 'Customer', type: 'text' }, + { key: 'search', label: 'Search reference or customer', type: 'text' }, + ], + + defaultSort: { key: 'createdAt', dir: 'DESC' }, + + scope(ctx, qb) { + const { params, directions } = ctx; + // andWhere, not where: `where()` resets any condition already on the + // builder, so scope() would silently drop anything a caller added first. + qb.andWhere('b.deleted_at IS NULL'); + + if (params.createdFrom) qb.andWhere('b.created_at >= :createdFrom', { createdFrom: params.createdFrom }); + if (params.createdTo) qb.andWhere('b.created_at < :createdTo', { createdTo: params.createdTo }); + + const statuses = params.statuses as string[] | null; + if (statuses?.length) qb.andWhere('b.status IN (:...statuses)', { statuses }); + + if (params.tradeDirection) qb.andWhere('b.trade_direction = :tradeDirection', { tradeDirection: params.tradeDirection }); + if (params.freightType) qb.andWhere('b.freight_type = :freightType', { freightType: params.freightType }); + if (params.paymentStatus) qb.andWhere('b.payment_status = :paymentStatus', { paymentStatus: params.paymentStatus }); + if (params.companyId) qb.andWhere('b.company_id = :companyId', { companyId: params.companyId }); + if (params.search) { + qb.andWhere('(b.reference ILIKE :search OR c.name ILIKE :search)', { search: `%${params.search as string}%` }); + } + + // Trade-direction ACL. Without this the export returns rows the user's own + // list page would not show them. + applyDirectionScope(qb, 'b.trade_direction', directions); + }, +}; diff --git a/apps/edr-freight-api/src/modules/exports/export-filter.util.ts b/apps/edr-freight-api/src/modules/exports/export-filter.util.ts new file mode 100644 index 000000000..c302d9d3b --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-filter.util.ts @@ -0,0 +1,81 @@ +import { DataSource } from 'typeorm'; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export type ExportFilterType = 'daterange' | 'date' | 'select' | 'multiselect' | 'text'; + +export interface ExportFilterOption { + value: string; + label: string; +} + +export interface ExportFilterDef { + key: string; + label: string; + type: ExportFilterType; + /** Static choices. Mutually exclusive with `optionsQuery`. */ + options?: ExportFilterOption[]; + /** Reference-data choices resolved from the DB and cached for the process. */ + optionsQuery?: (ds: DataSource) => Promise; +} + +/** Raw query-string bag. Per-registry filter keys, so `forbidNonWhitelisted` can't police it. */ +export type RawFilterQuery = Record; + +/** + * Coerce raw query strings into typed filter params per a filter declaration + * list. Unknown keys are dropped rather than rejected. + * + * Shared by the report runner and the export runner so the `daterange` + * handling in particular cannot drift between them: `To` is pushed forward a + * day because callers mean an INCLUSIVE end date while the SQL bound is + * exclusive (`created_at < :dateTo`). + */ +export function coerceFilterParams( + filters: ExportFilterDef[], + raw: RawFilterQuery, +): Record { + const params: Record = {}; + for (const filter of filters) { + if (filter.type === 'daterange') { + const from = raw[`${filter.key}From`]; + const to = raw[`${filter.key}To`]; + params[`${filter.key}From`] = from ? new Date(from).toISOString() : null; + params[`${filter.key}To`] = to + ? new Date(new Date(to).getTime() + DAY_MS).toISOString() + : null; + } else if (filter.type === 'multiselect') { + const csv = raw[filter.key]; + const items = csv?.split(',').map((s) => s.trim()).filter(Boolean) ?? []; + params[filter.key] = items.length ? items : null; + } else { + params[filter.key] = raw[filter.key]?.trim() || null; + } + } + return params; +} + +/** + * Process-lifetime cache for `optionsQuery` results — small, rarely-changing + * reference lists (23 stations, 18 cargo types) hit on every catalog load. + * + * ponytail: keyed by filter key alone, so two registries sharing a filter key + * share one option list. Key by `${registry}:${filterKey}` if that ever bites. + */ +const optionsCache = new Map(); + +export async function resolveFilterOptions( + filters: ExportFilterDef[], + ds: DataSource, +): Promise { + return Promise.all( + filters.map(async (filter) => { + if (!filter.optionsQuery) return filter; + const cached = optionsCache.get(filter.key); + if (cached) return { ...filter, options: cached, optionsQuery: undefined }; + const options = await filter.optionsQuery(ds); + optionsCache.set(filter.key, options); + return { ...filter, options, optionsQuery: undefined }; + }), + ); +} diff --git a/apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts b/apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts new file mode 100644 index 000000000..af6dd923d --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-query.builder.spec.ts @@ -0,0 +1,92 @@ +import { resolveJoins } from './export-query.builder'; +import { ExportDataset, ExportField } from './export.types'; + +const field = (key: string, requires?: string[]): ExportField => ({ + key, + label: key, + type: 'string', + group: 'g', + select: `x.${key}`, + requires, +}); + +/** Entities are never dereferenced by resolveJoins — only the alias graph matters. */ +const entity = {} as ExportDataset['joins'][number]['entity']; + +const dataset = ( + joins: ExportDataset['joins'], + alwaysJoin?: string[], +): ExportDataset => + ({ + key: 'test', + joins, + alwaysJoin, + fields: [], + }) as unknown as ExportDataset; + +describe('resolveJoins', () => { + it('pulls in only the joins the selected fields ask for', () => { + const ds = dataset([ + { alias: 'a', entity, on: 'a.id = b.a_id' }, + { alias: 'z', entity, on: 'z.id = b.z_id' }, + ]); + expect(resolveJoins(ds, [field('one', ['a'])]).map((j) => j.alias)).toEqual(['a']); + }); + + it('selecting nothing still applies alwaysJoin — the count query relies on this', () => { + const ds = dataset( + [ + { alias: 'a', entity, on: 'a.id = b.a_id' }, + { alias: 'z', entity, on: 'z.id = b.z_id' }, + ], + ['a'], + ); + expect(resolveJoins(ds, []).map((j) => j.alias)).toEqual(['a']); + }); + + it('resolves a transitive dependency, dependency first', () => { + const ds = dataset([ + { alias: 'ct', entity, on: 'ct.id = b.contract_id' }, + { alias: 'ctc', entity, on: 'ctc.id = ct.company_id', requires: ['ct'] }, + ]); + expect(resolveJoins(ds, [field('x', ['ctc'])]).map((j) => j.alias)).toEqual(['ct', 'ctc']); + }); + + it('resolves a multi-hop chain in order', () => { + const ds = dataset([ + { alias: 'a', entity, on: 'a.id = b.a_id' }, + { alias: 'bb', entity, on: 'bb.id = a.b_id', requires: ['a'] }, + { alias: 'cc', entity, on: 'cc.id = bb.c_id', requires: ['bb'] }, + ]); + expect(resolveJoins(ds, [field('x', ['cc'])]).map((j) => j.alias)).toEqual(['a', 'bb', 'cc']); + }); + + it('emits a shared join once, not per field that needs it', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]); + const joins = resolveJoins(ds, [field('one', ['a']), field('two', ['a'])]); + expect(joins.map((j) => j.alias)).toEqual(['a']); + }); + + it('does not duplicate a join already pulled in by alwaysJoin', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }], ['a']); + expect(resolveJoins(ds, [field('one', ['a'])]).map((j) => j.alias)).toEqual(['a']); + }); + + it('throws on a cycle rather than looping forever', () => { + const ds = dataset([ + { alias: 'a', entity, on: 'a.id = bb.a_id', requires: ['bb'] }, + { alias: 'bb', entity, on: 'bb.id = a.b_id', requires: ['a'] }, + ]); + expect(() => resolveJoins(ds, [field('x', ['a'])])).toThrow(/join cycle/); + }); + + it('throws on an undeclared alias — a typo must fail loudly, not silently 42P01', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]); + expect(() => resolveJoins(ds, [field('x', ['ghost'])])).toThrow(/unknown join alias "ghost"/); + }); + + it('a field with no requires pulls in no joins at all', () => { + const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]); + expect(resolveJoins(ds, [field('plain')])).toEqual([]); + }); +}); diff --git a/apps/edr-freight-api/src/modules/exports/export-query.builder.ts b/apps/edr-freight-api/src/modules/exports/export-query.builder.ts new file mode 100644 index 000000000..6f4c091d0 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-query.builder.ts @@ -0,0 +1,101 @@ +import { ObjectLiteral, SelectQueryBuilder } from 'typeorm'; + +import { ExportContext, ExportDataset, ExportField, ExportJoin } from './export.types'; + +/** + * Sort expression fallback: the SELECT alias TypeORM emitted, quoted. TypeORM + * double-quotes `addSelect` aliases (preserving case), so ordering by the bare + * key lets Postgres fold it to lowercase and 42703 on any camelCase alias. + */ +export const aliasSortExpr = (key: string): string => `"${key.replace(/"/g, '""')}"`; + +/** + * Selected fields -> the joins they need, transitively, dependencies first. + * DFS post-order over `requires`, memoized. Deterministic: `alwaysJoin` first, + * then fields in the dataset's own declaration order. + */ +export function resolveJoins(dataset: ExportDataset, fields: ExportField[]): ExportJoin[] { + const byAlias = new Map(dataset.joins.map((j) => [j.alias, j])); + const out: ExportJoin[] = []; + const done = new Set(); + const onStack = new Set(); + + const visit = (alias: string): void => { + if (done.has(alias)) return; + if (onStack.has(alias)) { + throw new Error(`export "${dataset.key}": join cycle at alias "${alias}"`); + } + const join = byAlias.get(alias); + if (!join) { + throw new Error(`export "${dataset.key}": unknown join alias "${alias}"`); + } + onStack.add(alias); + for (const dep of join.requires ?? []) visit(dep); + onStack.delete(alias); + done.add(alias); + out.push(join); + }; + + for (const alias of dataset.alwaysJoin ?? []) visit(alias); + for (const field of fields) for (const alias of field.requires ?? []) visit(alias); + return out; +} + +/** The download query: base + only the joins the selected fields need. */ +export function buildExportQuery( + dataset: ExportDataset, + fields: ExportField[], + ctx: ExportContext, +): SelectQueryBuilder { + const qb = ctx.ds.createQueryBuilder().from(dataset.base.entity, dataset.base.alias); + for (const join of resolveJoins(dataset, fields)) { + qb.leftJoin(join.entity, join.alias, join.on); + } + for (const field of fields) qb.addSelect(field.select, field.key); + dataset.scope(ctx, qb); + return qb; +} + +/** + * The count query: same base, same `scope()`, same WHERE — but no field joins + * and no selects. Exact rather than an estimate, because every lazy join is a + * left join to a to-one side and so cannot change the row count. + */ +export function buildExportCountQuery( + dataset: ExportDataset, + ctx: ExportContext, +): SelectQueryBuilder { + const qb = ctx.ds + .createQueryBuilder() + .select('COUNT(*)::int', 'total') + .from(dataset.base.entity, dataset.base.alias); + for (const join of resolveJoins(dataset, [])) { + qb.leftJoin(join.entity, join.alias, join.on); + } + dataset.scope(ctx, qb); + return qb; +} + +/** + * Resolve a requested sort against the SELECTED fields. Restricting to selected + * fields means a sort can never pull in a join the projection didn't already + * need — which is what keeps the count query's join set correct. + */ +export function resolveExportSort( + dataset: ExportDataset, + fields: ExportField[], + sortBy?: string, + sortOrder?: string, +): { expr: string; dir: 'ASC' | 'DESC' } | null { + const dir = sortOrder?.toUpperCase() === 'DESC' ? 'DESC' : 'ASC'; + const requested = sortBy && fields.find((f) => f.key === sortBy && f.sortExpr); + if (requested) return { expr: requested.sortExpr ?? aliasSortExpr(requested.key), dir }; + + if (!dataset.defaultSort) return null; + const fallback = fields.find((f) => f.key === dataset.defaultSort!.key); + if (!fallback) return null; + return { + expr: fallback.sortExpr ?? aliasSortExpr(fallback.key), + dir: dataset.defaultSort.dir, + }; +} diff --git a/apps/edr-freight-api/src/modules/exports/export-runner.service.ts b/apps/edr-freight-api/src/modules/exports/export-runner.service.ts new file mode 100644 index 000000000..816aeb38d --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export-runner.service.ts @@ -0,0 +1,69 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; +import { DataSource } from 'typeorm'; + +import { coerceFilterParams, RawFilterQuery } from './export-filter.util'; +import { + buildExportCountQuery, + buildExportQuery, + resolveExportSort, +} from './export-query.builder'; +import { ExportDataset, ExportField } from './export.types'; + +@Injectable() +export class ExportRunnerService { + constructor(@InjectDataSource() private readonly ds: DataSource) {} + + private context(dataset: ExportDataset, raw: RawFilterQuery, directions: string[] | null) { + return { ds: this.ds, params: coerceFilterParams(dataset.filters, raw), directions }; + } + + /** + * Exact row count for the current filters. Exact rather than estimated + * because lazy joins are all left joins to to-one sides, so the count cannot + * depend on which fields the caller picked. + */ + async count( + dataset: ExportDataset, + raw: RawFilterQuery, + directions: string[] | null, + ): Promise { + const qb = buildExportCountQuery(dataset, this.context(dataset, raw, directions)); + const row = await qb.getRawOne<{ total: number }>(); + return Number(row?.total ?? 0); + } + + /** + * Matching rows. + * + * `limit` is the caller's deliberate "first N" truncation — honoured + * silently, because they asked for it. `cap` is the format's hard ceiling — + * exceeding it throws, because a silently short file is worse than a clear + * error: nothing downstream reveals that rows are missing. + */ + async run( + dataset: ExportDataset, + fields: ExportField[], + raw: RawFilterQuery, + directions: string[] | null, + { cap, limit }: { cap: number; limit?: number }, + ): Promise[]> { + const ctx = this.context(dataset, raw, directions); + const qb = buildExportQuery(dataset, fields, ctx); + + const sort = resolveExportSort(dataset, fields, raw.sortBy, raw.sortOrder); + if (sort) qb.orderBy(sort.expr, sort.dir); + + const ceiling = limit ?? cap; + // ceiling + 1: fetching exactly `ceiling` cannot distinguish "there are + // exactly that many rows" from "there are more". + const items = await qb.limit(ceiling + 1).getRawMany(); + if (items.length <= ceiling) return items; + + // Asked to be truncated -> truncate. Hit the hard cap -> say so. + if (limit !== undefined) return items.slice(0, limit); + throw new BadRequestException( + `This export has more than ${cap.toLocaleString()} rows, the limit for this format. Narrow the filters, or export a smaller number of rows.`, + ); + } +} diff --git a/apps/edr-freight-api/src/modules/exports/export.registry.ts b/apps/edr-freight-api/src/modules/exports/export.registry.ts new file mode 100644 index 000000000..dc7a3c968 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export.registry.ts @@ -0,0 +1,15 @@ +import { bookingsDataset } from './datasets/bookings.dataset'; +import { ExportDataset } from './export.types'; + +/** + * Every exportable dataset. + * + * Adding one = a new file under `datasets/` + an entry here. No frontend edit, + * no route, no permission seed — the dialog is driven entirely by the catalog + * this registry serves, and a dataset reuses its module's existing `view` key. + */ +export const DATASETS: ExportDataset[] = [bookingsDataset]; + +const BY_KEY = new Map(DATASETS.map((d) => [d.key, d])); + +export const getDataset = (key: string): ExportDataset | undefined => BY_KEY.get(key); diff --git a/apps/edr-freight-api/src/modules/exports/export.types.ts b/apps/edr-freight-api/src/modules/exports/export.types.ts new file mode 100644 index 000000000..db12f0211 --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/export.types.ts @@ -0,0 +1,135 @@ +import { + DataSource, + EntityTarget, + ObjectLiteral, + ObjectType, + SelectQueryBuilder, +} from 'typeorm'; + +import { ExportFilterDef } from './export-filter.util'; +import { ExportFieldType } from './tabular-export.service'; + +export type { ExportFieldType }; + +/** + * A lazily-applied relation. + * + * There is deliberately no `kind: 'inner' | 'left'` here — every join is + * emitted as a LEFT JOIN, and the type makes anything else unrepresentable. + * An inner join added only because someone ticked a checkbox would silently + * change the rowset (ticking "Customer TIN" would drop every booking with a + * null company_id), so two exports of the same filters would disagree on their + * row count. Anything that genuinely must narrow rows belongs in `scope()`, + * where it is unconditional and visible. + * + * The payoff: because a left join to a to-one side can neither add nor remove + * rows, the row count is independent of which fields are selected — which is + * what lets the count endpoint be exact rather than an estimate. + */ +export interface ExportJoin { + /** Alias used by field `select` expressions and by `requires`. */ + alias: string; + /** Entity class. Narrower than `EntityTarget` to match TypeORM's join overload. */ + entity: ObjectType; + /** ON condition; may reference the base alias and any alias in `requires`. */ + on: string; + /** Other join aliases this join's ON clause depends on. Resolved transitively. */ + requires?: string[]; +} + +/** + * One exportable column. + * + * `select` must yield exactly ONE value per base row. To surface a one-to-many + * relation (a company's profiles, a booking's containers), aggregate inside a + * correlated subquery — `(SELECT string_agg(...) FROM ... WHERE ... = base.id)` + * — rather than adding a join, which would multiply rows and break the count. + * + * Sensitive columns are simply never declared: raw gateway payloads + * (payments.raw_initiation, client_action), signature/crypto blobs + * (invoices.eims_signed_qr), internal error dumps (eims_last_error), raw jsonb + * snapshots (pricing_breakdown, document_snapshot, financial_terms, + * attributes, business_license_files), bare internal user UUIDs, and internal + * review/rejection notes. Fields are opt-in, so omission is the whole + * enforcement mechanism. + */ +export interface ExportField { + /** Response key, sheet header id, and the picker's checkbox id. */ + key: string; + label: string; + type: ExportFieldType; + /** Scalar SQL projected as `key`. */ + select: string; + /** Join aliases `select` references. Omit for base-table-only fields. */ + requires?: string[]; + /** Picker group id; must exist in the dataset's `groups`. */ + group: string; + /** Pre-ticked when the dialog opens with no preset. */ + default?: boolean; + /** ORDER BY expression. Presence makes the field sortable. */ + sortExpr?: string; +} + +export interface ExportGroup { + id: string; + label: string; +} + +export interface ExportContext { + ds: DataSource; + /** Filter values, already coerced by `coerceFilterParams`. */ + params: Record; + /** Trade-scope directions. `null` = unrestricted, `[]` = show nothing. */ + directions: string[] | null; +} + +export interface ExportDataset { + key: string; + title: string; + description: string; + group: 'Commercial' | 'Operations' | 'Finance' | 'Fleet'; + /** + * Permission to export this dataset. Reuses the module's existing `view` + * key — if you may see these rows on their list page, you may export them. + * The export never returns a row the list endpoint would not. + */ + permission: string; + base: { entity: EntityTarget; alias: string }; + joins: ExportJoin[]; + /** + * Aliases applied unconditionally because `scope()` references them. This is + * the only reason a join is eager, and the count query applies exactly these. + */ + alwaysJoin?: string[]; + groups: ExportGroup[]; + fields: ExportField[]; + filters: ExportFilterDef[]; + /** Must name a field whose `sortExpr` references only the base alias. */ + defaultSort?: { key: string; dir: 'ASC' | 'DESC' }; + /** + * Base WHERE (soft-delete guard), filter application, and the trade-direction + * ACL. Runs identically for the count and download queries, so the row count + * the dialog shows is exactly what lands in the file. + * + * A dataset whose table carries a trade direction MUST apply it here, or the + * export leaks rows the user cannot see on the list page. + */ + scope(ctx: ExportContext, qb: SelectQueryBuilder): void; +} + +/** + * What `GET /exports` serves. `select` / `requires` / `sortExpr` are raw SQL + * and a map of the schema — they never leave the server. + */ +export interface ExportCatalogEntry { + key: string; + title: string; + description: string; + group: ExportDataset['group']; + groups: ExportGroup[]; + fields: Pick[]; + filters: ExportFilterDef[]; + formats: ('csv' | 'xlsx' | 'pdf')[]; + caps: { csv: number; xlsx: number; pdf: number }; + defaultSort?: { key: string; dir: 'ASC' | 'DESC' }; +} diff --git a/apps/edr-freight-api/src/modules/exports/exports.controller.ts b/apps/edr-freight-api/src/modules/exports/exports.controller.ts new file mode 100644 index 000000000..89ca7419a --- /dev/null +++ b/apps/edr-freight-api/src/modules/exports/exports.controller.ts @@ -0,0 +1,156 @@ +import { Controller, Get, NotFoundException, Param, Query, Res, UseGuards } from '@nestjs/common'; +import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'; +import { InjectDataSource } from '@nestjs/typeorm'; +import { DataSource } from 'typeorm'; +import { CurrentUser } from '@edr/api-common'; +import { JwtGuard } from '@tria-plc/api-common/modules/auth/services/jwt.guard'; +import type { Response } from 'express'; +import type { TCurrentUser } from '@tria-plc/api-common/modules/auth/types/current-user.type'; + +import { assertFreightPermission, hasFreightPermission } from '../../common/freight-permission.util'; +import { UserTradeAccessService } from '../user-trade-access/user-trade-access.service'; +import { resolveFilterOptions } from './export-filter.util'; +import { + EXPORT_MIME, + formatRowCap, + pickByKey, + resolveExportFormat, + resolveRowLimit, +} from './export-request.util'; +import { ExportRunnerService } from './export-runner.service'; +import { DATASETS, getDataset } from './export.registry'; +import { ExportCatalogEntry, ExportDataset, ExportField } from './export.types'; +import { CSV_ROW_CAP, PDF_ROW_CAP, TabularExportService, XLSX_ROW_CAP } from './tabular-export.service'; + +/** Raw query bag — filter keys are per-dataset, so DTO whitelisting can't police it. */ +type RawExportQuery = Record; + +const CAPS = { csv: CSV_ROW_CAP, xlsx: XLSX_ROW_CAP, pdf: PDF_ROW_CAP }; + +/** + * Metadata only. `select` / `requires` / `sortExpr` are raw SQL and a map of + * the schema — they never leave the server. + */ +const toCatalogEntry = (dataset: ExportDataset): ExportCatalogEntry => ({ + key: dataset.key, + title: dataset.title, + description: dataset.description, + group: dataset.group, + groups: dataset.groups, + fields: dataset.fields.map(({ key, label, type, group, default: isDefault }) => ({ + key, + label, + type, + group, + default: isDefault, + })), + filters: dataset.filters, + formats: ['csv', 'xlsx', 'pdf'], + caps: CAPS, + defaultSort: dataset.defaultSort, +}); + +/** + * Generic table export. One dataset per major table, each describing far more + * fields than its list page shows — including related-entity detail. + */ +@ApiTags('Exports') +@ApiBearerAuth() +@Controller('exports') +@UseGuards(JwtGuard) +export class ExportsController { + constructor( + private readonly runner: ExportRunnerService, + private readonly writer: TabularExportService, + private readonly userTradeAccessService: UserTradeAccessService, + @InjectDataSource() private readonly dataSource: DataSource, + ) {} + + @Get() + @ApiOperation({ summary: 'List datasets the caller has permission to export' }) + async catalog(@CurrentUser() user: TCurrentUser): Promise { + const allowed = DATASETS.filter((d) => hasFreightPermission(user, d.permission)); + return Promise.all( + allowed.map(async (d) => ({ + ...toCatalogEntry(d), + filters: await resolveFilterOptions(d.filters, this.dataSource), + })), + ); + } + + @Get(':key/count') + @ApiOperation({ summary: 'Exact row count for the given filters, plus the per-format caps' }) + async count( + @Param('key') key: string, + @Query() query: RawExportQuery, + @CurrentUser() user: TCurrentUser, + ): Promise<{ total: number; caps: typeof CAPS }> { + const dataset = this.resolve(key, user); + const directions = await this.userTradeAccessService.resolveAllowedDirections(user); + const total = await this.runner.count(dataset, query, directions); + return { total, caps: CAPS }; + } + + @Get(':key/download') + @ApiOperation({ summary: 'Export a dataset to csv, xlsx or pdf' }) + async download( + @Param('key') key: string, + @Query() query: RawExportQuery & { format?: string; fields?: string; limit?: string }, + @CurrentUser() user: TCurrentUser, + @Res() res: Response, + ): Promise { + const dataset = this.resolve(key, user); + const directions = await this.userTradeAccessService.resolveAllowedDirections(user); + const format = resolveExportFormat(query.format); + const fields = this.resolveFields(dataset, query.fields); + + const rows = await this.runner.run(dataset, fields, query, directions, { + cap: formatRowCap(format), + limit: resolveRowLimit(format, query.limit), + }); + const doc = { + title: dataset.title, + description: dataset.description, + label: `export:${dataset.key}`, + columns: fields.map(({ key: k, label, type }) => ({ key: k, label, type })), + rows, + }; + const buffer = + format === 'pdf' + ? await this.writer.toPdf(doc) + : format === 'csv' + ? await this.writer.toCsv(doc) + : await this.writer.toXlsx(doc); + + const mime = EXPORT_MIME[format]; + const stamp = new Date().toISOString().slice(0, 10); + res.setHeader('Content-Disposition', `attachment; filename="${dataset.key}-${stamp}.${mime.ext}"`); + res.setHeader('Content-Type', mime.type); + res.send(buffer); + } + + /** + * Requested fields, whitelisted against the dataset. No `fields=` means the + * DEFAULT set, not everything — a booking export has ~70 fields and dumping + * all of them on an unparameterised call is nobody's intent. + */ + private resolveFields(dataset: ExportDataset, raw: string | undefined): ExportField[] { + if (raw?.trim()) { + const picked = pickByKey(dataset.fields, raw); + // pickByKey falls back to everything when nothing matched; for a dataset + // the safer read of "all keys unknown" is still the default set. + if (picked.length !== dataset.fields.length) return picked; + } + const defaults = dataset.fields.filter((f) => f.default); + return defaults.length ? defaults : dataset.fields; + } + + private resolve(key: string, user: TCurrentUser): ExportDataset { + const dataset = getDataset(key); + if (!dataset) throw new NotFoundException(`Unknown export dataset: ${key}`); + // Export rides the dataset's own list-page view permission: if you may see + // these rows, you may export them. + assertFreightPermission(user, dataset.permission); + return dataset; + } +} diff --git a/apps/edr-freight-api/src/modules/exports/exports.module.ts b/apps/edr-freight-api/src/modules/exports/exports.module.ts index 6f56826bf..e3c64bd4c 100644 --- a/apps/edr-freight-api/src/modules/exports/exports.module.ts +++ b/apps/edr-freight-api/src/modules/exports/exports.module.ts @@ -1,17 +1,23 @@ import { Module } from '@nestjs/common'; import { DocumentsModule } from '../billing/documents/documents.module'; +import { UserTradeAccessModule } from '../user-trade-access/user-trade-access.module'; +import { ExportRunnerService } from './export-runner.service'; +import { ExportsController } from './exports.controller'; import { TabularExportService } from './tabular-export.service'; /** - * Export infrastructure. Currently just the shared tabular writer (xlsx / csv / - * pdf) that both the reports module and — once the dataset registry lands — the - * generic table exports write through. No domain dependencies, so any module can - * import it. + * Generic table export: a dataset registry describing far more fields than each + * list page shows (related-entity detail included), plus the shared tabular + * writer (csv / xlsx / pdf) the reports module also writes through. + * + * `TabularExportService` is exported so ReportsModule can reuse it without + * pulling in the dataset machinery. */ @Module({ - imports: [DocumentsModule], - providers: [TabularExportService], + imports: [DocumentsModule, UserTradeAccessModule], + controllers: [ExportsController], + providers: [TabularExportService, ExportRunnerService], exports: [TabularExportService], }) export class ExportsModule {} diff --git a/apps/edr-freight-api/src/scripts/validate-export-datasets.ts b/apps/edr-freight-api/src/scripts/validate-export-datasets.ts new file mode 100644 index 000000000..5ee036d3d --- /dev/null +++ b/apps/edr-freight-api/src/scripts/validate-export-datasets.ts @@ -0,0 +1,83 @@ +/** + * EXPLAIN-validates every export dataset against the real database. + * + * CLAUDE.md hard rule: raw SQL must be validated against a real DB before it + * ships. Every dataset is hand-written SQL expressions over wide tables where + * column drift is documented history, so a typo is a runtime 500 no type-check + * can catch. This builds each dataset's WIDEST query (all fields selected, so + * every join and every subquery is exercised) plus its count query, and runs + * both through EXPLAIN. + * + * npx ts-node -r tsconfig-paths/register src/scripts/validate-export-datasets.ts + */ +import 'dotenv/config'; + +import AppDataSource from '../data-source'; +import { buildExportCountQuery, buildExportQuery } from '../modules/exports/export-query.builder'; +import { DATASETS } from '../modules/exports/export.registry'; + +async function main(): Promise { + await AppDataSource.initialize(); + let failed = 0; + + for (const dataset of DATASETS) { + const ctx = { ds: AppDataSource, params: {}, directions: null }; + + const cases: [string, () => { sql: string; params: unknown[] }][] = [ + [ + `${dataset.key} (all ${dataset.fields.length} fields)`, + () => { + const qb = buildExportQuery(dataset, dataset.fields, ctx); + return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] }; + }, + ], + [ + `${dataset.key} (count)`, + () => { + const qb = buildExportCountQuery(dataset, ctx); + return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] }; + }, + ], + ]; + + // Each field ALONE. The all-fields query above cannot catch a field that + // references an alias it forgot to declare in `requires` — some other + // field's `requires` pulls that join in, so it only 42P01s when that one + // checkbox is ticked on its own. This is the check that finds it. + for (const field of dataset.fields) { + cases.push([ + `${dataset.key}.${field.key}`, + () => { + const qb = buildExportQuery(dataset, [field], ctx); + return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] }; + }, + ]); + } + + let fieldFailures = 0; + for (const [label, build] of cases) { + const isPerField = label.startsWith(`${dataset.key}.`); + try { + const { sql } = build(); + // Parameters are all optional filters and unset here, so the generated + // SQL carries no placeholders — EXPLAIN it directly. + await AppDataSource.query(`EXPLAIN ${sql}`); + if (!isPerField) console.log(` ok ${label}`); + } catch (error) { + failed += 1; + if (isPerField) fieldFailures += 1; + console.error(` FAIL ${label}`); + console.error(` ${(error as Error).message.split('\n')[0]}`); + } + } + if (!fieldFailures) { + console.log(` ok ${dataset.key} (each of ${dataset.fields.length} fields alone)`); + } + } + + await AppDataSource.destroy(); + console.log(failed ? `\n${failed} query/queries failed.` : '\nAll export dataset SQL validated.'); + process.exit(failed ? 1 : 0); +} + +void main();