mirror of
https://github.com/Tria-plc/edr-platform.git
synced 2026-08-26 18:42:49 +00:00
feat(exports): dataset-driven table export, starting with bookings
Adds a parallel export system the reports module can also draw on. A dataset describes a table's exportable fields — including related-entity detail the list page never shows — and the engine assembles a query from whichever fields the caller picked. GET /exports catalog (metadata only; select/requires never ship) GET /exports/:key/count exact row count + per-format caps GET /exports/:key/download csv | xlsx | pdf Two invariants carry the design: - Every lazy join is a LEFT join, and ExportJoin has no 'kind' field to make anything else expressible. An inner join added because a checkbox was ticked would change the rowset, so two exports of the same filters would disagree on their row count. - Because of that, the count cannot depend on field selection, so /count runs base + alwaysJoin only and is exact rather than an estimate. Verified: count and the delivered file both report 223 rows. One-to-many relations (a booking's containers) aggregate in a correlated subquery rather than joining, so a row can never multiply. Export rides each dataset's existing view permission — no new permission keys and no seeder change. Sensitive columns are simply never declared as fields: raw gateway payloads, signature blobs, error dumps, raw jsonb snapshots, internal user UUIDs and review notes are all absent by construction. bookings ships 77 fields across 10 groups. scripts/validate-export-datasets.ts EXPLAINs every dataset's widest query, its count query, and each field on its own against the real database — the per-field pass is what catches a field referencing a join it forgot to declare, which otherwise only fails when that one field is picked alone.
This commit is contained in:
Nathnael
@@ -100,6 +100,7 @@ import { RoutesModule } from "./modules/routes/routes.module";
|
||||
import { WarehousesModule } from "./modules/warehouses/warehouses.module";
|
||||
import { OverviewModule } from "./modules/overview/overview.module";
|
||||
import { ReportsModule } from "./modules/reports/reports.module";
|
||||
import { ExportsModule } from "./modules/exports/exports.module";
|
||||
import { UserTradeAccessModule } from "./modules/user-trade-access/user-trade-access.module";
|
||||
import { VehiclesModule } from "./modules/vehicles/vehicles.module";
|
||||
import { DriversModule } from "./modules/drivers/drivers.module";
|
||||
@@ -234,6 +235,7 @@ if (!process.env.APPLICATION_NAME) {
|
||||
WarehousesModule,
|
||||
OverviewModule,
|
||||
ReportsModule,
|
||||
ExportsModule,
|
||||
UserTradeAccessModule,
|
||||
VehiclesModule,
|
||||
DriversModule,
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import { FREIGHT_PERMS } from '../../../seed/freight-permissions.registry';
|
||||
import { Booking } from '../../bookings/entities/booking.entity';
|
||||
import { Company } from '../../companies/entities/company.entity';
|
||||
import { CompanyProfile } from '../../companies/entities/company-profile.entity';
|
||||
import { Contract } from '../../contracts/entities/contract.entity';
|
||||
import { CargoType } from '../../rule-engine/entities/cargo-type.entity';
|
||||
import { ServiceType } from '../../rule-engine/entities/service-type.entity';
|
||||
import { ShippingLine } from '../../rule-engine/entities/shipping-line.entity';
|
||||
import { Yard } from '../../rule-engine/entities/yard.entity';
|
||||
import { ShippingLineCompany } from '../../shipping-lines/entities/shipping-line-company.entity';
|
||||
import { Train } from '../../trains/entities/train.entity';
|
||||
import { applyDirectionScope } from '../../user-trade-access/trade-scope.util';
|
||||
import { ExportDataset } from '../export.types';
|
||||
|
||||
/**
|
||||
* Domain semantics shared with `reports/definitions/bookings-list.report.ts`.
|
||||
* Kept identical on purpose — for PER_ITEM bulk bookings `cargo_total_weight_vgm`
|
||||
* holds an item COUNT, not tonnage, and `adjusted_total_amount` silently
|
||||
* overrides `total_amount`. Getting either wrong misreports money or weight.
|
||||
*/
|
||||
const TONS = 'COALESCE(b.bulk_total_weight_tons, b.cargo_total_weight_vgm)';
|
||||
const REVENUE = 'COALESCE(b.adjusted_total_amount, b.total_amount)';
|
||||
|
||||
const STATUS_OPTIONS = [
|
||||
'DRAFT', 'SUBMITTED', 'UNDER_REVIEW', 'APPROVED', 'REJECTED',
|
||||
'CANCELLED', 'EXPIRED', 'SCHEDULED', 'LOADED', 'IN_TRANSIT',
|
||||
'ARRIVED', 'DELIVERED', 'COMPLETED',
|
||||
].map((v) => ({ value: v, label: v.replace(/_/g, ' ') }));
|
||||
|
||||
export const bookingsDataset: ExportDataset = {
|
||||
key: 'bookings',
|
||||
title: 'Bookings',
|
||||
description: 'Every booking, with customer, route, cargo, contract and payment detail',
|
||||
group: 'Commercial',
|
||||
permission: FREIGHT_PERMS.bookings.view,
|
||||
base: { entity: Booking, alias: 'b' },
|
||||
|
||||
// Every join is a LEFT join (see ExportJoin) — ticking a field must never
|
||||
// change which rows come back.
|
||||
joins: [
|
||||
{ alias: 'c', entity: Company, on: 'c.id = b.company_id' },
|
||||
{ alias: 'cp', entity: CompanyProfile, on: 'cp.id = b.company_profile_id' },
|
||||
{ alias: 'slc', entity: ShippingLineCompany, on: 'slc.id = b.shipping_line_company_id' },
|
||||
{ alias: 'o', entity: Yard, on: 'o.id = b.origin_yard_id' },
|
||||
{ alias: 'd', entity: Yard, on: 'd.id = b.destination_yard_id' },
|
||||
{ alias: 'cty', entity: CargoType, on: 'cty.id = b.cargo_type_id' },
|
||||
{ alias: 'st', entity: ServiceType, on: 'st.id = b.service_type_id' },
|
||||
{ alias: 'sl', entity: ShippingLine, on: 'sl.id = b.shipping_line_id' },
|
||||
{ alias: 'ct', entity: Contract, on: 'ct.id = b.contract_id' },
|
||||
{ alias: 't', entity: Train, on: 't.id = b.train_id' },
|
||||
// Transitive: the contract's own customer, reachable only once `ct` is in.
|
||||
{ alias: 'ctc', entity: Company, on: 'ctc.id = ct.company_id', requires: ['ct'] },
|
||||
],
|
||||
// `search` matches the customer name, so `c` is always present — which is
|
||||
// also why the count query joins it.
|
||||
alwaysJoin: ['c'],
|
||||
|
||||
groups: [
|
||||
{ id: 'booking', label: 'Booking' },
|
||||
{ id: 'customer', label: 'Customer' },
|
||||
{ id: 'route', label: 'Route' },
|
||||
{ id: 'cargo', label: 'Cargo' },
|
||||
{ id: 'payment', label: 'Payment' },
|
||||
{ id: 'scheduling', label: 'Scheduling' },
|
||||
{ id: 'contract', label: 'Contract' },
|
||||
{ id: 'firstMile', label: 'First mile' },
|
||||
{ id: 'lastMile', label: 'Last mile' },
|
||||
{ id: 'clearance', label: 'Clearance' },
|
||||
],
|
||||
|
||||
fields: [
|
||||
// ---- Booking -------------------------------------------------------
|
||||
{ key: 'reference', label: 'Reference', type: 'string', group: 'booking', default: true, select: 'b.reference', sortExpr: 'b.reference' },
|
||||
{ key: 'status', label: 'Status', type: 'string', group: 'booking', default: true, select: 'b.status', sortExpr: 'b.status' },
|
||||
{ key: 'bookingType', label: 'Booking type', type: 'string', group: 'booking', select: 'b.booking_type' },
|
||||
{ key: 'contractKind', label: 'Contract kind', type: 'string', group: 'booking', select: 'b.contract_kind' },
|
||||
{ key: 'createdAt', label: 'Created', type: 'datetime', group: 'booking', default: true, select: `to_char(b.created_at, 'YYYY-MM-DD HH24:MI')`, sortExpr: 'b.created_at' },
|
||||
{ key: 'updatedAt', label: 'Updated', type: 'datetime', group: 'booking', select: `to_char(b.updated_at, 'YYYY-MM-DD HH24:MI')`, sortExpr: 'b.updated_at' },
|
||||
{ key: 'expiresAt', label: 'Expires', type: 'date', group: 'booking', select: `to_char(b.expires_at, 'YYYY-MM-DD')` },
|
||||
{ key: 'createdByRole', label: 'Created by role', type: 'string', group: 'booking', select: 'b.created_by_role' },
|
||||
{ key: 'isSplit', label: 'Split booking', type: 'boolean', group: 'booking', select: 'b.is_split' },
|
||||
{ key: 'priorityScore', label: 'Priority score', type: 'number', group: 'booking', select: 'b.priority_score', sortExpr: 'b.priority_score' },
|
||||
{ key: 'versionNumber', label: 'Version', type: 'number', group: 'booking', select: 'b.version_number' },
|
||||
{ key: 'pnrCode', label: 'PNR code', type: 'string', group: 'booking', select: 'b.pnr_code' },
|
||||
|
||||
// ---- Customer (the "more than the UI shows" payload) ----------------
|
||||
{ key: 'customer', label: 'Customer', type: 'string', group: 'customer', default: true, requires: ['c'], select: 'c.name', sortExpr: 'c.name' },
|
||||
{ key: 'customerType', label: 'Customer type', type: 'string', group: 'customer', requires: ['c'], select: 'c.type' },
|
||||
{ key: 'customerKind', label: 'Customer kind', type: 'string', group: 'customer', requires: ['c'], select: 'c.kind' },
|
||||
{ key: 'customerStatus', label: 'Customer status', type: 'string', group: 'customer', requires: ['c'], select: 'c.status' },
|
||||
{ key: 'customerTin', label: 'Customer TIN', type: 'string', group: 'customer', requires: ['c'], select: 'c.tin' },
|
||||
{ key: 'customerVat', label: 'Customer VAT no.', type: 'string', group: 'customer', requires: ['c'], select: 'c.vat_number' },
|
||||
{ key: 'customerPhone', label: 'Customer phone', type: 'string', group: 'customer', requires: ['c'], select: 'c.phone' },
|
||||
{ key: 'customerEmail', label: 'Customer email', type: 'string', group: 'customer', requires: ['c'], select: 'c.email' },
|
||||
{ key: 'customerContact', label: 'Contact person', type: 'string', group: 'customer', requires: ['c'], select: 'c.contact_person_name' },
|
||||
{ key: 'customerContactPhone', label: 'Contact phone', type: 'string', group: 'customer', requires: ['c'], select: 'c.contact_person_phone' },
|
||||
{ key: 'customerAddress', label: 'Customer address', type: 'string', group: 'customer', requires: ['c'], select: 'c.address' },
|
||||
{ key: 'customerCountry', label: 'Customer country', type: 'string', group: 'customer', requires: ['c'], select: 'c.country' },
|
||||
{ key: 'customerRegion', label: 'Customer region', type: 'string', group: 'customer', requires: ['c'], select: 'c.region' },
|
||||
{ key: 'customerProfileRef', label: 'Profile reference', type: 'string', group: 'customer', requires: ['cp'], select: 'cp.reference' },
|
||||
{ key: 'customerProfileType', label: 'Profile type', type: 'string', group: 'customer', requires: ['cp'], select: 'cp.type' },
|
||||
{ key: 'isGovernment', label: 'Government', type: 'boolean', group: 'customer', select: 'b.is_government' },
|
||||
{ key: 'governmentInstitution', label: 'Government institution', type: 'string', group: 'customer', select: 'b.government_institution' },
|
||||
{ key: 'shippingLineCompany', label: 'Shipping line company', type: 'string', group: 'customer', requires: ['slc'], select: 'slc.name' },
|
||||
|
||||
// ---- Route ----------------------------------------------------------
|
||||
{ key: 'origin', label: 'Origin', type: 'string', group: 'route', default: true, requires: ['o'], select: 'o.label' },
|
||||
{ key: 'originCode', label: 'Origin code', type: 'string', group: 'route', requires: ['o'], select: 'o.code' },
|
||||
{ key: 'destination', label: 'Destination', type: 'string', group: 'route', default: true, requires: ['d'], select: 'd.label' },
|
||||
{ key: 'destinationCode', label: 'Destination code', type: 'string', group: 'route', requires: ['d'], select: 'd.code' },
|
||||
{ key: 'tradeDirection', label: 'Direction', type: 'string', group: 'route', default: true, select: 'b.trade_direction', sortExpr: 'b.trade_direction' },
|
||||
{ key: 'serviceType', label: 'Service type', type: 'string', group: 'route', requires: ['st'], select: 'st.service_name' },
|
||||
|
||||
// ---- Cargo -----------------------------------------------------------
|
||||
{ key: 'cargo', label: 'Cargo', type: 'string', group: 'cargo', default: true, requires: ['cty'], select: 'COALESCE(cty.cargo_type_name, b.cargo_free_text)' },
|
||||
{ key: 'freightType', label: 'Freight type', type: 'string', group: 'cargo', default: true, select: 'b.freight_type' },
|
||||
{ key: 'tons', label: 'Tonnage', type: 'tons', group: 'cargo', default: true, select: `ROUND(${TONS})::float8`, sortExpr: TONS },
|
||||
{ key: 'containerWeightVgm', label: 'Container VGM', type: 'number', group: 'cargo', select: 'b.cargo_total_weight_vgm' },
|
||||
{ key: 'bulkWeightTons', label: 'Bulk weight (t)', type: 'tons', group: 'cargo', select: 'b.bulk_total_weight_tons' },
|
||||
{ key: 'isHazardous', label: 'Hazardous', type: 'boolean', group: 'cargo', select: 'b.is_hazardous' },
|
||||
{ key: 'isReefer', label: 'Reefer', type: 'boolean', group: 'cargo', select: 'b.is_reefer' },
|
||||
{ key: 'shippingLine', label: 'Shipping line', type: 'string', group: 'cargo', requires: ['sl'], select: 'sl.label' },
|
||||
{
|
||||
// One-to-many, so it aggregates in a correlated subquery rather than a
|
||||
// join — a join here would multiply rows and break the count contract.
|
||||
key: 'containerNumbers', label: 'Container numbers', type: 'string', group: 'cargo',
|
||||
select: `(SELECT string_agg(bc.container_number, ' | ' ORDER BY bc.container_number)
|
||||
FROM freight.booking_container bc
|
||||
WHERE bc.booking_id = b.id AND bc.deleted_at IS NULL)`,
|
||||
},
|
||||
|
||||
// ---- Payment ----------------------------------------------------------
|
||||
{ key: 'amount', label: 'Amount', type: 'money', group: 'payment', default: true, select: `ROUND(${REVENUE}, 2)::float8`, sortExpr: REVENUE },
|
||||
{ key: 'totalAmount', label: 'Total amount (pre-adjustment)', type: 'money', group: 'payment', select: 'b.total_amount::float8' },
|
||||
{ key: 'adjustedTotalAmount', label: 'Adjusted total', type: 'money', group: 'payment', select: 'b.adjusted_total_amount::float8' },
|
||||
{ key: 'adjustmentReason', label: 'Adjustment reason', type: 'string', group: 'payment', select: 'b.adjustment_reason' },
|
||||
{ key: 'paymentStatus', label: 'Payment status', type: 'string', group: 'payment', default: true, select: 'b.payment_status', sortExpr: 'b.payment_status' },
|
||||
{ key: 'paymentCurrency', label: 'Currency', type: 'string', group: 'payment', select: 'b.payment_currency' },
|
||||
{ key: 'paymentDeadline', label: 'Payment deadline', type: 'datetime', group: 'payment', select: `to_char(b.payment_deadline, 'YYYY-MM-DD HH24:MI')` },
|
||||
|
||||
// ---- Scheduling --------------------------------------------------------
|
||||
{ key: 'scheduledDate', label: 'Scheduled date', type: 'date', group: 'scheduling', default: true, select: `to_char(b.scheduled_date, 'YYYY-MM-DD')`, sortExpr: 'b.scheduled_date' },
|
||||
{ key: 'schedulingStatus', label: 'Scheduling status', type: 'string', group: 'scheduling', select: 'b.scheduling_status' },
|
||||
{ key: 'wagonsRequired', label: 'Wagons required', type: 'number', group: 'scheduling', select: 'b.wagons_required' },
|
||||
{ key: 'trainCode', label: 'Train', type: 'string', group: 'scheduling', requires: ['t'], select: 't.code' },
|
||||
{ key: 'loadedAt', label: 'Loaded at', type: 'datetime', group: 'scheduling', select: `to_char(b.loaded_at, 'YYYY-MM-DD HH24:MI')` },
|
||||
{ key: 'arrivedAt', label: 'Arrived at', type: 'datetime', group: 'scheduling', select: `to_char(b.arrived_at, 'YYYY-MM-DD HH24:MI')` },
|
||||
|
||||
// ---- Contract ----------------------------------------------------------
|
||||
{ key: 'contractReference', label: 'Contract reference', type: 'string', group: 'contract', requires: ['ct'], select: 'ct.reference' },
|
||||
{ key: 'contractStatus', label: 'Contract status', type: 'string', group: 'contract', requires: ['ct'], select: 'ct.status' },
|
||||
{ key: 'contractCustomer', label: 'Contract customer', type: 'string', group: 'contract', requires: ['ctc'], select: 'ctc.name' },
|
||||
{ key: 'contractType', label: 'Contract type', type: 'string', group: 'contract', select: 'b.contract_type' },
|
||||
{ key: 'contractValidFrom', label: 'Contract valid from', type: 'date', group: 'contract', select: `to_char(b.contract_valid_from, 'YYYY-MM-DD')` },
|
||||
{ key: 'contractValidUntil', label: 'Contract valid until', type: 'date', group: 'contract', select: `to_char(b.contract_valid_until, 'YYYY-MM-DD')` },
|
||||
{ key: 'fullyExecutedAt', label: 'Fully executed at', type: 'datetime', group: 'contract', select: `to_char(b.fully_executed_at, 'YYYY-MM-DD HH24:MI')` },
|
||||
|
||||
// ---- First / last mile ---------------------------------------------------
|
||||
{ key: 'firstMileAddress', label: 'Pickup address', type: 'string', group: 'firstMile', select: 'b.first_mile_pickup_address' },
|
||||
{ key: 'lastMileAddress', label: 'Delivery address', type: 'string', group: 'lastMile', select: 'b.last_mile_delivery_address' },
|
||||
{ key: 'customerTruckPlate', label: 'Customer truck plate', type: 'string', group: 'lastMile', select: 'b.customer_truck_plate_number' },
|
||||
{ key: 'customerTruckDriver', label: 'Customer truck driver', type: 'string', group: 'lastMile', select: 'b.customer_truck_driver_name' },
|
||||
{ key: 'exportHandoverMode', label: 'Handover mode', type: 'string', group: 'lastMile', select: 'b.export_handover_mode' },
|
||||
|
||||
// ---- Clearance -------------------------------------------------------------
|
||||
{ key: 'customsClearingEnabled', label: 'Customs clearing', type: 'boolean', group: 'clearance', select: 'b.customs_clearing_enabled' },
|
||||
{ key: 'customsClearingAgent', label: 'Clearing agent', type: 'string', group: 'clearance', select: 'b.customs_clearing_agent' },
|
||||
{ key: 'clearancePhase', label: 'Clearance phase', type: 'string', group: 'clearance', select: 'b.clearance_current_phase' },
|
||||
{ key: 'dutyRequired', label: 'Duty required', type: 'boolean', group: 'clearance', select: 'b.duty_required' },
|
||||
{ key: 'vesselArrivalDate', label: 'Vessel arrival', type: 'date', group: 'clearance', select: `to_char(b.vessel_arrival_date, 'YYYY-MM-DD')` },
|
||||
{ key: 'doCollectedDate', label: 'DO collected', type: 'date', group: 'clearance', select: `to_char(b.do_collected_date, 'YYYY-MM-DD')` },
|
||||
{ key: 'doubleHandling', label: 'Double handling', type: 'boolean', group: 'clearance', select: 'b.double_handling' },
|
||||
],
|
||||
|
||||
filters: [
|
||||
{ key: 'created', label: 'Created', type: 'daterange' },
|
||||
{ key: 'statuses', label: 'Status', type: 'multiselect', options: STATUS_OPTIONS },
|
||||
{
|
||||
key: 'tradeDirection', label: 'Direction', type: 'select',
|
||||
options: ['IMPORT', 'EXPORT', 'DOMESTIC'].map((v) => ({ value: v, label: v })),
|
||||
},
|
||||
{
|
||||
key: 'freightType', label: 'Freight type', type: 'select',
|
||||
options: ['CONTAINER', 'BULK'].map((v) => ({ value: v, label: v })),
|
||||
},
|
||||
{ key: 'paymentStatus', label: 'Payment status', type: 'select', options: [
|
||||
{ value: 'PENDING', label: 'Pending' },
|
||||
{ value: 'PNR_GENERATED', label: 'PNR generated' },
|
||||
{ value: 'VERIFICATION_IN_PROGRESS', label: 'Verification in progress' },
|
||||
{ value: 'PAID', label: 'Paid' },
|
||||
{ value: 'FAILED', label: 'Failed' },
|
||||
] },
|
||||
{ key: 'companyId', label: 'Customer', type: 'text' },
|
||||
{ key: 'search', label: 'Search reference or customer', type: 'text' },
|
||||
],
|
||||
|
||||
defaultSort: { key: 'createdAt', dir: 'DESC' },
|
||||
|
||||
scope(ctx, qb) {
|
||||
const { params, directions } = ctx;
|
||||
// andWhere, not where: `where()` resets any condition already on the
|
||||
// builder, so scope() would silently drop anything a caller added first.
|
||||
qb.andWhere('b.deleted_at IS NULL');
|
||||
|
||||
if (params.createdFrom) qb.andWhere('b.created_at >= :createdFrom', { createdFrom: params.createdFrom });
|
||||
if (params.createdTo) qb.andWhere('b.created_at < :createdTo', { createdTo: params.createdTo });
|
||||
|
||||
const statuses = params.statuses as string[] | null;
|
||||
if (statuses?.length) qb.andWhere('b.status IN (:...statuses)', { statuses });
|
||||
|
||||
if (params.tradeDirection) qb.andWhere('b.trade_direction = :tradeDirection', { tradeDirection: params.tradeDirection });
|
||||
if (params.freightType) qb.andWhere('b.freight_type = :freightType', { freightType: params.freightType });
|
||||
if (params.paymentStatus) qb.andWhere('b.payment_status = :paymentStatus', { paymentStatus: params.paymentStatus });
|
||||
if (params.companyId) qb.andWhere('b.company_id = :companyId', { companyId: params.companyId });
|
||||
if (params.search) {
|
||||
qb.andWhere('(b.reference ILIKE :search OR c.name ILIKE :search)', { search: `%${params.search as string}%` });
|
||||
}
|
||||
|
||||
// Trade-direction ACL. Without this the export returns rows the user's own
|
||||
// list page would not show them.
|
||||
applyDirectionScope(qb, 'b.trade_direction', directions);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,81 @@
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
export type ExportFilterType = 'daterange' | 'date' | 'select' | 'multiselect' | 'text';
|
||||
|
||||
export interface ExportFilterOption {
|
||||
value: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ExportFilterDef {
|
||||
key: string;
|
||||
label: string;
|
||||
type: ExportFilterType;
|
||||
/** Static choices. Mutually exclusive with `optionsQuery`. */
|
||||
options?: ExportFilterOption[];
|
||||
/** Reference-data choices resolved from the DB and cached for the process. */
|
||||
optionsQuery?: (ds: DataSource) => Promise<ExportFilterOption[]>;
|
||||
}
|
||||
|
||||
/** Raw query-string bag. Per-registry filter keys, so `forbidNonWhitelisted` can't police it. */
|
||||
export type RawFilterQuery = Record<string, string | undefined>;
|
||||
|
||||
/**
|
||||
* Coerce raw query strings into typed filter params per a filter declaration
|
||||
* list. Unknown keys are dropped rather than rejected.
|
||||
*
|
||||
* Shared by the report runner and the export runner so the `daterange`
|
||||
* handling in particular cannot drift between them: `To` is pushed forward a
|
||||
* day because callers mean an INCLUSIVE end date while the SQL bound is
|
||||
* exclusive (`created_at < :dateTo`).
|
||||
*/
|
||||
export function coerceFilterParams(
|
||||
filters: ExportFilterDef[],
|
||||
raw: RawFilterQuery,
|
||||
): Record<string, unknown> {
|
||||
const params: Record<string, unknown> = {};
|
||||
for (const filter of filters) {
|
||||
if (filter.type === 'daterange') {
|
||||
const from = raw[`${filter.key}From`];
|
||||
const to = raw[`${filter.key}To`];
|
||||
params[`${filter.key}From`] = from ? new Date(from).toISOString() : null;
|
||||
params[`${filter.key}To`] = to
|
||||
? new Date(new Date(to).getTime() + DAY_MS).toISOString()
|
||||
: null;
|
||||
} else if (filter.type === 'multiselect') {
|
||||
const csv = raw[filter.key];
|
||||
const items = csv?.split(',').map((s) => s.trim()).filter(Boolean) ?? [];
|
||||
params[filter.key] = items.length ? items : null;
|
||||
} else {
|
||||
params[filter.key] = raw[filter.key]?.trim() || null;
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process-lifetime cache for `optionsQuery` results — small, rarely-changing
|
||||
* reference lists (23 stations, 18 cargo types) hit on every catalog load.
|
||||
*
|
||||
* ponytail: keyed by filter key alone, so two registries sharing a filter key
|
||||
* share one option list. Key by `${registry}:${filterKey}` if that ever bites.
|
||||
*/
|
||||
const optionsCache = new Map<string, ExportFilterOption[]>();
|
||||
|
||||
export async function resolveFilterOptions(
|
||||
filters: ExportFilterDef[],
|
||||
ds: DataSource,
|
||||
): Promise<ExportFilterDef[]> {
|
||||
return Promise.all(
|
||||
filters.map(async (filter) => {
|
||||
if (!filter.optionsQuery) return filter;
|
||||
const cached = optionsCache.get(filter.key);
|
||||
if (cached) return { ...filter, options: cached, optionsQuery: undefined };
|
||||
const options = await filter.optionsQuery(ds);
|
||||
optionsCache.set(filter.key, options);
|
||||
return { ...filter, options, optionsQuery: undefined };
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { resolveJoins } from './export-query.builder';
|
||||
import { ExportDataset, ExportField } from './export.types';
|
||||
|
||||
const field = (key: string, requires?: string[]): ExportField => ({
|
||||
key,
|
||||
label: key,
|
||||
type: 'string',
|
||||
group: 'g',
|
||||
select: `x.${key}`,
|
||||
requires,
|
||||
});
|
||||
|
||||
/** Entities are never dereferenced by resolveJoins — only the alias graph matters. */
|
||||
const entity = {} as ExportDataset['joins'][number]['entity'];
|
||||
|
||||
const dataset = (
|
||||
joins: ExportDataset['joins'],
|
||||
alwaysJoin?: string[],
|
||||
): ExportDataset =>
|
||||
({
|
||||
key: 'test',
|
||||
joins,
|
||||
alwaysJoin,
|
||||
fields: [],
|
||||
}) as unknown as ExportDataset;
|
||||
|
||||
describe('resolveJoins', () => {
|
||||
it('pulls in only the joins the selected fields ask for', () => {
|
||||
const ds = dataset([
|
||||
{ alias: 'a', entity, on: 'a.id = b.a_id' },
|
||||
{ alias: 'z', entity, on: 'z.id = b.z_id' },
|
||||
]);
|
||||
expect(resolveJoins(ds, [field('one', ['a'])]).map((j) => j.alias)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('selecting nothing still applies alwaysJoin — the count query relies on this', () => {
|
||||
const ds = dataset(
|
||||
[
|
||||
{ alias: 'a', entity, on: 'a.id = b.a_id' },
|
||||
{ alias: 'z', entity, on: 'z.id = b.z_id' },
|
||||
],
|
||||
['a'],
|
||||
);
|
||||
expect(resolveJoins(ds, []).map((j) => j.alias)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('resolves a transitive dependency, dependency first', () => {
|
||||
const ds = dataset([
|
||||
{ alias: 'ct', entity, on: 'ct.id = b.contract_id' },
|
||||
{ alias: 'ctc', entity, on: 'ctc.id = ct.company_id', requires: ['ct'] },
|
||||
]);
|
||||
expect(resolveJoins(ds, [field('x', ['ctc'])]).map((j) => j.alias)).toEqual(['ct', 'ctc']);
|
||||
});
|
||||
|
||||
it('resolves a multi-hop chain in order', () => {
|
||||
const ds = dataset([
|
||||
{ alias: 'a', entity, on: 'a.id = b.a_id' },
|
||||
{ alias: 'bb', entity, on: 'bb.id = a.b_id', requires: ['a'] },
|
||||
{ alias: 'cc', entity, on: 'cc.id = bb.c_id', requires: ['bb'] },
|
||||
]);
|
||||
expect(resolveJoins(ds, [field('x', ['cc'])]).map((j) => j.alias)).toEqual(['a', 'bb', 'cc']);
|
||||
});
|
||||
|
||||
it('emits a shared join once, not per field that needs it', () => {
|
||||
const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]);
|
||||
const joins = resolveJoins(ds, [field('one', ['a']), field('two', ['a'])]);
|
||||
expect(joins.map((j) => j.alias)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('does not duplicate a join already pulled in by alwaysJoin', () => {
|
||||
const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }], ['a']);
|
||||
expect(resolveJoins(ds, [field('one', ['a'])]).map((j) => j.alias)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('throws on a cycle rather than looping forever', () => {
|
||||
const ds = dataset([
|
||||
{ alias: 'a', entity, on: 'a.id = bb.a_id', requires: ['bb'] },
|
||||
{ alias: 'bb', entity, on: 'bb.id = a.b_id', requires: ['a'] },
|
||||
]);
|
||||
expect(() => resolveJoins(ds, [field('x', ['a'])])).toThrow(/join cycle/);
|
||||
});
|
||||
|
||||
it('throws on an undeclared alias — a typo must fail loudly, not silently 42P01', () => {
|
||||
const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]);
|
||||
expect(() => resolveJoins(ds, [field('x', ['ghost'])])).toThrow(/unknown join alias "ghost"/);
|
||||
});
|
||||
|
||||
it('a field with no requires pulls in no joins at all', () => {
|
||||
const ds = dataset([{ alias: 'a', entity, on: 'a.id = b.a_id' }]);
|
||||
expect(resolveJoins(ds, [field('plain')])).toEqual([]);
|
||||
});
|
||||
});
|
||||
101
apps/edr-freight-api/src/modules/exports/export-query.builder.ts
Normal file
101
apps/edr-freight-api/src/modules/exports/export-query.builder.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
import { ObjectLiteral, SelectQueryBuilder } from 'typeorm';
|
||||
|
||||
import { ExportContext, ExportDataset, ExportField, ExportJoin } from './export.types';
|
||||
|
||||
/**
|
||||
* Sort expression fallback: the SELECT alias TypeORM emitted, quoted. TypeORM
|
||||
* double-quotes `addSelect` aliases (preserving case), so ordering by the bare
|
||||
* key lets Postgres fold it to lowercase and 42703 on any camelCase alias.
|
||||
*/
|
||||
export const aliasSortExpr = (key: string): string => `"${key.replace(/"/g, '""')}"`;
|
||||
|
||||
/**
|
||||
* Selected fields -> the joins they need, transitively, dependencies first.
|
||||
* DFS post-order over `requires`, memoized. Deterministic: `alwaysJoin` first,
|
||||
* then fields in the dataset's own declaration order.
|
||||
*/
|
||||
export function resolveJoins(dataset: ExportDataset, fields: ExportField[]): ExportJoin[] {
|
||||
const byAlias = new Map(dataset.joins.map((j) => [j.alias, j]));
|
||||
const out: ExportJoin[] = [];
|
||||
const done = new Set<string>();
|
||||
const onStack = new Set<string>();
|
||||
|
||||
const visit = (alias: string): void => {
|
||||
if (done.has(alias)) return;
|
||||
if (onStack.has(alias)) {
|
||||
throw new Error(`export "${dataset.key}": join cycle at alias "${alias}"`);
|
||||
}
|
||||
const join = byAlias.get(alias);
|
||||
if (!join) {
|
||||
throw new Error(`export "${dataset.key}": unknown join alias "${alias}"`);
|
||||
}
|
||||
onStack.add(alias);
|
||||
for (const dep of join.requires ?? []) visit(dep);
|
||||
onStack.delete(alias);
|
||||
done.add(alias);
|
||||
out.push(join);
|
||||
};
|
||||
|
||||
for (const alias of dataset.alwaysJoin ?? []) visit(alias);
|
||||
for (const field of fields) for (const alias of field.requires ?? []) visit(alias);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The download query: base + only the joins the selected fields need. */
|
||||
export function buildExportQuery(
|
||||
dataset: ExportDataset,
|
||||
fields: ExportField[],
|
||||
ctx: ExportContext,
|
||||
): SelectQueryBuilder<ObjectLiteral> {
|
||||
const qb = ctx.ds.createQueryBuilder().from(dataset.base.entity, dataset.base.alias);
|
||||
for (const join of resolveJoins(dataset, fields)) {
|
||||
qb.leftJoin(join.entity, join.alias, join.on);
|
||||
}
|
||||
for (const field of fields) qb.addSelect(field.select, field.key);
|
||||
dataset.scope(ctx, qb);
|
||||
return qb;
|
||||
}
|
||||
|
||||
/**
|
||||
* The count query: same base, same `scope()`, same WHERE — but no field joins
|
||||
* and no selects. Exact rather than an estimate, because every lazy join is a
|
||||
* left join to a to-one side and so cannot change the row count.
|
||||
*/
|
||||
export function buildExportCountQuery(
|
||||
dataset: ExportDataset,
|
||||
ctx: ExportContext,
|
||||
): SelectQueryBuilder<ObjectLiteral> {
|
||||
const qb = ctx.ds
|
||||
.createQueryBuilder()
|
||||
.select('COUNT(*)::int', 'total')
|
||||
.from(dataset.base.entity, dataset.base.alias);
|
||||
for (const join of resolveJoins(dataset, [])) {
|
||||
qb.leftJoin(join.entity, join.alias, join.on);
|
||||
}
|
||||
dataset.scope(ctx, qb);
|
||||
return qb;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a requested sort against the SELECTED fields. Restricting to selected
|
||||
* fields means a sort can never pull in a join the projection didn't already
|
||||
* need — which is what keeps the count query's join set correct.
|
||||
*/
|
||||
export function resolveExportSort(
|
||||
dataset: ExportDataset,
|
||||
fields: ExportField[],
|
||||
sortBy?: string,
|
||||
sortOrder?: string,
|
||||
): { expr: string; dir: 'ASC' | 'DESC' } | null {
|
||||
const dir = sortOrder?.toUpperCase() === 'DESC' ? 'DESC' : 'ASC';
|
||||
const requested = sortBy && fields.find((f) => f.key === sortBy && f.sortExpr);
|
||||
if (requested) return { expr: requested.sortExpr ?? aliasSortExpr(requested.key), dir };
|
||||
|
||||
if (!dataset.defaultSort) return null;
|
||||
const fallback = fields.find((f) => f.key === dataset.defaultSort!.key);
|
||||
if (!fallback) return null;
|
||||
return {
|
||||
expr: fallback.sortExpr ?? aliasSortExpr(fallback.key),
|
||||
dir: dataset.defaultSort.dir,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
import { DataSource } from 'typeorm';
|
||||
|
||||
import { coerceFilterParams, RawFilterQuery } from './export-filter.util';
|
||||
import {
|
||||
buildExportCountQuery,
|
||||
buildExportQuery,
|
||||
resolveExportSort,
|
||||
} from './export-query.builder';
|
||||
import { ExportDataset, ExportField } from './export.types';
|
||||
|
||||
@Injectable()
|
||||
export class ExportRunnerService {
|
||||
constructor(@InjectDataSource() private readonly ds: DataSource) {}
|
||||
|
||||
private context(dataset: ExportDataset, raw: RawFilterQuery, directions: string[] | null) {
|
||||
return { ds: this.ds, params: coerceFilterParams(dataset.filters, raw), directions };
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact row count for the current filters. Exact rather than estimated
|
||||
* because lazy joins are all left joins to to-one sides, so the count cannot
|
||||
* depend on which fields the caller picked.
|
||||
*/
|
||||
async count(
|
||||
dataset: ExportDataset,
|
||||
raw: RawFilterQuery,
|
||||
directions: string[] | null,
|
||||
): Promise<number> {
|
||||
const qb = buildExportCountQuery(dataset, this.context(dataset, raw, directions));
|
||||
const row = await qb.getRawOne<{ total: number }>();
|
||||
return Number(row?.total ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Matching rows.
|
||||
*
|
||||
* `limit` is the caller's deliberate "first N" truncation — honoured
|
||||
* silently, because they asked for it. `cap` is the format's hard ceiling —
|
||||
* exceeding it throws, because a silently short file is worse than a clear
|
||||
* error: nothing downstream reveals that rows are missing.
|
||||
*/
|
||||
async run(
|
||||
dataset: ExportDataset,
|
||||
fields: ExportField[],
|
||||
raw: RawFilterQuery,
|
||||
directions: string[] | null,
|
||||
{ cap, limit }: { cap: number; limit?: number },
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const ctx = this.context(dataset, raw, directions);
|
||||
const qb = buildExportQuery(dataset, fields, ctx);
|
||||
|
||||
const sort = resolveExportSort(dataset, fields, raw.sortBy, raw.sortOrder);
|
||||
if (sort) qb.orderBy(sort.expr, sort.dir);
|
||||
|
||||
const ceiling = limit ?? cap;
|
||||
// ceiling + 1: fetching exactly `ceiling` cannot distinguish "there are
|
||||
// exactly that many rows" from "there are more".
|
||||
const items = await qb.limit(ceiling + 1).getRawMany();
|
||||
if (items.length <= ceiling) return items;
|
||||
|
||||
// Asked to be truncated -> truncate. Hit the hard cap -> say so.
|
||||
if (limit !== undefined) return items.slice(0, limit);
|
||||
throw new BadRequestException(
|
||||
`This export has more than ${cap.toLocaleString()} rows, the limit for this format. Narrow the filters, or export a smaller number of rows.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
15
apps/edr-freight-api/src/modules/exports/export.registry.ts
Normal file
15
apps/edr-freight-api/src/modules/exports/export.registry.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { bookingsDataset } from './datasets/bookings.dataset';
|
||||
import { ExportDataset } from './export.types';
|
||||
|
||||
/**
|
||||
* Every exportable dataset.
|
||||
*
|
||||
* Adding one = a new file under `datasets/` + an entry here. No frontend edit,
|
||||
* no route, no permission seed — the dialog is driven entirely by the catalog
|
||||
* this registry serves, and a dataset reuses its module's existing `view` key.
|
||||
*/
|
||||
export const DATASETS: ExportDataset[] = [bookingsDataset];
|
||||
|
||||
const BY_KEY = new Map(DATASETS.map((d) => [d.key, d]));
|
||||
|
||||
export const getDataset = (key: string): ExportDataset | undefined => BY_KEY.get(key);
|
||||
135
apps/edr-freight-api/src/modules/exports/export.types.ts
Normal file
135
apps/edr-freight-api/src/modules/exports/export.types.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
import {
|
||||
DataSource,
|
||||
EntityTarget,
|
||||
ObjectLiteral,
|
||||
ObjectType,
|
||||
SelectQueryBuilder,
|
||||
} from 'typeorm';
|
||||
|
||||
import { ExportFilterDef } from './export-filter.util';
|
||||
import { ExportFieldType } from './tabular-export.service';
|
||||
|
||||
export type { ExportFieldType };
|
||||
|
||||
/**
|
||||
* A lazily-applied relation.
|
||||
*
|
||||
* There is deliberately no `kind: 'inner' | 'left'` here — every join is
|
||||
* emitted as a LEFT JOIN, and the type makes anything else unrepresentable.
|
||||
* An inner join added only because someone ticked a checkbox would silently
|
||||
* change the rowset (ticking "Customer TIN" would drop every booking with a
|
||||
* null company_id), so two exports of the same filters would disagree on their
|
||||
* row count. Anything that genuinely must narrow rows belongs in `scope()`,
|
||||
* where it is unconditional and visible.
|
||||
*
|
||||
* The payoff: because a left join to a to-one side can neither add nor remove
|
||||
* rows, the row count is independent of which fields are selected — which is
|
||||
* what lets the count endpoint be exact rather than an estimate.
|
||||
*/
|
||||
export interface ExportJoin {
|
||||
/** Alias used by field `select` expressions and by `requires`. */
|
||||
alias: string;
|
||||
/** Entity class. Narrower than `EntityTarget` to match TypeORM's join overload. */
|
||||
entity: ObjectType<ObjectLiteral>;
|
||||
/** ON condition; may reference the base alias and any alias in `requires`. */
|
||||
on: string;
|
||||
/** Other join aliases this join's ON clause depends on. Resolved transitively. */
|
||||
requires?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* One exportable column.
|
||||
*
|
||||
* `select` must yield exactly ONE value per base row. To surface a one-to-many
|
||||
* relation (a company's profiles, a booking's containers), aggregate inside a
|
||||
* correlated subquery — `(SELECT string_agg(...) FROM ... WHERE ... = base.id)`
|
||||
* — rather than adding a join, which would multiply rows and break the count.
|
||||
*
|
||||
* Sensitive columns are simply never declared: raw gateway payloads
|
||||
* (payments.raw_initiation, client_action), signature/crypto blobs
|
||||
* (invoices.eims_signed_qr), internal error dumps (eims_last_error), raw jsonb
|
||||
* snapshots (pricing_breakdown, document_snapshot, financial_terms,
|
||||
* attributes, business_license_files), bare internal user UUIDs, and internal
|
||||
* review/rejection notes. Fields are opt-in, so omission is the whole
|
||||
* enforcement mechanism.
|
||||
*/
|
||||
export interface ExportField {
|
||||
/** Response key, sheet header id, and the picker's checkbox id. */
|
||||
key: string;
|
||||
label: string;
|
||||
type: ExportFieldType;
|
||||
/** Scalar SQL projected as `key`. */
|
||||
select: string;
|
||||
/** Join aliases `select` references. Omit for base-table-only fields. */
|
||||
requires?: string[];
|
||||
/** Picker group id; must exist in the dataset's `groups`. */
|
||||
group: string;
|
||||
/** Pre-ticked when the dialog opens with no preset. */
|
||||
default?: boolean;
|
||||
/** ORDER BY expression. Presence makes the field sortable. */
|
||||
sortExpr?: string;
|
||||
}
|
||||
|
||||
export interface ExportGroup {
|
||||
id: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ExportContext {
|
||||
ds: DataSource;
|
||||
/** Filter values, already coerced by `coerceFilterParams`. */
|
||||
params: Record<string, unknown>;
|
||||
/** Trade-scope directions. `null` = unrestricted, `[]` = show nothing. */
|
||||
directions: string[] | null;
|
||||
}
|
||||
|
||||
export interface ExportDataset {
|
||||
key: string;
|
||||
title: string;
|
||||
description: string;
|
||||
group: 'Commercial' | 'Operations' | 'Finance' | 'Fleet';
|
||||
/**
|
||||
* Permission to export this dataset. Reuses the module's existing `view`
|
||||
* key — if you may see these rows on their list page, you may export them.
|
||||
* The export never returns a row the list endpoint would not.
|
||||
*/
|
||||
permission: string;
|
||||
base: { entity: EntityTarget<ObjectLiteral>; alias: string };
|
||||
joins: ExportJoin[];
|
||||
/**
|
||||
* Aliases applied unconditionally because `scope()` references them. This is
|
||||
* the only reason a join is eager, and the count query applies exactly these.
|
||||
*/
|
||||
alwaysJoin?: string[];
|
||||
groups: ExportGroup[];
|
||||
fields: ExportField[];
|
||||
filters: ExportFilterDef[];
|
||||
/** Must name a field whose `sortExpr` references only the base alias. */
|
||||
defaultSort?: { key: string; dir: 'ASC' | 'DESC' };
|
||||
/**
|
||||
* Base WHERE (soft-delete guard), filter application, and the trade-direction
|
||||
* ACL. Runs identically for the count and download queries, so the row count
|
||||
* the dialog shows is exactly what lands in the file.
|
||||
*
|
||||
* A dataset whose table carries a trade direction MUST apply it here, or the
|
||||
* export leaks rows the user cannot see on the list page.
|
||||
*/
|
||||
scope(ctx: ExportContext, qb: SelectQueryBuilder<ObjectLiteral>): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* What `GET /exports` serves. `select` / `requires` / `sortExpr` are raw SQL
|
||||
* and a map of the schema — they never leave the server.
|
||||
*/
|
||||
export interface ExportCatalogEntry {
|
||||
key: string;
|
||||
title: string;
|
||||
description: string;
|
||||
group: ExportDataset['group'];
|
||||
groups: ExportGroup[];
|
||||
fields: Pick<ExportField, 'key' | 'label' | 'type' | 'group' | 'default'>[];
|
||||
filters: ExportFilterDef[];
|
||||
formats: ('csv' | 'xlsx' | 'pdf')[];
|
||||
caps: { csv: number; xlsx: number; pdf: number };
|
||||
defaultSort?: { key: string; dir: 'ASC' | 'DESC' };
|
||||
}
|
||||
156
apps/edr-freight-api/src/modules/exports/exports.controller.ts
Normal file
156
apps/edr-freight-api/src/modules/exports/exports.controller.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
import { Controller, Get, NotFoundException, Param, Query, Res, UseGuards } from '@nestjs/common';
|
||||
import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
import { DataSource } from 'typeorm';
|
||||
import { CurrentUser } from '@edr/api-common';
|
||||
import { JwtGuard } from '@tria-plc/api-common/modules/auth/services/jwt.guard';
|
||||
import type { Response } from 'express';
|
||||
import type { TCurrentUser } from '@tria-plc/api-common/modules/auth/types/current-user.type';
|
||||
|
||||
import { assertFreightPermission, hasFreightPermission } from '../../common/freight-permission.util';
|
||||
import { UserTradeAccessService } from '../user-trade-access/user-trade-access.service';
|
||||
import { resolveFilterOptions } from './export-filter.util';
|
||||
import {
|
||||
EXPORT_MIME,
|
||||
formatRowCap,
|
||||
pickByKey,
|
||||
resolveExportFormat,
|
||||
resolveRowLimit,
|
||||
} from './export-request.util';
|
||||
import { ExportRunnerService } from './export-runner.service';
|
||||
import { DATASETS, getDataset } from './export.registry';
|
||||
import { ExportCatalogEntry, ExportDataset, ExportField } from './export.types';
|
||||
import { CSV_ROW_CAP, PDF_ROW_CAP, TabularExportService, XLSX_ROW_CAP } from './tabular-export.service';
|
||||
|
||||
/** Raw query bag — filter keys are per-dataset, so DTO whitelisting can't police it. */
|
||||
type RawExportQuery = Record<string, string | undefined>;
|
||||
|
||||
const CAPS = { csv: CSV_ROW_CAP, xlsx: XLSX_ROW_CAP, pdf: PDF_ROW_CAP };
|
||||
|
||||
/**
|
||||
* Metadata only. `select` / `requires` / `sortExpr` are raw SQL and a map of
|
||||
* the schema — they never leave the server.
|
||||
*/
|
||||
const toCatalogEntry = (dataset: ExportDataset): ExportCatalogEntry => ({
|
||||
key: dataset.key,
|
||||
title: dataset.title,
|
||||
description: dataset.description,
|
||||
group: dataset.group,
|
||||
groups: dataset.groups,
|
||||
fields: dataset.fields.map(({ key, label, type, group, default: isDefault }) => ({
|
||||
key,
|
||||
label,
|
||||
type,
|
||||
group,
|
||||
default: isDefault,
|
||||
})),
|
||||
filters: dataset.filters,
|
||||
formats: ['csv', 'xlsx', 'pdf'],
|
||||
caps: CAPS,
|
||||
defaultSort: dataset.defaultSort,
|
||||
});
|
||||
|
||||
/**
|
||||
* Generic table export. One dataset per major table, each describing far more
|
||||
* fields than its list page shows — including related-entity detail.
|
||||
*/
|
||||
@ApiTags('Exports')
|
||||
@ApiBearerAuth()
|
||||
@Controller('exports')
|
||||
@UseGuards(JwtGuard)
|
||||
export class ExportsController {
|
||||
constructor(
|
||||
private readonly runner: ExportRunnerService,
|
||||
private readonly writer: TabularExportService,
|
||||
private readonly userTradeAccessService: UserTradeAccessService,
|
||||
@InjectDataSource() private readonly dataSource: DataSource,
|
||||
) {}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({ summary: 'List datasets the caller has permission to export' })
|
||||
async catalog(@CurrentUser() user: TCurrentUser): Promise<ExportCatalogEntry[]> {
|
||||
const allowed = DATASETS.filter((d) => hasFreightPermission(user, d.permission));
|
||||
return Promise.all(
|
||||
allowed.map(async (d) => ({
|
||||
...toCatalogEntry(d),
|
||||
filters: await resolveFilterOptions(d.filters, this.dataSource),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
@Get(':key/count')
|
||||
@ApiOperation({ summary: 'Exact row count for the given filters, plus the per-format caps' })
|
||||
async count(
|
||||
@Param('key') key: string,
|
||||
@Query() query: RawExportQuery,
|
||||
@CurrentUser() user: TCurrentUser,
|
||||
): Promise<{ total: number; caps: typeof CAPS }> {
|
||||
const dataset = this.resolve(key, user);
|
||||
const directions = await this.userTradeAccessService.resolveAllowedDirections(user);
|
||||
const total = await this.runner.count(dataset, query, directions);
|
||||
return { total, caps: CAPS };
|
||||
}
|
||||
|
||||
@Get(':key/download')
|
||||
@ApiOperation({ summary: 'Export a dataset to csv, xlsx or pdf' })
|
||||
async download(
|
||||
@Param('key') key: string,
|
||||
@Query() query: RawExportQuery & { format?: string; fields?: string; limit?: string },
|
||||
@CurrentUser() user: TCurrentUser,
|
||||
@Res() res: Response,
|
||||
): Promise<void> {
|
||||
const dataset = this.resolve(key, user);
|
||||
const directions = await this.userTradeAccessService.resolveAllowedDirections(user);
|
||||
const format = resolveExportFormat(query.format);
|
||||
const fields = this.resolveFields(dataset, query.fields);
|
||||
|
||||
const rows = await this.runner.run(dataset, fields, query, directions, {
|
||||
cap: formatRowCap(format),
|
||||
limit: resolveRowLimit(format, query.limit),
|
||||
});
|
||||
const doc = {
|
||||
title: dataset.title,
|
||||
description: dataset.description,
|
||||
label: `export:${dataset.key}`,
|
||||
columns: fields.map(({ key: k, label, type }) => ({ key: k, label, type })),
|
||||
rows,
|
||||
};
|
||||
const buffer =
|
||||
format === 'pdf'
|
||||
? await this.writer.toPdf(doc)
|
||||
: format === 'csv'
|
||||
? await this.writer.toCsv(doc)
|
||||
: await this.writer.toXlsx(doc);
|
||||
|
||||
const mime = EXPORT_MIME[format];
|
||||
const stamp = new Date().toISOString().slice(0, 10);
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${dataset.key}-${stamp}.${mime.ext}"`);
|
||||
res.setHeader('Content-Type', mime.type);
|
||||
res.send(buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Requested fields, whitelisted against the dataset. No `fields=` means the
|
||||
* DEFAULT set, not everything — a booking export has ~70 fields and dumping
|
||||
* all of them on an unparameterised call is nobody's intent.
|
||||
*/
|
||||
private resolveFields(dataset: ExportDataset, raw: string | undefined): ExportField[] {
|
||||
if (raw?.trim()) {
|
||||
const picked = pickByKey(dataset.fields, raw);
|
||||
// pickByKey falls back to everything when nothing matched; for a dataset
|
||||
// the safer read of "all keys unknown" is still the default set.
|
||||
if (picked.length !== dataset.fields.length) return picked;
|
||||
}
|
||||
const defaults = dataset.fields.filter((f) => f.default);
|
||||
return defaults.length ? defaults : dataset.fields;
|
||||
}
|
||||
|
||||
private resolve(key: string, user: TCurrentUser): ExportDataset {
|
||||
const dataset = getDataset(key);
|
||||
if (!dataset) throw new NotFoundException(`Unknown export dataset: ${key}`);
|
||||
// Export rides the dataset's own list-page view permission: if you may see
|
||||
// these rows, you may export them.
|
||||
assertFreightPermission(user, dataset.permission);
|
||||
return dataset;
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,23 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { DocumentsModule } from '../billing/documents/documents.module';
|
||||
import { UserTradeAccessModule } from '../user-trade-access/user-trade-access.module';
|
||||
import { ExportRunnerService } from './export-runner.service';
|
||||
import { ExportsController } from './exports.controller';
|
||||
import { TabularExportService } from './tabular-export.service';
|
||||
|
||||
/**
|
||||
* Export infrastructure. Currently just the shared tabular writer (xlsx / csv /
|
||||
* pdf) that both the reports module and — once the dataset registry lands — the
|
||||
* generic table exports write through. No domain dependencies, so any module can
|
||||
* import it.
|
||||
* Generic table export: a dataset registry describing far more fields than each
|
||||
* list page shows (related-entity detail included), plus the shared tabular
|
||||
* writer (csv / xlsx / pdf) the reports module also writes through.
|
||||
*
|
||||
* `TabularExportService` is exported so ReportsModule can reuse it without
|
||||
* pulling in the dataset machinery.
|
||||
*/
|
||||
@Module({
|
||||
imports: [DocumentsModule],
|
||||
providers: [TabularExportService],
|
||||
imports: [DocumentsModule, UserTradeAccessModule],
|
||||
controllers: [ExportsController],
|
||||
providers: [TabularExportService, ExportRunnerService],
|
||||
exports: [TabularExportService],
|
||||
})
|
||||
export class ExportsModule {}
|
||||
|
||||
83
apps/edr-freight-api/src/scripts/validate-export-datasets.ts
Normal file
83
apps/edr-freight-api/src/scripts/validate-export-datasets.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* EXPLAIN-validates every export dataset against the real database.
|
||||
*
|
||||
* CLAUDE.md hard rule: raw SQL must be validated against a real DB before it
|
||||
* ships. Every dataset is hand-written SQL expressions over wide tables where
|
||||
* column drift is documented history, so a typo is a runtime 500 no type-check
|
||||
* can catch. This builds each dataset's WIDEST query (all fields selected, so
|
||||
* every join and every subquery is exercised) plus its count query, and runs
|
||||
* both through EXPLAIN.
|
||||
*
|
||||
* npx ts-node -r tsconfig-paths/register src/scripts/validate-export-datasets.ts
|
||||
*/
|
||||
import 'dotenv/config';
|
||||
|
||||
import AppDataSource from '../data-source';
|
||||
import { buildExportCountQuery, buildExportQuery } from '../modules/exports/export-query.builder';
|
||||
import { DATASETS } from '../modules/exports/export.registry';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
await AppDataSource.initialize();
|
||||
let failed = 0;
|
||||
|
||||
for (const dataset of DATASETS) {
|
||||
const ctx = { ds: AppDataSource, params: {}, directions: null };
|
||||
|
||||
const cases: [string, () => { sql: string; params: unknown[] }][] = [
|
||||
[
|
||||
`${dataset.key} (all ${dataset.fields.length} fields)`,
|
||||
() => {
|
||||
const qb = buildExportQuery(dataset, dataset.fields, ctx);
|
||||
return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] };
|
||||
},
|
||||
],
|
||||
[
|
||||
`${dataset.key} (count)`,
|
||||
() => {
|
||||
const qb = buildExportCountQuery(dataset, ctx);
|
||||
return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] };
|
||||
},
|
||||
],
|
||||
];
|
||||
|
||||
// Each field ALONE. The all-fields query above cannot catch a field that
|
||||
// references an alias it forgot to declare in `requires` — some other
|
||||
// field's `requires` pulls that join in, so it only 42P01s when that one
|
||||
// checkbox is ticked on its own. This is the check that finds it.
|
||||
for (const field of dataset.fields) {
|
||||
cases.push([
|
||||
`${dataset.key}.${field.key}`,
|
||||
() => {
|
||||
const qb = buildExportQuery(dataset, [field], ctx);
|
||||
return { sql: qb.getQuery(), params: qb.getParameters() as unknown as unknown[] };
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
let fieldFailures = 0;
|
||||
for (const [label, build] of cases) {
|
||||
const isPerField = label.startsWith(`${dataset.key}.`);
|
||||
try {
|
||||
const { sql } = build();
|
||||
// Parameters are all optional filters and unset here, so the generated
|
||||
// SQL carries no placeholders — EXPLAIN it directly.
|
||||
await AppDataSource.query(`EXPLAIN ${sql}`);
|
||||
if (!isPerField) console.log(` ok ${label}`);
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
if (isPerField) fieldFailures += 1;
|
||||
console.error(` FAIL ${label}`);
|
||||
console.error(` ${(error as Error).message.split('\n')[0]}`);
|
||||
}
|
||||
}
|
||||
if (!fieldFailures) {
|
||||
console.log(` ok ${dataset.key} (each of ${dataset.fields.length} fields alone)`);
|
||||
}
|
||||
}
|
||||
|
||||
await AppDataSource.destroy();
|
||||
console.log(failed ? `\n${failed} query/queries failed.` : '\nAll export dataset SQL validated.');
|
||||
process.exit(failed ? 1 : 0);
|
||||
}
|
||||
|
||||
void main();
|
||||
Reference in New Issue
Block a user