From ea6eccbf098003d45907ea77f1c6c043d51407e8 Mon Sep 17 00:00:00 2001 From: marshalyordanos Date: Tue, 11 Aug 2026 09:26:23 +0300 Subject: [PATCH] feat(upload): increase document upload size limits to 50MB across the application --- .../src/common/document-upload.options.ts | 30 +++++++ apps/edr-freight-api/src/main.ts | 12 ++- ...0000000000-RaiseDocumentUploadSizeLimit.ts | 44 ++++++++++ .../modules/companies/companies.controller.ts | 3 +- .../entities/file-upload-field.entity.ts | 2 +- .../poa-delegation.constants.ts | 2 +- .../src/seed/file-upload-settings.seeder.ts | 24 +++--- docs/uploads.md | 86 +++++++++++++++++++ infrastructure/nginx/spa.conf | 5 ++ 9 files changed, 190 insertions(+), 18 deletions(-) create mode 100644 apps/edr-freight-api/src/common/document-upload.options.ts create mode 100644 apps/edr-freight-api/src/migrations/3380000000000-RaiseDocumentUploadSizeLimit.ts create mode 100644 docs/uploads.md diff --git a/apps/edr-freight-api/src/common/document-upload.options.ts b/apps/edr-freight-api/src/common/document-upload.options.ts new file mode 100644 index 000000000..736d8099b --- /dev/null +++ b/apps/edr-freight-api/src/common/document-upload.options.ts @@ -0,0 +1,30 @@ +import { MulterOptions } from "@nestjs/platform-express/multer/interfaces/multer-options.interface"; + +/** + * Ceiling for a single uploaded document, in bytes. + * + * Mirrors the 50MB `max_size_mb` the file-upload settings hand the portal, so + * the client-side gate and the server-side cap agree. Raising this alone is not + * enough to accept a 50MB upload: the reverse proxy in front of the API applies + * its own `client_max_body_size`, and nginx's 1MB default rejects the request + * with a 413 before it ever reaches Nest (see docs/uploads.md). + */ +export const DOCUMENT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024; + +/** Upper bound on parts in one multipart document post. */ +export const DOCUMENT_UPLOAD_MAX_FILES = 20; + +/** + * Multer caps for the document upload routes. + * + * Without an explicit `fileSize`, multer's default is unlimited and every byte + * is buffered in memory, so an oversized post is absorbed in full before + * anything can reject it. With the limit set, multer stops reading the socket + * at the ceiling instead. + */ +export const documentUploadMulterOptions: MulterOptions = { + limits: { + fileSize: DOCUMENT_UPLOAD_MAX_BYTES, + files: DOCUMENT_UPLOAD_MAX_FILES, + }, +}; diff --git a/apps/edr-freight-api/src/main.ts b/apps/edr-freight-api/src/main.ts index c9a718f5d..f26cd19e7 100644 --- a/apps/edr-freight-api/src/main.ts +++ b/apps/edr-freight-api/src/main.ts @@ -16,11 +16,17 @@ import { AppModule } from "./app.module"; /** * JSON body ceiling. Signing posts the signature AND the company stamp as - * base64 in one JSON body, and base64 inflates bytes by ~4/3 — a 10MB stamp is - * ~13.4MB on the wire. Express defaults to 100kb, which rejected any real stamp + * base64 in one JSON body, and base64 inflates bytes by ~4/3 — a 50MB asset is + * ~67MB on the wire. Express defaults to 100kb, which rejected any real stamp * image with a 413 "request entity too large". + * + * Sized to clear the 50MB per-document ceiling + * (`DOCUMENT_UPLOAD_MAX_BYTES`) after base64 inflation, with room for the + * surrounding JSON. Note that the reverse proxy applies its own + * `client_max_body_size` and rejects oversized bodies before Nest sees them — + * raising this alone does not lift the limit end to end (see docs/uploads.md). */ -const JSON_BODY_LIMIT = "20mb"; +const JSON_BODY_LIMIT = "100mb"; /** * Static /etc/hosts-style overrides from `DNS_HOST_OVERRIDES`, formatted as diff --git a/apps/edr-freight-api/src/migrations/3380000000000-RaiseDocumentUploadSizeLimit.ts b/apps/edr-freight-api/src/migrations/3380000000000-RaiseDocumentUploadSizeLimit.ts new file mode 100644 index 000000000..99d949e2e --- /dev/null +++ b/apps/edr-freight-api/src/migrations/3380000000000-RaiseDocumentUploadSizeLimit.ts @@ -0,0 +1,44 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +/** + * Raises the per-field document ceiling from 10MB to 50MB. + * + * `max_size_mb` is what the portal enforces client-side (SmartFileInput blocks + * the file and shows "File size exceeds the limit of NMB"), so the seeded 10 + * was the visible limit for every existing form even after the server-side caps + * were lifted. The seeder only writes these rows on first insert, so deployed + * environments keep their old value until this runs. + * + * Only rows still sitting at the old default are touched — a field an admin has + * deliberately tuned to something else keeps that value. + */ +export class RaiseDocumentUploadSizeLimit3380000000000 + implements MigrationInterface +{ + name = "RaiseDocumentUploadSizeLimit3380000000000"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE freight.file_upload_fields + ALTER COLUMN max_size_mb SET DEFAULT 50 + `); + await queryRunner.query(` + UPDATE freight.file_upload_fields + SET max_size_mb = 50 + WHERE max_size_mb = 10 + `); + } + + /** + * Restores the column default only. The old per-row values are not + * recoverable (10 and an admin-chosen 10 are indistinguishable after `up`), + * and shrinking a customer's limit back down would reject documents they have + * already uploaded, so the rows are deliberately left at 50. + */ + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE freight.file_upload_fields + ALTER COLUMN max_size_mb SET DEFAULT 10 + `); + } +} diff --git a/apps/edr-freight-api/src/modules/companies/companies.controller.ts b/apps/edr-freight-api/src/modules/companies/companies.controller.ts index de7a01484..1b14845b2 100644 --- a/apps/edr-freight-api/src/modules/companies/companies.controller.ts +++ b/apps/edr-freight-api/src/modules/companies/companies.controller.ts @@ -20,6 +20,7 @@ import { ApiOperation, ApiTags, ApiConsumes } from "@nestjs/swagger"; import { CurrentUser } from "@edr/api-common"; import type { TCurrentUser } from "@tria-plc/api-common/modules/auth/types/current-user.type"; import { BookingStaff, MixedAudience, PortalCustomer } from "../../common/booking-guards"; +import { documentUploadMulterOptions } from "../../common/document-upload.options"; import { assertFreightPermission, hasFreightPermission, @@ -726,7 +727,7 @@ export class CompaniesController { @Post(":companyId/documents") @MixedAudience(FREIGHT_PERMS.customers.update) - @UseInterceptors(AnyFilesInterceptor()) + @UseInterceptors(AnyFilesInterceptor(documentUploadMulterOptions)) @ApiConsumes("multipart/form-data") @ApiOperation({ summary: "Upload documents for a company (onboarding)" }) async uploadDocuments( diff --git a/apps/edr-freight-api/src/modules/file-upload-settings/entities/file-upload-field.entity.ts b/apps/edr-freight-api/src/modules/file-upload-settings/entities/file-upload-field.entity.ts index e76c5716b..097c1ec39 100644 --- a/apps/edr-freight-api/src/modules/file-upload-settings/entities/file-upload-field.entity.ts +++ b/apps/edr-freight-api/src/modules/file-upload-settings/entities/file-upload-field.entity.ts @@ -50,7 +50,7 @@ export class FileUploadField extends BaseEntity { }) allowedExtensions!: string[]; - @Column({ name: "max_size_mb", type: "integer", default: 10 }) + @Column({ name: "max_size_mb", type: "integer", default: 50 }) maxSizeMb!: number; @Column({ name: "display_order", type: "integer", default: 0 }) diff --git a/apps/edr-freight-api/src/modules/file-upload-settings/poa-delegation.constants.ts b/apps/edr-freight-api/src/modules/file-upload-settings/poa-delegation.constants.ts index 9085cc018..7f8a34175 100644 --- a/apps/edr-freight-api/src/modules/file-upload-settings/poa-delegation.constants.ts +++ b/apps/edr-freight-api/src/modules/file-upload-settings/poa-delegation.constants.ts @@ -46,7 +46,7 @@ export function poaDelegationField(displayOrder: number): FileUploadField { isMultiple: false, maxFiles: 1, allowedExtensions: ["pdf", "jpg", "jpeg", "png"], - maxSizeMb: 10, + maxSizeMb: 50, displayOrder, } as FileUploadField; } diff --git a/apps/edr-freight-api/src/seed/file-upload-settings.seeder.ts b/apps/edr-freight-api/src/seed/file-upload-settings.seeder.ts index 7bcae753b..512229706 100644 --- a/apps/edr-freight-api/src/seed/file-upload-settings.seeder.ts +++ b/apps/edr-freight-api/src/seed/file-upload-settings.seeder.ts @@ -37,7 +37,7 @@ const ETHIOPIAN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 1, }, { @@ -49,7 +49,7 @@ const ETHIOPIAN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 2, }, { @@ -60,7 +60,7 @@ const ETHIOPIAN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 3, }, poaDelegationDefault(4), @@ -76,7 +76,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 1, }, { @@ -87,7 +87,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 2, }, { @@ -98,7 +98,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 3, }, { @@ -109,7 +109,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ isMultiple: false, maxFiles: 1, allowedExtensions: DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 4, }, poaDelegationDefault(5), @@ -127,7 +127,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ // isMultiple: false, // maxFiles: 1, // allowedExtensions: DOC_EXTENSIONS, -// maxSizeMb: 10, +// maxSizeMb: 50, // displayOrder: 1, // }, // { @@ -138,7 +138,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ // isMultiple: false, // maxFiles: 1, // allowedExtensions: DOC_EXTENSIONS, -// maxSizeMb: 10, +// maxSizeMb: 50, // displayOrder: 2, // }, // { @@ -149,7 +149,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [ // isMultiple: false, // maxFiles: 1, // allowedExtensions: DOC_EXTENSIONS, -// maxSizeMb: 10, +// maxSizeMb: 50, // displayOrder: 3, // }, // ]; @@ -232,7 +232,7 @@ function clearanceField( isMultiple: false, maxFiles: 1, allowedExtensions: opts?.extensions ?? DOC_EXTENSIONS, - maxSizeMb: 10, + maxSizeMb: 50, displayOrder, }; } @@ -556,7 +556,7 @@ const DRIVER_DOCUMENT_FIELDS: OnboardingField[] = [ isMultiple: true, maxFiles: 20, allowedExtensions: ["pdf", "jpg", "jpeg", "png", "doc", "docx"], - maxSizeMb: 10, + maxSizeMb: 50, displayOrder: 1, }, ]; diff --git a/docs/uploads.md b/docs/uploads.md new file mode 100644 index 000000000..a41763d89 --- /dev/null +++ b/docs/uploads.md @@ -0,0 +1,86 @@ +# Upload size limits + +A document upload passes through four independent ceilings. The **smallest one +wins**, so raising a limit in application code changes nothing while a lower +limit sits in front of it. + +| # | Layer | Limit | Where it lives | +| - | ----- | ----- | -------------- | +| 1 | Reverse proxy (`client_max_body_size`) | **50m** | nginx config on the API host — **not in this repo** for deployed environments | +| 2 | Express JSON/urlencoded body | `JSON_BODY_LIMIT` = 80mb | `apps/edr-freight-api/src/main.ts` | +| 3 | Multer multipart (`fileSize`) | `DOCUMENT_UPLOAD_MAX_BYTES` = 50MB | `apps/edr-freight-api/src/common/document-upload.options.ts` | +| 4 | Per-field portal gate (`max_size_mb`) | 50 | `freight.file_upload_fields`, seeded by `file-upload-settings.seeder.ts` | + +Layer 4 is the only one the customer sees before uploading — `SmartFileInput` +blocks the file client-side with "File size exceeds the limit of NMB". Layers +1–3 produce a failed request after the fact. + +## The nginx layer is the one that bites + +nginx defaults `client_max_body_size` to **1m** and answers anything larger with +its own HTML error page: + +``` +HTTP/1.1 413 Request Entity Too Large +Server: nginx +Content-Type: text/html +``` + +Two tells that a 413 came from the proxy rather than the API: + +- the body is nginx's HTML page, not the API's JSON envelope, and +- `curl -w '%{size_upload}'` reports **0** — nginx rejects on the `Content-Length` + header, so the body is never transmitted. + +This is also why the failure looks like a CORS error in the browser: nginx's +error response carries no `Access-Control-Allow-Origin` header. + +### Applying it + +Add to the `server` (or `location`) block fronting the API and reload: + +```nginx +server { + server_name edrfreightapi-staging.edrsc.com; + + client_max_body_size 50m; + + location / { + proxy_pass http://freight-api:3001; + # Large uploads stream for a while; the default 60s read timeout can + # cut off a slow client mid-body. + proxy_read_timeout 300s; + proxy_request_buffering off; + } +} +``` + +```bash +nginx -t && nginx -s reload # -t first: a bad config that reloads takes the site down +``` + +On Kubernetes ingress-nginx this is an annotation on the Ingress instead: + +```yaml +nginx.ingress.kubernetes.io/proxy-body-size: 50m +``` + +Note that `client_max_body_size 0` disables the check entirely — do not use it. +An unbounded body is a denial-of-service vector, and layer 3 buffers uploads in +memory. + +## Verifying end to end + +512KB should pass the proxy and reach the API; 50MB should too. A `401` here is +a *success* for this purpose — it means the request got past nginx to the API's +auth layer. + +```bash +head -c 52428800 /dev/urandom > big.bin +curl -s -o /dev/null -w 'HTTP %{http_code} uploaded %{size_upload}\n' \ + -X POST "https://edrfreightapi-staging.edrsc.com/api/companies//documents" \ + -F "test=@big.bin" +``` + +- `413` with `uploaded 0` → the proxy is still capped; layer 1 was not applied. +- `401`/`200` with the full byte count → the body made it through. diff --git a/infrastructure/nginx/spa.conf b/infrastructure/nginx/spa.conf index 83629210c..0cc9b1de0 100644 --- a/infrastructure/nginx/spa.conf +++ b/infrastructure/nginx/spa.conf @@ -11,6 +11,11 @@ server { add_header Referrer-Policy "strict-origin-when-cross-origin" always; add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; + # nginx defaults to 1m and answers anything larger with its own HTML 413 + # before the request reaches an upstream. Kept in step with the 50MB + # per-document ceiling the API enforces (DOCUMENT_UPLOAD_MAX_BYTES). + client_max_body_size 50m; + gzip on; gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/wasm; gzip_min_length 1024;