mirror of
https://github.com/Tria-plc/edr-platform.git
synced 2026-08-26 12:41:04 +00:00
feat(upload): increase document upload size limits to 50MB across the application
This commit is contained in:
30
apps/edr-freight-api/src/common/document-upload.options.ts
Normal file
30
apps/edr-freight-api/src/common/document-upload.options.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import { MulterOptions } from "@nestjs/platform-express/multer/interfaces/multer-options.interface";
|
||||
|
||||
/**
|
||||
* Ceiling for a single uploaded document, in bytes.
|
||||
*
|
||||
* Mirrors the 50MB `max_size_mb` the file-upload settings hand the portal, so
|
||||
* the client-side gate and the server-side cap agree. Raising this alone is not
|
||||
* enough to accept a 50MB upload: the reverse proxy in front of the API applies
|
||||
* its own `client_max_body_size`, and nginx's 1MB default rejects the request
|
||||
* with a 413 before it ever reaches Nest (see docs/uploads.md).
|
||||
*/
|
||||
export const DOCUMENT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
|
||||
|
||||
/** Upper bound on parts in one multipart document post. */
|
||||
export const DOCUMENT_UPLOAD_MAX_FILES = 20;
|
||||
|
||||
/**
|
||||
* Multer caps for the document upload routes.
|
||||
*
|
||||
* Without an explicit `fileSize`, multer's default is unlimited and every byte
|
||||
* is buffered in memory, so an oversized post is absorbed in full before
|
||||
* anything can reject it. With the limit set, multer stops reading the socket
|
||||
* at the ceiling instead.
|
||||
*/
|
||||
export const documentUploadMulterOptions: MulterOptions = {
|
||||
limits: {
|
||||
fileSize: DOCUMENT_UPLOAD_MAX_BYTES,
|
||||
files: DOCUMENT_UPLOAD_MAX_FILES,
|
||||
},
|
||||
};
|
||||
@@ -16,11 +16,17 @@ import { AppModule } from "./app.module";
|
||||
|
||||
/**
|
||||
* JSON body ceiling. Signing posts the signature AND the company stamp as
|
||||
* base64 in one JSON body, and base64 inflates bytes by ~4/3 — a 10MB stamp is
|
||||
* ~13.4MB on the wire. Express defaults to 100kb, which rejected any real stamp
|
||||
* base64 in one JSON body, and base64 inflates bytes by ~4/3 — a 50MB asset is
|
||||
* ~67MB on the wire. Express defaults to 100kb, which rejected any real stamp
|
||||
* image with a 413 "request entity too large".
|
||||
*
|
||||
* Sized to clear the 50MB per-document ceiling
|
||||
* (`DOCUMENT_UPLOAD_MAX_BYTES`) after base64 inflation, with room for the
|
||||
* surrounding JSON. Note that the reverse proxy applies its own
|
||||
* `client_max_body_size` and rejects oversized bodies before Nest sees them —
|
||||
* raising this alone does not lift the limit end to end (see docs/uploads.md).
|
||||
*/
|
||||
const JSON_BODY_LIMIT = "20mb";
|
||||
const JSON_BODY_LIMIT = "100mb";
|
||||
|
||||
/**
|
||||
* Static /etc/hosts-style overrides from `DNS_HOST_OVERRIDES`, formatted as
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { MigrationInterface, QueryRunner } from "typeorm";
|
||||
|
||||
/**
|
||||
* Raises the per-field document ceiling from 10MB to 50MB.
|
||||
*
|
||||
* `max_size_mb` is what the portal enforces client-side (SmartFileInput blocks
|
||||
* the file and shows "File size exceeds the limit of NMB"), so the seeded 10
|
||||
* was the visible limit for every existing form even after the server-side caps
|
||||
* were lifted. The seeder only writes these rows on first insert, so deployed
|
||||
* environments keep their old value until this runs.
|
||||
*
|
||||
* Only rows still sitting at the old default are touched — a field an admin has
|
||||
* deliberately tuned to something else keeps that value.
|
||||
*/
|
||||
export class RaiseDocumentUploadSizeLimit3380000000000
|
||||
implements MigrationInterface
|
||||
{
|
||||
name = "RaiseDocumentUploadSizeLimit3380000000000";
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE freight.file_upload_fields
|
||||
ALTER COLUMN max_size_mb SET DEFAULT 50
|
||||
`);
|
||||
await queryRunner.query(`
|
||||
UPDATE freight.file_upload_fields
|
||||
SET max_size_mb = 50
|
||||
WHERE max_size_mb = 10
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restores the column default only. The old per-row values are not
|
||||
* recoverable (10 and an admin-chosen 10 are indistinguishable after `up`),
|
||||
* and shrinking a customer's limit back down would reject documents they have
|
||||
* already uploaded, so the rows are deliberately left at 50.
|
||||
*/
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE freight.file_upload_fields
|
||||
ALTER COLUMN max_size_mb SET DEFAULT 10
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import { ApiOperation, ApiTags, ApiConsumes } from "@nestjs/swagger";
|
||||
import { CurrentUser } from "@edr/api-common";
|
||||
import type { TCurrentUser } from "@tria-plc/api-common/modules/auth/types/current-user.type";
|
||||
import { BookingStaff, MixedAudience, PortalCustomer } from "../../common/booking-guards";
|
||||
import { documentUploadMulterOptions } from "../../common/document-upload.options";
|
||||
import {
|
||||
assertFreightPermission,
|
||||
hasFreightPermission,
|
||||
@@ -726,7 +727,7 @@ export class CompaniesController {
|
||||
|
||||
@Post(":companyId/documents")
|
||||
@MixedAudience(FREIGHT_PERMS.customers.update)
|
||||
@UseInterceptors(AnyFilesInterceptor())
|
||||
@UseInterceptors(AnyFilesInterceptor(documentUploadMulterOptions))
|
||||
@ApiConsumes("multipart/form-data")
|
||||
@ApiOperation({ summary: "Upload documents for a company (onboarding)" })
|
||||
async uploadDocuments(
|
||||
|
||||
@@ -50,7 +50,7 @@ export class FileUploadField extends BaseEntity {
|
||||
})
|
||||
allowedExtensions!: string[];
|
||||
|
||||
@Column({ name: "max_size_mb", type: "integer", default: 10 })
|
||||
@Column({ name: "max_size_mb", type: "integer", default: 50 })
|
||||
maxSizeMb!: number;
|
||||
|
||||
@Column({ name: "display_order", type: "integer", default: 0 })
|
||||
|
||||
@@ -46,7 +46,7 @@ export function poaDelegationField(displayOrder: number): FileUploadField {
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: ["pdf", "jpg", "jpeg", "png"],
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder,
|
||||
} as FileUploadField;
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ const ETHIOPIAN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 1,
|
||||
},
|
||||
{
|
||||
@@ -49,7 +49,7 @@ const ETHIOPIAN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 2,
|
||||
},
|
||||
{
|
||||
@@ -60,7 +60,7 @@ const ETHIOPIAN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 3,
|
||||
},
|
||||
poaDelegationDefault(4),
|
||||
@@ -76,7 +76,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 1,
|
||||
},
|
||||
{
|
||||
@@ -87,7 +87,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 2,
|
||||
},
|
||||
{
|
||||
@@ -98,7 +98,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 3,
|
||||
},
|
||||
{
|
||||
@@ -109,7 +109,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 4,
|
||||
},
|
||||
poaDelegationDefault(5),
|
||||
@@ -127,7 +127,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
// isMultiple: false,
|
||||
// maxFiles: 1,
|
||||
// allowedExtensions: DOC_EXTENSIONS,
|
||||
// maxSizeMb: 10,
|
||||
// maxSizeMb: 50,
|
||||
// displayOrder: 1,
|
||||
// },
|
||||
// {
|
||||
@@ -138,7 +138,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
// isMultiple: false,
|
||||
// maxFiles: 1,
|
||||
// allowedExtensions: DOC_EXTENSIONS,
|
||||
// maxSizeMb: 10,
|
||||
// maxSizeMb: 50,
|
||||
// displayOrder: 2,
|
||||
// },
|
||||
// {
|
||||
@@ -149,7 +149,7 @@ const FOREIGN_ONBOARDING_FIELDS: OnboardingField[] = [
|
||||
// isMultiple: false,
|
||||
// maxFiles: 1,
|
||||
// allowedExtensions: DOC_EXTENSIONS,
|
||||
// maxSizeMb: 10,
|
||||
// maxSizeMb: 50,
|
||||
// displayOrder: 3,
|
||||
// },
|
||||
// ];
|
||||
@@ -232,7 +232,7 @@ function clearanceField(
|
||||
isMultiple: false,
|
||||
maxFiles: 1,
|
||||
allowedExtensions: opts?.extensions ?? DOC_EXTENSIONS,
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder,
|
||||
};
|
||||
}
|
||||
@@ -556,7 +556,7 @@ const DRIVER_DOCUMENT_FIELDS: OnboardingField[] = [
|
||||
isMultiple: true,
|
||||
maxFiles: 20,
|
||||
allowedExtensions: ["pdf", "jpg", "jpeg", "png", "doc", "docx"],
|
||||
maxSizeMb: 10,
|
||||
maxSizeMb: 50,
|
||||
displayOrder: 1,
|
||||
},
|
||||
];
|
||||
|
||||
86
docs/uploads.md
Normal file
86
docs/uploads.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# Upload size limits
|
||||
|
||||
A document upload passes through four independent ceilings. The **smallest one
|
||||
wins**, so raising a limit in application code changes nothing while a lower
|
||||
limit sits in front of it.
|
||||
|
||||
| # | Layer | Limit | Where it lives |
|
||||
| - | ----- | ----- | -------------- |
|
||||
| 1 | Reverse proxy (`client_max_body_size`) | **50m** | nginx config on the API host — **not in this repo** for deployed environments |
|
||||
| 2 | Express JSON/urlencoded body | `JSON_BODY_LIMIT` = 80mb | `apps/edr-freight-api/src/main.ts` |
|
||||
| 3 | Multer multipart (`fileSize`) | `DOCUMENT_UPLOAD_MAX_BYTES` = 50MB | `apps/edr-freight-api/src/common/document-upload.options.ts` |
|
||||
| 4 | Per-field portal gate (`max_size_mb`) | 50 | `freight.file_upload_fields`, seeded by `file-upload-settings.seeder.ts` |
|
||||
|
||||
Layer 4 is the only one the customer sees before uploading — `SmartFileInput`
|
||||
blocks the file client-side with "File size exceeds the limit of NMB". Layers
|
||||
1–3 produce a failed request after the fact.
|
||||
|
||||
## The nginx layer is the one that bites
|
||||
|
||||
nginx defaults `client_max_body_size` to **1m** and answers anything larger with
|
||||
its own HTML error page:
|
||||
|
||||
```
|
||||
HTTP/1.1 413 Request Entity Too Large
|
||||
Server: nginx
|
||||
Content-Type: text/html
|
||||
```
|
||||
|
||||
Two tells that a 413 came from the proxy rather than the API:
|
||||
|
||||
- the body is nginx's HTML page, not the API's JSON envelope, and
|
||||
- `curl -w '%{size_upload}'` reports **0** — nginx rejects on the `Content-Length`
|
||||
header, so the body is never transmitted.
|
||||
|
||||
This is also why the failure looks like a CORS error in the browser: nginx's
|
||||
error response carries no `Access-Control-Allow-Origin` header.
|
||||
|
||||
### Applying it
|
||||
|
||||
Add to the `server` (or `location`) block fronting the API and reload:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
server_name edrfreightapi-staging.edrsc.com;
|
||||
|
||||
client_max_body_size 50m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://freight-api:3001;
|
||||
# Large uploads stream for a while; the default 60s read timeout can
|
||||
# cut off a slow client mid-body.
|
||||
proxy_read_timeout 300s;
|
||||
proxy_request_buffering off;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
nginx -t && nginx -s reload # -t first: a bad config that reloads takes the site down
|
||||
```
|
||||
|
||||
On Kubernetes ingress-nginx this is an annotation on the Ingress instead:
|
||||
|
||||
```yaml
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: 50m
|
||||
```
|
||||
|
||||
Note that `client_max_body_size 0` disables the check entirely — do not use it.
|
||||
An unbounded body is a denial-of-service vector, and layer 3 buffers uploads in
|
||||
memory.
|
||||
|
||||
## Verifying end to end
|
||||
|
||||
512KB should pass the proxy and reach the API; 50MB should too. A `401` here is
|
||||
a *success* for this purpose — it means the request got past nginx to the API's
|
||||
auth layer.
|
||||
|
||||
```bash
|
||||
head -c 52428800 /dev/urandom > big.bin
|
||||
curl -s -o /dev/null -w 'HTTP %{http_code} uploaded %{size_upload}\n' \
|
||||
-X POST "https://edrfreightapi-staging.edrsc.com/api/companies/<id>/documents" \
|
||||
-F "test=@big.bin"
|
||||
```
|
||||
|
||||
- `413` with `uploaded 0` → the proxy is still capped; layer 1 was not applied.
|
||||
- `401`/`200` with the full byte count → the body made it through.
|
||||
@@ -11,6 +11,11 @@ server {
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||
|
||||
# nginx defaults to 1m and answers anything larger with its own HTML 413
|
||||
# before the request reaches an upstream. Kept in step with the 50MB
|
||||
# per-document ceiling the API enforces (DOCUMENT_UPLOAD_MAX_BYTES).
|
||||
client_max_body_size 50m;
|
||||
|
||||
gzip on;
|
||||
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/wasm;
|
||||
gzip_min_length 1024;
|
||||
|
||||
Reference in New Issue
Block a user