mirror of
https://github.com/Tria-plc/edr-platform.git
synced 2026-09-06 11:15:03 +00:00
feat(eims): resolve buyer geography from the MoR location master
BuyerDetails Country/Region/City/Wereda now resolve from the Ministry's own EIMS_COUNTRY_REGION_VW master instead of the EIMS_BUYER_*_CODES env maps and the ethiopia-geo-codes table. Both invented their codes and looked names up globally, so KERSA/GORO/BABILE/BURE — each present in several zones with different LOCALITY_NOs — could be filed against the wrong jurisdiction. Resolution is hierarchical and refuses to guess: an unknown or ambiguous address raises a local validation error naming the level that failed, and never selects the first matching row. Spelling differences between EDR and MoR live in a reviewed, parent-scoped alias layer; the dataset itself stays verbatim so it remains traceable to the Ministry sheet. Resolution now runs before the counter reservation in both the single and bulk paths, so a bad company address no longer burns an EIMS sequence number. Adds eims:import-locations to regenerate the dataset from a future workbook, reporting duplicate rows and same-hierarchy code conflicts.
This commit is contained in:
278
apps/edr-freight-api/src/scripts/import-mor-locations.ts
Normal file
278
apps/edr-freight-api/src/scripts/import-mor-locations.ts
Normal file
@@ -0,0 +1,278 @@
|
||||
/**
|
||||
* Converts a Ministry of Revenues EIMS location workbook into `src/config/mor-locations.data.ts`.
|
||||
*
|
||||
* pnpm --filter @edr/freight-api eims:import-locations <workbook.xlsx> [--sheet SHEET_NAME]
|
||||
*
|
||||
* Exists so a future MoR workbook replaces the dataset by rerunning one command and reviewing the
|
||||
* diff, instead of anyone hand-editing a thousand rows of tax reference data. Production never
|
||||
* parses the workbook: the committed TypeScript is what ships.
|
||||
*
|
||||
* The generated file reproduces the Ministry's own labels and IDs verbatim. This script validates
|
||||
* and reports; it does not correct. Spelling compatibility with EDR/e-Trade names lives in
|
||||
* `mor-location.resolver.ts`, so the dataset stays traceable back to the source sheet.
|
||||
*/
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { Workbook } from "exceljs";
|
||||
|
||||
const DEFAULT_SHEET = "EIMS_COUNTRY_REGION_VW";
|
||||
const OUTPUT = resolve(__dirname, "../config/mor-locations.data.ts");
|
||||
|
||||
const COLUMNS = [
|
||||
"COUNTRY_NO",
|
||||
"COUNTRY_NAME",
|
||||
"PARISH_NO",
|
||||
"PARISH_NAME",
|
||||
"CITY_NO",
|
||||
"CITY_NAME",
|
||||
"LOCALITY_NO",
|
||||
"LOCALITY_DESC",
|
||||
] as const;
|
||||
|
||||
type Column = (typeof COLUMNS)[number];
|
||||
type Row = [number, string, number, string, number, string, number, string];
|
||||
|
||||
/** Header cells arrive with stray casing, spaces and non-breaking spaces; compare on this form. */
|
||||
const headerKey = (value: unknown): string =>
|
||||
String(value ?? "")
|
||||
.toUpperCase()
|
||||
.replace(/[^A-Z0-9]+/g, "_")
|
||||
.replace(/^_+|_+$/g, "");
|
||||
|
||||
/**
|
||||
* Cell text with only the transport-layer damage removed (Excel's non-breaking spaces, and the
|
||||
* CR/LF a wrapped cell leaves behind). Deliberately keeps the Ministry's own leading/trailing
|
||||
* spaces and spelling — the resolver normalizes at comparison time, the dataset stays as supplied.
|
||||
*/
|
||||
const cellText = (value: unknown): string => {
|
||||
if (value === null || value === undefined) return "";
|
||||
const raw =
|
||||
typeof value === "object" && "text" in (value as object)
|
||||
? String((value as { text: unknown }).text ?? "")
|
||||
: String(value);
|
||||
return raw.replace(/\u00a0/g, " ").replace(/\r?\n/g, " ");
|
||||
};
|
||||
|
||||
const cellNumber = (value: unknown): number | null => {
|
||||
const text = cellText(value).trim();
|
||||
if (!/^[0-9]+$/.test(text)) return null;
|
||||
return Number(text);
|
||||
};
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
const sheetFlag = args.indexOf("--sheet");
|
||||
const sheetName = sheetFlag >= 0 ? args[sheetFlag + 1] : DEFAULT_SHEET;
|
||||
const workbookPath = args.find((arg, i) => !arg.startsWith("--") && i !== sheetFlag + 1);
|
||||
|
||||
if (!workbookPath) {
|
||||
throw new Error(
|
||||
"Usage: eims:import-locations <workbook.xlsx> [--sheet SHEET_NAME]\n" +
|
||||
`Defaults to sheet "${DEFAULT_SHEET}".`,
|
||||
);
|
||||
}
|
||||
|
||||
const workbook = new Workbook();
|
||||
await workbook.xlsx.readFile(resolve(process.cwd(), workbookPath));
|
||||
|
||||
const sheet = workbook.getWorksheet(sheetName);
|
||||
if (!sheet) {
|
||||
const available = workbook.worksheets.map((w) => w.name).join(", ");
|
||||
throw new Error(`Sheet "${sheetName}" not found. Sheets in this workbook: ${available}`);
|
||||
}
|
||||
|
||||
// The header is not guaranteed to be row 1 — find the first row carrying every required column.
|
||||
let headerRow = 0;
|
||||
let columnIndex: Partial<Record<Column, number>> = {};
|
||||
for (let r = 1; r <= Math.min(sheet.rowCount, 20); r++) {
|
||||
const found: Partial<Record<Column, number>> = {};
|
||||
sheet.getRow(r).eachCell({ includeEmpty: false }, (cell, colNumber) => {
|
||||
const key = headerKey(cell.value) as Column;
|
||||
if (COLUMNS.includes(key) && found[key] === undefined) found[key] = colNumber;
|
||||
});
|
||||
if (COLUMNS.every((c) => found[c] !== undefined)) {
|
||||
headerRow = r;
|
||||
columnIndex = found;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (headerRow === 0) {
|
||||
throw new Error(
|
||||
`Sheet "${sheetName}" has no header row containing all required columns: ${COLUMNS.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const rows: Row[] = [];
|
||||
const problems: string[] = [];
|
||||
|
||||
for (let r = headerRow + 1; r <= sheet.rowCount; r++) {
|
||||
const sheetRow = sheet.getRow(r);
|
||||
const at = (column: Column): unknown => sheetRow.getCell(columnIndex[column]!).value;
|
||||
|
||||
// A sheet exported from a view is usually padded with blank rows at the end; skip silently.
|
||||
if (COLUMNS.every((c) => cellText(at(c)).trim() === "")) continue;
|
||||
|
||||
const countryNo = cellNumber(at("COUNTRY_NO"));
|
||||
const parishNo = cellNumber(at("PARISH_NO"));
|
||||
const cityNo = cellNumber(at("CITY_NO"));
|
||||
const localityNo = cellNumber(at("LOCALITY_NO"));
|
||||
const countryName = cellText(at("COUNTRY_NAME"));
|
||||
const parishName = cellText(at("PARISH_NAME"));
|
||||
const cityName = cellText(at("CITY_NAME"));
|
||||
const localityDesc = cellText(at("LOCALITY_DESC"));
|
||||
|
||||
const missingIds = (
|
||||
[
|
||||
["COUNTRY_NO", countryNo],
|
||||
["PARISH_NO", parishNo],
|
||||
["CITY_NO", cityNo],
|
||||
["LOCALITY_NO", localityNo],
|
||||
] as const
|
||||
)
|
||||
.filter(([, value]) => value === null)
|
||||
.map(([name]) => name);
|
||||
const blankNames = (
|
||||
[
|
||||
["COUNTRY_NAME", countryName],
|
||||
["PARISH_NAME", parishName],
|
||||
["CITY_NAME", cityName],
|
||||
["LOCALITY_DESC", localityDesc],
|
||||
] as const
|
||||
)
|
||||
.filter(([, value]) => value.trim() === "")
|
||||
.map(([name]) => name);
|
||||
|
||||
if (missingIds.length > 0 || blankNames.length > 0) {
|
||||
problems.push(
|
||||
`row ${r}: ${[
|
||||
missingIds.length ? `non-numeric/missing ${missingIds.join(", ")}` : "",
|
||||
blankNames.length ? `blank ${blankNames.join(", ")}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("; ")}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.push([
|
||||
countryNo!,
|
||||
countryName,
|
||||
parishNo!,
|
||||
parishName,
|
||||
cityNo!,
|
||||
cityName,
|
||||
localityNo!,
|
||||
localityDesc,
|
||||
]);
|
||||
}
|
||||
|
||||
if (problems.length > 0) {
|
||||
throw new Error(
|
||||
`Sheet "${sheetName}" has ${problems.length} unusable row(s); nothing was written:\n ` +
|
||||
problems.slice(0, 25).join("\n ") +
|
||||
(problems.length > 25 ? `\n ... and ${problems.length - 25} more` : ""),
|
||||
);
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
throw new Error(`Sheet "${sheetName}" has a valid header but no data rows.`);
|
||||
}
|
||||
|
||||
// Exact duplicates carry no information and only inflate the file — collapsed here, and the
|
||||
// count reported, so the collapse is a stated decision rather than a silent one. IDs are never
|
||||
// touched: only whole identical rows are dropped.
|
||||
const seen = new Map<string, Row>();
|
||||
for (const row of rows) {
|
||||
const key = JSON.stringify(row);
|
||||
if (!seen.has(key)) seen.set(key, row);
|
||||
}
|
||||
const unique = [...seen.values()];
|
||||
const duplicates = rows.length - unique.length;
|
||||
|
||||
// Deterministic output: same workbook in, byte-identical file out, so a regeneration diff shows
|
||||
// only what the Ministry actually changed.
|
||||
unique.sort(
|
||||
(a, b) =>
|
||||
a[0] - b[0] ||
|
||||
a[2] - b[2] ||
|
||||
a[4] - b[4] ||
|
||||
a[6] - b[6] ||
|
||||
a[7].localeCompare(b[7]) ||
|
||||
a[5].localeCompare(b[5]),
|
||||
);
|
||||
|
||||
// A name that resolves to two different codes under the same parent cannot be resolved by any
|
||||
// amount of normalization — the resolver refuses it as ambiguous at filing time rather than
|
||||
// picking one. Reported here so it can be raised with MoR instead of surfacing on a live invoice.
|
||||
const byPath = new Map<string, Set<number>>();
|
||||
const collide = (level: string, path: string, name: string, no: number): void => {
|
||||
const key = [level, path, name.trim().toUpperCase().replace(/\s+/g, " ")].join(" | ");
|
||||
if (!byPath.has(key)) byPath.set(key, new Set());
|
||||
byPath.get(key)!.add(no);
|
||||
};
|
||||
for (const [cNo, cName, pNo, pName, tNo, tName, lNo, lName] of unique) {
|
||||
collide("COUNTRY_NAME", "", cName, cNo);
|
||||
collide("PARISH_NAME", String(cNo), pName, pNo);
|
||||
collide("CITY_NAME", `${cNo}/${pNo}`, tName, tNo);
|
||||
collide("LOCALITY_DESC", `${cNo}/${pNo}/${tNo}`, lName, lNo);
|
||||
}
|
||||
const conflicts = [...byPath.entries()]
|
||||
.filter(([, codes]) => codes.size > 1)
|
||||
.map(([key, codes]) => {
|
||||
const [level, path, name] = key.split(" | ");
|
||||
const where = path ? ` under ${path}` : "";
|
||||
return ` ${level} "${name}"${where} -> codes ${[...codes].sort((a, b) => a - b).join(", ")}`;
|
||||
})
|
||||
.sort();
|
||||
|
||||
const header = `/**
|
||||
* GENERATED FILE — do not hand-edit.
|
||||
*
|
||||
* MoR EIMS location master (\`${sheetName}\`), the Ministry's own geographic reference data.
|
||||
* Regenerate from a supplied workbook with:
|
||||
*
|
||||
* pnpm --filter @edr/freight-api eims:import-locations <path-to.xlsx>
|
||||
*
|
||||
* Values are reproduced verbatim from the Ministry sheet — original spelling, original casing,
|
||||
* original numbering. Nothing here is cleaned up or renumbered: this file is the traceable copy of
|
||||
* the source. Spelling compatibility between EDR/e-Trade names and MoR names belongs in
|
||||
* \`mor-location.resolver.ts\`'s normalization and alias layer, never here.
|
||||
*/
|
||||
|
||||
/** \`[COUNTRY_NO, COUNTRY_NAME, PARISH_NO, PARISH_NAME, CITY_NO, CITY_NAME, LOCALITY_NO, LOCALITY_DESC]\` */
|
||||
export type MorLocationTuple = [number, string, number, string, number, string, number, string];
|
||||
|
||||
export const MOR_LOCATIONS: MorLocationTuple[] = [
|
||||
`;
|
||||
const body = unique
|
||||
.map(
|
||||
([cNo, cName, pNo, pName, tNo, tName, lNo, lName]) =>
|
||||
` [${cNo}, ${JSON.stringify(cName)}, ${pNo}, ${JSON.stringify(pName)}, ${tNo}, ` +
|
||||
`${JSON.stringify(tName)}, ${lNo}, ${JSON.stringify(lName)}],`,
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
writeFileSync(OUTPUT, `${header}${body}\n];\n`, "utf8");
|
||||
|
||||
const countries = new Set(unique.map((r) => r[0])).size;
|
||||
const regions = new Set(unique.map((r) => `${r[0]}/${r[2]}`)).size;
|
||||
const zones = new Set(unique.map((r) => `${r[0]}/${r[2]}/${r[4]}`)).size;
|
||||
|
||||
console.log(`Wrote ${OUTPUT}`);
|
||||
console.log(
|
||||
` ${unique.length} rows - ${countries} countries, ${regions} regions, ${zones} zones` +
|
||||
(duplicates > 0 ? `; collapsed ${duplicates} exact duplicate row(s)` : ""),
|
||||
);
|
||||
if (conflicts.length > 0) {
|
||||
console.log(
|
||||
` ${conflicts.length} same-hierarchy name conflict(s) - these resolve to an ambiguity ` +
|
||||
"error at filing time, never to a guess:",
|
||||
);
|
||||
console.log(conflicts.join("\n"));
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err: Error) => {
|
||||
console.error(err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user