diff --git a/.claude/skills/edr-db/SKILL.md b/.claude/skills/edr-db/SKILL.md index 47333128d..dde5f4b8a 100644 --- a/.claude/skills/edr-db/SKILL.md +++ b/.claude/skills/edr-db/SKILL.md @@ -11,7 +11,7 @@ One script, runs from anywhere in the repo (resolves `pg` from `apps/edr-freight node .claude/skills/edr-db/query.cjs "SELECT ... " # run SQL, console.table output node .claude/skills/edr-db/query.cjs explain "SELECT ..." # EXPLAIN-validate only (no rows touched) node .claude/skills/edr-db/query.cjs columns # freight.
column list -node .claude/skills/edr-db/query.cjs migrations [like] # public.migrations rows (newest first) +node .claude/skills/edr-db/query.cjs migrations [like] # migration rows, newest first (freight.migrations + iam.typeorm_migrations) node .claude/skills/edr-db/query.cjs drift
# bare column names, for diffing vs the entity ``` @@ -31,7 +31,11 @@ defaulting to the shared dev database (`edr_dev`). (`WHERE NOT EXISTS` guards) — watch-mode API instances race `migrationsRun`, and non-idempotent statements have double-run here before. - Timestamps for new migrations: must be unique across `src/migrations/` AND - higher than `SELECT max(timestamp) FROM public.migrations`. + higher than `SELECT max(timestamp) FROM freight.migrations`. +- Freight and IAM keep separate histories: `freight.migrations` for + `apps/edr-freight-api/src/migrations/*`, `iam.typeorm_migrations` for the + `@tria-plc/iamapi-common` migrations. `public.migrations` is the pre-split + table, left in place for rollback — never write to it. ## Diagnosing a pasted 400/500 (the recurring loop) diff --git a/.claude/skills/edr-db/query.cjs b/.claude/skills/edr-db/query.cjs index b4797bd45..6f41f555b 100644 --- a/.claude/skills/edr-db/query.cjs +++ b/.claude/skills/edr-db/query.cjs @@ -8,7 +8,7 @@ * node .claude/skills/edr-db/query.cjs "SELECT ... " run SQL (console.table) * node .claude/skills/edr-db/query.cjs explain "SELECT..." EXPLAIN-validate only * node .claude/skills/edr-db/query.cjs columns
list freight.
columns - * node .claude/skills/edr-db/query.cjs migrations [like] public.migrations rows + * node .claude/skills/edr-db/query.cjs migrations [like] freight/iam migration rows * node .claude/skills/edr-db/query.cjs drift
columns vs entity check helper * * Connection: DB_HOST/DB_PORT/DB_USER/DB_PASSWORD/DB_NAME env vars, falling @@ -52,12 +52,27 @@ async function main() { console.table(r.rows); } else if (first === 'migrations') { const like = rest[0] ? `%${rest[0]}%` : '%'; - const r = await c.query( - `SELECT id, timestamp, name FROM public.migrations - WHERE name ILIKE $1 ORDER BY id DESC LIMIT 40`, - [like], - ); - console.table(r.rows); + // Histories are split per owner: freight.migrations (this app) and + // iam.typeorm_migrations (@tria-plc/iamapi-common). public.migrations is the + // pre-split table, kept for rollback — read it only if the split has not + // been applied to this DB yet. + const sources = [ + ['freight', 'freight.migrations'], + ['iam', 'iam.typeorm_migrations'], + ['legacy', 'public.migrations'], + ]; + const rows = []; + for (const [owner, table] of sources) { + const present = await c.query(`SELECT to_regclass($1) IS NOT NULL AS ok`, [table]); + if (!present.rows[0].ok) continue; + const r = await c.query( + `SELECT id, timestamp, name FROM ${table} + WHERE name ILIKE $1 ORDER BY id DESC LIMIT 40`, + [like], + ); + rows.push(...r.rows.map((row) => ({ owner, ...row }))); + } + console.table(rows); } else if (first === 'drift') { // Quick drift signal: DB columns for the table. Compare by eye against // the entity's @Column names; a recorded-but-absent column = drift. diff --git a/apps/edr-freight-api/src/config/database.config.ts b/apps/edr-freight-api/src/config/database.config.ts index 0591cd0b1..f699b193d 100644 --- a/apps/edr-freight-api/src/config/database.config.ts +++ b/apps/edr-freight-api/src/config/database.config.ts @@ -118,30 +118,91 @@ const iamMigrationsGlob = join( ); const freightMigrationsGlob = join(__dirname, "../migrations/*.js"); -export function buildDataSourceOptions(): DataSourceOptions { +/** + * Migration history is split per owner instead of sharing one `public.migrations` + * table: + * + * - IAM migrations ship with `@tria-plc/iamapi-common`, target the `iam` schema + * and are recorded in `iam.typeorm_migrations` — the same table the package's + * own CLI (`pnpm iam:migration:run|show|revert`) uses, so both paths agree on + * what has been applied. + * - Freight migrations are recorded in `freight.migrations`. + * + * `public.migrations` is the pre-split table; `src/scripts/migrate.ts` adopts its + * rows into the two tables above on first run and then leaves it untouched. + */ +export const IAM_MIGRATIONS = { + schema: "iam", + table: "typeorm_migrations", +} as const; + +export const FREIGHT_MIGRATIONS = { + schema: "freight", + table: "migrations", +} as const; + +export const LEGACY_MIGRATIONS = { + schema: "public", + table: "migrations", +} as const; + +function buildConnectionOptions() { return { - type: "postgres", + type: "postgres" as const, host: process.env.DB_HOST ?? "localhost", port: parseInt(process.env.DB_PORT ?? "5433", 10), username: process.env.DB_USER ?? "postgres", password: process.env.DB_PASSWORD ?? "", database: process.env.DB_NAME ?? "edr_freight", - schema: "public", // NOTE: do NOT pass `extra.options: '-c search_path=...'`. That sends the // Postgres startup `options` parameter, which connection poolers (PgBouncer / // proxies fronting the remote edr_dev DB) reject with // `08P01 unsupported startup parameter in options: search_path`. // The search_path is instead applied per-connection via a pool `connect` - // handler in app.module.ts (see setPoolSearchPath). - entities: [__dirname + "/../**/*.entity.{ts,js}", ...iamEntities], - migrations: [ - iamMigrationsGlob, - freightMigrationsGlob, - ], - migrationsTransactionMode: "each", + // handler (app.module.ts `setPoolSearchPath`, migrate.ts `applySearchPath`). synchronize: false, logging: - process.env.TYPEORM_LOGGING === "true" ? true : ["error", "warn"], + process.env.TYPEORM_LOGGING === "true" + ? true + : (["error", "warn"] as DataSourceOptions["logging"]), + }; +} + +/** + * Runtime options for the API (and the seed scripts using `AppDataSource`). + * Carries no migrations: migrations run only through `src/scripts/migrate.ts`, + * which uses the two dedicated DataSources below. + */ +export function buildDataSourceOptions(): DataSourceOptions { + return { + ...buildConnectionOptions(), + schema: "public", + entities: [__dirname + "/../**/*.entity.{ts,js}", ...iamEntities], + migrations: [], + }; +} + +/** IAM migrations only, recorded in `iam.typeorm_migrations`. */ +export function buildIamMigrationDataSourceOptions(): DataSourceOptions { + return { + ...buildConnectionOptions(), + schema: IAM_MIGRATIONS.schema, + entities: [], + migrations: [iamMigrationsGlob], + migrationsTableName: IAM_MIGRATIONS.table, + migrationsTransactionMode: "each", + }; +} + +/** Freight migrations only, recorded in `freight.migrations`. */ +export function buildFreightMigrationDataSourceOptions(): DataSourceOptions { + return { + ...buildConnectionOptions(), + schema: FREIGHT_MIGRATIONS.schema, + entities: [], + migrations: [freightMigrationsGlob], + migrationsTableName: FREIGHT_MIGRATIONS.table, + migrationsTransactionMode: "each", }; } diff --git a/apps/edr-freight-api/src/scripts/migrate.ts b/apps/edr-freight-api/src/scripts/migrate.ts index 7f331e1d3..ac8a97b28 100644 --- a/apps/edr-freight-api/src/scripts/migrate.ts +++ b/apps/edr-freight-api/src/scripts/migrate.ts @@ -1,24 +1,179 @@ import "dotenv/config"; -import { AppDataSource } from "../data-source"; -import { ensurePostgresSchemas } from "../config/ensure-postgres-schemas"; -import { buildDataSourceOptions } from "../config/database.config"; +import { DataSource, DataSourceOptions } from "typeorm"; +import { + ensurePostgresSchemas, + APPLICATION_SEARCH_PATH, +} from "../config/ensure-postgres-schemas"; +import { + buildDataSourceOptions, + buildIamMigrationDataSourceOptions, + buildFreightMigrationDataSourceOptions, + FREIGHT_MIGRATIONS, + IAM_MIGRATIONS, + LEGACY_MIGRATIONS, +} from "../config/database.config"; -async function main(): Promise { +/** + * Migrations run as two independent histories: + * + * 1. IAM — migrations shipped by `@tria-plc/iamapi-common`, recorded in + * `iam.typeorm_migrations`. + * 2. Freight — this app's `src/migrations/*`, recorded in `freight.migrations`. + * + * They used to share `public.migrations`, which meant a single ordered history + * across two independently versioned sources: dropping in a new IAM package + * release interleaved its migrations with freight's by timestamp, and + * `migration:revert` could not tell the two apart. `adoptLegacyHistory()` below + * moves the existing rows into the two tables on the first run after this change, + * so nothing re-runs. + */ + +/** + * The pooler in front of the remote DB rejects the Postgres `options` startup + * parameter, so search_path is set per physical connection instead. pg queues the + * SET on the client before the pool hands it to a caller, so every query a + * migration issues already sees the full schema search order. + */ +function applySearchPath(dataSource: DataSource): void { + const pool = (dataSource.driver as { master?: unknown }).master as + | { on?: (event: string, cb: (client: unknown) => void) => void } + | undefined; + + pool?.on?.("connect", (client) => { + (client as { query: (sql: string) => Promise }) + .query(`SET search_path TO ${APPLICATION_SEARCH_PATH}`) + .catch(() => { + /* connection will be validated on first real query */ + }); + }); +} + +async function initialize(options: DataSourceOptions): Promise { + const dataSource = new DataSource(options); + await dataSource.initialize(); + applySearchPath(dataSource); + return dataSource; +} + +function qualified({ + schema, + table, +}: { + schema: string; + table: string; +}): string { + return `"${schema}"."${table}"`; +} + +/** Same shape TypeORM's MigrationExecutor creates for a Postgres history table. */ +async function createHistoryTable( + dataSource: DataSource, + target: { schema: string; table: string }, +): Promise { + await dataSource.query( + `CREATE TABLE IF NOT EXISTS ${qualified(target)} ( + "id" SERIAL NOT NULL, + "timestamp" bigint NOT NULL, + "name" character varying NOT NULL, + CONSTRAINT "PK_${target.schema}_${target.table}" PRIMARY KEY ("id") + )`, + ); +} + +async function countRows( + dataSource: DataSource, + target: { schema: string; table: string }, +): Promise { + const [{ count }] = (await dataSource.query( + `SELECT count(*)::int AS count FROM ${qualified(target)}`, + )) as [{ count: number }]; + return count; +} + +/** + * Copy the pre-split `public.migrations` rows into the per-owner tables. + * + * Only fills a table that is still empty — once a history is live it is the + * source of truth, and re-copying would resurrect rows that a deliberate + * `migration:revert` removed. `public.migrations` is intentionally left in place: + * it is the rollback path if an older build of this app is redeployed. + */ +async function adoptLegacyHistory( + dataSource: DataSource, + iamMigrationNames: string[], +): Promise { + const legacyExists = (await dataSource.query( + `SELECT to_regclass($1) IS NOT NULL AS present`, + [`${LEGACY_MIGRATIONS.schema}.${LEGACY_MIGRATIONS.table}`], + )) as [{ present: boolean }]; + + if (!legacyExists[0].present) return; + + await createHistoryTable(dataSource, IAM_MIGRATIONS); + await createHistoryTable(dataSource, FREIGHT_MIGRATIONS); + + const adopt = async ( + target: { schema: string; table: string }, + /** true → rows whose name is an IAM migration; false → everything else. */ + isIam: boolean, + ): Promise => { + if ((await countRows(dataSource, target)) > 0) return; + + const inserted = (await dataSource.query( + `INSERT INTO ${qualified(target)} ("timestamp", "name") + SELECT legacy."timestamp", legacy."name" + FROM ${qualified(LEGACY_MIGRATIONS)} legacy + WHERE legacy."name" ${isIam ? "= ANY" : "<> ALL"}($1::text[]) + ORDER BY legacy."timestamp" + RETURNING "name"`, + [iamMigrationNames], + )) as unknown[]; + + console.log( + `adopted ${inserted.length} row(s) from ${qualified(LEGACY_MIGRATIONS)} into ${qualified(target)}`, + ); + }; + + await adopt(IAM_MIGRATIONS, true); + await adopt(FREIGHT_MIGRATIONS, false); +} + +async function runMigrations( + label: string, + dataSource: DataSource, +): Promise { + const applied = await dataSource.runMigrations(); + for (const migration of applied) { + console.log(`applied [${label}]: ${migration.name}`); + } + if (applied.length === 0) console.log(`no pending ${label} migrations`); +} + +export async function runAllMigrations(): Promise { await ensurePostgresSchemas(buildDataSourceOptions()); - await AppDataSource.initialize(); + const iam = await initialize(buildIamMigrationDataSourceOptions()); try { - const applied = await AppDataSource.runMigrations(); - for (const migration of applied) { - console.log(`applied: ${migration.name}`); - } - if (applied.length === 0) console.log("no pending migrations"); + const iamMigrationNames = iam.migrations.map( + (migration) => migration.name ?? migration.constructor.name, + ); + await adoptLegacyHistory(iam, iamMigrationNames); + await runMigrations("iam", iam); } finally { - await AppDataSource.destroy(); + await iam.destroy(); + } + + const freight = await initialize(buildFreightMigrationDataSourceOptions()); + try { + await runMigrations("freight", freight); + } finally { + await freight.destroy(); } } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +if (require.main === module) { + runAllMigrations().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/apps/edr-freight-api/src/scripts/run-migrations.ts b/apps/edr-freight-api/src/scripts/run-migrations.ts index b5cb23078..33934f5d5 100644 --- a/apps/edr-freight-api/src/scripts/run-migrations.ts +++ b/apps/edr-freight-api/src/scripts/run-migrations.ts @@ -1,21 +1,11 @@ -import { AppDataSource } from '../data-source'; +import { runAllMigrations } from './migrate'; -async function runMigrations() { - try { - console.log('Initializing datasource...'); - await AppDataSource.initialize(); - console.log('Datasource initialized. Running migrations...'); - const migrations = await AppDataSource.runMigrations(); - console.log(`Applied ${migrations.length} migrations.`); - await AppDataSource.destroy(); - process.exit(0); - } catch (err) { +// Alias for `migrate.ts` — the real logic lives there. IAM and freight now run as +// two separate migration histories, so this must not go back to +// `AppDataSource.runMigrations()`: the runtime DataSource carries no migrations. +runAllMigrations() + .then(() => process.exit(0)) + .catch((err) => { console.error('Migration run failed:', err); - try { - await AppDataSource.destroy(); - } catch {} process.exit(1); - } -} - -runMigrations(); + });