This commit is contained in:
Nathnael
2026-08-31 11:45:55 +00:00
parent 313e0053c3
commit 0ac85ebc1f
8 changed files with 451 additions and 8 deletions

View File

@@ -0,0 +1,100 @@
import 'reflect-metadata';
import type { Response } from 'express';
import type { DataSource } from 'typeorm';
import type { MatrixClient } from '../chat/matrix.client';
import type { EmailClientService } from '../notifications/email-client.service';
import type { SmsClientService } from '../notifications/sms-client.service';
import { HealthController } from './health.controller';
type ReadinessBody = {
status: string;
checks: {
chat: { status: string; enabled: boolean; actingAs?: string; error?: string };
};
};
/** Captures what the controller wrote, in place of an express Response. */
function recorder() {
const sent: { code?: number; body?: ReadinessBody } = {};
const res = {
status(code: number) {
sent.code = code;
return this;
},
json(body: ReadinessBody) {
sent.body = body;
return this;
},
};
return { sent, res: res as unknown as Response };
}
function controllerWith(matrix: Partial<MatrixClient>) {
const dataSource = { query: jest.fn(async () => [{ '?column?': 1 }]) };
return new HealthController(
dataSource as unknown as DataSource,
{ brokerConnected: true } as unknown as SmsClientService,
{ brokerConnected: true } as unknown as EmailClientService,
matrix as MatrixClient,
);
}
describe('HealthController readiness — chat check', () => {
it('reports degraded, not 503, when MATRIX_ADMIN_TOKEN is not a server admin', async () => {
// The dev outage. Chat is broken, but chat is not worth pulling the pod
// out of the load balancer for — bookings and billing still work.
const controller = controllerWith({
enabled: true,
adminCheck: jest.fn(async () => ({
ok: false,
actingAs: '@super-admin.f15347:matrixdev.edrsc.com',
error: 'Matrix GET /_synapse/admin/v2/users?limit=1 -> 403: not a server admin',
})),
});
const { sent, res } = recorder();
await controller.readiness(res);
expect(sent.code).toBe(200);
expect(sent.body?.status).toBe('degraded');
expect(sent.body?.checks.chat.status).toBe('error');
// The account name is the actionable half — it says *which* token is wired up.
expect(sent.body?.checks.chat.actingAs).toBe(
'@super-admin.f15347:matrixdev.edrsc.com',
);
});
it('reports ok when the token really is a server admin', async () => {
const controller = controllerWith({
enabled: true,
adminCheck: jest.fn(async () => ({
ok: true,
actingAs: '@edrbot:matrixdev.edrsc.com',
})),
});
const { sent, res } = recorder();
await controller.readiness(res);
expect(sent.body?.status).toBe('ok');
expect(sent.body?.checks.chat).toMatchObject({
status: 'ok',
enabled: true,
actingAs: '@edrbot:matrixdev.edrsc.com',
});
});
it('does not call Synapse, or degrade, when chat is switched off', async () => {
const adminCheck = jest.fn();
const controller = controllerWith({ enabled: false, adminCheck });
const { sent, res } = recorder();
await controller.readiness(res);
expect(adminCheck).not.toHaveBeenCalled();
expect(sent.body?.status).toBe('ok');
expect(sent.body?.checks.chat).toEqual({ status: 'unknown', enabled: false });
});
});

View File

@@ -7,6 +7,7 @@ import { Public } from "@edr/api-common";
import { Response } from "express";
import { DataSource } from "typeorm";
import { MatrixClient } from "../chat/matrix.client";
import { EmailClientService } from "../notifications/email-client.service";
import { SmsClientService } from "../notifications/sms-client.service";
@@ -32,6 +33,7 @@ export class HealthController {
private readonly dataSource: DataSource,
private readonly smsClient: SmsClientService,
private readonly emailClient: EmailClientService,
private readonly matrix: MatrixClient,
) {}
@Get()
@@ -45,7 +47,7 @@ export class HealthController {
@Public()
@ApiOperation({
summary:
"Readiness probe — database plus SMS/email broker connectivity. Broker failures report as degraded unless READINESS_REQUIRES_BROKER=true.",
"Readiness probe — database, SMS/email broker connectivity, and the Matrix admin token. Broker failures report as degraded unless READINESS_REQUIRES_BROKER=true; chat failures always report as degraded.",
})
async readiness(@Res() res: Response) {
const startedAt = Date.now();
@@ -76,23 +78,55 @@ export class HealthController {
enabled: process.env.RABBITMQ_ENABLED !== "false",
};
const chat = await this.chatCheck();
const brokerDown =
broker.sms.status === "error" || broker.email.status === "error";
const failed =
database.status === "error" ||
(READINESS_REQUIRES_BROKER && brokerDown);
const status = failed ? "error" : brokerDown ? "degraded" : "ok";
const status = failed
? "error"
: brokerDown || chat.status === "error"
? "degraded"
: "ok";
return res
.status(failed ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK)
.json({
status,
timestamp: new Date().toISOString(),
checks: { database, broker },
checks: { database, broker, chat },
});
}
/**
* Chat provisioning runs entirely on MATRIX_ADMIN_TOKEN, and a token that is
* valid but not *server admin* fails only the `/_synapse/admin` half: rooms
* are never created, joins never happen, and the sole symptom is an empty
* Element for every employee. Nothing else in the probe would catch that.
*
* Degraded, never a 503 — chat is not worth pulling the pod out of the load
* balancer for, by the same reasoning as the broker check above. `unknown`
* when MATRIX_ENABLED is off: a feature that is switched off is not a fault.
*/
private async chatCheck(): Promise<{
status: CheckStatus;
enabled: boolean;
actingAs?: string;
error?: string;
}> {
if (!this.matrix.enabled) return { status: "unknown", enabled: false };
const check = await this.matrix.adminCheck();
return {
status: check.ok ? "ok" : "error",
enabled: true,
actingAs: check.actingAs,
error: check.error,
};
}
@Get("info")
@Public()
@ApiOperation({ summary: "App info — version, environment, uptime" })

View File

@@ -2,13 +2,15 @@
import { Module } from "@nestjs/common";
import { ChatModule } from "../chat/chat.module";
import { HealthController } from "./health.controller";
import { NotificationsModule } from "../notifications/notifications.module";
@Module({
// NotificationsModule exports the SMS/email clients; the readiness probe reads
// their broker connection state rather than opening a second connection.
imports: [NotificationsModule],
// ChatModule exports MatrixClient for the MATRIX_ADMIN_TOKEN check.
imports: [NotificationsModule, ChatModule],
controllers: [HealthController],
})
export class HealthModule {}