fix: matrix

apps/edr-freight-api/src/modules/chat/matrix.client.spec.ts

@@ -121,6 +121,39 @@ describe('MatrixClient.verifyServerAdmin', () => {
   });
 });
 
+describe('MatrixClient rate limiting', () => {
+  it('retries a 429 after the delay Synapse asks for', async () => {
+    // The dev outage: a reconcile is a burst of writes, Synapse throttled an
+    // m.space.child PUT, and one un-retried 429 threw the whole run away.
+    fetchMock
+      .mockResolvedValueOnce(response(200, { user_id: '@edrbot:matrix.test' }))
+      .mockResolvedValueOnce(
+        response(429, {
+          errcode: 'M_LIMIT_EXCEEDED',
+          error: 'Too Many Requests',
+          retry_after_ms: 1,
+        }),
+      )
+      .mockResolvedValueOnce(response(200, { users: [] }));
+
+    const check = await new MatrixClient(config).verifyServerAdmin();
+
+    expect(check.ok).toBe(true);
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+  });
+
+  it('gives up rather than hanging on a homeserver that only ever 429s', async () => {
+    fetchMock.mockResolvedValue(
+      response(429, { errcode: 'M_LIMIT_EXCEEDED', retry_after_ms: 1 }),
+    );
+
+    const check = await new MatrixClient(config).verifyServerAdmin();
+
+    expect(check.ok).toBe(false);
+    expect(check.error).toContain('429');
+  });
+});
+
 describe('MatrixClient.adminCheck', () => {
   it('does not re-hit Synapse on every readiness probe', async () => {
     fetchMock

apps/edr-freight-api/src/modules/chat/matrix.client.ts

@@ -57,6 +57,9 @@ export class MatrixClient implements OnApplicationBootstrap {
   /** The token is a deploy-time fact and the readiness probe runs every few
    *  seconds, so {@link adminCheck} memoises for this long. */
   private static readonly ADMIN_CHECK_TTL_MS = 5 * 60_000;
+  /** Enough to ride out Synapse's limiter; short enough that a genuinely
+   *  wedged homeserver still fails the run rather than hanging it. */
+  private static readonly MAX_RATE_LIMIT_RETRIES = 5;
   private adminCheckCache?: { at: number; result: AdminCheck };
 
   constructor(
@@ -95,13 +98,43 @@ export class MatrixClient implements OnApplicationBootstrap {
     return this.config.enabled;
   }
 
+  /**
+   * Synapse answers a burst of writes with 429 + `retry_after_ms`, and a
+   * reconcile is nothing but a burst of writes — one run creates the space,
+   * #general and a room per position, then force-joins every holder into each.
+   * The first run against dev tripped the limiter on an `m.space.child` PUT,
+   * and because nothing retried, that single 429 threw the whole reconcile
+   * away mid-flight. On the sign-in path ChatSsoService swallows the throw, so
+   * the only visible symptom was an empty Element.
+   *
+   * Honour the delay Synapse asks for rather than guessing at one.
+   */
+  private async fetchWithRetry(
+    url: string,
+    init: Parameters<typeof fetch>[1],
+  ): Promise<Awaited<ReturnType<typeof fetch>>> {
+    for (let attempt = 0; ; attempt++) {
+      const res = await fetch(url, init);
+      if (res.status !== 429 || attempt >= MatrixClient.MAX_RATE_LIMIT_RETRIES) {
+        return res;
+      }
+      // Body is discarded either way — this response is being retried.
+      const body = (await res.json().catch(() => ({}))) as {
+        retry_after_ms?: number;
+      };
+      await new Promise((resolve) =>
+        setTimeout(resolve, (Number(body.retry_after_ms) || 1000) + 100),
+      );
+    }
+  }
+
   private async request<T>(
     method: string,
     path: string,
     body?: unknown,
     token: string = this.config.adminToken,
   ): Promise<T> {
-    const res = await fetch(`${this.config.baseUrl}${path}`, {
+    const res = await this.fetchWithRetry(`${this.config.baseUrl}${path}`, {
       method,
       headers: {
         'Content-Type': 'application/json',
@@ -125,7 +158,7 @@ export class MatrixClient implements OnApplicationBootstrap {
     path: string,
     body: unknown,
   ): Promise<T> {
-    const res = await fetch(`${this.config.baseUrl}${path}`, {
+    const res = await this.fetchWithRetry(`${this.config.baseUrl}${path}`, {
       method,
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(body),
@@ -145,7 +178,7 @@ export class MatrixClient implements OnApplicationBootstrap {
     path: string,
     token?: string,
   ): Promise<T | null> {
-    const res = await fetch(`${this.config.baseUrl}${path}`, {
+    const res = await this.fetchWithRetry(`${this.config.baseUrl}${path}`, {
       method,
       headers: { Authorization: `Bearer ${token ?? this.config.adminToken}` },
     });