diff --git a/src/Utils/bounded-retry.ts b/src/Utils/bounded-retry.ts new file mode 100644 index 00000000..d1d85e24 --- /dev/null +++ b/src/Utils/bounded-retry.ts @@ -0,0 +1,225 @@ +/** + * Bounded retry — WhatsApp-aligned per-operation retry without global state. + * + * Replaces the circuit breaker pattern, which was causing cascading failures + * (5 timeouts in 60s -> all queries blocked for 30s). Empirical capture of + * WhatsApp Android's behavior shows it uses per-operation exponential backoff + * with a delay cap, not a state-machine circuit breaker: + * + * delays observed (Frida trace): 3000 -> 10000 -> 60000 -> ~64000 -> 120000 ms + * memory profile during 5min disconnect: dropped 53MB and stabilised + * FDs during disconnect: closed (305 -> 295), did not accumulate + * recovery on reconnect: ~10s, no retry storm + * + * Design: + * - Each operation has its own independent retry timer + * - Failures in one operation do NOT block other operations + * - Bounded by maxAttempts + ttlMs (eventual give-up to prevent unbounded + * accumulation under prolonged outages) + * - Per-attempt timeout (independent of total ttl) + * + * @module Utils/bounded-retry + */ + +import { metrics } from './prometheus-metrics.js' + +/** + * Default delay sequence (milliseconds) — matches WhatsApp Android empirical + * behavior. After exhausting the sequence, the last value is used (cap). + */ +export const WHATSAPP_BACKOFF_DELAYS = [3000, 10000, 60000, 60000, 120000] as const + +/** + * Default jitter (+/- 15%) to prevent thundering-herd retries. + */ +export const DEFAULT_JITTER_FACTOR = 0.15 as const + +/** + * Default time-to-live for retries: 10 minutes. + * + * Empirical justification: WhatsApp Android stabilises memory in ~3-5 min + * during a network outage; 10 min gives a generous safety buffer while + * preventing unbounded retry accumulation. + */ +export const DEFAULT_TTL_MS = 10 * 60 * 1000 + +/** + * Default per-attempt timeout: 30 seconds. + * + * Most WhatsApp queries respond within seconds. A 30s timeout is generous + * enough for slow networks but prevents a single hang from blocking retries. + */ +export const DEFAULT_PER_ATTEMPT_TIMEOUT_MS = 30000 + +export interface BoundedRetryOptions { + /** Operation name for logging/metrics */ + name?: string + /** Sequence of delays (ms). Last value is used as cap. */ + delays?: readonly number[] + /** Jitter factor 0..1 (default 0.15) */ + jitter?: number + /** Total wall-clock budget — gives up after this. Default 10 min. */ + ttlMs?: number + /** Per-attempt timeout (ms). Default 30s. */ + perAttemptTimeoutMs?: number + /** Predicate: should we retry on this error? Default: always */ + shouldRetry?: (err: Error, attempt: number) => boolean + /** Hook fired before each retry */ + onRetry?: (err: Error, attempt: number, delayMs: number) => void + /** AbortSignal — if aborted, give up immediately */ + signal?: AbortSignal +} + +export class BoundedRetryGiveUpError extends Error { + constructor( + public readonly opName: string, + public readonly attempts: number, + public readonly elapsedMs: number, + public readonly lastError: Error + ) { + super( + `bounded-retry "${opName}" gave up after ${attempts} attempts ` + + `(${elapsedMs}ms elapsed). Last error: ${lastError.message}` + ) + this.name = 'BoundedRetryGiveUpError' + } +} + +export class BoundedRetryAbortedError extends Error { + constructor(public readonly opName: string) { + super(`bounded-retry "${opName}" aborted via signal`) + this.name = 'BoundedRetryAbortedError' + } +} + +/** + * Apply jitter to a delay: returns delay * (1 +/- jitter) + */ +function withJitter(delayMs: number, jitter: number): number { + if (jitter <= 0) return delayMs + const factor = 1 + (Math.random() * 2 - 1) * jitter + return Math.max(0, Math.round(delayMs * factor)) +} + +/** + * Pick the delay for a given attempt index. Falls back to the last value + * (cap) once the sequence is exhausted. + */ +function pickDelay(attempt: number, delays: readonly number[]): number { + if (delays.length === 0) return 0 + if (attempt < delays.length) return delays[attempt]! + return delays[delays.length - 1]! +} + +/** + * Wrap a promise with a per-attempt timeout. Rejects if the promise does + * not settle before timeoutMs elapses. + */ +function withTimeout(promise: Promise, timeoutMs: number, name: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`bounded-retry "${name}" attempt timed out after ${timeoutMs}ms`)) + }, timeoutMs) + promise + .then(value => { + clearTimeout(timer) + resolve(value) + }) + .catch(err => { + clearTimeout(timer) + reject(err) + }) + }) +} + +/** + * Sleep with abort support. + */ +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new Error('aborted')) + return + } + const timer = setTimeout(resolve, ms) + if (signal) { + const onAbort = () => { + clearTimeout(timer) + reject(new Error('aborted')) + } + signal.addEventListener('abort', onAbort, { once: true }) + } + }) +} + +/** + * Run an async operation with bounded exponential-backoff retry. + * + * Independent per-call: no global state, no cross-operation interaction. + * Memory bound: at most one outstanding retry timer per call. + * + * @example + * ```ts + * const result = await withBoundedRetry( + * () => assertSessions([jid], true), + * { name: 'assertSessions', ttlMs: 5 * 60_000 } + * ) + * ``` + */ +export async function withBoundedRetry( + operation: () => Promise, + options: BoundedRetryOptions = {} +): Promise { + const name = options.name ?? 'bounded-retry' + const delays = options.delays ?? WHATSAPP_BACKOFF_DELAYS + const jitter = options.jitter ?? DEFAULT_JITTER_FACTOR + const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS + const perAttemptTimeoutMs = options.perAttemptTimeoutMs ?? DEFAULT_PER_ATTEMPT_TIMEOUT_MS + const shouldRetry = options.shouldRetry ?? (() => true) + + const start = Date.now() + let lastError: Error = new Error('unknown') + let attempt = 0 + + while (true) { + if (options.signal?.aborted) { + throw new BoundedRetryAbortedError(name) + } + + try { + const result = await withTimeout(operation(), perAttemptTimeoutMs, name) + if (attempt > 0) { + metrics.socketEvents.inc({ event: 'bounded_retry_recovered' }) + } + return result + } catch (err) { + lastError = err as Error + attempt++ + + const elapsed = Date.now() - start + if (elapsed >= ttlMs) { + metrics.errors.inc({ category: 'bounded_retry', code: 'ttl_exceeded' }) + throw new BoundedRetryGiveUpError(name, attempt, elapsed, lastError) + } + + if (!shouldRetry(lastError, attempt)) { + metrics.errors.inc({ category: 'bounded_retry', code: 'predicate_no_retry' }) + throw lastError + } + + const baseDelay = pickDelay(attempt - 1, delays) + // Cap remaining delay so we do not blow past ttlMs + const remainingBudget = Math.max(0, ttlMs - elapsed) + const delayMs = Math.min(withJitter(baseDelay, jitter), remainingBudget) + + options.onRetry?.(lastError, attempt, delayMs) + metrics.socketEvents.inc({ event: 'bounded_retry_attempt' }) + + try { + await sleep(delayMs, options.signal) + } catch { + throw new BoundedRetryAbortedError(name) + } + } + } +} diff --git a/src/__tests__/Utils/bounded-retry.test.ts b/src/__tests__/Utils/bounded-retry.test.ts new file mode 100644 index 00000000..41c760b8 --- /dev/null +++ b/src/__tests__/Utils/bounded-retry.test.ts @@ -0,0 +1,213 @@ +import { jest } from '@jest/globals' +import { + withBoundedRetry, + BoundedRetryGiveUpError, + BoundedRetryAbortedError, + WHATSAPP_BACKOFF_DELAYS, + DEFAULT_TTL_MS +} from '../../Utils/bounded-retry' + +// All tests use very short real delays (1-50ms) so they run fast. +// This sidesteps the fragility of jest fake timers + async chains. + +describe('bounded-retry — WhatsApp-aligned per-operation retry', () => { + test('default delay sequence matches WhatsApp Android empirical capture', () => { + // Frida trace: 3000 -> 10000 -> 60000 -> ~64000 -> 120000 ms + expect(WHATSAPP_BACKOFF_DELAYS).toEqual([3000, 10000, 60000, 60000, 120000]) + }) + + test('default TTL is 10 minutes (matches empirical stabilisation + safety buffer)', () => { + expect(DEFAULT_TTL_MS).toBe(600_000) + }) + + test('first-attempt success returns immediately, no retries', async () => { + const op = jest.fn<() => Promise>().mockResolvedValue('ok') + const result = await withBoundedRetry(op, { name: 'fast-success' }) + expect(result).toBe('ok') + expect(op).toHaveBeenCalledTimes(1) + }) + + test('retries with the configured delay sequence after failures', async () => { + const op = jest + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('try1')) + .mockRejectedValueOnce(new Error('try2')) + .mockResolvedValueOnce(42) + + const onRetry = jest.fn<(err: Error, attempt: number, delayMs: number) => void>() + const result = await withBoundedRetry(op, { + name: 'three-tries', + delays: [10, 20, 40], + jitter: 0, + onRetry + }) + + expect(result).toBe(42) + expect(op).toHaveBeenCalledTimes(3) + expect(onRetry).toHaveBeenCalledTimes(2) + expect(onRetry.mock.calls[0]![2]).toBe(10) + expect(onRetry.mock.calls[1]![2]).toBe(20) + }) + + test('TTL exhaustion throws BoundedRetryGiveUpError (no infinite retries)', async () => { + const op = jest.fn<() => Promise>().mockRejectedValue(new Error('always-fails')) + + await expect( + withBoundedRetry(op, { + name: 'ttl-test', + delays: [10], + jitter: 0, + ttlMs: 50 + }) + ).rejects.toBeInstanceOf(BoundedRetryGiveUpError) + + // Should have tried multiple times before giving up + expect(op.mock.calls.length).toBeGreaterThanOrEqual(2) + }) + + test('per-attempt timeout fires for slow operations', async () => { + const slowOp = jest.fn<() => Promise>(() => new Promise(() => {})) // hangs + + await expect( + withBoundedRetry(slowOp, { + name: 'slow', + delays: [5], + jitter: 0, + ttlMs: 200, + perAttemptTimeoutMs: 30 + }) + ).rejects.toBeInstanceOf(BoundedRetryGiveUpError) + + // Multiple attempts should have been tried (each timed out at 30ms) + expect(slowOp.mock.calls.length).toBeGreaterThan(1) + }) + + test('shouldRetry predicate can short-circuit retries', async () => { + const op = jest + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('transient')) + .mockRejectedValueOnce(new Error('FATAL: do not retry')) + + await expect( + withBoundedRetry(op, { + name: 'predicate-test', + delays: [1], + jitter: 0, + shouldRetry: err => !err.message.startsWith('FATAL') + }) + ).rejects.toThrow('FATAL: do not retry') + + expect(op).toHaveBeenCalledTimes(2) + }) + + test('AbortSignal cancels in-flight retry sleep', async () => { + const op = jest.fn<() => Promise>().mockRejectedValue(new Error('fail')) + const ctrl = new AbortController() + + const promise = withBoundedRetry(op, { + name: 'abort-test', + delays: [10_000], // long sleep so abort can happen during it + jitter: 0, + signal: ctrl.signal + }) + + // First attempt fires immediately; sleep starts + await new Promise(resolve => setTimeout(resolve, 10)) + ctrl.abort() + + await expect(promise).rejects.toBeInstanceOf(BoundedRetryAbortedError) + }) + + test('two concurrent retry chains do NOT interfere with each other (per-op isolation)', async () => { + // Key advantage over circuit breaker: failures in one chain do not + // block another chain. With circuit breaker, 5 failures from chain A + // would block chain B too. With bounded-retry, B is independent. + + const flakyOp = jest + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('a1')) + .mockResolvedValueOnce('a-ok') + + const fastOp = jest.fn<() => Promise>().mockResolvedValue('b-ok') + + // Start a (will retry once after 30ms) and b (immediate) + const aPromise = withBoundedRetry(flakyOp, { name: 'a', delays: [30], jitter: 0 }) + const bPromise = withBoundedRetry(fastOp, { name: 'b', delays: [30], jitter: 0 }) + + // b should resolve right away + const bResult = await bPromise + expect(bResult).toBe('b-ok') + expect(fastOp).toHaveBeenCalledTimes(1) + + // a still running — wait for it + const aResult = await aPromise + expect(aResult).toBe('a-ok') + expect(flakyOp).toHaveBeenCalledTimes(2) + }) + + test('delay sequence caps at last value (no unbounded growth)', async () => { + const op = jest.fn<() => Promise>().mockRejectedValue(new Error('fail')) + const onRetry = jest.fn<(err: Error, attempt: number, delayMs: number) => void>() + + await expect( + withBoundedRetry(op, { + name: 'cap-test', + delays: [5, 10, 15], // 3 distinct steps; 4th+ retry should reuse 15 + jitter: 0, + ttlMs: 300, + onRetry + }) + ).rejects.toBeInstanceOf(BoundedRetryGiveUpError) + + // Find delays >= 4th retry — all should be capped at 15 + const lateRetries = onRetry.mock.calls.filter(c => c[1] >= 4) + expect(lateRetries.length).toBeGreaterThan(0) + for (const call of lateRetries) { + expect(call[2]).toBeLessThanOrEqual(15) + } + }) + + test('jitter randomises delay within +/- factor', async () => { + const op = jest.fn<() => Promise>().mockRejectedValue(new Error('fail')) + const observed: number[] = [] + + // Single run, capture multiple onRetry calls (each with different jitter) + await withBoundedRetry(op, { + name: 'jitter-test', + delays: [50], // base delay + jitter: 0.3, // +/- 30% -> [35, 65] + ttlMs: 1000, // ttl high enough for ~10 retries + perAttemptTimeoutMs: 5, + onRetry: (_, __, d) => observed.push(d) + }).catch(() => {}) + + expect(observed.length).toBeGreaterThanOrEqual(3) + const unique = new Set(observed) + expect(unique.size).toBeGreaterThan(1) + // Every observed delay must be within jitter window — but the LAST one + // may be capped by remaining ttl budget, so check all but the last + const inWindow = observed.slice(0, -1) + for (const d of inWindow) { + expect(d).toBeGreaterThanOrEqual(35) + expect(d).toBeLessThanOrEqual(65) + } + }) + + test('delay does not exceed remaining TTL budget', async () => { + const op = jest.fn<() => Promise>().mockRejectedValue(new Error('fail')) + const onRetry = jest.fn<(err: Error, attempt: number, delayMs: number) => void>() + + await withBoundedRetry(op, { + name: 'budget-cap', + delays: [200], // would exceed ttl + jitter: 0, + ttlMs: 50, // very short + onRetry + }).catch(() => {}) + + // All scheduled delays should be capped to remaining budget + for (const call of onRetry.mock.calls) { + expect(call[2]).toBeLessThanOrEqual(50) + } + }) +})