From 931ef712505e4997c7e8801af0c3a5f1aa5e5459 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 04:06:58 +0000 Subject: [PATCH 01/14] fix: prevent duplicate metric registration error Check if nodejs_eventloop_lag_seconds metric already exists before creating it in SystemMetricsCollector. This prevents conflicts when collectDefaultMetrics from prom-client has already registered metrics with the same name. Added helper functions metricExists() and getExistingMetric() to safely check and retrieve existing metrics from the custom registry. --- src/Utils/prometheus-metrics.ts | 38 +++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 378ce449..3c94b16f 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -95,6 +95,31 @@ export function getConfiguredPrefix(): string { return configuredPrefix } +/** + * Check if a metric with the given name already exists in the custom registry + */ +function metricExists(name: string): boolean { + const fullName = getFullMetricName(name) + try { + const existing = customRegistry.getSingleMetric(fullName) + return existing !== undefined + } catch { + return false + } +} + +/** + * Get an existing metric from the custom registry + */ +function getExistingMetric(name: string): T | undefined { + const fullName = getFullMetricName(name) + try { + return customRegistry.getSingleMetric(fullName) as T | undefined + } catch { + return undefined + } +} + // ============================================ // Configuration // ============================================ @@ -1051,10 +1076,15 @@ export class SystemMetricsCollector { new Gauge('system_load_average', 'System load average', ['period']) ) - // Event loop lag - this.eventLoopLag = registry.register( - new Histogram('nodejs_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS) - ) + // Event loop lag - check if already exists (may be created by collectDefaultMetrics) + const existingEventLoopLag = getExistingMetric('nodejs_eventloop_lag_seconds') + if (existingEventLoopLag) { + this.eventLoopLag = existingEventLoopLag + } else { + this.eventLoopLag = registry.register( + new Histogram('nodejs_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS) + ) + } // Initialize CPU baseline this.lastCpuUsage = process.cpuUsage() From b5b43b5fd6c717d789d6e645de7d3bd10a906d03 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 04:13:42 +0000 Subject: [PATCH 02/14] fix: use different metric name to avoid conflict with collectDefaultMetrics --- src/Utils/prometheus-metrics.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 3c94b16f..dc2b5dab 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1076,15 +1076,12 @@ export class SystemMetricsCollector { new Gauge('system_load_average', 'System load average', ['period']) ) - // Event loop lag - check if already exists (may be created by collectDefaultMetrics) - const existingEventLoopLag = getExistingMetric('nodejs_eventloop_lag_seconds') - if (existingEventLoopLag) { - this.eventLoopLag = existingEventLoopLag - } else { - this.eventLoopLag = registry.register( - new Histogram('nodejs_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS) - ) - } + // Event loop lag - use different name to avoid conflict with collectDefaultMetrics + // prom-client's collectDefaultMetrics creates nodejs_eventloop_lag_seconds + // We use system_eventloop_lag_seconds to avoid duplicate registration + this.eventLoopLag = registry.register( + new Histogram('system_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS) + ) // Initialize CPU baseline this.lastCpuUsage = process.cpuUsage() From ee80d2cdf8adc1054aa84a296544007eb520d7a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 11:57:23 +0000 Subject: [PATCH 03/14] feat: add missing metrics for dashboard compatibility Added new metrics: - active_connections: Number of active WhatsApp connections - connection_errors_total: Total connection errors by type - buffer_destroyed_total: Total buffers destroyed - buffer_final_flush_total: Final flushes during destruction - buffer_cache_cleanup_total: Cache cleanup operations - buffer_cache_size: Current buffer cache size - adaptive_health_status: System health (0=unhealthy, 1=healthy) - adaptive_event_rate: Current event rate per second These metrics are required for the Grafana dashboard to display all panels correctly. --- src/Utils/prometheus-metrics.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index dc2b5dab..8d9f2d0d 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1391,6 +1391,12 @@ export const metrics = { connectionLatency: baileysMetrics.register( new Histogram('connection_latency_ms', 'Connection establishment latency in ms', [], [100, 250, 500, 1000, 2500, 5000, 10000]) ), + activeConnections: baileysMetrics.register( + new Gauge('active_connections', 'Number of active WhatsApp connections') + ), + connectionErrors: baileysMetrics.register( + new Counter('connection_errors_total', 'Total connection errors', ['error_type']) + ), // ========== Message Metrics ========== messagesSent: baileysMetrics.register( @@ -1472,6 +1478,18 @@ export const metrics = { eventsProcessed: baileysMetrics.register( new Counter('events_processed_total', 'Total events processed from buffer', ['event_type']) ), + bufferDestroyed: baileysMetrics.register( + new Counter('buffer_destroyed_total', 'Total buffers destroyed', ['reason', 'had_pending_flush']) + ), + bufferFinalFlush: baileysMetrics.register( + new Counter('buffer_final_flush_total', 'Total final flushes during destruction') + ), + bufferCacheCleanup: baileysMetrics.register( + new Counter('buffer_cache_cleanup_total', 'Total cache cleanup operations') + ), + bufferCacheSize: baileysMetrics.register( + new Gauge('buffer_cache_size', 'Current buffer cache size') + ), // ========== Adaptive Flush Metrics ========== adaptiveFlushInterval: baileysMetrics.register( @@ -1489,6 +1507,12 @@ export const metrics = { adaptiveFlushEfficiency: baileysMetrics.register( new Gauge('adaptive_flush_efficiency_percent', 'Flush efficiency percentage') ), + adaptiveHealthStatus: baileysMetrics.register( + new Gauge('adaptive_health_status', 'Adaptive system health status (0=unhealthy, 1=healthy)') + ), + adaptiveEventRate: baileysMetrics.register( + new Gauge('adaptive_event_rate', 'Current event rate per second') + ), // ========== Error Metrics ========== errors: baileysMetrics.register( From e4ffbb4b5c7886f0048c5f3ad000888034f633d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 12:50:02 +0000 Subject: [PATCH 04/14] fix: enable buffer metrics automatically when Prometheus is enabled Previously, buffer metrics required a separate BAILEYS_BUFFER_METRICS=true env variable even when BAILEYS_PROMETHEUS_ENABLED=true was set. This caused the buffer_flushes_total metric to have no data even though flushes were occurring. Now buffer metrics are enabled automatically when either BAILEYS_BUFFER_METRICS or BAILEYS_PROMETHEUS_ENABLED is set to true. --- src/Utils/event-buffer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utils/event-buffer.ts b/src/Utils/event-buffer.ts index 15a0f47c..fb61aec8 100644 --- a/src/Utils/event-buffer.ts +++ b/src/Utils/event-buffer.ts @@ -61,7 +61,7 @@ export function loadBufferConfig(): BufferConfig { maxBufferSize: parseInt(process.env.BAILEYS_BUFFER_MAX_SIZE || '5000', 10), flushDebounceMs: parseInt(process.env.BAILEYS_BUFFER_FLUSH_DEBOUNCE_MS || '100', 10), enableAdaptiveTimeout: process.env.BAILEYS_BUFFER_ADAPTIVE_TIMEOUT !== 'false', - enableMetrics: process.env.BAILEYS_BUFFER_METRICS === 'true', + enableMetrics: process.env.BAILEYS_BUFFER_METRICS === 'true' || process.env.BAILEYS_PROMETHEUS_ENABLED === 'true', lruCleanupRatio: parseFloat(process.env.BAILEYS_BUFFER_LRU_CLEANUP_RATIO || '0.2'), bufferWarnThreshold: parseFloat(process.env.BAILEYS_BUFFER_WARN_THRESHOLD || '0.8') } From b1687389cd62f54defbcccd06e8c2ee5f96f8e01 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 12:57:44 +0000 Subject: [PATCH 05/14] fix: resolve buffer_flushes_total metric label mismatch The buffer_flushes_total metric was registered twice with different labels: - In main metrics object: ['type', 'reason'] - In getEventBufferMetrics(): ['forced'] This caused recordBufferFlush() to silently fail because it tried to increment with { forced } labels but the registered metric expected { type, reason } labels. Fix: - Remove duplicate bufferFlushes from getEventBufferMetrics() - Update recordBufferFlush() to use metrics.bufferFlushes with correct labels - Update recordEventBuffered() to use main metrics.eventsBuffered - Rename local variable to avoid shadowing global metrics object --- src/Utils/prometheus-metrics.ts | 50 +++++++++------------------------ 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 8d9f2d0d..6315e958 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1798,31 +1798,19 @@ export function trackOperation( // Event Buffer Metrics Integration // ============================================ -// Event buffer metrics (lazy initialized) +// Event buffer metrics (lazy initialized) - for detailed buffer tracking +// Note: bufferFlushes is NOT here - use metrics.bufferFlushes from main metrics object let eventBufferMetrics: { - eventsBuffered: Counter | null - bufferFlushes: Counter | null bufferFlushEvents: Histogram | null bufferCurrentSize: Gauge | null bufferPeakSize: Gauge | null bufferHistoryCacheSize: Gauge | null - bufferOverflows: Counter | null bufferLruCleanups: Counter | null } | null = null function getEventBufferMetrics() { if (!eventBufferMetrics) { eventBufferMetrics = { - eventsBuffered: new Counter( - 'events_buffered_total', - 'Total number of events buffered', - ['event_type'] - ), - bufferFlushes: new Counter( - 'buffer_flushes_total', - 'Total number of buffer flushes', - ['forced'] - ), bufferFlushEvents: new Histogram( 'buffer_flush_events', 'Number of events per buffer flush', @@ -1841,24 +1829,11 @@ function getEventBufferMetrics() { 'buffer_history_cache_size', 'Current size of history cache' ), - bufferOverflows: new Counter( - 'buffer_overflows_total', - 'Total number of buffer overflows detected' - ), bufferLruCleanups: new Counter( 'buffer_lru_cleanups_total', 'Total number of LRU cache cleanups performed' ) } - // Register all metrics - if (eventBufferMetrics.eventsBuffered) baileysMetrics.register(eventBufferMetrics.eventsBuffered) - if (eventBufferMetrics.bufferFlushes) baileysMetrics.register(eventBufferMetrics.bufferFlushes) - if (eventBufferMetrics.bufferFlushEvents) baileysMetrics.register(eventBufferMetrics.bufferFlushEvents) - if (eventBufferMetrics.bufferCurrentSize) baileysMetrics.register(eventBufferMetrics.bufferCurrentSize) - if (eventBufferMetrics.bufferPeakSize) baileysMetrics.register(eventBufferMetrics.bufferPeakSize) - if (eventBufferMetrics.bufferHistoryCacheSize) baileysMetrics.register(eventBufferMetrics.bufferHistoryCacheSize) - if (eventBufferMetrics.bufferOverflows) baileysMetrics.register(eventBufferMetrics.bufferOverflows) - if (eventBufferMetrics.bufferLruCleanups) baileysMetrics.register(eventBufferMetrics.bufferLruCleanups) } return eventBufferMetrics } @@ -1869,7 +1844,7 @@ function getEventBufferMetrics() { */ export function recordEventBuffered(eventType: string, count: number = 1): void { try { - const metrics = getEventBufferMetrics() + // Use the main metrics object which has eventsBuffered with label ['event_type'] metrics.eventsBuffered?.inc({ event_type: eventType }, count) } catch { // Metrics not initialized, ignore silently @@ -1882,10 +1857,13 @@ export function recordEventBuffered(eventType: string, count: number = 1): void */ export function recordBufferFlush(eventCount: number, forced: boolean): void { try { - const metrics = getEventBufferMetrics() - metrics.bufferFlushes?.inc({ forced: forced ? 'true' : 'false' }) - metrics.bufferFlushEvents?.observe({}, eventCount) - metrics.bufferCurrentSize?.set({}, 0) // Reset after flush + // Use the main metrics object which has the correct labels ['type', 'reason'] + metrics.bufferFlushes?.inc({ type: 'event', reason: forced ? 'forced' : 'normal' }) + + // Also update the lazy-loaded event buffer metrics for detailed tracking + const ebMetrics = getEventBufferMetrics() + ebMetrics.bufferFlushEvents?.observe({}, eventCount) + ebMetrics.bufferCurrentSize?.set({}, 0) // Reset after flush } catch { // Metrics not initialized, ignore silently } @@ -1903,10 +1881,10 @@ export function updateBufferStatistics(stats: { lruCleanups: number }): void { try { - const metrics = getEventBufferMetrics() - metrics.bufferCurrentSize?.set({}, stats.currentSize) - metrics.bufferPeakSize?.set({}, stats.peakSize) - metrics.bufferHistoryCacheSize?.set({}, stats.historyCacheSize) + const ebMetrics = getEventBufferMetrics() + ebMetrics.bufferCurrentSize?.set({}, stats.currentSize) + ebMetrics.bufferPeakSize?.set({}, stats.peakSize) + ebMetrics.bufferHistoryCacheSize?.set({}, stats.historyCacheSize) } catch { // Metrics not initialized, ignore silently } From 68f6a2ae884dfd8706d913a62507de1a5252c4c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 13:28:25 +0000 Subject: [PATCH 06/14] fix: update buffer_cache_size metric with actual history cache size The buffer_cache_size metric was defined but never updated, always showing 0. Fix: - Add historyCacheSize parameter to recordBufferFlush function - Pass historyCache.size from event-buffer.ts when recording flush metrics - Update both bufferCacheSize and bufferHistoryCacheSize metrics --- src/Utils/event-buffer.ts | 6 +++--- src/Utils/prometheus-metrics.ts | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Utils/event-buffer.ts b/src/Utils/event-buffer.ts index fb61aec8..b701542f 100644 --- a/src/Utils/event-buffer.ts +++ b/src/Utils/event-buffer.ts @@ -410,9 +410,9 @@ export const makeEventBuffer = ( } } - const recordFlushMetrics = (eventCount: number, forced: boolean) => { + const recordFlushMetrics = (eventCount: number, forced: boolean, cacheSize: number) => { if (metricsModule) { - metricsModule.recordBufferFlush(eventCount, forced) + metricsModule.recordBufferFlush(eventCount, forced, cacheSize) } } @@ -553,7 +553,7 @@ export const makeEventBuffer = ( stats.historyCacheSize = historyCache.size // Record metrics - recordFlushMetrics(eventCount, force) + recordFlushMetrics(eventCount, force, historyCache.size) // Log with [BAILEYS] prefix - use getMode() to avoid duplicating mode calculation logic const flushDuration = Date.now() - flushStartTime diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 6315e958..d5690329 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1855,15 +1855,23 @@ export function recordEventBuffered(eventType: string, count: number = 1): void * Record a buffer flush operation * Used by event-buffer.ts for metrics integration */ -export function recordBufferFlush(eventCount: number, forced: boolean): void { +export function recordBufferFlush(eventCount: number, forced: boolean, historyCacheSize?: number): void { try { // Use the main metrics object which has the correct labels ['type', 'reason'] metrics.bufferFlushes?.inc({ type: 'event', reason: forced ? 'forced' : 'normal' }) + // Update buffer cache size if provided + if (typeof historyCacheSize === 'number') { + metrics.bufferCacheSize?.set({}, historyCacheSize) + } + // Also update the lazy-loaded event buffer metrics for detailed tracking const ebMetrics = getEventBufferMetrics() ebMetrics.bufferFlushEvents?.observe({}, eventCount) ebMetrics.bufferCurrentSize?.set({}, 0) // Reset after flush + if (typeof historyCacheSize === 'number') { + ebMetrics.bufferHistoryCacheSize?.set({}, historyCacheSize) + } } catch { // Metrics not initialized, ignore silently } From 6570a1fc6eca3118dfd4ec332fcad6319718ee3c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 13:31:32 +0000 Subject: [PATCH 07/14] fix: add buffer_cache_cleanup_total metric increment The metric was defined but never incremented when LRU cleanup happened. Added recordCacheCleanup() function and call it from cleanupHistoryCache() when cache entries are removed due to exceeding maxHistoryCacheSize. --- src/Utils/event-buffer.ts | 4 ++++ src/Utils/prometheus-metrics.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/Utils/event-buffer.ts b/src/Utils/event-buffer.ts index b701542f..5ae8614b 100644 --- a/src/Utils/event-buffer.ts +++ b/src/Utils/event-buffer.ts @@ -482,6 +482,10 @@ export const makeEventBuffer = ( removed: removed.length, remaining: historyCache.size }) + // Record metrics for cache cleanup + if (metricsModule) { + metricsModule.recordCacheCleanup(removed.length) + } } } diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index d5690329..3708f456 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1898,6 +1898,18 @@ export function updateBufferStatistics(stats: { } } +/** + * Record a cache cleanup operation + * Used by event-buffer.ts when LRU cleanup is performed + */ +export function recordCacheCleanup(removedCount: number): void { + try { + metrics.bufferCacheCleanup?.inc({}) + } catch { + // Metrics not initialized, ignore silently + } +} + // ============================================ // Global Instance // ============================================ From 6412286c73cf35ef036114fc27540a149a96580a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 13:40:31 +0000 Subject: [PATCH 08/14] fix: add buffer_overflows_total metric increment The metric was defined but never incremented when buffer overflow occurred. Added recordBufferOverflow() function and call it from checkBufferOverflow() when buffer exceeds maxBufferSize (default 5000 events). --- src/Utils/event-buffer.ts | 4 ++++ src/Utils/prometheus-metrics.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/Utils/event-buffer.ts b/src/Utils/event-buffer.ts index 5ae8614b..262eb3c0 100644 --- a/src/Utils/event-buffer.ts +++ b/src/Utils/event-buffer.ts @@ -449,6 +449,10 @@ export const makeEventBuffer = ( currentSize: currentEventCount, maxSize: config.maxBufferSize }) + // Record overflow metric + if (metricsModule) { + metricsModule.recordBufferOverflow() + } flush(true) return true } diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 3708f456..6bf20816 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1910,6 +1910,18 @@ export function recordCacheCleanup(removedCount: number): void { } } +/** + * Record a buffer overflow event + * Used by event-buffer.ts when buffer exceeds max size + */ +export function recordBufferOverflow(): void { + try { + metrics.bufferOverflows?.inc({ type: 'event' }) + } catch { + // Metrics not initialized, ignore silently + } +} + // ============================================ // Global Instance // ============================================ From 8d0c03cc2ad2ee1500a5d6fa03f9d9cad8f7c076 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 13:43:31 +0000 Subject: [PATCH 09/14] feat: add connection_errors_total metric tracking Added recordConnectionError() function and integrated it into socket.ts to track connection errors by type: - connection_closed - connection_lost - connection_replaced - timed_out - logged_out - bad_session - restart_required - multidevice_mismatch - error_{code} for other status codes --- src/Socket/socket.ts | 36 +++++++++++++++++++++++++++++++++ src/Utils/prometheus-metrics.ts | 12 +++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/Socket/socket.ts b/src/Socket/socket.ts index 08f45ac4..4542d288 100644 --- a/src/Socket/socket.ts +++ b/src/Socket/socket.ts @@ -54,6 +54,7 @@ import { jidEncode, S_WHATSAPP_NET } from '../WABinary' +import { recordConnectionError } from '../Utils/prometheus-metrics' import { BinaryInfo } from '../WAM/BinaryInfo.js' import { USyncQuery, USyncUser } from '../WAUSync/' import { WebSocketClient } from './Client' @@ -730,6 +731,41 @@ export const makeSocket = (config: SocketConfig) => { closed = true logger.info({ trace: error?.stack }, error ? 'connection errored' : 'connection closed') + // Record connection error metric + if (error) { + const statusCode = (error as Boom)?.output?.statusCode || 0 + let errorType = 'unknown' + switch (statusCode) { + case DisconnectReason.connectionClosed: + errorType = 'connection_closed' + break + case DisconnectReason.connectionLost: + errorType = 'connection_lost' + break + case DisconnectReason.connectionReplaced: + errorType = 'connection_replaced' + break + case DisconnectReason.timedOut: + errorType = 'timed_out' + break + case DisconnectReason.loggedOut: + errorType = 'logged_out' + break + case DisconnectReason.badSession: + errorType = 'bad_session' + break + case DisconnectReason.restartRequired: + errorType = 'restart_required' + break + case DisconnectReason.multideviceMismatch: + errorType = 'multidevice_mismatch' + break + default: + errorType = `error_${statusCode}` + } + recordConnectionError(errorType) + } + clearInterval(keepAliveReq) clearTimeout(qrTimer) diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 6bf20816..d9728b20 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1922,6 +1922,18 @@ export function recordBufferOverflow(): void { } } +/** + * Record a connection error + * Used by socket.ts when connection fails + */ +export function recordConnectionError(errorType: string): void { + try { + metrics.connectionErrors?.inc({ error_type: errorType }) + } catch { + // Metrics not initialized, ignore silently + } +} + // ============================================ // Global Instance // ============================================ From 5b8ea3bb2bdba9ce1d03ef0a2a84661d684bd4f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 13:47:48 +0000 Subject: [PATCH 10/14] fix: add buffer_destroyed_total and buffer_final_flush_total metrics Added recordBufferDestroyed() and recordBufferFinalFlush() functions and integrated into destroy() function in event-buffer.ts. Tracks: - Buffer destruction with reason and whether there was pending flush - Final flushes that occur during buffer destruction --- src/Utils/event-buffer.ts | 12 +++++++++++- src/Utils/prometheus-metrics.ts | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/Utils/event-buffer.ts b/src/Utils/event-buffer.ts index 262eb3c0..c5f67c1b 100644 --- a/src/Utils/event-buffer.ts +++ b/src/Utils/event-buffer.ts @@ -607,11 +607,16 @@ export const makeEventBuffer = ( } logger.debug('Destroying event buffer') + const hadPendingFlush = isBuffering destroyed = true // Flush any remaining events - if (isBuffering) { + if (hadPendingFlush) { flush(true) + // Record final flush metric + if (metricsModule) { + metricsModule.recordBufferFinalFlush() + } } // Clear all timers @@ -624,6 +629,11 @@ export const makeEventBuffer = ( // Remove all event listeners ev.removeAllListeners() + // Record buffer destroyed metric + if (metricsModule) { + metricsModule.recordBufferDestroyed('normal', hadPendingFlush) + } + logger.debug('Event buffer destroyed successfully') } diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index d9728b20..7eadbf3d 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1934,6 +1934,30 @@ export function recordConnectionError(errorType: string): void { } } +/** + * Record buffer destruction + * Used by event-buffer.ts when buffer is destroyed + */ +export function recordBufferDestroyed(reason: string, hadPendingFlush: boolean): void { + try { + metrics.bufferDestroyed?.inc({ reason, had_pending_flush: hadPendingFlush ? 'true' : 'false' }) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record final flush during buffer destruction + * Used by event-buffer.ts when buffer flushes remaining events on destroy + */ +export function recordBufferFinalFlush(): void { + try { + metrics.bufferFinalFlush?.inc({}) + } catch { + // Metrics not initialized, ignore silently + } +} + // ============================================ // Global Instance // ============================================ From f9ed1dc16f308465ea206f0d2fbcf053066e3bf6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 13:57:34 +0000 Subject: [PATCH 11/14] feat: add adaptive metrics and circuit breaker trips tracking Added metrics tracking for: - circuit_breaker_trips_total: Incremented when circuit opens - adaptive_health_status: 1 if healthy (not in aggressive mode), 0 otherwise - adaptive_event_rate: Current events per second Added methods to AdaptiveTimeoutCalculator: - getEventRate(): Returns current event rate in events/second - isHealthy(): Returns true if not in aggressive mode Metrics are updated on each buffer flush when adaptive timeout is enabled. --- src/Utils/circuit-breaker.ts | 5 +++++ src/Utils/event-buffer.ts | 30 ++++++++++++++++++++++++++++++ src/Utils/prometheus-metrics.ts | 13 +++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/Utils/circuit-breaker.ts b/src/Utils/circuit-breaker.ts index fc20e135..143f0729 100644 --- a/src/Utils/circuit-breaker.ts +++ b/src/Utils/circuit-breaker.ts @@ -380,6 +380,11 @@ export class CircuitBreaker extends EventEmitter { this.options.onOpen() this.emit('open') + // Record circuit breaker trip metric + if (this.options.collectMetrics) { + metrics.circuitBreakerTrips?.inc({ name: this.options.name }) + } + // Schedule transition to HALF_OPEN this.resetTimer = setTimeout(() => { this.transitionTo('half-open') diff --git a/src/Utils/event-buffer.ts b/src/Utils/event-buffer.ts index c5f67c1b..64aec7b2 100644 --- a/src/Utils/event-buffer.ts +++ b/src/Utils/event-buffer.ts @@ -329,6 +329,31 @@ class AdaptiveTimeoutCalculator { } return 'balanced' } + + /** + * Get current event rate in events per second + */ + getEventRate(): number { + if (this.eventTimestamps.length < 2) { + return 0 + } + const oldest = this.eventTimestamps[0]! + const newest = this.eventTimestamps[this.eventTimestamps.length - 1]! + const timeSpan = newest - oldest + if (timeSpan === 0) return 0 + return (this.eventTimestamps.length / timeSpan) * 1000 + } + + /** + * Check if system is healthy based on current mode + * Conservative mode with low event rate is healthy + * Aggressive mode might indicate high load + */ + isHealthy(): boolean { + const mode = this.getMode() + // System is considered healthy if not in aggressive mode (high load) + return mode !== 'aggressive' + } } // ============================================================================ @@ -563,6 +588,11 @@ export const makeEventBuffer = ( // Record metrics recordFlushMetrics(eventCount, force, historyCache.size) + // Update adaptive metrics + if (config.enableAdaptiveTimeout && metricsModule) { + metricsModule.updateAdaptiveMetrics(adaptiveTimeout.getEventRate(), adaptiveTimeout.isHealthy()) + } + // Log with [BAILEYS] prefix - use getMode() to avoid duplicating mode calculation logic const flushDuration = Date.now() - flushStartTime logEventBuffer('buffer_flush', { diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 7eadbf3d..85b1bec1 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1958,6 +1958,19 @@ export function recordBufferFinalFlush(): void { } } +/** + * Update adaptive system metrics + * Used by event-buffer.ts to report adaptive timeout health and event rate + */ +export function updateAdaptiveMetrics(eventRate: number, isHealthy: boolean): void { + try { + metrics.adaptiveEventRate?.set({}, eventRate) + metrics.adaptiveHealthStatus?.set({}, isHealthy ? 1 : 0) + } catch { + // Metrics not initialized, ignore silently + } +} + // ============================================ // Global Instance // ============================================ From de5988ba8fc88e9022b512cb8e8668874dc9be0f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 14:20:17 +0000 Subject: [PATCH 12/14] feat: implement WhatsApp connection and message metrics Added tracking for all WhatsApp-related metrics: Connection metrics: - active_connections: Gauge tracking active connections (inc on open, dec on close) - connection_attempts_total: Counter for connection attempts (success/failure) Message metrics: - messages_sent_total: Counter with type label (text, image, video, audio, etc) - messages_received_total: Counter with type label - history_sync_messages_total: Counter for history sync messages Added helper functions in prometheus-metrics.ts: - incrementActiveConnections(), decrementActiveConnections(), setActiveConnections() - recordConnectionAttempt(status) - recordMessageSent(type), recordMessageReceived(type) - recordMessageRetry(type), recordMessageFailure(type, reason) - setMessagesQueued(count, priority) - recordHistorySyncMessages(count) --- src/Socket/messages-recv.ts | 14 +++- src/Socket/messages-send.ts | 12 ++++ src/Socket/socket.ts | 15 ++++- src/Utils/process-message.ts | 7 +- src/Utils/prometheus-metrics.ts | 114 ++++++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 3 deletions(-) diff --git a/src/Socket/messages-recv.ts b/src/Socket/messages-recv.ts index 707a884d..36deaf09 100644 --- a/src/Socket/messages-recv.ts +++ b/src/Socket/messages-recv.ts @@ -4,7 +4,7 @@ import { randomBytes } from 'crypto' import Long from 'long' import { proto } from '../../WAProto/index.js' import { DEFAULT_CACHE_TTLS, KEY_BUNDLE_TYPE, MIN_PREKEY_COUNT } from '../Defaults' -import { metrics } from '../Utils/prometheus-metrics.js' +import { metrics, recordMessageReceived, recordHistorySyncMessages } from '../Utils/prometheus-metrics.js' import type { GroupParticipant, MessageReceiptType, @@ -1387,6 +1387,18 @@ export const makeMessagesRecvSocket = (config: SocketConfig) => { if (msg.key.id && msg.key.remoteJid) { logMessageReceived(msg.key.id, msg.key.remoteJid) } + + // Record message received metric + const msgContent = msg.message + const msgType = msgContent?.conversation ? 'text' + : msgContent?.imageMessage ? 'image' + : msgContent?.videoMessage ? 'video' + : msgContent?.audioMessage ? 'audio' + : msgContent?.documentMessage ? 'document' + : msgContent?.stickerMessage ? 'sticker' + : msgContent?.reactionMessage ? 'reaction' + : 'other' + recordMessageReceived(msgType) }) } catch (error) { logger.error({ error, node: binaryNodeToString(node) }, 'error in handling message') diff --git a/src/Socket/messages-send.ts b/src/Socket/messages-send.ts index d95102c5..8e24aedc 100644 --- a/src/Socket/messages-send.ts +++ b/src/Socket/messages-send.ts @@ -56,6 +56,7 @@ import { S_WHATSAPP_NET } from '../WABinary' import { USyncQuery, USyncUser } from '../WAUSync' +import { recordMessageSent, recordMessageRetry, recordMessageFailure } from '../Utils/prometheus-metrics' import { makeNewsletterSocket } from './newsletter' export const makeMessagesSocket = (config: SocketConfig) => { @@ -1038,6 +1039,17 @@ export const makeMessagesSocket = (config: SocketConfig) => { // Log with [BAILEYS] prefix logMessageSent(msgId, destinationJid) + // Record message sent metric + const msgType = message.conversation ? 'text' + : message.imageMessage ? 'image' + : message.videoMessage ? 'video' + : message.audioMessage ? 'audio' + : message.documentMessage ? 'document' + : message.stickerMessage ? 'sticker' + : message.reactionMessage ? 'reaction' + : 'other' + recordMessageSent(msgType) + // Add message to retry cache if enabled if (messageRetryManager && !participant) { messageRetryManager.addRecentMessage(destinationJid, msgId, message) diff --git a/src/Socket/socket.ts b/src/Socket/socket.ts index 4542d288..2d3b9f2c 100644 --- a/src/Socket/socket.ts +++ b/src/Socket/socket.ts @@ -54,7 +54,12 @@ import { jidEncode, S_WHATSAPP_NET } from '../WABinary' -import { recordConnectionError } from '../Utils/prometheus-metrics' +import { + recordConnectionError, + recordConnectionAttempt, + incrementActiveConnections, + decrementActiveConnections +} from '../Utils/prometheus-metrics' import { BinaryInfo } from '../WAM/BinaryInfo.js' import { USyncQuery, USyncUser } from '../WAUSync/' import { WebSocketClient } from './Client' @@ -764,8 +769,12 @@ export const makeSocket = (config: SocketConfig) => { errorType = `error_${statusCode}` } recordConnectionError(errorType) + recordConnectionAttempt('failure') } + // Decrement active connections + decrementActiveConnections() + clearInterval(keepAliveReq) clearTimeout(qrTimer) @@ -1080,6 +1089,10 @@ export const makeSocket = (config: SocketConfig) => { ev.emit('connection.update', { connection: 'open' }) + // Record successful connection metrics + recordConnectionAttempt('success') + incrementActiveConnections() + if (node.attrs.lid && authState.creds.me?.id) { const myLID = node.attrs.lid process.nextTick(async () => { diff --git a/src/Utils/process-message.ts b/src/Utils/process-message.ts index 3487232b..b1743cca 100644 --- a/src/Utils/process-message.ts +++ b/src/Utils/process-message.ts @@ -16,7 +16,7 @@ import type { WAMessage, WAMessageKey } from '../Types' -import { metrics } from './prometheus-metrics.js' +import { metrics, recordHistorySyncMessages } from './prometheus-metrics.js' import { WAMessageStubType } from '../Types' import { getContentType, normalizeMessageContent } from '../Utils/messages' import { @@ -353,6 +353,11 @@ const processMessage = async ( isLatest: histNotification.syncType !== proto.HistorySync.HistorySyncType.ON_DEMAND ? isLatest : undefined, peerDataRequestSessionId: histNotification.peerDataRequestSessionId }) + + // Record history sync metrics + if (data.messages?.length) { + recordHistorySyncMessages(data.messages.length) + } } break diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 85b1bec1..4543d383 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -1971,6 +1971,120 @@ export function updateAdaptiveMetrics(eventRate: number, isHealthy: boolean): vo } } +// ============================================ +// WhatsApp Connection & Message Metrics +// ============================================ + +/** + * Increment active connections gauge + */ +export function incrementActiveConnections(): void { + try { + metrics.activeConnections?.inc({}) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Decrement active connections gauge + */ +export function decrementActiveConnections(): void { + try { + metrics.activeConnections?.dec({}) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Set active connections to specific value + */ +export function setActiveConnections(count: number): void { + try { + metrics.activeConnections?.set({}, count) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record a connection attempt + */ +export function recordConnectionAttempt(status: 'success' | 'failure'): void { + try { + metrics.connectionAttempts?.inc({ status }) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record a message sent + */ +export function recordMessageSent(type: string = 'text'): void { + try { + metrics.messagesSent?.inc({ type }) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record a message received + */ +export function recordMessageReceived(type: string = 'text'): void { + try { + metrics.messagesReceived?.inc({ type }) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record a message retry attempt + */ +export function recordMessageRetry(type: string = 'text'): void { + try { + metrics.messageRetries?.inc({ type }) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record a message failure + */ +export function recordMessageFailure(type: string = 'text', reason: string = 'unknown'): void { + try { + metrics.messageFailures?.inc({ type, reason }) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Update messages queued gauge + */ +export function setMessagesQueued(count: number, priority: string = 'normal'): void { + try { + metrics.messagesQueued?.set({ priority }, count) + } catch { + // Metrics not initialized, ignore silently + } +} + +/** + * Record history sync messages + */ +export function recordHistorySyncMessages(count: number = 1): void { + try { + metrics.historySyncMessages?.inc({}, count) + } catch { + // Metrics not initialized, ignore silently + } +} + // ============================================ // Global Instance // ============================================ From b3917299cd0cdb2adaf34b8aa91279c02ab859a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 14:42:22 +0000 Subject: [PATCH 13/14] feat: add message retry and failure metrics tracking Added metrics tracking for: - message_retries_total: Incremented when message retry is attempted - message_failures_total: Incremented when max retries reached or encryption fails Note: messages_queued metric is not applicable as this implementation sends messages directly without an explicit queue system. --- src/Socket/messages-recv.ts | 6 +++++- src/Socket/messages-send.ts | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Socket/messages-recv.ts b/src/Socket/messages-recv.ts index 36deaf09..4be626e9 100644 --- a/src/Socket/messages-recv.ts +++ b/src/Socket/messages-recv.ts @@ -4,7 +4,7 @@ import { randomBytes } from 'crypto' import Long from 'long' import { proto } from '../../WAProto/index.js' import { DEFAULT_CACHE_TTLS, KEY_BUNDLE_TYPE, MIN_PREKEY_COUNT } from '../Defaults' -import { metrics, recordMessageReceived, recordHistorySyncMessages } from '../Utils/prometheus-metrics.js' +import { metrics, recordMessageReceived, recordHistorySyncMessages, recordMessageRetry, recordMessageFailure } from '../Utils/prometheus-metrics.js' import type { GroupParticipant, MessageReceiptType, @@ -409,11 +409,13 @@ export const makeMessagesRecvSocket = (config: SocketConfig) => { if (messageRetryManager.hasExceededMaxRetries(msgId)) { logger.debug({ msgId }, 'reached retry limit with new retry manager, clearing') messageRetryManager.markRetryFailed(msgId) + recordMessageFailure('retry', 'max_retries_reached') return } // Increment retry count using new system const retryCount = messageRetryManager.incrementRetryCount(msgId) + recordMessageRetry('retry') // Use the new retry count for the rest of the logic const key = `${msgId}:${msgKey?.participant}` @@ -425,11 +427,13 @@ export const makeMessagesRecvSocket = (config: SocketConfig) => { if (retryCount >= maxMsgRetryCount) { logger.debug({ retryCount, msgId }, 'reached retry limit, clearing') await msgRetryCache.del(key) + recordMessageFailure('retry', 'max_retries_reached') return } retryCount += 1 await msgRetryCache.set(key, retryCount) + recordMessageRetry('retry') } const key = `${msgId}:${msgKey?.participant}` diff --git a/src/Socket/messages-send.ts b/src/Socket/messages-send.ts index 8e24aedc..ae2be48d 100644 --- a/src/Socket/messages-send.ts +++ b/src/Socket/messages-send.ts @@ -596,6 +596,7 @@ export const makeMessagesSocket = (config: SocketConfig) => { const nodes = (await Promise.all(encryptionPromises)).filter(node => node !== null) as BinaryNode[] if (recipientJids.length > 0 && nodes.length === 0) { + recordMessageFailure('send', 'encryption_failed') throw new Boom('All encryptions failed', { statusCode: 500 }) } From bc7fcca45facc863688f1ceabcdbb14d58a6232f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Jan 2026 15:14:29 +0000 Subject: [PATCH 14/14] fix: initialize metrics with labels to show zero values in Prometheus Metrics with labels only appear in Prometheus output after being incremented at least once. This fix pre-initializes all labeled metrics with zero values so they appear in Grafana dashboards immediately, including buffer_destroyed_total which was showing "No data". --- src/Utils/prometheus-metrics.ts | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/Utils/prometheus-metrics.ts b/src/Utils/prometheus-metrics.ts index 4543d383..c24a43fe 100644 --- a/src/Utils/prometheus-metrics.ts +++ b/src/Utils/prometheus-metrics.ts @@ -2104,12 +2104,59 @@ export function getMetricsManager(config?: Partial): PrometheusMe return globalMetricsManager } +/** + * Initialize metrics with labels to zero values + * This ensures they appear in Prometheus output even before first increment + */ +function initializeMetricsWithLabels(): void { + try { + // Buffer destroyed metric + metrics.bufferDestroyed?.inc({ reason: 'normal', had_pending_flush: 'true' }, 0) + metrics.bufferDestroyed?.inc({ reason: 'normal', had_pending_flush: 'false' }, 0) + metrics.bufferDestroyed?.inc({ reason: 'error', had_pending_flush: 'true' }, 0) + metrics.bufferDestroyed?.inc({ reason: 'error', had_pending_flush: 'false' }, 0) + + // Buffer flushes metric + metrics.bufferFlushes?.inc({ type: 'event', reason: 'normal' }, 0) + metrics.bufferFlushes?.inc({ type: 'event', reason: 'forced' }, 0) + + // Connection metrics + metrics.connectionAttempts?.inc({ status: 'success' }, 0) + metrics.connectionAttempts?.inc({ status: 'failure' }, 0) + metrics.connectionErrors?.inc({ error_type: 'timeout' }, 0) + metrics.connectionErrors?.inc({ error_type: 'closed' }, 0) + metrics.connectionErrors?.inc({ error_type: 'unknown' }, 0) + + // Message metrics + metrics.messagesSent?.inc({ type: 'text' }, 0) + metrics.messagesSent?.inc({ type: 'media' }, 0) + metrics.messagesSent?.inc({ type: 'other' }, 0) + metrics.messagesReceived?.inc({ type: 'text' }, 0) + metrics.messagesReceived?.inc({ type: 'media' }, 0) + metrics.messagesReceived?.inc({ type: 'notification' }, 0) + metrics.messagesReceived?.inc({ type: 'other' }, 0) + metrics.messageRetries?.inc({ type: 'text' }, 0) + metrics.messageRetries?.inc({ type: 'other' }, 0) + metrics.messageFailures?.inc({ type: 'text', reason: 'max_retries' }, 0) + metrics.messageFailures?.inc({ type: 'other', reason: 'max_retries' }, 0) + + // Circuit breaker metric + metrics.circuitBreakerTrips?.inc({ name: 'main' }, 0) + + console.log('[Prometheus] Initialized metrics with labels') + } catch (error) { + console.error('[Prometheus] Error initializing metrics with labels:', error) + } +} + /** * Initialize global metrics (call once at application startup) */ export async function initializeMetrics(config?: Partial): Promise { const manager = getMetricsManager(config) await manager.initialize() + // Initialize metrics with labels to zero values + initializeMetricsWithLabels() return manager }