fix prometheus server
fix prometheus server
This commit is contained in:
@@ -4,7 +4,7 @@ import { randomBytes } from 'crypto'
|
|||||||
import Long from 'long'
|
import Long from 'long'
|
||||||
import { proto } from '../../WAProto/index.js'
|
import { proto } from '../../WAProto/index.js'
|
||||||
import { DEFAULT_CACHE_TTLS, KEY_BUNDLE_TYPE, MIN_PREKEY_COUNT } from '../Defaults'
|
import { DEFAULT_CACHE_TTLS, KEY_BUNDLE_TYPE, MIN_PREKEY_COUNT } from '../Defaults'
|
||||||
import { metrics } from '../Utils/prometheus-metrics.js'
|
import { metrics, recordMessageReceived, recordHistorySyncMessages, recordMessageRetry, recordMessageFailure } from '../Utils/prometheus-metrics.js'
|
||||||
import type {
|
import type {
|
||||||
GroupParticipant,
|
GroupParticipant,
|
||||||
MessageReceiptType,
|
MessageReceiptType,
|
||||||
@@ -409,11 +409,13 @@ export const makeMessagesRecvSocket = (config: SocketConfig) => {
|
|||||||
if (messageRetryManager.hasExceededMaxRetries(msgId)) {
|
if (messageRetryManager.hasExceededMaxRetries(msgId)) {
|
||||||
logger.debug({ msgId }, 'reached retry limit with new retry manager, clearing')
|
logger.debug({ msgId }, 'reached retry limit with new retry manager, clearing')
|
||||||
messageRetryManager.markRetryFailed(msgId)
|
messageRetryManager.markRetryFailed(msgId)
|
||||||
|
recordMessageFailure('retry', 'max_retries_reached')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Increment retry count using new system
|
// Increment retry count using new system
|
||||||
const retryCount = messageRetryManager.incrementRetryCount(msgId)
|
const retryCount = messageRetryManager.incrementRetryCount(msgId)
|
||||||
|
recordMessageRetry('retry')
|
||||||
|
|
||||||
// Use the new retry count for the rest of the logic
|
// Use the new retry count for the rest of the logic
|
||||||
const key = `${msgId}:${msgKey?.participant}`
|
const key = `${msgId}:${msgKey?.participant}`
|
||||||
@@ -425,11 +427,13 @@ export const makeMessagesRecvSocket = (config: SocketConfig) => {
|
|||||||
if (retryCount >= maxMsgRetryCount) {
|
if (retryCount >= maxMsgRetryCount) {
|
||||||
logger.debug({ retryCount, msgId }, 'reached retry limit, clearing')
|
logger.debug({ retryCount, msgId }, 'reached retry limit, clearing')
|
||||||
await msgRetryCache.del(key)
|
await msgRetryCache.del(key)
|
||||||
|
recordMessageFailure('retry', 'max_retries_reached')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
retryCount += 1
|
retryCount += 1
|
||||||
await msgRetryCache.set(key, retryCount)
|
await msgRetryCache.set(key, retryCount)
|
||||||
|
recordMessageRetry('retry')
|
||||||
}
|
}
|
||||||
|
|
||||||
const key = `${msgId}:${msgKey?.participant}`
|
const key = `${msgId}:${msgKey?.participant}`
|
||||||
@@ -1387,6 +1391,18 @@ export const makeMessagesRecvSocket = (config: SocketConfig) => {
|
|||||||
if (msg.key.id && msg.key.remoteJid) {
|
if (msg.key.id && msg.key.remoteJid) {
|
||||||
logMessageReceived(msg.key.id, msg.key.remoteJid)
|
logMessageReceived(msg.key.id, msg.key.remoteJid)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record message received metric
|
||||||
|
const msgContent = msg.message
|
||||||
|
const msgType = msgContent?.conversation ? 'text'
|
||||||
|
: msgContent?.imageMessage ? 'image'
|
||||||
|
: msgContent?.videoMessage ? 'video'
|
||||||
|
: msgContent?.audioMessage ? 'audio'
|
||||||
|
: msgContent?.documentMessage ? 'document'
|
||||||
|
: msgContent?.stickerMessage ? 'sticker'
|
||||||
|
: msgContent?.reactionMessage ? 'reaction'
|
||||||
|
: 'other'
|
||||||
|
recordMessageReceived(msgType)
|
||||||
})
|
})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error({ error, node: binaryNodeToString(node) }, 'error in handling message')
|
logger.error({ error, node: binaryNodeToString(node) }, 'error in handling message')
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ import {
|
|||||||
S_WHATSAPP_NET
|
S_WHATSAPP_NET
|
||||||
} from '../WABinary'
|
} from '../WABinary'
|
||||||
import { USyncQuery, USyncUser } from '../WAUSync'
|
import { USyncQuery, USyncUser } from '../WAUSync'
|
||||||
|
import { recordMessageSent, recordMessageRetry, recordMessageFailure } from '../Utils/prometheus-metrics'
|
||||||
import { makeNewsletterSocket } from './newsletter'
|
import { makeNewsletterSocket } from './newsletter'
|
||||||
|
|
||||||
export const makeMessagesSocket = (config: SocketConfig) => {
|
export const makeMessagesSocket = (config: SocketConfig) => {
|
||||||
@@ -595,6 +596,7 @@ export const makeMessagesSocket = (config: SocketConfig) => {
|
|||||||
const nodes = (await Promise.all(encryptionPromises)).filter(node => node !== null) as BinaryNode[]
|
const nodes = (await Promise.all(encryptionPromises)).filter(node => node !== null) as BinaryNode[]
|
||||||
|
|
||||||
if (recipientJids.length > 0 && nodes.length === 0) {
|
if (recipientJids.length > 0 && nodes.length === 0) {
|
||||||
|
recordMessageFailure('send', 'encryption_failed')
|
||||||
throw new Boom('All encryptions failed', { statusCode: 500 })
|
throw new Boom('All encryptions failed', { statusCode: 500 })
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1038,6 +1040,17 @@ export const makeMessagesSocket = (config: SocketConfig) => {
|
|||||||
// Log with [BAILEYS] prefix
|
// Log with [BAILEYS] prefix
|
||||||
logMessageSent(msgId, destinationJid)
|
logMessageSent(msgId, destinationJid)
|
||||||
|
|
||||||
|
// Record message sent metric
|
||||||
|
const msgType = message.conversation ? 'text'
|
||||||
|
: message.imageMessage ? 'image'
|
||||||
|
: message.videoMessage ? 'video'
|
||||||
|
: message.audioMessage ? 'audio'
|
||||||
|
: message.documentMessage ? 'document'
|
||||||
|
: message.stickerMessage ? 'sticker'
|
||||||
|
: message.reactionMessage ? 'reaction'
|
||||||
|
: 'other'
|
||||||
|
recordMessageSent(msgType)
|
||||||
|
|
||||||
// Add message to retry cache if enabled
|
// Add message to retry cache if enabled
|
||||||
if (messageRetryManager && !participant) {
|
if (messageRetryManager && !participant) {
|
||||||
messageRetryManager.addRecentMessage(destinationJid, msgId, message)
|
messageRetryManager.addRecentMessage(destinationJid, msgId, message)
|
||||||
|
|||||||
@@ -54,6 +54,12 @@ import {
|
|||||||
jidEncode,
|
jidEncode,
|
||||||
S_WHATSAPP_NET
|
S_WHATSAPP_NET
|
||||||
} from '../WABinary'
|
} from '../WABinary'
|
||||||
|
import {
|
||||||
|
recordConnectionError,
|
||||||
|
recordConnectionAttempt,
|
||||||
|
incrementActiveConnections,
|
||||||
|
decrementActiveConnections
|
||||||
|
} from '../Utils/prometheus-metrics'
|
||||||
import { BinaryInfo } from '../WAM/BinaryInfo.js'
|
import { BinaryInfo } from '../WAM/BinaryInfo.js'
|
||||||
import { USyncQuery, USyncUser } from '../WAUSync/'
|
import { USyncQuery, USyncUser } from '../WAUSync/'
|
||||||
import { WebSocketClient } from './Client'
|
import { WebSocketClient } from './Client'
|
||||||
@@ -730,6 +736,45 @@ export const makeSocket = (config: SocketConfig) => {
|
|||||||
closed = true
|
closed = true
|
||||||
logger.info({ trace: error?.stack }, error ? 'connection errored' : 'connection closed')
|
logger.info({ trace: error?.stack }, error ? 'connection errored' : 'connection closed')
|
||||||
|
|
||||||
|
// Record connection error metric
|
||||||
|
if (error) {
|
||||||
|
const statusCode = (error as Boom)?.output?.statusCode || 0
|
||||||
|
let errorType = 'unknown'
|
||||||
|
switch (statusCode) {
|
||||||
|
case DisconnectReason.connectionClosed:
|
||||||
|
errorType = 'connection_closed'
|
||||||
|
break
|
||||||
|
case DisconnectReason.connectionLost:
|
||||||
|
errorType = 'connection_lost'
|
||||||
|
break
|
||||||
|
case DisconnectReason.connectionReplaced:
|
||||||
|
errorType = 'connection_replaced'
|
||||||
|
break
|
||||||
|
case DisconnectReason.timedOut:
|
||||||
|
errorType = 'timed_out'
|
||||||
|
break
|
||||||
|
case DisconnectReason.loggedOut:
|
||||||
|
errorType = 'logged_out'
|
||||||
|
break
|
||||||
|
case DisconnectReason.badSession:
|
||||||
|
errorType = 'bad_session'
|
||||||
|
break
|
||||||
|
case DisconnectReason.restartRequired:
|
||||||
|
errorType = 'restart_required'
|
||||||
|
break
|
||||||
|
case DisconnectReason.multideviceMismatch:
|
||||||
|
errorType = 'multidevice_mismatch'
|
||||||
|
break
|
||||||
|
default:
|
||||||
|
errorType = `error_${statusCode}`
|
||||||
|
}
|
||||||
|
recordConnectionError(errorType)
|
||||||
|
recordConnectionAttempt('failure')
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrement active connections
|
||||||
|
decrementActiveConnections()
|
||||||
|
|
||||||
clearInterval(keepAliveReq)
|
clearInterval(keepAliveReq)
|
||||||
clearTimeout(qrTimer)
|
clearTimeout(qrTimer)
|
||||||
|
|
||||||
@@ -1044,6 +1089,10 @@ export const makeSocket = (config: SocketConfig) => {
|
|||||||
|
|
||||||
ev.emit('connection.update', { connection: 'open' })
|
ev.emit('connection.update', { connection: 'open' })
|
||||||
|
|
||||||
|
// Record successful connection metrics
|
||||||
|
recordConnectionAttempt('success')
|
||||||
|
incrementActiveConnections()
|
||||||
|
|
||||||
if (node.attrs.lid && authState.creds.me?.id) {
|
if (node.attrs.lid && authState.creds.me?.id) {
|
||||||
const myLID = node.attrs.lid
|
const myLID = node.attrs.lid
|
||||||
process.nextTick(async () => {
|
process.nextTick(async () => {
|
||||||
|
|||||||
@@ -380,6 +380,11 @@ export class CircuitBreaker extends EventEmitter {
|
|||||||
this.options.onOpen()
|
this.options.onOpen()
|
||||||
this.emit('open')
|
this.emit('open')
|
||||||
|
|
||||||
|
// Record circuit breaker trip metric
|
||||||
|
if (this.options.collectMetrics) {
|
||||||
|
metrics.circuitBreakerTrips?.inc({ name: this.options.name })
|
||||||
|
}
|
||||||
|
|
||||||
// Schedule transition to HALF_OPEN
|
// Schedule transition to HALF_OPEN
|
||||||
this.resetTimer = setTimeout(() => {
|
this.resetTimer = setTimeout(() => {
|
||||||
this.transitionTo('half-open')
|
this.transitionTo('half-open')
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ export function loadBufferConfig(): BufferConfig {
|
|||||||
maxBufferSize: parseInt(process.env.BAILEYS_BUFFER_MAX_SIZE || '5000', 10),
|
maxBufferSize: parseInt(process.env.BAILEYS_BUFFER_MAX_SIZE || '5000', 10),
|
||||||
flushDebounceMs: parseInt(process.env.BAILEYS_BUFFER_FLUSH_DEBOUNCE_MS || '100', 10),
|
flushDebounceMs: parseInt(process.env.BAILEYS_BUFFER_FLUSH_DEBOUNCE_MS || '100', 10),
|
||||||
enableAdaptiveTimeout: process.env.BAILEYS_BUFFER_ADAPTIVE_TIMEOUT !== 'false',
|
enableAdaptiveTimeout: process.env.BAILEYS_BUFFER_ADAPTIVE_TIMEOUT !== 'false',
|
||||||
enableMetrics: process.env.BAILEYS_BUFFER_METRICS === 'true',
|
enableMetrics: process.env.BAILEYS_BUFFER_METRICS === 'true' || process.env.BAILEYS_PROMETHEUS_ENABLED === 'true',
|
||||||
lruCleanupRatio: parseFloat(process.env.BAILEYS_BUFFER_LRU_CLEANUP_RATIO || '0.2'),
|
lruCleanupRatio: parseFloat(process.env.BAILEYS_BUFFER_LRU_CLEANUP_RATIO || '0.2'),
|
||||||
bufferWarnThreshold: parseFloat(process.env.BAILEYS_BUFFER_WARN_THRESHOLD || '0.8')
|
bufferWarnThreshold: parseFloat(process.env.BAILEYS_BUFFER_WARN_THRESHOLD || '0.8')
|
||||||
}
|
}
|
||||||
@@ -329,6 +329,31 @@ class AdaptiveTimeoutCalculator {
|
|||||||
}
|
}
|
||||||
return 'balanced'
|
return 'balanced'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current event rate in events per second
|
||||||
|
*/
|
||||||
|
getEventRate(): number {
|
||||||
|
if (this.eventTimestamps.length < 2) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
const oldest = this.eventTimestamps[0]!
|
||||||
|
const newest = this.eventTimestamps[this.eventTimestamps.length - 1]!
|
||||||
|
const timeSpan = newest - oldest
|
||||||
|
if (timeSpan === 0) return 0
|
||||||
|
return (this.eventTimestamps.length / timeSpan) * 1000
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if system is healthy based on current mode
|
||||||
|
* Conservative mode with low event rate is healthy
|
||||||
|
* Aggressive mode might indicate high load
|
||||||
|
*/
|
||||||
|
isHealthy(): boolean {
|
||||||
|
const mode = this.getMode()
|
||||||
|
// System is considered healthy if not in aggressive mode (high load)
|
||||||
|
return mode !== 'aggressive'
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -410,9 +435,9 @@ export const makeEventBuffer = (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const recordFlushMetrics = (eventCount: number, forced: boolean) => {
|
const recordFlushMetrics = (eventCount: number, forced: boolean, cacheSize: number) => {
|
||||||
if (metricsModule) {
|
if (metricsModule) {
|
||||||
metricsModule.recordBufferFlush(eventCount, forced)
|
metricsModule.recordBufferFlush(eventCount, forced, cacheSize)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -449,6 +474,10 @@ export const makeEventBuffer = (
|
|||||||
currentSize: currentEventCount,
|
currentSize: currentEventCount,
|
||||||
maxSize: config.maxBufferSize
|
maxSize: config.maxBufferSize
|
||||||
})
|
})
|
||||||
|
// Record overflow metric
|
||||||
|
if (metricsModule) {
|
||||||
|
metricsModule.recordBufferOverflow()
|
||||||
|
}
|
||||||
flush(true)
|
flush(true)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -482,6 +511,10 @@ export const makeEventBuffer = (
|
|||||||
removed: removed.length,
|
removed: removed.length,
|
||||||
remaining: historyCache.size
|
remaining: historyCache.size
|
||||||
})
|
})
|
||||||
|
// Record metrics for cache cleanup
|
||||||
|
if (metricsModule) {
|
||||||
|
metricsModule.recordCacheCleanup(removed.length)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -553,7 +586,12 @@ export const makeEventBuffer = (
|
|||||||
stats.historyCacheSize = historyCache.size
|
stats.historyCacheSize = historyCache.size
|
||||||
|
|
||||||
// Record metrics
|
// Record metrics
|
||||||
recordFlushMetrics(eventCount, force)
|
recordFlushMetrics(eventCount, force, historyCache.size)
|
||||||
|
|
||||||
|
// Update adaptive metrics
|
||||||
|
if (config.enableAdaptiveTimeout && metricsModule) {
|
||||||
|
metricsModule.updateAdaptiveMetrics(adaptiveTimeout.getEventRate(), adaptiveTimeout.isHealthy())
|
||||||
|
}
|
||||||
|
|
||||||
// Log with [BAILEYS] prefix - use getMode() to avoid duplicating mode calculation logic
|
// Log with [BAILEYS] prefix - use getMode() to avoid duplicating mode calculation logic
|
||||||
const flushDuration = Date.now() - flushStartTime
|
const flushDuration = Date.now() - flushStartTime
|
||||||
@@ -599,11 +637,16 @@ export const makeEventBuffer = (
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger.debug('Destroying event buffer')
|
logger.debug('Destroying event buffer')
|
||||||
|
const hadPendingFlush = isBuffering
|
||||||
destroyed = true
|
destroyed = true
|
||||||
|
|
||||||
// Flush any remaining events
|
// Flush any remaining events
|
||||||
if (isBuffering) {
|
if (hadPendingFlush) {
|
||||||
flush(true)
|
flush(true)
|
||||||
|
// Record final flush metric
|
||||||
|
if (metricsModule) {
|
||||||
|
metricsModule.recordBufferFinalFlush()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear all timers
|
// Clear all timers
|
||||||
@@ -616,6 +659,11 @@ export const makeEventBuffer = (
|
|||||||
// Remove all event listeners
|
// Remove all event listeners
|
||||||
ev.removeAllListeners()
|
ev.removeAllListeners()
|
||||||
|
|
||||||
|
// Record buffer destroyed metric
|
||||||
|
if (metricsModule) {
|
||||||
|
metricsModule.recordBufferDestroyed('normal', hadPendingFlush)
|
||||||
|
}
|
||||||
|
|
||||||
logger.debug('Event buffer destroyed successfully')
|
logger.debug('Event buffer destroyed successfully')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import type {
|
|||||||
WAMessage,
|
WAMessage,
|
||||||
WAMessageKey
|
WAMessageKey
|
||||||
} from '../Types'
|
} from '../Types'
|
||||||
import { metrics } from './prometheus-metrics.js'
|
import { metrics, recordHistorySyncMessages } from './prometheus-metrics.js'
|
||||||
import { WAMessageStubType } from '../Types'
|
import { WAMessageStubType } from '../Types'
|
||||||
import { getContentType, normalizeMessageContent } from '../Utils/messages'
|
import { getContentType, normalizeMessageContent } from '../Utils/messages'
|
||||||
import {
|
import {
|
||||||
@@ -353,6 +353,11 @@ const processMessage = async (
|
|||||||
isLatest: histNotification.syncType !== proto.HistorySync.HistorySyncType.ON_DEMAND ? isLatest : undefined,
|
isLatest: histNotification.syncType !== proto.HistorySync.HistorySyncType.ON_DEMAND ? isLatest : undefined,
|
||||||
peerDataRequestSessionId: histNotification.peerDataRequestSessionId
|
peerDataRequestSessionId: histNotification.peerDataRequestSessionId
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Record history sync metrics
|
||||||
|
if (data.messages?.length) {
|
||||||
|
recordHistorySyncMessages(data.messages.length)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
break
|
break
|
||||||
|
|||||||
+310
-39
@@ -95,6 +95,31 @@ export function getConfiguredPrefix(): string {
|
|||||||
return configuredPrefix
|
return configuredPrefix
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a metric with the given name already exists in the custom registry
|
||||||
|
*/
|
||||||
|
function metricExists(name: string): boolean {
|
||||||
|
const fullName = getFullMetricName(name)
|
||||||
|
try {
|
||||||
|
const existing = customRegistry.getSingleMetric(fullName)
|
||||||
|
return existing !== undefined
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get an existing metric from the custom registry
|
||||||
|
*/
|
||||||
|
function getExistingMetric<T>(name: string): T | undefined {
|
||||||
|
const fullName = getFullMetricName(name)
|
||||||
|
try {
|
||||||
|
return customRegistry.getSingleMetric(fullName) as T | undefined
|
||||||
|
} catch {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// Configuration
|
// Configuration
|
||||||
// ============================================
|
// ============================================
|
||||||
@@ -1051,9 +1076,11 @@ export class SystemMetricsCollector {
|
|||||||
new Gauge('system_load_average', 'System load average', ['period'])
|
new Gauge('system_load_average', 'System load average', ['period'])
|
||||||
)
|
)
|
||||||
|
|
||||||
// Event loop lag
|
// Event loop lag - use different name to avoid conflict with collectDefaultMetrics
|
||||||
|
// prom-client's collectDefaultMetrics creates nodejs_eventloop_lag_seconds
|
||||||
|
// We use system_eventloop_lag_seconds to avoid duplicate registration
|
||||||
this.eventLoopLag = registry.register(
|
this.eventLoopLag = registry.register(
|
||||||
new Histogram('nodejs_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS)
|
new Histogram('system_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS)
|
||||||
)
|
)
|
||||||
|
|
||||||
// Initialize CPU baseline
|
// Initialize CPU baseline
|
||||||
@@ -1364,6 +1391,12 @@ export const metrics = {
|
|||||||
connectionLatency: baileysMetrics.register(
|
connectionLatency: baileysMetrics.register(
|
||||||
new Histogram('connection_latency_ms', 'Connection establishment latency in ms', [], [100, 250, 500, 1000, 2500, 5000, 10000])
|
new Histogram('connection_latency_ms', 'Connection establishment latency in ms', [], [100, 250, 500, 1000, 2500, 5000, 10000])
|
||||||
),
|
),
|
||||||
|
activeConnections: baileysMetrics.register(
|
||||||
|
new Gauge('active_connections', 'Number of active WhatsApp connections')
|
||||||
|
),
|
||||||
|
connectionErrors: baileysMetrics.register(
|
||||||
|
new Counter('connection_errors_total', 'Total connection errors', ['error_type'])
|
||||||
|
),
|
||||||
|
|
||||||
// ========== Message Metrics ==========
|
// ========== Message Metrics ==========
|
||||||
messagesSent: baileysMetrics.register(
|
messagesSent: baileysMetrics.register(
|
||||||
@@ -1445,6 +1478,18 @@ export const metrics = {
|
|||||||
eventsProcessed: baileysMetrics.register(
|
eventsProcessed: baileysMetrics.register(
|
||||||
new Counter('events_processed_total', 'Total events processed from buffer', ['event_type'])
|
new Counter('events_processed_total', 'Total events processed from buffer', ['event_type'])
|
||||||
),
|
),
|
||||||
|
bufferDestroyed: baileysMetrics.register(
|
||||||
|
new Counter('buffer_destroyed_total', 'Total buffers destroyed', ['reason', 'had_pending_flush'])
|
||||||
|
),
|
||||||
|
bufferFinalFlush: baileysMetrics.register(
|
||||||
|
new Counter('buffer_final_flush_total', 'Total final flushes during destruction')
|
||||||
|
),
|
||||||
|
bufferCacheCleanup: baileysMetrics.register(
|
||||||
|
new Counter('buffer_cache_cleanup_total', 'Total cache cleanup operations')
|
||||||
|
),
|
||||||
|
bufferCacheSize: baileysMetrics.register(
|
||||||
|
new Gauge('buffer_cache_size', 'Current buffer cache size')
|
||||||
|
),
|
||||||
|
|
||||||
// ========== Adaptive Flush Metrics ==========
|
// ========== Adaptive Flush Metrics ==========
|
||||||
adaptiveFlushInterval: baileysMetrics.register(
|
adaptiveFlushInterval: baileysMetrics.register(
|
||||||
@@ -1462,6 +1507,12 @@ export const metrics = {
|
|||||||
adaptiveFlushEfficiency: baileysMetrics.register(
|
adaptiveFlushEfficiency: baileysMetrics.register(
|
||||||
new Gauge('adaptive_flush_efficiency_percent', 'Flush efficiency percentage')
|
new Gauge('adaptive_flush_efficiency_percent', 'Flush efficiency percentage')
|
||||||
),
|
),
|
||||||
|
adaptiveHealthStatus: baileysMetrics.register(
|
||||||
|
new Gauge('adaptive_health_status', 'Adaptive system health status (0=unhealthy, 1=healthy)')
|
||||||
|
),
|
||||||
|
adaptiveEventRate: baileysMetrics.register(
|
||||||
|
new Gauge('adaptive_event_rate', 'Current event rate per second')
|
||||||
|
),
|
||||||
|
|
||||||
// ========== Error Metrics ==========
|
// ========== Error Metrics ==========
|
||||||
errors: baileysMetrics.register(
|
errors: baileysMetrics.register(
|
||||||
@@ -1747,31 +1798,19 @@ export function trackOperation(
|
|||||||
// Event Buffer Metrics Integration
|
// Event Buffer Metrics Integration
|
||||||
// ============================================
|
// ============================================
|
||||||
|
|
||||||
// Event buffer metrics (lazy initialized)
|
// Event buffer metrics (lazy initialized) - for detailed buffer tracking
|
||||||
|
// Note: bufferFlushes is NOT here - use metrics.bufferFlushes from main metrics object
|
||||||
let eventBufferMetrics: {
|
let eventBufferMetrics: {
|
||||||
eventsBuffered: Counter | null
|
|
||||||
bufferFlushes: Counter | null
|
|
||||||
bufferFlushEvents: Histogram | null
|
bufferFlushEvents: Histogram | null
|
||||||
bufferCurrentSize: Gauge | null
|
bufferCurrentSize: Gauge | null
|
||||||
bufferPeakSize: Gauge | null
|
bufferPeakSize: Gauge | null
|
||||||
bufferHistoryCacheSize: Gauge | null
|
bufferHistoryCacheSize: Gauge | null
|
||||||
bufferOverflows: Counter | null
|
|
||||||
bufferLruCleanups: Counter | null
|
bufferLruCleanups: Counter | null
|
||||||
} | null = null
|
} | null = null
|
||||||
|
|
||||||
function getEventBufferMetrics() {
|
function getEventBufferMetrics() {
|
||||||
if (!eventBufferMetrics) {
|
if (!eventBufferMetrics) {
|
||||||
eventBufferMetrics = {
|
eventBufferMetrics = {
|
||||||
eventsBuffered: new Counter(
|
|
||||||
'events_buffered_total',
|
|
||||||
'Total number of events buffered',
|
|
||||||
['event_type']
|
|
||||||
),
|
|
||||||
bufferFlushes: new Counter(
|
|
||||||
'buffer_flushes_total',
|
|
||||||
'Total number of buffer flushes',
|
|
||||||
['forced']
|
|
||||||
),
|
|
||||||
bufferFlushEvents: new Histogram(
|
bufferFlushEvents: new Histogram(
|
||||||
'buffer_flush_events',
|
'buffer_flush_events',
|
||||||
'Number of events per buffer flush',
|
'Number of events per buffer flush',
|
||||||
@@ -1790,24 +1829,11 @@ function getEventBufferMetrics() {
|
|||||||
'buffer_history_cache_size',
|
'buffer_history_cache_size',
|
||||||
'Current size of history cache'
|
'Current size of history cache'
|
||||||
),
|
),
|
||||||
bufferOverflows: new Counter(
|
|
||||||
'buffer_overflows_total',
|
|
||||||
'Total number of buffer overflows detected'
|
|
||||||
),
|
|
||||||
bufferLruCleanups: new Counter(
|
bufferLruCleanups: new Counter(
|
||||||
'buffer_lru_cleanups_total',
|
'buffer_lru_cleanups_total',
|
||||||
'Total number of LRU cache cleanups performed'
|
'Total number of LRU cache cleanups performed'
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
// Register all metrics
|
|
||||||
if (eventBufferMetrics.eventsBuffered) baileysMetrics.register(eventBufferMetrics.eventsBuffered)
|
|
||||||
if (eventBufferMetrics.bufferFlushes) baileysMetrics.register(eventBufferMetrics.bufferFlushes)
|
|
||||||
if (eventBufferMetrics.bufferFlushEvents) baileysMetrics.register(eventBufferMetrics.bufferFlushEvents)
|
|
||||||
if (eventBufferMetrics.bufferCurrentSize) baileysMetrics.register(eventBufferMetrics.bufferCurrentSize)
|
|
||||||
if (eventBufferMetrics.bufferPeakSize) baileysMetrics.register(eventBufferMetrics.bufferPeakSize)
|
|
||||||
if (eventBufferMetrics.bufferHistoryCacheSize) baileysMetrics.register(eventBufferMetrics.bufferHistoryCacheSize)
|
|
||||||
if (eventBufferMetrics.bufferOverflows) baileysMetrics.register(eventBufferMetrics.bufferOverflows)
|
|
||||||
if (eventBufferMetrics.bufferLruCleanups) baileysMetrics.register(eventBufferMetrics.bufferLruCleanups)
|
|
||||||
}
|
}
|
||||||
return eventBufferMetrics
|
return eventBufferMetrics
|
||||||
}
|
}
|
||||||
@@ -1818,7 +1844,7 @@ function getEventBufferMetrics() {
|
|||||||
*/
|
*/
|
||||||
export function recordEventBuffered(eventType: string, count: number = 1): void {
|
export function recordEventBuffered(eventType: string, count: number = 1): void {
|
||||||
try {
|
try {
|
||||||
const metrics = getEventBufferMetrics()
|
// Use the main metrics object which has eventsBuffered with label ['event_type']
|
||||||
metrics.eventsBuffered?.inc({ event_type: eventType }, count)
|
metrics.eventsBuffered?.inc({ event_type: eventType }, count)
|
||||||
} catch {
|
} catch {
|
||||||
// Metrics not initialized, ignore silently
|
// Metrics not initialized, ignore silently
|
||||||
@@ -1829,12 +1855,23 @@ export function recordEventBuffered(eventType: string, count: number = 1): void
|
|||||||
* Record a buffer flush operation
|
* Record a buffer flush operation
|
||||||
* Used by event-buffer.ts for metrics integration
|
* Used by event-buffer.ts for metrics integration
|
||||||
*/
|
*/
|
||||||
export function recordBufferFlush(eventCount: number, forced: boolean): void {
|
export function recordBufferFlush(eventCount: number, forced: boolean, historyCacheSize?: number): void {
|
||||||
try {
|
try {
|
||||||
const metrics = getEventBufferMetrics()
|
// Use the main metrics object which has the correct labels ['type', 'reason']
|
||||||
metrics.bufferFlushes?.inc({ forced: forced ? 'true' : 'false' })
|
metrics.bufferFlushes?.inc({ type: 'event', reason: forced ? 'forced' : 'normal' })
|
||||||
metrics.bufferFlushEvents?.observe({}, eventCount)
|
|
||||||
metrics.bufferCurrentSize?.set({}, 0) // Reset after flush
|
// Update buffer cache size if provided
|
||||||
|
if (typeof historyCacheSize === 'number') {
|
||||||
|
metrics.bufferCacheSize?.set({}, historyCacheSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also update the lazy-loaded event buffer metrics for detailed tracking
|
||||||
|
const ebMetrics = getEventBufferMetrics()
|
||||||
|
ebMetrics.bufferFlushEvents?.observe({}, eventCount)
|
||||||
|
ebMetrics.bufferCurrentSize?.set({}, 0) // Reset after flush
|
||||||
|
if (typeof historyCacheSize === 'number') {
|
||||||
|
ebMetrics.bufferHistoryCacheSize?.set({}, historyCacheSize)
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Metrics not initialized, ignore silently
|
// Metrics not initialized, ignore silently
|
||||||
}
|
}
|
||||||
@@ -1852,10 +1889,197 @@ export function updateBufferStatistics(stats: {
|
|||||||
lruCleanups: number
|
lruCleanups: number
|
||||||
}): void {
|
}): void {
|
||||||
try {
|
try {
|
||||||
const metrics = getEventBufferMetrics()
|
const ebMetrics = getEventBufferMetrics()
|
||||||
metrics.bufferCurrentSize?.set({}, stats.currentSize)
|
ebMetrics.bufferCurrentSize?.set({}, stats.currentSize)
|
||||||
metrics.bufferPeakSize?.set({}, stats.peakSize)
|
ebMetrics.bufferPeakSize?.set({}, stats.peakSize)
|
||||||
metrics.bufferHistoryCacheSize?.set({}, stats.historyCacheSize)
|
ebMetrics.bufferHistoryCacheSize?.set({}, stats.historyCacheSize)
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a cache cleanup operation
|
||||||
|
* Used by event-buffer.ts when LRU cleanup is performed
|
||||||
|
*/
|
||||||
|
export function recordCacheCleanup(removedCount: number): void {
|
||||||
|
try {
|
||||||
|
metrics.bufferCacheCleanup?.inc({})
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a buffer overflow event
|
||||||
|
* Used by event-buffer.ts when buffer exceeds max size
|
||||||
|
*/
|
||||||
|
export function recordBufferOverflow(): void {
|
||||||
|
try {
|
||||||
|
metrics.bufferOverflows?.inc({ type: 'event' })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a connection error
|
||||||
|
* Used by socket.ts when connection fails
|
||||||
|
*/
|
||||||
|
export function recordConnectionError(errorType: string): void {
|
||||||
|
try {
|
||||||
|
metrics.connectionErrors?.inc({ error_type: errorType })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record buffer destruction
|
||||||
|
* Used by event-buffer.ts when buffer is destroyed
|
||||||
|
*/
|
||||||
|
export function recordBufferDestroyed(reason: string, hadPendingFlush: boolean): void {
|
||||||
|
try {
|
||||||
|
metrics.bufferDestroyed?.inc({ reason, had_pending_flush: hadPendingFlush ? 'true' : 'false' })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record final flush during buffer destruction
|
||||||
|
* Used by event-buffer.ts when buffer flushes remaining events on destroy
|
||||||
|
*/
|
||||||
|
export function recordBufferFinalFlush(): void {
|
||||||
|
try {
|
||||||
|
metrics.bufferFinalFlush?.inc({})
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update adaptive system metrics
|
||||||
|
* Used by event-buffer.ts to report adaptive timeout health and event rate
|
||||||
|
*/
|
||||||
|
export function updateAdaptiveMetrics(eventRate: number, isHealthy: boolean): void {
|
||||||
|
try {
|
||||||
|
metrics.adaptiveEventRate?.set({}, eventRate)
|
||||||
|
metrics.adaptiveHealthStatus?.set({}, isHealthy ? 1 : 0)
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// WhatsApp Connection & Message Metrics
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increment active connections gauge
|
||||||
|
*/
|
||||||
|
export function incrementActiveConnections(): void {
|
||||||
|
try {
|
||||||
|
metrics.activeConnections?.inc({})
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decrement active connections gauge
|
||||||
|
*/
|
||||||
|
export function decrementActiveConnections(): void {
|
||||||
|
try {
|
||||||
|
metrics.activeConnections?.dec({})
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set active connections to specific value
|
||||||
|
*/
|
||||||
|
export function setActiveConnections(count: number): void {
|
||||||
|
try {
|
||||||
|
metrics.activeConnections?.set({}, count)
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a connection attempt
|
||||||
|
*/
|
||||||
|
export function recordConnectionAttempt(status: 'success' | 'failure'): void {
|
||||||
|
try {
|
||||||
|
metrics.connectionAttempts?.inc({ status })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a message sent
|
||||||
|
*/
|
||||||
|
export function recordMessageSent(type: string = 'text'): void {
|
||||||
|
try {
|
||||||
|
metrics.messagesSent?.inc({ type })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a message received
|
||||||
|
*/
|
||||||
|
export function recordMessageReceived(type: string = 'text'): void {
|
||||||
|
try {
|
||||||
|
metrics.messagesReceived?.inc({ type })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a message retry attempt
|
||||||
|
*/
|
||||||
|
export function recordMessageRetry(type: string = 'text'): void {
|
||||||
|
try {
|
||||||
|
metrics.messageRetries?.inc({ type })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record a message failure
|
||||||
|
*/
|
||||||
|
export function recordMessageFailure(type: string = 'text', reason: string = 'unknown'): void {
|
||||||
|
try {
|
||||||
|
metrics.messageFailures?.inc({ type, reason })
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update messages queued gauge
|
||||||
|
*/
|
||||||
|
export function setMessagesQueued(count: number, priority: string = 'normal'): void {
|
||||||
|
try {
|
||||||
|
metrics.messagesQueued?.set({ priority }, count)
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record history sync messages
|
||||||
|
*/
|
||||||
|
export function recordHistorySyncMessages(count: number = 1): void {
|
||||||
|
try {
|
||||||
|
metrics.historySyncMessages?.inc({}, count)
|
||||||
} catch {
|
} catch {
|
||||||
// Metrics not initialized, ignore silently
|
// Metrics not initialized, ignore silently
|
||||||
}
|
}
|
||||||
@@ -1880,12 +2104,59 @@ export function getMetricsManager(config?: Partial<MetricsConfig>): PrometheusMe
|
|||||||
return globalMetricsManager
|
return globalMetricsManager
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize metrics with labels to zero values
|
||||||
|
* This ensures they appear in Prometheus output even before first increment
|
||||||
|
*/
|
||||||
|
function initializeMetricsWithLabels(): void {
|
||||||
|
try {
|
||||||
|
// Buffer destroyed metric
|
||||||
|
metrics.bufferDestroyed?.inc({ reason: 'normal', had_pending_flush: 'true' }, 0)
|
||||||
|
metrics.bufferDestroyed?.inc({ reason: 'normal', had_pending_flush: 'false' }, 0)
|
||||||
|
metrics.bufferDestroyed?.inc({ reason: 'error', had_pending_flush: 'true' }, 0)
|
||||||
|
metrics.bufferDestroyed?.inc({ reason: 'error', had_pending_flush: 'false' }, 0)
|
||||||
|
|
||||||
|
// Buffer flushes metric
|
||||||
|
metrics.bufferFlushes?.inc({ type: 'event', reason: 'normal' }, 0)
|
||||||
|
metrics.bufferFlushes?.inc({ type: 'event', reason: 'forced' }, 0)
|
||||||
|
|
||||||
|
// Connection metrics
|
||||||
|
metrics.connectionAttempts?.inc({ status: 'success' }, 0)
|
||||||
|
metrics.connectionAttempts?.inc({ status: 'failure' }, 0)
|
||||||
|
metrics.connectionErrors?.inc({ error_type: 'timeout' }, 0)
|
||||||
|
metrics.connectionErrors?.inc({ error_type: 'closed' }, 0)
|
||||||
|
metrics.connectionErrors?.inc({ error_type: 'unknown' }, 0)
|
||||||
|
|
||||||
|
// Message metrics
|
||||||
|
metrics.messagesSent?.inc({ type: 'text' }, 0)
|
||||||
|
metrics.messagesSent?.inc({ type: 'media' }, 0)
|
||||||
|
metrics.messagesSent?.inc({ type: 'other' }, 0)
|
||||||
|
metrics.messagesReceived?.inc({ type: 'text' }, 0)
|
||||||
|
metrics.messagesReceived?.inc({ type: 'media' }, 0)
|
||||||
|
metrics.messagesReceived?.inc({ type: 'notification' }, 0)
|
||||||
|
metrics.messagesReceived?.inc({ type: 'other' }, 0)
|
||||||
|
metrics.messageRetries?.inc({ type: 'text' }, 0)
|
||||||
|
metrics.messageRetries?.inc({ type: 'other' }, 0)
|
||||||
|
metrics.messageFailures?.inc({ type: 'text', reason: 'max_retries' }, 0)
|
||||||
|
metrics.messageFailures?.inc({ type: 'other', reason: 'max_retries' }, 0)
|
||||||
|
|
||||||
|
// Circuit breaker metric
|
||||||
|
metrics.circuitBreakerTrips?.inc({ name: 'main' }, 0)
|
||||||
|
|
||||||
|
console.log('[Prometheus] Initialized metrics with labels')
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[Prometheus] Error initializing metrics with labels:', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize global metrics (call once at application startup)
|
* Initialize global metrics (call once at application startup)
|
||||||
*/
|
*/
|
||||||
export async function initializeMetrics(config?: Partial<MetricsConfig>): Promise<PrometheusMetricsManager> {
|
export async function initializeMetrics(config?: Partial<MetricsConfig>): Promise<PrometheusMetricsManager> {
|
||||||
const manager = getMetricsManager(config)
|
const manager = getMetricsManager(config)
|
||||||
await manager.initialize()
|
await manager.initialize()
|
||||||
|
// Initialize metrics with labels to zero values
|
||||||
|
initializeMetricsWithLabels()
|
||||||
return manager
|
return manager
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user