feat: Enterprise-grade observability with Circuit Breaker and Prometheus Metrics
## Summary
- Integrate Circuit Breaker pattern into Socket operations for resilience
- Enhance Prometheus Metrics to enterprise-grade (60+ metrics)
- Add environment variable configuration for metrics
## Changes
### Circuit Breaker Integration
- 3 Circuit Breakers: `query`, `connection`, `preKey`
- Automatic failure detection and recovery
- Configurable thresholds and timeouts
- `getCircuitBreakerStats()` and `resetCircuitBreakers()` utilities
### Prometheus Metrics Enhancement
- Increased from 18 to 60+ metrics
- HTTP server endpoint at `:9092/metrics`
- System metrics (CPU, memory, load average)
- Event Buffer, Cache, Retry, Socket metrics
- Environment configuration via `BAILEYS_PROMETHEUS_*`
### Configuration
```env
BAILEYS_PROMETHEUS_ENABLED=true
BAILEYS_PROMETHEUS_PORT=9092
BAILEYS_PROMETHEUS_PREFIX=baileys
BAILEYS_PROMETHEUS_LABELS={"environment":"production","service":"whatsapp-api"}
This commit is contained in:
Renato Alcara
+756
-28
@@ -1,21 +1,86 @@
|
||||
/**
|
||||
* Prometheus Metrics Exposition
|
||||
* Prometheus Metrics Exposition - Enterprise Grade
|
||||
*
|
||||
* Provides:
|
||||
* - Counters for event counting
|
||||
* - Gauges for instantaneous values
|
||||
* - Histograms for value distribution
|
||||
* - Summaries for percentiles
|
||||
* - /metrics endpoint ready for scraping
|
||||
* - HTTP /metrics endpoint ready for scraping
|
||||
* - Dynamic labels
|
||||
* - Baileys events integration
|
||||
* - Event buffer metrics
|
||||
* - System metrics (memory, CPU, uptime)
|
||||
* - Circuit breaker metrics
|
||||
* - Environment variable configuration
|
||||
*
|
||||
* Note: Works standalone without prom-client,
|
||||
* but can be integrated with prom-client if available.
|
||||
* Configuration via environment variables:
|
||||
* - BAILEYS_PROMETHEUS_ENABLED: Enable/disable metrics (default: false)
|
||||
* - BAILEYS_PROMETHEUS_PORT: Port for HTTP metrics server (default: 9092)
|
||||
* - BAILEYS_PROMETHEUS_PATH: Path for metrics endpoint (default: /metrics)
|
||||
* - BAILEYS_PROMETHEUS_PREFIX: Prefix for all metrics (default: baileys)
|
||||
* - BAILEYS_PROMETHEUS_LABELS: JSON string with default labels (e.g. {"environment":"production"})
|
||||
* - BAILEYS_PROMETHEUS_COLLECT_DEFAULT: Collect default Node.js metrics (default: true)
|
||||
*
|
||||
* @module Utils/prometheus-metrics
|
||||
*/
|
||||
|
||||
import { createServer, IncomingMessage, ServerResponse, Server } from 'http'
|
||||
import * as os from 'os'
|
||||
|
||||
// ============================================
|
||||
// Configuration
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Metrics configuration from environment
|
||||
*/
|
||||
export interface MetricsConfig {
|
||||
enabled: boolean
|
||||
port: number
|
||||
path: string
|
||||
prefix: string
|
||||
defaultLabels: Labels
|
||||
includeSystem: boolean
|
||||
collectDefaultMetrics: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse JSON labels from environment variable
|
||||
*/
|
||||
function parseLabelsFromEnv(envValue: string | undefined): Labels {
|
||||
if (!envValue) return {}
|
||||
try {
|
||||
const parsed = JSON.parse(envValue)
|
||||
if (typeof parsed === 'object' && parsed !== null) {
|
||||
return parsed as Labels
|
||||
}
|
||||
return {}
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from environment variables
|
||||
* Supports both BAILEYS_PROMETHEUS_* and METRICS_* prefixes for compatibility
|
||||
*/
|
||||
export function loadMetricsConfig(): MetricsConfig {
|
||||
return {
|
||||
enabled: (process.env.BAILEYS_PROMETHEUS_ENABLED ?? process.env.METRICS_ENABLED) === 'true',
|
||||
port: parseInt(process.env.BAILEYS_PROMETHEUS_PORT || process.env.METRICS_PORT || '9092', 10),
|
||||
path: process.env.BAILEYS_PROMETHEUS_PATH || process.env.METRICS_PATH || '/metrics',
|
||||
prefix: process.env.BAILEYS_PROMETHEUS_PREFIX || process.env.METRICS_PREFIX || 'baileys',
|
||||
defaultLabels: parseLabelsFromEnv(process.env.BAILEYS_PROMETHEUS_LABELS),
|
||||
includeSystem: (process.env.BAILEYS_PROMETHEUS_COLLECT_DEFAULT ?? process.env.METRICS_INCLUDE_SYSTEM) !== 'false',
|
||||
collectDefaultMetrics: (process.env.BAILEYS_PROMETHEUS_COLLECT_DEFAULT ?? process.env.METRICS_COLLECT_DEFAULT) !== 'false',
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Types
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Metric type
|
||||
*/
|
||||
@@ -31,6 +96,16 @@ export type Labels = Record<string, string>
|
||||
*/
|
||||
export const DEFAULT_BUCKETS = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000]
|
||||
|
||||
/**
|
||||
* Default latency buckets (in seconds)
|
||||
*/
|
||||
export const DEFAULT_LATENCY_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
|
||||
|
||||
/**
|
||||
* Default size buckets (in bytes)
|
||||
*/
|
||||
export const DEFAULT_SIZE_BUCKETS = [1024, 10240, 102400, 1048576, 10485760, 104857600]
|
||||
|
||||
/**
|
||||
* Default summary percentiles
|
||||
*/
|
||||
@@ -75,8 +150,15 @@ export interface SummaryValue {
|
||||
count: number
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Counter Class
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Counter class - monotonically increasing
|
||||
* Counter class - monotonically increasing metric
|
||||
*
|
||||
* Counters only go up (or reset to zero). They are used for
|
||||
* counting events like requests, errors, tasks completed, etc.
|
||||
*/
|
||||
export class Counter implements BaseMetric {
|
||||
readonly type = 'counter' as const
|
||||
@@ -102,6 +184,10 @@ export class Counter implements BaseMetric {
|
||||
incValue = value ?? 1
|
||||
}
|
||||
|
||||
if (incValue < 0) {
|
||||
throw new Error('Counter cannot be decremented')
|
||||
}
|
||||
|
||||
const key = this.labelsToKey(labels)
|
||||
const existing = this.values.get(key)
|
||||
|
||||
@@ -158,8 +244,15 @@ export class Counter implements BaseMetric {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Gauge Class
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Gauge class - value that can increase and decrease
|
||||
*
|
||||
* Gauges represent a snapshot of a value at a point in time.
|
||||
* Examples: temperature, current memory usage, queue size.
|
||||
*/
|
||||
export class Gauge implements BaseMetric {
|
||||
readonly type = 'gauge' as const
|
||||
@@ -301,8 +394,15 @@ export class Gauge implements BaseMetric {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Histogram Class
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Histogram class - distribution of values in buckets
|
||||
*
|
||||
* Histograms sample observations and count them in configurable buckets.
|
||||
* They also provide sum and count of observations.
|
||||
*/
|
||||
export class Histogram implements BaseMetric {
|
||||
readonly type = 'histogram' as const
|
||||
@@ -345,7 +445,7 @@ export class Histogram implements BaseMetric {
|
||||
this.values.set(key, histValue)
|
||||
}
|
||||
|
||||
// Increment appropriate buckets
|
||||
// Increment appropriate buckets (cumulative)
|
||||
for (const bucket of this.buckets) {
|
||||
if (observeValue <= bucket) {
|
||||
histValue.buckets.set(bucket, (histValue.buckets.get(bucket) ?? 0) + 1)
|
||||
@@ -357,7 +457,7 @@ export class Histogram implements BaseMetric {
|
||||
}
|
||||
|
||||
/**
|
||||
* Timer helper
|
||||
* Timer helper - measures duration in milliseconds
|
||||
*/
|
||||
startTimer(labels: Labels = {}): () => number {
|
||||
const start = process.hrtime.bigint()
|
||||
@@ -368,6 +468,18 @@ export class Histogram implements BaseMetric {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Timer helper - measures duration in seconds
|
||||
*/
|
||||
startTimerSeconds(labels: Labels = {}): () => number {
|
||||
const start = process.hrtime.bigint()
|
||||
return () => {
|
||||
const duration = Number(process.hrtime.bigint() - start) / 1_000_000_000 // seconds
|
||||
this.observe(labels, duration)
|
||||
return duration
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get histogram values
|
||||
*/
|
||||
@@ -420,8 +532,15 @@ export class Histogram implements BaseMetric {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Summary Class
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Summary class - value percentiles
|
||||
* Summary class - value percentiles (quantiles)
|
||||
*
|
||||
* Summaries calculate quantiles over a sliding time window.
|
||||
* Useful for tracking latency distributions.
|
||||
*/
|
||||
export class Summary implements BaseMetric {
|
||||
readonly type = 'summary' as const
|
||||
@@ -472,7 +591,7 @@ export class Summary implements BaseMetric {
|
||||
summaryValue.sum += observeValue
|
||||
summaryValue.count++
|
||||
|
||||
// Limit size
|
||||
// Limit size to prevent memory leaks
|
||||
if (summaryValue.values.length > this.maxSize) {
|
||||
const removed = summaryValue.values.shift()
|
||||
if (removed !== undefined) {
|
||||
@@ -562,8 +681,15 @@ export class Summary implements BaseMetric {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Metrics Registry
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Metrics registry
|
||||
* Metrics registry - manages collection of metrics
|
||||
*
|
||||
* The registry holds all metrics and provides methods to
|
||||
* retrieve, reset, and export them in Prometheus format.
|
||||
*/
|
||||
export class MetricsRegistry {
|
||||
private metricsMap: Map<string, Counter | Gauge | Histogram | Summary> = new Map()
|
||||
@@ -585,13 +711,21 @@ export class MetricsRegistry {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a metric
|
||||
* Get a metric by name
|
||||
*/
|
||||
get(name: string): Counter | Gauge | Histogram | Summary | undefined {
|
||||
const fullName = this.prefix ? `${this.prefix}_${name}` : name
|
||||
return this.metricsMap.get(fullName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a metric exists
|
||||
*/
|
||||
has(name: string): boolean {
|
||||
const fullName = this.prefix ? `${this.prefix}_${name}` : name
|
||||
return this.metricsMap.has(fullName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a metric
|
||||
*/
|
||||
@@ -600,6 +734,13 @@ export class MetricsRegistry {
|
||||
return this.metricsMap.delete(fullName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered metrics
|
||||
*/
|
||||
getAll(): Map<string, Counter | Gauge | Histogram | Summary> {
|
||||
return new Map(this.metricsMap)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all metrics
|
||||
*/
|
||||
@@ -610,7 +751,14 @@ export class MetricsRegistry {
|
||||
}
|
||||
|
||||
/**
|
||||
* Return metrics in Prometheus format
|
||||
* Set default labels that will be added to all metrics
|
||||
*/
|
||||
setDefaultLabels(labels: Labels): void {
|
||||
this.defaultLabels = { ...labels }
|
||||
}
|
||||
|
||||
/**
|
||||
* Return metrics in Prometheus exposition format
|
||||
*/
|
||||
async getMetricsOutput(): Promise<string> {
|
||||
const lines: string[] = []
|
||||
@@ -638,7 +786,7 @@ export class MetricsRegistry {
|
||||
lines.push(`${name}_bucket${bucketLabels} ${value.buckets.get(bucket) ?? 0}`)
|
||||
}
|
||||
|
||||
// +Inf bucket
|
||||
// +Inf bucket (total count)
|
||||
const infLabels = this.formatLabels({
|
||||
...this.defaultLabels,
|
||||
...value.labels,
|
||||
@@ -695,18 +843,256 @@ export class MetricsRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
// === Default Baileys Metrics ===
|
||||
// ============================================
|
||||
// System Metrics Collection
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* System metrics collector
|
||||
* Collects Node.js process and system-level metrics
|
||||
*/
|
||||
export class SystemMetricsCollector {
|
||||
private processStartTime: number
|
||||
private registry: MetricsRegistry
|
||||
|
||||
// Process metrics
|
||||
public readonly processUptime: Gauge
|
||||
public readonly processCpuUsage: Gauge
|
||||
public readonly processMemoryUsage: Gauge
|
||||
public readonly processMemoryExternal: Gauge
|
||||
public readonly processMemoryHeapTotal: Gauge
|
||||
public readonly processMemoryHeapUsed: Gauge
|
||||
public readonly processMemoryRss: Gauge
|
||||
|
||||
// System metrics
|
||||
public readonly systemCpuUsage: Gauge
|
||||
public readonly systemMemoryTotal: Gauge
|
||||
public readonly systemMemoryFree: Gauge
|
||||
public readonly systemLoadAverage: Gauge
|
||||
|
||||
// Event loop metrics
|
||||
public readonly eventLoopLag: Histogram
|
||||
|
||||
constructor(registry: MetricsRegistry) {
|
||||
this.registry = registry
|
||||
this.processStartTime = Date.now()
|
||||
|
||||
// Initialize process metrics
|
||||
this.processUptime = registry.register(
|
||||
new Gauge('process_uptime_seconds', 'Process uptime in seconds')
|
||||
)
|
||||
this.processCpuUsage = registry.register(
|
||||
new Gauge('process_cpu_usage_percent', 'Process CPU usage percentage', ['type'])
|
||||
)
|
||||
this.processMemoryUsage = registry.register(
|
||||
new Gauge('process_memory_bytes', 'Process memory usage in bytes', ['type'])
|
||||
)
|
||||
this.processMemoryExternal = registry.register(
|
||||
new Gauge('process_memory_external_bytes', 'External memory used by C++ objects')
|
||||
)
|
||||
this.processMemoryHeapTotal = registry.register(
|
||||
new Gauge('process_memory_heap_total_bytes', 'Total heap memory')
|
||||
)
|
||||
this.processMemoryHeapUsed = registry.register(
|
||||
new Gauge('process_memory_heap_used_bytes', 'Used heap memory')
|
||||
)
|
||||
this.processMemoryRss = registry.register(
|
||||
new Gauge('process_memory_rss_bytes', 'Resident set size')
|
||||
)
|
||||
|
||||
// Initialize system metrics
|
||||
this.systemCpuUsage = registry.register(
|
||||
new Gauge('system_cpu_usage_percent', 'System CPU usage percentage')
|
||||
)
|
||||
this.systemMemoryTotal = registry.register(
|
||||
new Gauge('system_memory_total_bytes', 'Total system memory')
|
||||
)
|
||||
this.systemMemoryFree = registry.register(
|
||||
new Gauge('system_memory_free_bytes', 'Free system memory')
|
||||
)
|
||||
this.systemLoadAverage = registry.register(
|
||||
new Gauge('system_load_average', 'System load average', ['period'])
|
||||
)
|
||||
|
||||
// Event loop lag
|
||||
this.eventLoopLag = registry.register(
|
||||
new Histogram('nodejs_eventloop_lag_seconds', 'Event loop lag in seconds', [], DEFAULT_LATENCY_BUCKETS)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect all system metrics
|
||||
*/
|
||||
collect(): void {
|
||||
// Process uptime
|
||||
this.processUptime.set((Date.now() - this.processStartTime) / 1000)
|
||||
|
||||
// CPU usage
|
||||
const cpuUsage = process.cpuUsage()
|
||||
this.processCpuUsage.set({ type: 'user' }, cpuUsage.user / 1000000) // Convert to seconds
|
||||
this.processCpuUsage.set({ type: 'system' }, cpuUsage.system / 1000000)
|
||||
|
||||
// Memory usage
|
||||
const memUsage = process.memoryUsage()
|
||||
this.processMemoryRss.set(memUsage.rss)
|
||||
this.processMemoryHeapTotal.set(memUsage.heapTotal)
|
||||
this.processMemoryHeapUsed.set(memUsage.heapUsed)
|
||||
this.processMemoryExternal.set(memUsage.external)
|
||||
this.processMemoryUsage.set({ type: 'rss' }, memUsage.rss)
|
||||
this.processMemoryUsage.set({ type: 'heapTotal' }, memUsage.heapTotal)
|
||||
this.processMemoryUsage.set({ type: 'heapUsed' }, memUsage.heapUsed)
|
||||
this.processMemoryUsage.set({ type: 'external' }, memUsage.external)
|
||||
|
||||
// System metrics
|
||||
this.systemMemoryTotal.set(os.totalmem())
|
||||
this.systemMemoryFree.set(os.freemem())
|
||||
|
||||
// Load average
|
||||
const loadAvg = os.loadavg()
|
||||
this.systemLoadAverage.set({ period: '1m' }, loadAvg[0])
|
||||
this.systemLoadAverage.set({ period: '5m' }, loadAvg[1])
|
||||
this.systemLoadAverage.set({ period: '15m' }, loadAvg[2])
|
||||
|
||||
// Event loop lag measurement
|
||||
this.measureEventLoopLag()
|
||||
}
|
||||
|
||||
private measureEventLoopLag(): void {
|
||||
const start = process.hrtime.bigint()
|
||||
setImmediate(() => {
|
||||
const lag = Number(process.hrtime.bigint() - start) / 1_000_000_000 // seconds
|
||||
this.eventLoopLag.observe(lag)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HTTP Metrics Server
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* HTTP server for exposing metrics endpoint
|
||||
*/
|
||||
export class MetricsServer {
|
||||
private server: Server | null = null
|
||||
private registry: MetricsRegistry
|
||||
private systemCollector: SystemMetricsCollector | null = null
|
||||
private collectInterval: NodeJS.Timeout | null = null
|
||||
private config: MetricsConfig
|
||||
|
||||
constructor(registry: MetricsRegistry, config?: Partial<MetricsConfig>) {
|
||||
this.registry = registry
|
||||
this.config = { ...loadMetricsConfig(), ...config }
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the metrics HTTP server
|
||||
*/
|
||||
start(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!this.config.enabled) {
|
||||
resolve()
|
||||
return
|
||||
}
|
||||
|
||||
if (this.server) {
|
||||
resolve()
|
||||
return
|
||||
}
|
||||
|
||||
// Initialize system collector if enabled
|
||||
if (this.config.includeSystem) {
|
||||
this.systemCollector = new SystemMetricsCollector(this.registry)
|
||||
this.collectInterval = setInterval(() => {
|
||||
this.systemCollector?.collect()
|
||||
}, 10000) // Collect every 10 seconds
|
||||
}
|
||||
|
||||
this.server = createServer(async (req: IncomingMessage, res: ServerResponse) => {
|
||||
if (req.url === this.config.path && req.method === 'GET') {
|
||||
try {
|
||||
// Collect system metrics before responding
|
||||
this.systemCollector?.collect()
|
||||
|
||||
const metricsOutput = await this.registry.getMetricsOutput()
|
||||
res.writeHead(200, { 'Content-Type': this.registry.contentType() })
|
||||
res.end(metricsOutput)
|
||||
} catch (error) {
|
||||
res.writeHead(500, { 'Content-Type': 'text/plain' })
|
||||
res.end('Error collecting metrics')
|
||||
}
|
||||
} else if (req.url === '/health' && req.method === 'GET') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
||||
res.end(JSON.stringify({ status: 'ok', timestamp: new Date().toISOString() }))
|
||||
} else {
|
||||
res.writeHead(404, { 'Content-Type': 'text/plain' })
|
||||
res.end('Not Found')
|
||||
}
|
||||
})
|
||||
|
||||
this.server.on('error', reject)
|
||||
this.server.listen(this.config.port, () => {
|
||||
const labelsInfo = Object.keys(this.config.defaultLabels).length > 0
|
||||
? ` with labels: ${JSON.stringify(this.config.defaultLabels)}`
|
||||
: ''
|
||||
console.log(`[Prometheus] Metrics server listening on http://0.0.0.0:${this.config.port}${this.config.path}${labelsInfo}`)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the metrics HTTP server
|
||||
*/
|
||||
stop(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (this.collectInterval) {
|
||||
clearInterval(this.collectInterval)
|
||||
this.collectInterval = null
|
||||
}
|
||||
|
||||
if (this.server) {
|
||||
this.server.close(() => {
|
||||
this.server = null
|
||||
resolve()
|
||||
})
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if server is running
|
||||
*/
|
||||
isRunning(): boolean {
|
||||
return this.server !== null
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Pre-defined Baileys Metrics
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Load metrics configuration from environment
|
||||
*/
|
||||
const metricsConfig = loadMetricsConfig()
|
||||
|
||||
/**
|
||||
* Global registry for Baileys metrics
|
||||
* Uses prefix and default labels from environment variables
|
||||
*/
|
||||
export const baileysMetrics = new MetricsRegistry({ prefix: 'baileys' })
|
||||
export const baileysMetrics = new MetricsRegistry({
|
||||
prefix: metricsConfig.prefix,
|
||||
defaultLabels: metricsConfig.defaultLabels
|
||||
})
|
||||
|
||||
/**
|
||||
* Pre-defined metrics for Baileys
|
||||
*/
|
||||
export const metrics = {
|
||||
// Connection
|
||||
// ========== Connection Metrics ==========
|
||||
connectionAttempts: baileysMetrics.register(
|
||||
new Counter('connection_attempts_total', 'Total connection attempts', ['status'])
|
||||
),
|
||||
@@ -716,8 +1102,14 @@ export const metrics = {
|
||||
connectionDuration: baileysMetrics.register(
|
||||
new Gauge('connection_duration_seconds', 'Current connection duration in seconds', ['instance'])
|
||||
),
|
||||
reconnectAttempts: baileysMetrics.register(
|
||||
new Counter('reconnect_attempts_total', 'Total reconnection attempts', ['reason'])
|
||||
),
|
||||
connectionLatency: baileysMetrics.register(
|
||||
new Histogram('connection_latency_ms', 'Connection establishment latency in ms', [], [100, 250, 500, 1000, 2500, 5000, 10000])
|
||||
),
|
||||
|
||||
// Messages
|
||||
// ========== Message Metrics ==========
|
||||
messagesSent: baileysMetrics.register(
|
||||
new Counter('messages_sent_total', 'Total messages sent', ['type'])
|
||||
),
|
||||
@@ -727,8 +1119,17 @@ export const metrics = {
|
||||
messageLatency: baileysMetrics.register(
|
||||
new Histogram('message_latency_ms', 'Message send latency in ms', ['type'], [10, 50, 100, 250, 500, 1000, 2500, 5000])
|
||||
),
|
||||
messageRetries: baileysMetrics.register(
|
||||
new Counter('message_retries_total', 'Total message retry attempts', ['type'])
|
||||
),
|
||||
messageFailures: baileysMetrics.register(
|
||||
new Counter('message_failures_total', 'Total message send failures', ['type', 'reason'])
|
||||
),
|
||||
messagesQueued: baileysMetrics.register(
|
||||
new Gauge('messages_queued', 'Current number of messages in queue', ['priority'])
|
||||
),
|
||||
|
||||
// Media
|
||||
// ========== Media Metrics ==========
|
||||
mediaUploads: baileysMetrics.register(
|
||||
new Counter('media_uploads_total', 'Total media uploads', ['type', 'status'])
|
||||
),
|
||||
@@ -736,43 +1137,262 @@ export const metrics = {
|
||||
new Counter('media_downloads_total', 'Total media downloads', ['type', 'status'])
|
||||
),
|
||||
mediaSize: baileysMetrics.register(
|
||||
new Histogram('media_size_bytes', 'Media size in bytes', ['type', 'direction'], [1024, 10240, 102400, 1048576, 10485760])
|
||||
new Histogram('media_size_bytes', 'Media size in bytes', ['type', 'direction'], DEFAULT_SIZE_BUCKETS)
|
||||
),
|
||||
mediaLatency: baileysMetrics.register(
|
||||
new Histogram('media_latency_ms', 'Media upload/download latency in ms', ['type', 'direction'], [100, 500, 1000, 2500, 5000, 10000, 30000])
|
||||
),
|
||||
|
||||
// Errors
|
||||
// ========== Event Buffer Metrics ==========
|
||||
bufferSize: baileysMetrics.register(
|
||||
new Gauge('buffer_size', 'Current event buffer size', ['type'])
|
||||
),
|
||||
bufferCapacity: baileysMetrics.register(
|
||||
new Gauge('buffer_capacity', 'Maximum event buffer capacity', ['type'])
|
||||
),
|
||||
bufferUtilization: baileysMetrics.register(
|
||||
new Gauge('buffer_utilization_percent', 'Event buffer utilization percentage', ['type'])
|
||||
),
|
||||
bufferFlushes: baileysMetrics.register(
|
||||
new Counter('buffer_flushes_total', 'Total buffer flush operations', ['type', 'reason'])
|
||||
),
|
||||
bufferOverflows: baileysMetrics.register(
|
||||
new Counter('buffer_overflows_total', 'Total buffer overflow events', ['type'])
|
||||
),
|
||||
eventsBuffered: baileysMetrics.register(
|
||||
new Counter('events_buffered_total', 'Total events added to buffer', ['event_type'])
|
||||
),
|
||||
eventsDropped: baileysMetrics.register(
|
||||
new Counter('events_dropped_total', 'Total events dropped due to buffer full', ['event_type'])
|
||||
),
|
||||
bufferFlushLatency: baileysMetrics.register(
|
||||
new Histogram('buffer_flush_latency_ms', 'Buffer flush operation latency in ms', ['type'], [1, 5, 10, 25, 50, 100, 250])
|
||||
),
|
||||
eventsProcessed: baileysMetrics.register(
|
||||
new Counter('events_processed_total', 'Total events processed from buffer', ['event_type'])
|
||||
),
|
||||
|
||||
// ========== Adaptive Flush Metrics ==========
|
||||
adaptiveFlushInterval: baileysMetrics.register(
|
||||
new Gauge('adaptive_flush_interval_ms', 'Current adaptive flush interval in ms')
|
||||
),
|
||||
adaptiveFlushAdjustments: baileysMetrics.register(
|
||||
new Counter('adaptive_flush_adjustments_total', 'Total adaptive flush interval adjustments', ['direction'])
|
||||
),
|
||||
adaptiveFlushThroughput: baileysMetrics.register(
|
||||
new Gauge('adaptive_flush_throughput', 'Events processed per second by adaptive flush')
|
||||
),
|
||||
adaptiveFlushBackpressure: baileysMetrics.register(
|
||||
new Gauge('adaptive_flush_backpressure', 'Backpressure indicator (0-1)')
|
||||
),
|
||||
adaptiveFlushEfficiency: baileysMetrics.register(
|
||||
new Gauge('adaptive_flush_efficiency_percent', 'Flush efficiency percentage')
|
||||
),
|
||||
|
||||
// ========== Error Metrics ==========
|
||||
errors: baileysMetrics.register(
|
||||
new Counter('errors_total', 'Total errors', ['category', 'code'])
|
||||
),
|
||||
errorRate: baileysMetrics.register(
|
||||
new Gauge('error_rate', 'Current error rate per minute', ['category'])
|
||||
),
|
||||
|
||||
// Retries
|
||||
// ========== Retry Metrics ==========
|
||||
retries: baileysMetrics.register(
|
||||
new Counter('retries_total', 'Total retries', ['operation'])
|
||||
),
|
||||
retryLatency: baileysMetrics.register(
|
||||
new Histogram('retry_latency_ms', 'Retry latency in ms', ['operation'])
|
||||
),
|
||||
retrySuccess: baileysMetrics.register(
|
||||
new Counter('retry_success_total', 'Successful retries', ['operation'])
|
||||
),
|
||||
retryExhausted: baileysMetrics.register(
|
||||
new Counter('retry_exhausted_total', 'Exhausted retry attempts', ['operation'])
|
||||
),
|
||||
|
||||
// Socket
|
||||
// ========== Socket Metrics ==========
|
||||
socketEvents: baileysMetrics.register(
|
||||
new Counter('socket_events_total', 'Total socket events', ['event'])
|
||||
),
|
||||
socketLatency: baileysMetrics.register(
|
||||
new Histogram('socket_latency_ms', 'Socket operation latency in ms', ['operation'])
|
||||
),
|
||||
socketBytesReceived: baileysMetrics.register(
|
||||
new Counter('socket_bytes_received_total', 'Total bytes received through socket')
|
||||
),
|
||||
socketBytesSent: baileysMetrics.register(
|
||||
new Counter('socket_bytes_sent_total', 'Total bytes sent through socket')
|
||||
),
|
||||
socketReconnects: baileysMetrics.register(
|
||||
new Counter('socket_reconnects_total', 'Total socket reconnection attempts', ['reason'])
|
||||
),
|
||||
|
||||
// Encryption
|
||||
// ========== Circuit Breaker Metrics ==========
|
||||
circuitBreakerState: baileysMetrics.register(
|
||||
new Gauge('circuit_breaker_state', 'Circuit breaker state (0=closed, 1=open, 2=half-open)', ['name'])
|
||||
),
|
||||
circuitBreakerTrips: baileysMetrics.register(
|
||||
new Counter('circuit_breaker_trips_total', 'Total circuit breaker trips', ['name'])
|
||||
),
|
||||
circuitBreakerRecoveries: baileysMetrics.register(
|
||||
new Counter('circuit_breaker_recoveries_total', 'Total circuit breaker recoveries', ['name'])
|
||||
),
|
||||
circuitBreakerRejections: baileysMetrics.register(
|
||||
new Counter('circuit_breaker_rejections_total', 'Total requests rejected by circuit breaker', ['name'])
|
||||
),
|
||||
circuitBreakerSuccesses: baileysMetrics.register(
|
||||
new Counter('circuit_breaker_successes_total', 'Total successful requests through circuit breaker', ['name'])
|
||||
),
|
||||
circuitBreakerFailures: baileysMetrics.register(
|
||||
new Counter('circuit_breaker_failures_total', 'Total failed requests through circuit breaker', ['name'])
|
||||
),
|
||||
|
||||
// ========== Encryption Metrics ==========
|
||||
encryptionOperations: baileysMetrics.register(
|
||||
new Counter('encryption_operations_total', 'Total encryption operations', ['operation'])
|
||||
),
|
||||
encryptionLatency: baileysMetrics.register(
|
||||
new Histogram('encryption_latency_ms', 'Encryption operation latency in ms', ['operation'], [1, 5, 10, 25, 50, 100])
|
||||
),
|
||||
keyExchanges: baileysMetrics.register(
|
||||
new Counter('key_exchanges_total', 'Total key exchange operations', ['type'])
|
||||
),
|
||||
preKeyCount: baileysMetrics.register(
|
||||
new Gauge('prekey_count', 'Current prekey count')
|
||||
),
|
||||
|
||||
// Cache
|
||||
cacheHits: baileysMetrics.register(new Counter('cache_hits_total', 'Total cache hits', ['cache'])),
|
||||
cacheMisses: baileysMetrics.register(new Counter('cache_misses_total', 'Total cache misses', ['cache'])),
|
||||
cacheSize: baileysMetrics.register(new Gauge('cache_size', 'Current cache size', ['cache'])),
|
||||
// ========== Cache Metrics ==========
|
||||
cacheHits: baileysMetrics.register(
|
||||
new Counter('cache_hits_total', 'Total cache hits', ['cache'])
|
||||
),
|
||||
cacheMisses: baileysMetrics.register(
|
||||
new Counter('cache_misses_total', 'Total cache misses', ['cache'])
|
||||
),
|
||||
cacheSize: baileysMetrics.register(
|
||||
new Gauge('cache_size', 'Current cache size', ['cache'])
|
||||
),
|
||||
cacheEvictions: baileysMetrics.register(
|
||||
new Counter('cache_evictions_total', 'Total cache evictions', ['cache', 'reason'])
|
||||
),
|
||||
cacheHitRate: baileysMetrics.register(
|
||||
new Gauge('cache_hit_rate', 'Cache hit rate (0-1)', ['cache'])
|
||||
),
|
||||
|
||||
// ========== Query Metrics ==========
|
||||
queryLatency: baileysMetrics.register(
|
||||
new Histogram('query_latency_ms', 'Query latency in ms', ['query_type'], [10, 50, 100, 250, 500, 1000, 2500, 5000])
|
||||
),
|
||||
queryCount: baileysMetrics.register(
|
||||
new Counter('query_count_total', 'Total queries executed', ['query_type', 'status'])
|
||||
),
|
||||
queryTimeouts: baileysMetrics.register(
|
||||
new Counter('query_timeouts_total', 'Total query timeouts', ['query_type'])
|
||||
),
|
||||
|
||||
// ========== Presence Metrics ==========
|
||||
presenceUpdates: baileysMetrics.register(
|
||||
new Counter('presence_updates_total', 'Total presence updates received', ['type'])
|
||||
),
|
||||
presenceSubscriptions: baileysMetrics.register(
|
||||
new Gauge('presence_subscriptions', 'Current presence subscriptions')
|
||||
),
|
||||
|
||||
// ========== Group Metrics ==========
|
||||
groupOperations: baileysMetrics.register(
|
||||
new Counter('group_operations_total', 'Total group operations', ['operation', 'status'])
|
||||
),
|
||||
groupMetadataFetches: baileysMetrics.register(
|
||||
new Counter('group_metadata_fetches_total', 'Total group metadata fetches', ['status'])
|
||||
),
|
||||
|
||||
// ========== History Sync Metrics ==========
|
||||
historySyncEvents: baileysMetrics.register(
|
||||
new Counter('history_sync_events_total', 'Total history sync events', ['type'])
|
||||
),
|
||||
historySyncMessages: baileysMetrics.register(
|
||||
new Counter('history_sync_messages_total', 'Total messages synced from history')
|
||||
),
|
||||
historySyncDuration: baileysMetrics.register(
|
||||
new Histogram('history_sync_duration_ms', 'History sync duration in ms', ['type'], [1000, 5000, 10000, 30000, 60000])
|
||||
),
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Metrics Manager
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Helper to create HTTP metrics endpoint
|
||||
* PrometheusMetricsManager - High-level manager for all metrics
|
||||
*
|
||||
* Provides a unified interface for managing metrics, HTTP server,
|
||||
* and system metrics collection.
|
||||
*/
|
||||
export class PrometheusMetricsManager {
|
||||
public readonly registry: MetricsRegistry
|
||||
public readonly metrics: typeof metrics
|
||||
public readonly server: MetricsServer
|
||||
private systemCollector: SystemMetricsCollector | null = null
|
||||
private config: MetricsConfig
|
||||
|
||||
constructor(config?: Partial<MetricsConfig>) {
|
||||
this.config = { ...loadMetricsConfig(), ...config }
|
||||
this.registry = baileysMetrics
|
||||
this.metrics = metrics
|
||||
this.server = new MetricsServer(this.registry, this.config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the metrics manager
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (!this.config.enabled) {
|
||||
return
|
||||
}
|
||||
|
||||
if (this.config.includeSystem) {
|
||||
this.systemCollector = new SystemMetricsCollector(this.registry)
|
||||
}
|
||||
|
||||
await this.server.start()
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the metrics manager
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
await this.server.stop()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metrics output in Prometheus format
|
||||
*/
|
||||
async getMetricsOutput(): Promise<string> {
|
||||
this.systemCollector?.collect()
|
||||
return this.registry.getMetricsOutput()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all metrics
|
||||
*/
|
||||
resetAll(): void {
|
||||
this.registry.resetAll()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if metrics are enabled
|
||||
*/
|
||||
isEnabled(): boolean {
|
||||
return this.config.enabled
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Helper Functions
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Helper to create HTTP metrics endpoint handler
|
||||
*/
|
||||
export function createMetricsHandler(registry: MetricsRegistry = baileysMetrics) {
|
||||
return async (_req: unknown, res: { setHeader: (name: string, value: string) => void; end: (body: string) => void }) => {
|
||||
@@ -782,4 +1402,112 @@ export function createMetricsHandler(registry: MetricsRegistry = baileysMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Express middleware for metrics endpoint
|
||||
*/
|
||||
export function createExpressMetricsMiddleware(registry: MetricsRegistry = baileysMetrics) {
|
||||
return async (_req: unknown, res: { set: (name: string, value: string) => void; send: (body: string) => void }) => {
|
||||
const metricsOutput = await registry.getMetricsOutput()
|
||||
res.set('Content-Type', registry.contentType())
|
||||
res.send(metricsOutput)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track operation duration using histogram
|
||||
*/
|
||||
export function trackDuration<T>(
|
||||
histogram: Histogram,
|
||||
labels: Labels,
|
||||
operation: () => T
|
||||
): T {
|
||||
const endTimer = histogram.startTimer(labels)
|
||||
try {
|
||||
const result = operation()
|
||||
if (result instanceof Promise) {
|
||||
return result.finally(() => endTimer()) as T
|
||||
}
|
||||
endTimer()
|
||||
return result
|
||||
} catch (error) {
|
||||
endTimer()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track operation duration using async histogram
|
||||
*/
|
||||
export async function trackDurationAsync<T>(
|
||||
histogram: Histogram,
|
||||
labels: Labels,
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const endTimer = histogram.startTimer(labels)
|
||||
try {
|
||||
const result = await operation()
|
||||
endTimer()
|
||||
return result
|
||||
} catch (error) {
|
||||
endTimer()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment counter with automatic error tracking
|
||||
*/
|
||||
export function trackOperation(
|
||||
successCounter: Counter,
|
||||
errorCounter: Counter,
|
||||
labels: Labels
|
||||
) {
|
||||
return {
|
||||
success: () => successCounter.inc(labels),
|
||||
failure: (errorCode?: string) => errorCounter.inc({ ...labels, code: errorCode || 'unknown' }),
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Global Instance
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Global metrics manager instance
|
||||
*/
|
||||
let globalMetricsManager: PrometheusMetricsManager | null = null
|
||||
|
||||
/**
|
||||
* Get or create global metrics manager
|
||||
*/
|
||||
export function getMetricsManager(config?: Partial<MetricsConfig>): PrometheusMetricsManager {
|
||||
if (!globalMetricsManager) {
|
||||
globalMetricsManager = new PrometheusMetricsManager(config)
|
||||
}
|
||||
return globalMetricsManager
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize global metrics (call once at application startup)
|
||||
*/
|
||||
export async function initializeMetrics(config?: Partial<MetricsConfig>): Promise<PrometheusMetricsManager> {
|
||||
const manager = getMetricsManager(config)
|
||||
await manager.initialize()
|
||||
return manager
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown global metrics (call at application shutdown)
|
||||
*/
|
||||
export async function shutdownMetrics(): Promise<void> {
|
||||
if (globalMetricsManager) {
|
||||
await globalMetricsManager.shutdown()
|
||||
globalMetricsManager = null
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Default Export
|
||||
// ============================================
|
||||
|
||||
export default baileysMetrics
|
||||
|
||||
Reference in New Issue
Block a user