feat: add adaptive metrics and circuit breaker trips tracking
Added metrics tracking for: - circuit_breaker_trips_total: Incremented when circuit opens - adaptive_health_status: 1 if healthy (not in aggressive mode), 0 otherwise - adaptive_event_rate: Current events per second Added methods to AdaptiveTimeoutCalculator: - getEventRate(): Returns current event rate in events/second - isHealthy(): Returns true if not in aggressive mode Metrics are updated on each buffer flush when adaptive timeout is enabled.
This commit is contained in:
@@ -380,6 +380,11 @@ export class CircuitBreaker extends EventEmitter {
|
|||||||
this.options.onOpen()
|
this.options.onOpen()
|
||||||
this.emit('open')
|
this.emit('open')
|
||||||
|
|
||||||
|
// Record circuit breaker trip metric
|
||||||
|
if (this.options.collectMetrics) {
|
||||||
|
metrics.circuitBreakerTrips?.inc({ name: this.options.name })
|
||||||
|
}
|
||||||
|
|
||||||
// Schedule transition to HALF_OPEN
|
// Schedule transition to HALF_OPEN
|
||||||
this.resetTimer = setTimeout(() => {
|
this.resetTimer = setTimeout(() => {
|
||||||
this.transitionTo('half-open')
|
this.transitionTo('half-open')
|
||||||
|
|||||||
@@ -329,6 +329,31 @@ class AdaptiveTimeoutCalculator {
|
|||||||
}
|
}
|
||||||
return 'balanced'
|
return 'balanced'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current event rate in events per second
|
||||||
|
*/
|
||||||
|
getEventRate(): number {
|
||||||
|
if (this.eventTimestamps.length < 2) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
const oldest = this.eventTimestamps[0]!
|
||||||
|
const newest = this.eventTimestamps[this.eventTimestamps.length - 1]!
|
||||||
|
const timeSpan = newest - oldest
|
||||||
|
if (timeSpan === 0) return 0
|
||||||
|
return (this.eventTimestamps.length / timeSpan) * 1000
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if system is healthy based on current mode
|
||||||
|
* Conservative mode with low event rate is healthy
|
||||||
|
* Aggressive mode might indicate high load
|
||||||
|
*/
|
||||||
|
isHealthy(): boolean {
|
||||||
|
const mode = this.getMode()
|
||||||
|
// System is considered healthy if not in aggressive mode (high load)
|
||||||
|
return mode !== 'aggressive'
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -563,6 +588,11 @@ export const makeEventBuffer = (
|
|||||||
// Record metrics
|
// Record metrics
|
||||||
recordFlushMetrics(eventCount, force, historyCache.size)
|
recordFlushMetrics(eventCount, force, historyCache.size)
|
||||||
|
|
||||||
|
// Update adaptive metrics
|
||||||
|
if (config.enableAdaptiveTimeout && metricsModule) {
|
||||||
|
metricsModule.updateAdaptiveMetrics(adaptiveTimeout.getEventRate(), adaptiveTimeout.isHealthy())
|
||||||
|
}
|
||||||
|
|
||||||
// Log with [BAILEYS] prefix - use getMode() to avoid duplicating mode calculation logic
|
// Log with [BAILEYS] prefix - use getMode() to avoid duplicating mode calculation logic
|
||||||
const flushDuration = Date.now() - flushStartTime
|
const flushDuration = Date.now() - flushStartTime
|
||||||
logEventBuffer('buffer_flush', {
|
logEventBuffer('buffer_flush', {
|
||||||
|
|||||||
@@ -1958,6 +1958,19 @@ export function recordBufferFinalFlush(): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update adaptive system metrics
|
||||||
|
* Used by event-buffer.ts to report adaptive timeout health and event rate
|
||||||
|
*/
|
||||||
|
export function updateAdaptiveMetrics(eventRate: number, isHealthy: boolean): void {
|
||||||
|
try {
|
||||||
|
metrics.adaptiveEventRate?.set({}, eventRate)
|
||||||
|
metrics.adaptiveHealthStatus?.set({}, isHealthy ? 1 : 0)
|
||||||
|
} catch {
|
||||||
|
// Metrics not initialized, ignore silently
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// Global Instance
|
// Global Instance
|
||||||
// ============================================
|
// ============================================
|
||||||
|
|||||||
Reference in New Issue
Block a user