c6c6f0233d
* Expand the linters and types of warnings to improve consistency and safety * Fail lint workflow if there are errors * golint has been replaced by revive * Hand-pick some of the default exclude list * Ignore error when trying to delete preview gif * Ignore linter warning opening playlist path * Rename user field Id -> ID * A bunch of renames to address linter warnings * Rename ChatClient -> Client per linter suggestion best practice * Rename ChatServer -> Server per linter suggestion best practice * More linter warning fixes * Add missing comments to all exported functions and properties
85 lines
1.9 KiB
Go
85 lines
1.9 KiB
Go
package metrics
|
|
|
|
import (
|
|
"time"
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
const maxCPUAlertingThresholdPCT = 85
|
|
const maxRAMAlertingThresholdPCT = 85
|
|
const maxDiskAlertingThresholdPCT = 90
|
|
|
|
var inCPUAlertingState = false
|
|
var inRAMAlertingState = false
|
|
var inDiskAlertingState = false
|
|
|
|
var errorResetDuration = time.Minute * 5
|
|
|
|
const alertingError = "The %s utilization of %d%% could cause problems with video generation and delivery. Visit the documentation at http://owncast.online/docs/troubleshooting/ if you are experiencing issues."
|
|
|
|
func handleAlerting() {
|
|
handleCPUAlerting()
|
|
handleRAMAlerting()
|
|
handleDiskAlerting()
|
|
}
|
|
|
|
func handleCPUAlerting() {
|
|
if len(Metrics.CPUUtilizations) < 2 {
|
|
return
|
|
}
|
|
|
|
avg := recentAverage(Metrics.CPUUtilizations)
|
|
if avg > maxCPUAlertingThresholdPCT && !inCPUAlertingState {
|
|
log.Warnf(alertingError, "CPU", avg)
|
|
inCPUAlertingState = true
|
|
|
|
resetTimer := time.NewTimer(errorResetDuration)
|
|
go func() {
|
|
<-resetTimer.C
|
|
inCPUAlertingState = false
|
|
}()
|
|
}
|
|
}
|
|
|
|
func handleRAMAlerting() {
|
|
if len(Metrics.RAMUtilizations) < 2 {
|
|
return
|
|
}
|
|
|
|
avg := recentAverage(Metrics.RAMUtilizations)
|
|
if avg > maxRAMAlertingThresholdPCT && !inRAMAlertingState {
|
|
log.Warnf(alertingError, "memory", avg)
|
|
inRAMAlertingState = true
|
|
|
|
resetTimer := time.NewTimer(errorResetDuration)
|
|
go func() {
|
|
<-resetTimer.C
|
|
inRAMAlertingState = false
|
|
}()
|
|
}
|
|
}
|
|
|
|
func handleDiskAlerting() {
|
|
if len(Metrics.DiskUtilizations) < 2 {
|
|
return
|
|
}
|
|
|
|
avg := recentAverage(Metrics.DiskUtilizations)
|
|
|
|
if avg > maxDiskAlertingThresholdPCT && !inDiskAlertingState {
|
|
log.Warnf(alertingError, "disk", avg)
|
|
inDiskAlertingState = true
|
|
|
|
resetTimer := time.NewTimer(errorResetDuration)
|
|
go func() {
|
|
<-resetTimer.C
|
|
inDiskAlertingState = false
|
|
}()
|
|
}
|
|
}
|
|
|
|
func recentAverage(values []timestampedValue) int {
|
|
return (values[len(values)-1].Value + values[len(values)-2].Value) / 2
|
|
}
|