refactor: 移除 Ops 监控模块
移除未完成的运维监控功能，简化系统架构： - 删除 ops_handler, ops_service, ops_repo 等后端代码 - 删除 ops 相关数据库迁移文件 - 删除前端 OpsDashboard 页面和 API
This commit is contained in:
ianshaw
@@ -1,99 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrorLog represents an ops error log item for list queries.
|
||||
//
|
||||
// Field naming matches docs/API-运维监控中心2.0.md (L3 根因追踪 - 错误日志列表).
|
||||
type ErrorLog struct {
|
||||
ID int64 `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
|
||||
Level string `json:"level,omitempty"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
AccountID string `json:"account_id,omitempty"`
|
||||
APIPath string `json:"api_path,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
HTTPCode int `json:"http_code,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
|
||||
DurationMs *int `json:"duration_ms,omitempty"`
|
||||
RetryCount *int `json:"retry_count,omitempty"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
}
|
||||
|
||||
// ErrorLogFilter describes optional filters and pagination for listing ops error logs.
|
||||
type ErrorLogFilter struct {
|
||||
StartTime *time.Time
|
||||
EndTime *time.Time
|
||||
|
||||
ErrorCode *int
|
||||
Provider string
|
||||
AccountID *int64
|
||||
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
func (f *ErrorLogFilter) normalize() (page, pageSize int) {
|
||||
page = 1
|
||||
pageSize = 20
|
||||
if f == nil {
|
||||
return page, pageSize
|
||||
}
|
||||
|
||||
if f.Page > 0 {
|
||||
page = f.Page
|
||||
}
|
||||
if f.PageSize > 0 {
|
||||
pageSize = f.PageSize
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
return page, pageSize
|
||||
}
|
||||
|
||||
type ErrorLogListResponse struct {
|
||||
Errors []*ErrorLog `json:"errors"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
}
|
||||
|
||||
func (s *OpsService) GetErrorLogs(ctx context.Context, filter *ErrorLogFilter) (*ErrorLogListResponse, error) {
|
||||
if s == nil || s.repo == nil {
|
||||
return &ErrorLogListResponse{
|
||||
Errors: []*ErrorLog{},
|
||||
Total: 0,
|
||||
Page: 1,
|
||||
PageSize: 20,
|
||||
}, nil
|
||||
}
|
||||
|
||||
page, pageSize := filter.normalize()
|
||||
if filter == nil {
|
||||
filter = &ErrorLogFilter{}
|
||||
}
|
||||
filter.Page = page
|
||||
filter.PageSize = pageSize
|
||||
|
||||
items, total, err := s.repo.ListErrorLogs(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if items == nil {
|
||||
items = []*ErrorLog{}
|
||||
}
|
||||
|
||||
return &ErrorLogListResponse{
|
||||
Errors: items,
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,834 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type OpsAlertService struct {
|
||||
opsService *OpsService
|
||||
userService *UserService
|
||||
emailService *EmailService
|
||||
httpClient *http.Client
|
||||
|
||||
interval time.Duration
|
||||
|
||||
startOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
stopCtx context.Context
|
||||
stop context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// opsAlertEvalInterval defines how often OpsAlertService evaluates alert rules.
|
||||
//
|
||||
// Production uses opsMetricsInterval. Tests may override this variable to keep
|
||||
// integration tests fast without changing production defaults.
|
||||
var opsAlertEvalInterval = opsMetricsInterval
|
||||
|
||||
func NewOpsAlertService(opsService *OpsService, userService *UserService, emailService *EmailService) *OpsAlertService {
|
||||
return &OpsAlertService{
|
||||
opsService: opsService,
|
||||
userService: userService,
|
||||
emailService: emailService,
|
||||
httpClient: &http.Client{Timeout: 10 * time.Second},
|
||||
interval: opsAlertEvalInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches the background alert evaluation loop.
|
||||
//
|
||||
// Stop must be called during shutdown to ensure the goroutine exits.
|
||||
func (s *OpsAlertService) Start() {
|
||||
s.StartWithContext(context.Background())
|
||||
}
|
||||
|
||||
// StartWithContext is like Start but allows the caller to provide a parent context.
|
||||
// When the parent context is canceled, the service stops automatically.
|
||||
func (s *OpsAlertService) StartWithContext(ctx context.Context) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
s.startOnce.Do(func() {
|
||||
if s.interval <= 0 {
|
||||
s.interval = opsAlertEvalInterval
|
||||
}
|
||||
|
||||
s.stopCtx, s.stop = context.WithCancel(ctx)
|
||||
s.wg.Add(1)
|
||||
go s.run()
|
||||
})
|
||||
}
|
||||
|
||||
// Stop gracefully stops the background goroutine started by Start/StartWithContext.
|
||||
// It is safe to call Stop multiple times.
|
||||
func (s *OpsAlertService) Stop() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
|
||||
s.stopOnce.Do(func() {
|
||||
if s.stop != nil {
|
||||
s.stop()
|
||||
}
|
||||
})
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) run() {
|
||||
defer s.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(s.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.evaluateOnce()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
s.evaluateOnce()
|
||||
case <-s.stopCtx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) evaluateOnce() {
|
||||
ctx, cancel := context.WithTimeout(s.stopCtx, opsAlertEvaluateTimeout)
|
||||
defer cancel()
|
||||
|
||||
s.Evaluate(ctx, time.Now())
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) Evaluate(ctx context.Context, now time.Time) {
|
||||
if s == nil || s.opsService == nil {
|
||||
return
|
||||
}
|
||||
|
||||
rules, err := s.opsService.ListAlertRules(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] failed to list rules: %v", err)
|
||||
return
|
||||
}
|
||||
if len(rules) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
maxSustainedByWindow := make(map[int]int)
|
||||
for _, rule := range rules {
|
||||
if !rule.Enabled {
|
||||
continue
|
||||
}
|
||||
window := rule.WindowMinutes
|
||||
if window <= 0 {
|
||||
window = 1
|
||||
}
|
||||
sustained := rule.SustainedMinutes
|
||||
if sustained <= 0 {
|
||||
sustained = 1
|
||||
}
|
||||
if sustained > maxSustainedByWindow[window] {
|
||||
maxSustainedByWindow[window] = sustained
|
||||
}
|
||||
}
|
||||
|
||||
metricsByWindow := make(map[int][]OpsMetrics)
|
||||
for window, limit := range maxSustainedByWindow {
|
||||
metrics, err := s.opsService.ListRecentSystemMetrics(ctx, window, limit)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] failed to load metrics window=%dm: %v", window, err)
|
||||
continue
|
||||
}
|
||||
metricsByWindow[window] = metrics
|
||||
}
|
||||
|
||||
for _, rule := range rules {
|
||||
if !rule.Enabled {
|
||||
continue
|
||||
}
|
||||
window := rule.WindowMinutes
|
||||
if window <= 0 {
|
||||
window = 1
|
||||
}
|
||||
sustained := rule.SustainedMinutes
|
||||
if sustained <= 0 {
|
||||
sustained = 1
|
||||
}
|
||||
|
||||
metrics := metricsByWindow[window]
|
||||
selected, ok := selectContiguousMetrics(metrics, sustained, now)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
breached, latestValue, ok := evaluateRule(rule, selected)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
activeEvent, err := s.opsService.GetActiveAlertEvent(ctx, rule.ID)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] failed to get active event (rule=%d): %v", rule.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if breached {
|
||||
if activeEvent != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
lastEvent, err := s.opsService.GetLatestAlertEvent(ctx, rule.ID)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] failed to get latest event (rule=%d): %v", rule.ID, err)
|
||||
continue
|
||||
}
|
||||
if lastEvent != nil && rule.CooldownMinutes > 0 {
|
||||
cooldown := time.Duration(rule.CooldownMinutes) * time.Minute
|
||||
if now.Sub(lastEvent.FiredAt) < cooldown {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
event := &OpsAlertEvent{
|
||||
RuleID: rule.ID,
|
||||
Severity: rule.Severity,
|
||||
Status: OpsAlertStatusFiring,
|
||||
Title: fmt.Sprintf("%s: %s", rule.Severity, rule.Name),
|
||||
Description: buildAlertDescription(rule, latestValue),
|
||||
MetricValue: latestValue,
|
||||
ThresholdValue: rule.Threshold,
|
||||
FiredAt: now,
|
||||
CreatedAt: now,
|
||||
}
|
||||
|
||||
if err := s.opsService.CreateAlertEvent(ctx, event); err != nil {
|
||||
log.Printf("[OpsAlert] failed to create event (rule=%d): %v", rule.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
emailSent, webhookSent := s.dispatchNotifications(ctx, rule, event)
|
||||
if emailSent || webhookSent {
|
||||
if err := s.opsService.UpdateAlertEventNotifications(ctx, event.ID, emailSent, webhookSent); err != nil {
|
||||
log.Printf("[OpsAlert] failed to update notification flags (event=%d): %v", event.ID, err)
|
||||
}
|
||||
}
|
||||
} else if activeEvent != nil {
|
||||
resolvedAt := now
|
||||
if err := s.opsService.UpdateAlertEventStatus(ctx, activeEvent.ID, OpsAlertStatusResolved, &resolvedAt); err != nil {
|
||||
log.Printf("[OpsAlert] failed to resolve event (event=%d): %v", activeEvent.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const opsMetricsContinuityTolerance = 20 * time.Second
|
||||
|
||||
// selectContiguousMetrics picks the newest N metrics and verifies they are continuous.
|
||||
//
|
||||
// This prevents a sustained rule from triggering when metrics sampling has gaps
|
||||
// (e.g. collector downtime) and avoids evaluating "stale" data.
|
||||
//
|
||||
// Assumptions:
|
||||
// - Metrics are ordered by UpdatedAt DESC (newest first).
|
||||
// - Metrics are expected to be collected at opsMetricsInterval cadence.
|
||||
func selectContiguousMetrics(metrics []OpsMetrics, needed int, now time.Time) ([]OpsMetrics, bool) {
|
||||
if needed <= 0 {
|
||||
return nil, false
|
||||
}
|
||||
if len(metrics) < needed {
|
||||
return nil, false
|
||||
}
|
||||
newest := metrics[0].UpdatedAt
|
||||
if newest.IsZero() {
|
||||
return nil, false
|
||||
}
|
||||
if now.Sub(newest) > opsMetricsInterval+opsMetricsContinuityTolerance {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
selected := metrics[:needed]
|
||||
for i := 0; i < len(selected)-1; i++ {
|
||||
a := selected[i].UpdatedAt
|
||||
b := selected[i+1].UpdatedAt
|
||||
if a.IsZero() || b.IsZero() {
|
||||
return nil, false
|
||||
}
|
||||
gap := a.Sub(b)
|
||||
if gap < opsMetricsInterval-opsMetricsContinuityTolerance || gap > opsMetricsInterval+opsMetricsContinuityTolerance {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
return selected, true
|
||||
}
|
||||
|
||||
func evaluateRule(rule OpsAlertRule, metrics []OpsMetrics) (bool, float64, bool) {
|
||||
if len(metrics) == 0 {
|
||||
return false, 0, false
|
||||
}
|
||||
|
||||
latestValue, ok := metricValue(metrics[0], rule.MetricType)
|
||||
if !ok {
|
||||
return false, 0, false
|
||||
}
|
||||
|
||||
for _, metric := range metrics {
|
||||
value, ok := metricValue(metric, rule.MetricType)
|
||||
if !ok || !compareMetric(value, rule.Operator, rule.Threshold) {
|
||||
return false, latestValue, true
|
||||
}
|
||||
}
|
||||
|
||||
return true, latestValue, true
|
||||
}
|
||||
|
||||
func metricValue(metric OpsMetrics, metricType string) (float64, bool) {
|
||||
switch metricType {
|
||||
case OpsMetricSuccessRate:
|
||||
if metric.RequestCount == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return metric.SuccessRate, true
|
||||
case OpsMetricErrorRate:
|
||||
if metric.RequestCount == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return metric.ErrorRate, true
|
||||
case OpsMetricP95LatencyMs:
|
||||
return float64(metric.P95LatencyMs), true
|
||||
case OpsMetricP99LatencyMs:
|
||||
return float64(metric.P99LatencyMs), true
|
||||
case OpsMetricHTTP2Errors:
|
||||
return float64(metric.HTTP2Errors), true
|
||||
case OpsMetricCPUUsagePercent:
|
||||
return metric.CPUUsagePercent, true
|
||||
case OpsMetricMemoryUsagePercent:
|
||||
return metric.MemoryUsagePercent, true
|
||||
case OpsMetricQueueDepth:
|
||||
return float64(metric.ConcurrencyQueueDepth), true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func compareMetric(value float64, operator string, threshold float64) bool {
|
||||
switch operator {
|
||||
case ">":
|
||||
return value > threshold
|
||||
case ">=":
|
||||
return value >= threshold
|
||||
case "<":
|
||||
return value < threshold
|
||||
case "<=":
|
||||
return value <= threshold
|
||||
case "==":
|
||||
return value == threshold
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func buildAlertDescription(rule OpsAlertRule, value float64) string {
|
||||
window := rule.WindowMinutes
|
||||
if window <= 0 {
|
||||
window = 1
|
||||
}
|
||||
return fmt.Sprintf("Rule %s triggered: %s %s %.2f (current %.2f) over last %dm",
|
||||
rule.Name,
|
||||
rule.MetricType,
|
||||
rule.Operator,
|
||||
rule.Threshold,
|
||||
value,
|
||||
window,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) dispatchNotifications(ctx context.Context, rule OpsAlertRule, event *OpsAlertEvent) (bool, bool) {
|
||||
emailSent := false
|
||||
webhookSent := false
|
||||
|
||||
notifyCtx, cancel := s.notificationContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
if rule.NotifyEmail {
|
||||
emailSent = s.sendEmailNotification(notifyCtx, rule, event)
|
||||
}
|
||||
if rule.NotifyWebhook && rule.WebhookURL != "" {
|
||||
webhookSent = s.sendWebhookNotification(notifyCtx, rule, event)
|
||||
}
|
||||
// Fallback channel: if email is enabled but ultimately fails, try webhook even if the
|
||||
// webhook toggle is off (as long as a webhook URL is configured).
|
||||
if rule.NotifyEmail && !emailSent && !rule.NotifyWebhook && rule.WebhookURL != "" {
|
||||
log.Printf("[OpsAlert] email failed; attempting webhook fallback (rule=%d)", rule.ID)
|
||||
webhookSent = s.sendWebhookNotification(notifyCtx, rule, event)
|
||||
}
|
||||
|
||||
return emailSent, webhookSent
|
||||
}
|
||||
|
||||
const (
|
||||
opsAlertEvaluateTimeout = 45 * time.Second
|
||||
opsAlertNotificationTimeout = 30 * time.Second
|
||||
opsAlertEmailMaxRetries = 3
|
||||
)
|
||||
|
||||
var opsAlertEmailBackoff = []time.Duration{
|
||||
1 * time.Second,
|
||||
2 * time.Second,
|
||||
4 * time.Second,
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) notificationContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
parent := ctx
|
||||
if s != nil && s.stopCtx != nil {
|
||||
parent = s.stopCtx
|
||||
}
|
||||
if parent == nil {
|
||||
parent = context.Background()
|
||||
}
|
||||
return context.WithTimeout(parent, opsAlertNotificationTimeout)
|
||||
}
|
||||
|
||||
var opsAlertSleep = sleepWithContext
|
||||
|
||||
func sleepWithContext(ctx context.Context, d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
time.Sleep(d)
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func retryWithBackoff(
|
||||
ctx context.Context,
|
||||
maxRetries int,
|
||||
backoff []time.Duration,
|
||||
fn func() error,
|
||||
onError func(attempt int, total int, nextDelay time.Duration, err error),
|
||||
) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if maxRetries < 0 {
|
||||
maxRetries = 0
|
||||
}
|
||||
totalAttempts := maxRetries + 1
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= totalAttempts; attempt++ {
|
||||
if attempt > 1 {
|
||||
backoffIdx := attempt - 2
|
||||
if backoffIdx < len(backoff) {
|
||||
if err := opsAlertSleep(ctx, backoff[backoffIdx]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := fn(); err != nil {
|
||||
lastErr = err
|
||||
nextDelay := time.Duration(0)
|
||||
if attempt < totalAttempts {
|
||||
nextIdx := attempt - 1
|
||||
if nextIdx < len(backoff) {
|
||||
nextDelay = backoff[nextIdx]
|
||||
}
|
||||
}
|
||||
if onError != nil {
|
||||
onError(attempt, totalAttempts, nextDelay, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) sendEmailNotification(ctx context.Context, rule OpsAlertRule, event *OpsAlertEvent) bool {
|
||||
if s.emailService == nil || s.userService == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
admin, err := s.userService.GetFirstAdmin(ctx)
|
||||
if err != nil || admin == nil || admin.Email == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
subject := fmt.Sprintf("[Ops Alert][%s] %s", rule.Severity, rule.Name)
|
||||
body := fmt.Sprintf(
|
||||
"Alert triggered: %s\n\nMetric: %s\nThreshold: %.2f\nCurrent: %.2f\nWindow: %dm\nStatus: %s\nTime: %s",
|
||||
rule.Name,
|
||||
rule.MetricType,
|
||||
rule.Threshold,
|
||||
event.MetricValue,
|
||||
rule.WindowMinutes,
|
||||
event.Status,
|
||||
event.FiredAt.Format(time.RFC3339),
|
||||
)
|
||||
|
||||
config, err := s.emailService.GetSMTPConfig(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] email config load failed: %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
if err := retryWithBackoff(
|
||||
ctx,
|
||||
opsAlertEmailMaxRetries,
|
||||
opsAlertEmailBackoff,
|
||||
func() error {
|
||||
return s.emailService.SendEmailWithConfig(config, admin.Email, subject, body)
|
||||
},
|
||||
func(attempt int, total int, nextDelay time.Duration, err error) {
|
||||
if attempt < total {
|
||||
log.Printf("[OpsAlert] email send failed (attempt=%d/%d), retrying in %s: %v", attempt, total, nextDelay, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[OpsAlert] email send failed (attempt=%d/%d), giving up: %v", attempt, total, err)
|
||||
},
|
||||
); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
log.Printf("[OpsAlert] email send canceled: %v", err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *OpsAlertService) sendWebhookNotification(ctx context.Context, rule OpsAlertRule, event *OpsAlertEvent) bool {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
webhookTarget, err := validateWebhookURL(ctx, rule.WebhookURL)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] invalid webhook url (rule=%d): %v", rule.ID, err)
|
||||
return false
|
||||
}
|
||||
|
||||
payload := map[string]any{
|
||||
"rule_id": rule.ID,
|
||||
"rule_name": rule.Name,
|
||||
"severity": rule.Severity,
|
||||
"status": event.Status,
|
||||
"metric_type": rule.MetricType,
|
||||
"metric_value": event.MetricValue,
|
||||
"threshold_value": rule.Threshold,
|
||||
"window_minutes": rule.WindowMinutes,
|
||||
"fired_at": event.FiredAt.Format(time.RFC3339),
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookTarget.URL.String(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := buildWebhookHTTPClient(s.httpClient, webhookTarget).Do(req)
|
||||
if err != nil {
|
||||
log.Printf("[OpsAlert] webhook send failed: %v", err)
|
||||
return false
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
log.Printf("[OpsAlert] webhook returned status %d", resp.StatusCode)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const webhookHTTPClientTimeout = 10 * time.Second
|
||||
|
||||
func buildWebhookHTTPClient(base *http.Client, webhookTarget *validatedWebhookTarget) *http.Client {
|
||||
var client http.Client
|
||||
if base != nil {
|
||||
client = *base
|
||||
}
|
||||
if client.Timeout <= 0 {
|
||||
client.Timeout = webhookHTTPClientTimeout
|
||||
}
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
if webhookTarget != nil {
|
||||
client.Transport = buildWebhookTransport(client.Transport, webhookTarget)
|
||||
}
|
||||
return &client
|
||||
}
|
||||
|
||||
var disallowedWebhookIPNets = []net.IPNet{
|
||||
// "this host on this network" / unspecified.
|
||||
mustParseCIDR("0.0.0.0/8"),
|
||||
mustParseCIDR("127.0.0.0/8"), // loopback (includes 127.0.0.1)
|
||||
mustParseCIDR("10.0.0.0/8"), // RFC1918
|
||||
mustParseCIDR("192.168.0.0/16"), // RFC1918
|
||||
mustParseCIDR("172.16.0.0/12"), // RFC1918 (172.16.0.0 - 172.31.255.255)
|
||||
mustParseCIDR("100.64.0.0/10"), // RFC6598 (carrier-grade NAT)
|
||||
mustParseCIDR("169.254.0.0/16"), // IPv4 link-local (includes 169.254.169.254 metadata IP on many clouds)
|
||||
mustParseCIDR("198.18.0.0/15"), // RFC2544 benchmark testing
|
||||
mustParseCIDR("224.0.0.0/4"), // IPv4 multicast
|
||||
mustParseCIDR("240.0.0.0/4"), // IPv4 reserved
|
||||
mustParseCIDR("::/128"), // IPv6 unspecified
|
||||
mustParseCIDR("::1/128"), // IPv6 loopback
|
||||
mustParseCIDR("fc00::/7"), // IPv6 unique local
|
||||
mustParseCIDR("fe80::/10"), // IPv6 link-local
|
||||
mustParseCIDR("ff00::/8"), // IPv6 multicast
|
||||
}
|
||||
|
||||
func mustParseCIDR(cidr string) net.IPNet {
|
||||
_, block, err := net.ParseCIDR(cidr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return *block
|
||||
}
|
||||
|
||||
var lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
}
|
||||
|
||||
type validatedWebhookTarget struct {
|
||||
URL *url.URL
|
||||
|
||||
host string
|
||||
port string
|
||||
pinnedIPs []net.IP
|
||||
}
|
||||
|
||||
var webhookBaseDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialer := net.Dialer{
|
||||
Timeout: 5 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
}
|
||||
|
||||
func buildWebhookTransport(base http.RoundTripper, webhookTarget *validatedWebhookTarget) http.RoundTripper {
|
||||
if webhookTarget == nil || webhookTarget.URL == nil {
|
||||
return base
|
||||
}
|
||||
|
||||
var transport *http.Transport
|
||||
switch typed := base.(type) {
|
||||
case *http.Transport:
|
||||
if typed != nil {
|
||||
transport = typed.Clone()
|
||||
}
|
||||
}
|
||||
if transport == nil {
|
||||
if defaultTransport, ok := http.DefaultTransport.(*http.Transport); ok && defaultTransport != nil {
|
||||
transport = defaultTransport.Clone()
|
||||
} else {
|
||||
transport = (&http.Transport{}).Clone()
|
||||
}
|
||||
}
|
||||
|
||||
webhookHost := webhookTarget.host
|
||||
webhookPort := webhookTarget.port
|
||||
pinnedIPs := append([]net.IP(nil), webhookTarget.pinnedIPs...)
|
||||
|
||||
transport.Proxy = nil
|
||||
transport.DialTLSContext = nil
|
||||
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil || host == "" || port == "" {
|
||||
return nil, fmt.Errorf("webhook dial target is invalid: %q", addr)
|
||||
}
|
||||
|
||||
canonicalHost := strings.TrimSuffix(strings.ToLower(host), ".")
|
||||
if canonicalHost != webhookHost || port != webhookPort {
|
||||
return nil, fmt.Errorf("webhook dial target mismatch: %q", addr)
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, ip := range pinnedIPs {
|
||||
if isDisallowedWebhookIP(ip) {
|
||||
lastErr = fmt.Errorf("webhook target resolves to a disallowed ip")
|
||||
continue
|
||||
}
|
||||
|
||||
dialAddr := net.JoinHostPort(ip.String(), port)
|
||||
conn, err := webhookBaseDialContext(ctx, network, dialAddr)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = errors.New("webhook target has no resolved addresses")
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
return transport
|
||||
}
|
||||
|
||||
func validateWebhookURL(ctx context.Context, raw string) (*validatedWebhookTarget, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil, errors.New("webhook url is empty")
|
||||
}
|
||||
// Avoid request smuggling / header injection vectors.
|
||||
if strings.ContainsAny(raw, "\r\n") {
|
||||
return nil, errors.New("webhook url contains invalid characters")
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return nil, errors.New("webhook url format is invalid")
|
||||
}
|
||||
if !strings.EqualFold(parsed.Scheme, "https") {
|
||||
return nil, errors.New("webhook url scheme must be https")
|
||||
}
|
||||
parsed.Scheme = "https"
|
||||
if parsed.Host == "" || parsed.Hostname() == "" {
|
||||
return nil, errors.New("webhook url must include host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return nil, errors.New("webhook url must not include userinfo")
|
||||
}
|
||||
if parsed.Port() != "" {
|
||||
port, err := strconv.Atoi(parsed.Port())
|
||||
if err != nil || port < 1 || port > 65535 {
|
||||
return nil, errors.New("webhook url port is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
host := strings.TrimSuffix(strings.ToLower(parsed.Hostname()), ".")
|
||||
if host == "localhost" {
|
||||
return nil, errors.New("webhook url host must not be localhost")
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if isDisallowedWebhookIP(ip) {
|
||||
return nil, errors.New("webhook url host resolves to a disallowed ip")
|
||||
}
|
||||
return &validatedWebhookTarget{
|
||||
URL: parsed,
|
||||
host: host,
|
||||
port: portForScheme(parsed),
|
||||
pinnedIPs: []net.IP{ip},
|
||||
}, nil
|
||||
}
|
||||
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
ips, err := lookupIPAddrs(ctx, host)
|
||||
if err != nil || len(ips) == 0 {
|
||||
return nil, errors.New("webhook url host cannot be resolved")
|
||||
}
|
||||
pinned := make([]net.IP, 0, len(ips))
|
||||
for _, addr := range ips {
|
||||
if isDisallowedWebhookIP(addr.IP) {
|
||||
return nil, errors.New("webhook url host resolves to a disallowed ip")
|
||||
}
|
||||
if addr.IP != nil {
|
||||
pinned = append(pinned, addr.IP)
|
||||
}
|
||||
}
|
||||
|
||||
if len(pinned) == 0 {
|
||||
return nil, errors.New("webhook url host cannot be resolved")
|
||||
}
|
||||
|
||||
return &validatedWebhookTarget{
|
||||
URL: parsed,
|
||||
host: host,
|
||||
port: portForScheme(parsed),
|
||||
pinnedIPs: uniqueResolvedIPs(pinned),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isDisallowedWebhookIP(ip net.IP) bool {
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
ip = ip4
|
||||
} else if ip16 := ip.To16(); ip16 != nil {
|
||||
ip = ip16
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
|
||||
// Disallow non-public addresses even if they're not explicitly covered by the CIDR list.
|
||||
// This provides defense-in-depth against SSRF targets such as link-local, multicast, and
|
||||
// unspecified addresses, and ensures any "pinned" IP is still blocked at dial time.
|
||||
if ip.IsUnspecified() ||
|
||||
ip.IsLoopback() ||
|
||||
ip.IsMulticast() ||
|
||||
ip.IsLinkLocalUnicast() ||
|
||||
ip.IsLinkLocalMulticast() ||
|
||||
ip.IsPrivate() {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, block := range disallowedWebhookIPNets {
|
||||
if block.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func portForScheme(u *url.URL) string {
|
||||
if u != nil && u.Port() != "" {
|
||||
return u.Port()
|
||||
}
|
||||
return "443"
|
||||
}
|
||||
|
||||
func uniqueResolvedIPs(ips []net.IP) []net.IP {
|
||||
seen := make(map[string]struct{}, len(ips))
|
||||
out := make([]net.IP, 0, len(ips))
|
||||
for _, ip := range ips {
|
||||
if ip == nil {
|
||||
continue
|
||||
}
|
||||
key := ip.String()
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, ip)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,271 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// This integration test protects the DI startup contract for OpsAlertService.
|
||||
//
|
||||
// Background:
|
||||
// - OpsMetricsCollector previously called alertService.Start()/Evaluate() directly.
|
||||
// - Those direct calls were removed, so OpsAlertService must now start via DI
|
||||
// (ProvideOpsAlertService in wire.go) and run its own evaluation ticker.
|
||||
//
|
||||
// What we validate here:
|
||||
// 1. When we construct via the Wire provider functions (ProvideOpsAlertService +
|
||||
// ProvideOpsMetricsCollector), OpsAlertService starts automatically.
|
||||
// 2. Its evaluation loop continues to tick even if OpsMetricsCollector is stopped,
|
||||
// proving the alert evaluator is independent.
|
||||
// 3. The evaluation path can trigger alert logic (CreateAlertEvent called).
|
||||
func TestOpsAlertService_StartedViaWireProviders_RunsIndependentTicker(t *testing.T) {
|
||||
oldInterval := opsAlertEvalInterval
|
||||
opsAlertEvalInterval = 25 * time.Millisecond
|
||||
t.Cleanup(func() { opsAlertEvalInterval = oldInterval })
|
||||
|
||||
repo := newFakeOpsRepository()
|
||||
opsService := NewOpsService(repo, nil)
|
||||
|
||||
// Start via the Wire provider function (the production DI path).
|
||||
alertService := ProvideOpsAlertService(opsService, nil, nil)
|
||||
t.Cleanup(alertService.Stop)
|
||||
|
||||
// Construct via ProvideOpsMetricsCollector (wire.go). Stop immediately to ensure
|
||||
// the alert ticker keeps running without the metrics collector.
|
||||
collector := ProvideOpsMetricsCollector(opsService, NewConcurrencyService(nil))
|
||||
collector.Stop()
|
||||
|
||||
// Wait for at least one evaluation (run() calls evaluateOnce immediately).
|
||||
require.Eventually(t, func() bool {
|
||||
return repo.listRulesCalls.Load() >= 1
|
||||
}, 1*time.Second, 5*time.Millisecond)
|
||||
|
||||
// Confirm the evaluation loop keeps ticking after the metrics collector is stopped.
|
||||
callsAfterCollectorStop := repo.listRulesCalls.Load()
|
||||
require.Eventually(t, func() bool {
|
||||
return repo.listRulesCalls.Load() >= callsAfterCollectorStop+2
|
||||
}, 1*time.Second, 5*time.Millisecond)
|
||||
|
||||
// Confirm the evaluation logic actually fires an alert event at least once.
|
||||
select {
|
||||
case <-repo.eventCreatedCh:
|
||||
// ok
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("expected OpsAlertService to create an alert event, but none was created (ListAlertRules calls=%d)", repo.listRulesCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func newFakeOpsRepository() *fakeOpsRepository {
|
||||
return &fakeOpsRepository{
|
||||
eventCreatedCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// fakeOpsRepository is a lightweight in-memory stub of OpsRepository for integration tests.
|
||||
// It avoids real DB/Redis usage and provides deterministic responses fast.
|
||||
type fakeOpsRepository struct {
|
||||
listRulesCalls atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
activeEvent *OpsAlertEvent
|
||||
latestEvent *OpsAlertEvent
|
||||
nextEventID int64
|
||||
eventCreatedCh chan struct{}
|
||||
eventOnce sync.Once
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) CreateErrorLog(ctx context.Context, log *OpsErrorLog) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) ListErrorLogsLegacy(ctx context.Context, filters OpsErrorLogFilters) ([]OpsErrorLog, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) ListErrorLogs(ctx context.Context, filter *ErrorLogFilter) ([]*ErrorLog, int64, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetLatestSystemMetric(ctx context.Context) (*OpsMetrics, error) {
|
||||
return &OpsMetrics{WindowMinutes: 1}, sql.ErrNoRows
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) CreateSystemMetric(ctx context.Context, metric *OpsMetrics) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetWindowStats(ctx context.Context, startTime, endTime time.Time) (*OpsWindowStats, error) {
|
||||
return &OpsWindowStats{}, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetProviderStats(ctx context.Context, startTime, endTime time.Time) ([]*ProviderStats, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetLatencyHistogram(ctx context.Context, startTime, endTime time.Time) ([]*LatencyHistogramItem, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetErrorDistribution(ctx context.Context, startTime, endTime time.Time) ([]*ErrorDistributionItem, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) ListRecentSystemMetrics(ctx context.Context, windowMinutes, limit int) ([]OpsMetrics, error) {
|
||||
if limit <= 0 {
|
||||
limit = 1
|
||||
}
|
||||
now := time.Now()
|
||||
metrics := make([]OpsMetrics, 0, limit)
|
||||
for i := 0; i < limit; i++ {
|
||||
metrics = append(metrics, OpsMetrics{
|
||||
WindowMinutes: windowMinutes,
|
||||
CPUUsagePercent: 99,
|
||||
UpdatedAt: now.Add(-time.Duration(i) * opsMetricsInterval),
|
||||
})
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) ListSystemMetricsRange(ctx context.Context, windowMinutes int, startTime, endTime time.Time, limit int) ([]OpsMetrics, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) ListAlertRules(ctx context.Context) ([]OpsAlertRule, error) {
|
||||
call := r.listRulesCalls.Add(1)
|
||||
// Delay enabling rules slightly so the test can stop OpsMetricsCollector first,
|
||||
// then observe the alert evaluator ticking independently.
|
||||
if call < 5 {
|
||||
return nil, nil
|
||||
}
|
||||
return []OpsAlertRule{
|
||||
{
|
||||
ID: 1,
|
||||
Name: "cpu too high (test)",
|
||||
Enabled: true,
|
||||
MetricType: OpsMetricCPUUsagePercent,
|
||||
Operator: ">",
|
||||
Threshold: 0,
|
||||
WindowMinutes: 1,
|
||||
SustainedMinutes: 1,
|
||||
Severity: "P1",
|
||||
NotifyEmail: false,
|
||||
NotifyWebhook: false,
|
||||
CooldownMinutes: 0,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetActiveAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.activeEvent == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if r.activeEvent.RuleID != ruleID {
|
||||
return nil, nil
|
||||
}
|
||||
if r.activeEvent.Status != OpsAlertStatusFiring {
|
||||
return nil, nil
|
||||
}
|
||||
clone := *r.activeEvent
|
||||
return &clone, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetLatestAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.latestEvent == nil || r.latestEvent.RuleID != ruleID {
|
||||
return nil, nil
|
||||
}
|
||||
clone := *r.latestEvent
|
||||
return &clone, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) CreateAlertEvent(ctx context.Context, event *OpsAlertEvent) error {
|
||||
if event == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
r.nextEventID++
|
||||
event.ID = r.nextEventID
|
||||
|
||||
clone := *event
|
||||
r.latestEvent = &clone
|
||||
if clone.Status == OpsAlertStatusFiring {
|
||||
r.activeEvent = &clone
|
||||
}
|
||||
|
||||
r.eventOnce.Do(func() { close(r.eventCreatedCh) })
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) UpdateAlertEventStatus(ctx context.Context, eventID int64, status string, resolvedAt *time.Time) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.activeEvent != nil && r.activeEvent.ID == eventID {
|
||||
r.activeEvent.Status = status
|
||||
r.activeEvent.ResolvedAt = resolvedAt
|
||||
}
|
||||
if r.latestEvent != nil && r.latestEvent.ID == eventID {
|
||||
r.latestEvent.Status = status
|
||||
r.latestEvent.ResolvedAt = resolvedAt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) UpdateAlertEventNotifications(ctx context.Context, eventID int64, emailSent, webhookSent bool) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.activeEvent != nil && r.activeEvent.ID == eventID {
|
||||
r.activeEvent.EmailSent = emailSent
|
||||
r.activeEvent.WebhookSent = webhookSent
|
||||
}
|
||||
if r.latestEvent != nil && r.latestEvent.ID == eventID {
|
||||
r.latestEvent.EmailSent = emailSent
|
||||
r.latestEvent.WebhookSent = webhookSent
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) CountActiveAlerts(ctx context.Context) (int, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.activeEvent == nil {
|
||||
return 0, nil
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetOverviewStats(ctx context.Context, startTime, endTime time.Time) (*OverviewStats, error) {
|
||||
return &OverviewStats{}, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetCachedLatestSystemMetric(ctx context.Context) (*OpsMetrics, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) SetCachedLatestSystemMetric(ctx context.Context, metric *OpsMetrics) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) GetCachedDashboardOverview(ctx context.Context, timeRange string) (*DashboardOverviewData, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) SetCachedDashboardOverview(ctx context.Context, timeRange string, data *DashboardOverviewData, ttl time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *fakeOpsRepository) PingRedis(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,315 +0,0 @@
|
||||
//go:build unit || opsalert_unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSelectContiguousMetrics_Contiguous(t *testing.T) {
|
||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
metrics := []OpsMetrics{
|
||||
{UpdatedAt: now},
|
||||
{UpdatedAt: now.Add(-1 * time.Minute)},
|
||||
{UpdatedAt: now.Add(-2 * time.Minute)},
|
||||
}
|
||||
|
||||
selected, ok := selectContiguousMetrics(metrics, 3, now)
|
||||
require.True(t, ok)
|
||||
require.Len(t, selected, 3)
|
||||
}
|
||||
|
||||
func TestSelectContiguousMetrics_GapFails(t *testing.T) {
|
||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
metrics := []OpsMetrics{
|
||||
{UpdatedAt: now},
|
||||
// Missing the -1m sample (gap ~=2m).
|
||||
{UpdatedAt: now.Add(-2 * time.Minute)},
|
||||
{UpdatedAt: now.Add(-3 * time.Minute)},
|
||||
}
|
||||
|
||||
_, ok := selectContiguousMetrics(metrics, 3, now)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestSelectContiguousMetrics_StaleNewestFails(t *testing.T) {
|
||||
now := time.Date(2026, 1, 1, 0, 10, 0, 0, time.UTC)
|
||||
metrics := []OpsMetrics{
|
||||
{UpdatedAt: now.Add(-10 * time.Minute)},
|
||||
{UpdatedAt: now.Add(-11 * time.Minute)},
|
||||
}
|
||||
|
||||
_, ok := selectContiguousMetrics(metrics, 2, now)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestMetricValue_SuccessRate_NoTrafficIsNoData(t *testing.T) {
|
||||
metric := OpsMetrics{
|
||||
RequestCount: 0,
|
||||
SuccessRate: 0,
|
||||
}
|
||||
value, ok := metricValue(metric, OpsMetricSuccessRate)
|
||||
require.False(t, ok)
|
||||
require.Equal(t, 0.0, value)
|
||||
}
|
||||
|
||||
func TestOpsAlertService_StopWithoutStart_NoPanic(t *testing.T) {
|
||||
s := NewOpsAlertService(nil, nil, nil)
|
||||
require.NotPanics(t, func() { s.Stop() })
|
||||
}
|
||||
|
||||
func TestOpsAlertService_StartStop_Graceful(t *testing.T) {
|
||||
s := NewOpsAlertService(nil, nil, nil)
|
||||
s.interval = 5 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
s.StartWithContext(ctx)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
s.Stop()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// ok
|
||||
case <-time.After(1 * time.Second):
|
||||
t.Fatal("Stop did not return; background goroutine likely stuck")
|
||||
}
|
||||
|
||||
require.NotPanics(t, func() { s.Stop() })
|
||||
}
|
||||
|
||||
func TestBuildWebhookHTTPClient_DefaultTimeout(t *testing.T) {
|
||||
client := buildWebhookHTTPClient(nil, nil)
|
||||
require.Equal(t, webhookHTTPClientTimeout, client.Timeout)
|
||||
require.NotNil(t, client.CheckRedirect)
|
||||
require.ErrorIs(t, client.CheckRedirect(nil, nil), http.ErrUseLastResponse)
|
||||
|
||||
base := &http.Client{}
|
||||
client = buildWebhookHTTPClient(base, nil)
|
||||
require.Equal(t, webhookHTTPClientTimeout, client.Timeout)
|
||||
require.NotNil(t, client.CheckRedirect)
|
||||
|
||||
base = &http.Client{Timeout: 2 * time.Second}
|
||||
client = buildWebhookHTTPClient(base, nil)
|
||||
require.Equal(t, 2*time.Second, client.Timeout)
|
||||
require.NotNil(t, client.CheckRedirect)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RequiresHTTPS(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
|
||||
_, err := validateWebhookURL(context.Background(), "http://example.com/webhook")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_InvalidFormatRejected(t *testing.T) {
|
||||
_, err := validateWebhookURL(context.Background(), "https://[::1")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RejectsUserinfo(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
|
||||
_, err := validateWebhookURL(context.Background(), "https://user:pass@example.com/webhook")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RejectsLocalhost(t *testing.T) {
|
||||
_, err := validateWebhookURL(context.Background(), "https://localhost/webhook")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RejectsPrivateIPLiteral(t *testing.T) {
|
||||
cases := []string{
|
||||
"https://0.0.0.0/webhook",
|
||||
"https://127.0.0.1/webhook",
|
||||
"https://10.0.0.1/webhook",
|
||||
"https://192.168.1.2/webhook",
|
||||
"https://172.16.0.1/webhook",
|
||||
"https://172.31.255.255/webhook",
|
||||
"https://100.64.0.1/webhook",
|
||||
"https://169.254.169.254/webhook",
|
||||
"https://198.18.0.1/webhook",
|
||||
"https://224.0.0.1/webhook",
|
||||
"https://240.0.0.1/webhook",
|
||||
"https://[::]/webhook",
|
||||
"https://[::1]/webhook",
|
||||
"https://[ff02::1]/webhook",
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc, func(t *testing.T) {
|
||||
_, err := validateWebhookURL(context.Background(), tc)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RejectsPrivateIPViaDNS(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
require.Equal(t, "internal.example", host)
|
||||
return []net.IPAddr{{IP: net.ParseIP("10.0.0.2")}}, nil
|
||||
}
|
||||
|
||||
_, err := validateWebhookURL(context.Background(), "https://internal.example/webhook")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RejectsLinkLocalIPViaDNS(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
require.Equal(t, "metadata.example", host)
|
||||
return []net.IPAddr{{IP: net.ParseIP("169.254.169.254")}}, nil
|
||||
}
|
||||
|
||||
_, err := validateWebhookURL(context.Background(), "https://metadata.example/webhook")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_AllowsPublicHostViaDNS(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
require.Equal(t, "example.com", host)
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
|
||||
target, err := validateWebhookURL(context.Background(), "https://example.com:443/webhook")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "https", target.URL.Scheme)
|
||||
require.Equal(t, "example.com", target.URL.Hostname())
|
||||
require.Equal(t, "443", target.URL.Port())
|
||||
}
|
||||
|
||||
func TestValidateWebhookURL_RejectsInvalidPort(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
|
||||
_, err := validateWebhookURL(context.Background(), "https://example.com:99999/webhook")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestWebhookTransport_UsesPinnedIP_NoDNSRebinding(t *testing.T) {
|
||||
oldLookup := lookupIPAddrs
|
||||
oldDial := webhookBaseDialContext
|
||||
t.Cleanup(func() {
|
||||
lookupIPAddrs = oldLookup
|
||||
webhookBaseDialContext = oldDial
|
||||
})
|
||||
|
||||
lookupCalls := 0
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
lookupCalls++
|
||||
require.Equal(t, "example.com", host)
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
|
||||
target, err := validateWebhookURL(context.Background(), "https://example.com/webhook")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, lookupCalls)
|
||||
|
||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
lookupCalls++
|
||||
return []net.IPAddr{{IP: net.ParseIP("10.0.0.1")}}, nil
|
||||
}
|
||||
|
||||
var dialAddrs []string
|
||||
webhookBaseDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialAddrs = append(dialAddrs, addr)
|
||||
return nil, errors.New("dial blocked in test")
|
||||
}
|
||||
|
||||
client := buildWebhookHTTPClient(nil, target)
|
||||
transport, ok := client.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
|
||||
_, err = transport.DialContext(context.Background(), "tcp", "example.com:443")
|
||||
require.Error(t, err)
|
||||
require.Equal(t, []string{"93.184.216.34:443"}, dialAddrs)
|
||||
require.Equal(t, 1, lookupCalls, "dial path must not re-resolve DNS")
|
||||
}
|
||||
|
||||
func TestRetryWithBackoff_SucceedsAfterRetries(t *testing.T) {
|
||||
oldSleep := opsAlertSleep
|
||||
t.Cleanup(func() { opsAlertSleep = oldSleep })
|
||||
|
||||
var slept []time.Duration
|
||||
opsAlertSleep = func(ctx context.Context, d time.Duration) error {
|
||||
slept = append(slept, d)
|
||||
return nil
|
||||
}
|
||||
|
||||
attempts := 0
|
||||
err := retryWithBackoff(
|
||||
context.Background(),
|
||||
3,
|
||||
[]time.Duration{time.Second, 2 * time.Second, 4 * time.Second},
|
||||
func() error {
|
||||
attempts++
|
||||
if attempts <= 3 {
|
||||
return errors.New("send failed")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, attempts)
|
||||
require.Equal(t, []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}, slept)
|
||||
}
|
||||
|
||||
func TestRetryWithBackoff_ContextCanceledStopsRetries(t *testing.T) {
|
||||
oldSleep := opsAlertSleep
|
||||
t.Cleanup(func() { opsAlertSleep = oldSleep })
|
||||
|
||||
var slept []time.Duration
|
||||
opsAlertSleep = func(ctx context.Context, d time.Duration) error {
|
||||
slept = append(slept, d)
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
attempts := 0
|
||||
err := retryWithBackoff(
|
||||
ctx,
|
||||
3,
|
||||
[]time.Duration{time.Second, 2 * time.Second, 4 * time.Second},
|
||||
func() error {
|
||||
attempts++
|
||||
return errors.New("send failed")
|
||||
},
|
||||
func(attempt int, total int, nextDelay time.Duration, err error) {
|
||||
if attempt == 1 {
|
||||
cancel()
|
||||
}
|
||||
},
|
||||
)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Equal(t, 1, attempts)
|
||||
require.Equal(t, []time.Duration{time.Second}, slept)
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
OpsAlertStatusFiring = "firing"
|
||||
OpsAlertStatusResolved = "resolved"
|
||||
)
|
||||
|
||||
const (
|
||||
OpsMetricSuccessRate = "success_rate"
|
||||
OpsMetricErrorRate = "error_rate"
|
||||
OpsMetricP95LatencyMs = "p95_latency_ms"
|
||||
OpsMetricP99LatencyMs = "p99_latency_ms"
|
||||
OpsMetricHTTP2Errors = "http2_errors"
|
||||
OpsMetricCPUUsagePercent = "cpu_usage_percent"
|
||||
OpsMetricMemoryUsagePercent = "memory_usage_percent"
|
||||
OpsMetricQueueDepth = "concurrency_queue_depth"
|
||||
)
|
||||
|
||||
type OpsAlertRule struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Enabled bool `json:"enabled"`
|
||||
MetricType string `json:"metric_type"`
|
||||
Operator string `json:"operator"`
|
||||
Threshold float64 `json:"threshold"`
|
||||
WindowMinutes int `json:"window_minutes"`
|
||||
SustainedMinutes int `json:"sustained_minutes"`
|
||||
Severity string `json:"severity"`
|
||||
NotifyEmail bool `json:"notify_email"`
|
||||
NotifyWebhook bool `json:"notify_webhook"`
|
||||
WebhookURL string `json:"webhook_url"`
|
||||
CooldownMinutes int `json:"cooldown_minutes"`
|
||||
DimensionFilters map[string]any `json:"dimension_filters,omitempty"`
|
||||
NotifyChannels []string `json:"notify_channels,omitempty"`
|
||||
NotifyConfig map[string]any `json:"notify_config,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type OpsAlertEvent struct {
|
||||
ID int64 `json:"id"`
|
||||
RuleID int64 `json:"rule_id"`
|
||||
Severity string `json:"severity"`
|
||||
Status string `json:"status"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
MetricValue float64 `json:"metric_value"`
|
||||
ThresholdValue float64 `json:"threshold_value"`
|
||||
FiredAt time.Time `json:"fired_at"`
|
||||
ResolvedAt *time.Time `json:"resolved_at"`
|
||||
EmailSent bool `json:"email_sent"`
|
||||
WebhookSent bool `json:"webhook_sent"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (s *OpsService) ListAlertRules(ctx context.Context) ([]OpsAlertRule, error) {
|
||||
return s.repo.ListAlertRules(ctx)
|
||||
}
|
||||
|
||||
func (s *OpsService) GetActiveAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
||||
return s.repo.GetActiveAlertEvent(ctx, ruleID)
|
||||
}
|
||||
|
||||
func (s *OpsService) GetLatestAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
||||
return s.repo.GetLatestAlertEvent(ctx, ruleID)
|
||||
}
|
||||
|
||||
func (s *OpsService) CreateAlertEvent(ctx context.Context, event *OpsAlertEvent) error {
|
||||
return s.repo.CreateAlertEvent(ctx, event)
|
||||
}
|
||||
|
||||
func (s *OpsService) UpdateAlertEventStatus(ctx context.Context, eventID int64, status string, resolvedAt *time.Time) error {
|
||||
return s.repo.UpdateAlertEventStatus(ctx, eventID, status, resolvedAt)
|
||||
}
|
||||
|
||||
func (s *OpsService) UpdateAlertEventNotifications(ctx context.Context, eventID int64, emailSent, webhookSent bool) error {
|
||||
return s.repo.UpdateAlertEventNotifications(ctx, eventID, emailSent, webhookSent)
|
||||
}
|
||||
|
||||
func (s *OpsService) ListRecentSystemMetrics(ctx context.Context, windowMinutes, limit int) ([]OpsMetrics, error) {
|
||||
return s.repo.ListRecentSystemMetrics(ctx, windowMinutes, limit)
|
||||
}
|
||||
|
||||
func (s *OpsService) CountActiveAlerts(ctx context.Context) (int, error) {
|
||||
return s.repo.CountActiveAlerts(ctx)
|
||||
}
|
||||
@@ -1,203 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/shirou/gopsutil/v4/cpu"
|
||||
"github.com/shirou/gopsutil/v4/mem"
|
||||
)
|
||||
|
||||
const (
|
||||
opsMetricsInterval = 1 * time.Minute
|
||||
opsMetricsCollectTimeout = 10 * time.Second
|
||||
|
||||
opsMetricsWindowShortMinutes = 1
|
||||
opsMetricsWindowLongMinutes = 5
|
||||
|
||||
bytesPerMB = 1024 * 1024
|
||||
cpuUsageSampleInterval = 0 * time.Second
|
||||
|
||||
percentScale = 100
|
||||
)
|
||||
|
||||
type OpsMetricsCollector struct {
|
||||
opsService *OpsService
|
||||
concurrencyService *ConcurrencyService
|
||||
interval time.Duration
|
||||
lastGCPauseTotal uint64
|
||||
lastGCPauseMu sync.Mutex
|
||||
stopCh chan struct{}
|
||||
startOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
}
|
||||
|
||||
func NewOpsMetricsCollector(opsService *OpsService, concurrencyService *ConcurrencyService) *OpsMetricsCollector {
|
||||
return &OpsMetricsCollector{
|
||||
opsService: opsService,
|
||||
concurrencyService: concurrencyService,
|
||||
interval: opsMetricsInterval,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) Start() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.startOnce.Do(func() {
|
||||
if c.stopCh == nil {
|
||||
c.stopCh = make(chan struct{})
|
||||
}
|
||||
go c.run()
|
||||
})
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.stopOnce.Do(func() {
|
||||
if c.stopCh != nil {
|
||||
close(c.stopCh)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) run() {
|
||||
ticker := time.NewTicker(c.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
c.collectOnce()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
c.collectOnce()
|
||||
case <-c.stopCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) collectOnce() {
|
||||
if c.opsService == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), opsMetricsCollectTimeout)
|
||||
defer cancel()
|
||||
|
||||
now := time.Now()
|
||||
systemStats := c.collectSystemStats(ctx)
|
||||
queueDepth := c.collectQueueDepth(ctx)
|
||||
activeAlerts := c.collectActiveAlerts(ctx)
|
||||
|
||||
for _, window := range []int{opsMetricsWindowShortMinutes, opsMetricsWindowLongMinutes} {
|
||||
startTime := now.Add(-time.Duration(window) * time.Minute)
|
||||
windowStats, err := c.opsService.GetWindowStats(ctx, startTime, now)
|
||||
if err != nil {
|
||||
log.Printf("[OpsMetrics] failed to get window stats (%dm): %v", window, err)
|
||||
continue
|
||||
}
|
||||
|
||||
successRate, errorRate := computeRates(windowStats.SuccessCount, windowStats.ErrorCount)
|
||||
requestCount := windowStats.SuccessCount + windowStats.ErrorCount
|
||||
metric := &OpsMetrics{
|
||||
WindowMinutes: window,
|
||||
RequestCount: requestCount,
|
||||
SuccessCount: windowStats.SuccessCount,
|
||||
ErrorCount: windowStats.ErrorCount,
|
||||
SuccessRate: successRate,
|
||||
ErrorRate: errorRate,
|
||||
P95LatencyMs: windowStats.P95LatencyMs,
|
||||
P99LatencyMs: windowStats.P99LatencyMs,
|
||||
HTTP2Errors: windowStats.HTTP2Errors,
|
||||
ActiveAlerts: activeAlerts,
|
||||
CPUUsagePercent: systemStats.cpuUsage,
|
||||
MemoryUsedMB: systemStats.memoryUsedMB,
|
||||
MemoryTotalMB: systemStats.memoryTotalMB,
|
||||
MemoryUsagePercent: systemStats.memoryUsagePercent,
|
||||
HeapAllocMB: systemStats.heapAllocMB,
|
||||
GCPauseMs: systemStats.gcPauseMs,
|
||||
ConcurrencyQueueDepth: queueDepth,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
if err := c.opsService.RecordMetrics(ctx, metric); err != nil {
|
||||
log.Printf("[OpsMetrics] failed to record metrics (%dm): %v", window, err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func computeRates(successCount, errorCount int64) (float64, float64) {
|
||||
total := successCount + errorCount
|
||||
if total == 0 {
|
||||
// No traffic => no data. Rates are kept at 0 and request_count will be 0.
|
||||
// The UI should render this as N/A instead of "100% success".
|
||||
return 0, 0
|
||||
}
|
||||
successRate := float64(successCount) / float64(total) * percentScale
|
||||
errorRate := float64(errorCount) / float64(total) * percentScale
|
||||
return successRate, errorRate
|
||||
}
|
||||
|
||||
type opsSystemStats struct {
|
||||
cpuUsage float64
|
||||
memoryUsedMB int64
|
||||
memoryTotalMB int64
|
||||
memoryUsagePercent float64
|
||||
heapAllocMB int64
|
||||
gcPauseMs float64
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) collectSystemStats(ctx context.Context) opsSystemStats {
|
||||
stats := opsSystemStats{}
|
||||
|
||||
if percents, err := cpu.PercentWithContext(ctx, cpuUsageSampleInterval, false); err == nil && len(percents) > 0 {
|
||||
stats.cpuUsage = percents[0]
|
||||
}
|
||||
|
||||
if vm, err := mem.VirtualMemoryWithContext(ctx); err == nil {
|
||||
stats.memoryUsedMB = int64(vm.Used / bytesPerMB)
|
||||
stats.memoryTotalMB = int64(vm.Total / bytesPerMB)
|
||||
stats.memoryUsagePercent = vm.UsedPercent
|
||||
}
|
||||
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
stats.heapAllocMB = int64(memStats.HeapAlloc / bytesPerMB)
|
||||
c.lastGCPauseMu.Lock()
|
||||
if c.lastGCPauseTotal != 0 && memStats.PauseTotalNs >= c.lastGCPauseTotal {
|
||||
stats.gcPauseMs = float64(memStats.PauseTotalNs-c.lastGCPauseTotal) / float64(time.Millisecond)
|
||||
}
|
||||
c.lastGCPauseTotal = memStats.PauseTotalNs
|
||||
c.lastGCPauseMu.Unlock()
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) collectQueueDepth(ctx context.Context) int {
|
||||
if c.concurrencyService == nil {
|
||||
return 0
|
||||
}
|
||||
depth, err := c.concurrencyService.GetTotalWaitCount(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[OpsMetrics] failed to get queue depth: %v", err)
|
||||
return 0
|
||||
}
|
||||
return depth
|
||||
}
|
||||
|
||||
func (c *OpsMetricsCollector) collectActiveAlerts(ctx context.Context) int {
|
||||
if c.opsService == nil {
|
||||
return 0
|
||||
}
|
||||
count, err := c.opsService.CountActiveAlerts(ctx)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return count
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user