refactor: 移除 Ops 监控模块
移除未完成的运维监控功能，简化系统架构： - 删除 ops_handler, ops_service, ops_repo 等后端代码 - 删除 ops 相关数据库迁移文件 - 删除前端 OpsDashboard 页面和 API
This commit is contained in:
ianshaw
@@ -1,402 +0,0 @@
|
|||||||
package admin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Wei-Shaw/sub2api/internal/pkg/response"
|
|
||||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
|
||||||
"github.com/gin-gonic/gin"
|
|
||||||
)
|
|
||||||
|
|
||||||
// OpsHandler handles ops dashboard endpoints.
|
|
||||||
type OpsHandler struct {
|
|
||||||
opsService *service.OpsService
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewOpsHandler creates a new OpsHandler.
|
|
||||||
func NewOpsHandler(opsService *service.OpsService) *OpsHandler {
|
|
||||||
return &OpsHandler{opsService: opsService}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetMetrics returns the latest ops metrics snapshot.
|
|
||||||
// GET /api/v1/admin/ops/metrics
|
|
||||||
func (h *OpsHandler) GetMetrics(c *gin.Context) {
|
|
||||||
metrics, err := h.opsService.GetLatestMetrics(c.Request.Context())
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to get ops metrics")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response.Success(c, metrics)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListMetricsHistory returns a time-range slice of metrics for charts.
|
|
||||||
// GET /api/v1/admin/ops/metrics/history
|
|
||||||
//
|
|
||||||
// Query params:
|
|
||||||
// - window_minutes: int (default 1)
|
|
||||||
// - minutes: int (lookback; optional)
|
|
||||||
// - start_time/end_time: RFC3339 timestamps (optional; overrides minutes when provided)
|
|
||||||
// - limit: int (optional; max 100, default 300 for backward compatibility)
|
|
||||||
func (h *OpsHandler) ListMetricsHistory(c *gin.Context) {
|
|
||||||
windowMinutes := 1
|
|
||||||
if v := c.Query("window_minutes"); v != "" {
|
|
||||||
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
|
|
||||||
windowMinutes = parsed
|
|
||||||
} else {
|
|
||||||
response.BadRequest(c, "Invalid window_minutes")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
limit := 300
|
|
||||||
limitProvided := false
|
|
||||||
if v := c.Query("limit"); v != "" {
|
|
||||||
parsed, err := strconv.Atoi(v)
|
|
||||||
if err != nil || parsed <= 0 || parsed > 5000 {
|
|
||||||
response.BadRequest(c, "Invalid limit (must be 1-5000)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
limit = parsed
|
|
||||||
limitProvided = true
|
|
||||||
}
|
|
||||||
|
|
||||||
endTime := time.Now()
|
|
||||||
startTime := time.Time{}
|
|
||||||
|
|
||||||
if startTimeStr := c.Query("start_time"); startTimeStr != "" {
|
|
||||||
parsed, err := time.Parse(time.RFC3339, startTimeStr)
|
|
||||||
if err != nil {
|
|
||||||
response.BadRequest(c, "Invalid start_time format (RFC3339)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
startTime = parsed
|
|
||||||
}
|
|
||||||
if endTimeStr := c.Query("end_time"); endTimeStr != "" {
|
|
||||||
parsed, err := time.Parse(time.RFC3339, endTimeStr)
|
|
||||||
if err != nil {
|
|
||||||
response.BadRequest(c, "Invalid end_time format (RFC3339)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
endTime = parsed
|
|
||||||
}
|
|
||||||
|
|
||||||
// If explicit range not provided, use lookback minutes.
|
|
||||||
if startTime.IsZero() {
|
|
||||||
if v := c.Query("minutes"); v != "" {
|
|
||||||
minutes, err := strconv.Atoi(v)
|
|
||||||
if err != nil || minutes <= 0 {
|
|
||||||
response.BadRequest(c, "Invalid minutes")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if minutes > 60*24*7 {
|
|
||||||
minutes = 60 * 24 * 7
|
|
||||||
}
|
|
||||||
startTime = endTime.Add(-time.Duration(minutes) * time.Minute)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default time range: last 24 hours.
|
|
||||||
if startTime.IsZero() {
|
|
||||||
startTime = endTime.Add(-24 * time.Hour)
|
|
||||||
if !limitProvided {
|
|
||||||
// Metrics are collected at 1-minute cadence; 24h requires ~1440 points.
|
|
||||||
limit = 24 * 60
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if startTime.After(endTime) {
|
|
||||||
response.BadRequest(c, "Invalid time range: start_time must be <= end_time")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
items, err := h.opsService.ListMetricsHistory(c.Request.Context(), windowMinutes, startTime, endTime, limit)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to list ops metrics history")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response.Success(c, gin.H{"items": items})
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListErrorLogs lists recent error logs with optional filters.
|
|
||||||
// GET /api/v1/admin/ops/error-logs
|
|
||||||
//
|
|
||||||
// Query params:
|
|
||||||
// - start_time/end_time: RFC3339 timestamps (optional)
|
|
||||||
// - platform: string (optional)
|
|
||||||
// - phase: string (optional)
|
|
||||||
// - severity: string (optional)
|
|
||||||
// - q: string (optional; fuzzy match)
|
|
||||||
// - limit: int (optional; default 100; max 500)
|
|
||||||
func (h *OpsHandler) ListErrorLogs(c *gin.Context) {
|
|
||||||
var filters service.OpsErrorLogFilters
|
|
||||||
|
|
||||||
if startTimeStr := c.Query("start_time"); startTimeStr != "" {
|
|
||||||
startTime, err := time.Parse(time.RFC3339, startTimeStr)
|
|
||||||
if err != nil {
|
|
||||||
response.BadRequest(c, "Invalid start_time format (RFC3339)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filters.StartTime = &startTime
|
|
||||||
}
|
|
||||||
if endTimeStr := c.Query("end_time"); endTimeStr != "" {
|
|
||||||
endTime, err := time.Parse(time.RFC3339, endTimeStr)
|
|
||||||
if err != nil {
|
|
||||||
response.BadRequest(c, "Invalid end_time format (RFC3339)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filters.EndTime = &endTime
|
|
||||||
}
|
|
||||||
|
|
||||||
if filters.StartTime != nil && filters.EndTime != nil && filters.StartTime.After(*filters.EndTime) {
|
|
||||||
response.BadRequest(c, "Invalid time range: start_time must be <= end_time")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
filters.Platform = c.Query("platform")
|
|
||||||
filters.Phase = c.Query("phase")
|
|
||||||
filters.Severity = c.Query("severity")
|
|
||||||
filters.Query = c.Query("q")
|
|
||||||
|
|
||||||
filters.Limit = 100
|
|
||||||
if limitStr := c.Query("limit"); limitStr != "" {
|
|
||||||
limit, err := strconv.Atoi(limitStr)
|
|
||||||
if err != nil || limit <= 0 || limit > 500 {
|
|
||||||
response.BadRequest(c, "Invalid limit (must be 1-500)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filters.Limit = limit
|
|
||||||
}
|
|
||||||
|
|
||||||
items, total, err := h.opsService.ListErrorLogs(c.Request.Context(), filters)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to list error logs")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
response.Success(c, gin.H{
|
|
||||||
"items": items,
|
|
||||||
"total": total,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetDashboardOverview returns realtime ops dashboard overview.
|
|
||||||
// GET /api/v1/admin/ops/dashboard/overview
|
|
||||||
//
|
|
||||||
// Query params:
|
|
||||||
// - time_range: string (optional; default "1h") one of: 5m, 30m, 1h, 6h, 24h
|
|
||||||
func (h *OpsHandler) GetDashboardOverview(c *gin.Context) {
|
|
||||||
timeRange := c.Query("time_range")
|
|
||||||
if timeRange == "" {
|
|
||||||
timeRange = "1h"
|
|
||||||
}
|
|
||||||
|
|
||||||
switch timeRange {
|
|
||||||
case "5m", "30m", "1h", "6h", "24h":
|
|
||||||
default:
|
|
||||||
response.BadRequest(c, "Invalid time_range (supported: 5m, 30m, 1h, 6h, 24h)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := h.opsService.GetDashboardOverview(c.Request.Context(), timeRange)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to get dashboard overview")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response.Success(c, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetProviderHealth returns upstream provider health comparison data.
|
|
||||||
// GET /api/v1/admin/ops/dashboard/providers
|
|
||||||
//
|
|
||||||
// Query params:
|
|
||||||
// - time_range: string (optional; default "1h") one of: 5m, 30m, 1h, 6h, 24h
|
|
||||||
func (h *OpsHandler) GetProviderHealth(c *gin.Context) {
|
|
||||||
timeRange := c.Query("time_range")
|
|
||||||
if timeRange == "" {
|
|
||||||
timeRange = "1h"
|
|
||||||
}
|
|
||||||
|
|
||||||
switch timeRange {
|
|
||||||
case "5m", "30m", "1h", "6h", "24h":
|
|
||||||
default:
|
|
||||||
response.BadRequest(c, "Invalid time_range (supported: 5m, 30m, 1h, 6h, 24h)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
providers, err := h.opsService.GetProviderHealth(c.Request.Context(), timeRange)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to get provider health")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var totalRequests int64
|
|
||||||
var weightedSuccess float64
|
|
||||||
var bestProvider string
|
|
||||||
var worstProvider string
|
|
||||||
var bestRate float64
|
|
||||||
var worstRate float64
|
|
||||||
hasRate := false
|
|
||||||
|
|
||||||
for _, p := range providers {
|
|
||||||
if p == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
totalRequests += p.RequestCount
|
|
||||||
weightedSuccess += (p.SuccessRate / 100) * float64(p.RequestCount)
|
|
||||||
|
|
||||||
if p.RequestCount <= 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !hasRate {
|
|
||||||
bestProvider = p.Name
|
|
||||||
worstProvider = p.Name
|
|
||||||
bestRate = p.SuccessRate
|
|
||||||
worstRate = p.SuccessRate
|
|
||||||
hasRate = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.SuccessRate > bestRate {
|
|
||||||
bestProvider = p.Name
|
|
||||||
bestRate = p.SuccessRate
|
|
||||||
}
|
|
||||||
if p.SuccessRate < worstRate {
|
|
||||||
worstProvider = p.Name
|
|
||||||
worstRate = p.SuccessRate
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
avgSuccessRate := 0.0
|
|
||||||
if totalRequests > 0 {
|
|
||||||
avgSuccessRate = (weightedSuccess / float64(totalRequests)) * 100
|
|
||||||
avgSuccessRate = math.Round(avgSuccessRate*100) / 100
|
|
||||||
}
|
|
||||||
|
|
||||||
response.Success(c, gin.H{
|
|
||||||
"providers": providers,
|
|
||||||
"summary": gin.H{
|
|
||||||
"total_requests": totalRequests,
|
|
||||||
"avg_success_rate": avgSuccessRate,
|
|
||||||
"best_provider": bestProvider,
|
|
||||||
"worst_provider": worstProvider,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetErrorLogs returns a paginated error log list with multi-dimensional filters.
|
|
||||||
// GET /api/v1/admin/ops/errors
|
|
||||||
func (h *OpsHandler) GetErrorLogs(c *gin.Context) {
|
|
||||||
page, pageSize := response.ParsePagination(c)
|
|
||||||
|
|
||||||
filter := &service.ErrorLogFilter{
|
|
||||||
Page: page,
|
|
||||||
PageSize: pageSize,
|
|
||||||
}
|
|
||||||
|
|
||||||
if startTimeStr := c.Query("start_time"); startTimeStr != "" {
|
|
||||||
startTime, err := time.Parse(time.RFC3339, startTimeStr)
|
|
||||||
if err != nil {
|
|
||||||
response.BadRequest(c, "Invalid start_time format (RFC3339)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filter.StartTime = &startTime
|
|
||||||
}
|
|
||||||
if endTimeStr := c.Query("end_time"); endTimeStr != "" {
|
|
||||||
endTime, err := time.Parse(time.RFC3339, endTimeStr)
|
|
||||||
if err != nil {
|
|
||||||
response.BadRequest(c, "Invalid end_time format (RFC3339)")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filter.EndTime = &endTime
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.StartTime != nil && filter.EndTime != nil && filter.StartTime.After(*filter.EndTime) {
|
|
||||||
response.BadRequest(c, "Invalid time range: start_time must be <= end_time")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if errorCodeStr := c.Query("error_code"); errorCodeStr != "" {
|
|
||||||
code, err := strconv.Atoi(errorCodeStr)
|
|
||||||
if err != nil || code < 0 {
|
|
||||||
response.BadRequest(c, "Invalid error_code")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filter.ErrorCode = &code
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keep both parameter names for compatibility: provider (docs) and platform (legacy).
|
|
||||||
filter.Provider = c.Query("provider")
|
|
||||||
if filter.Provider == "" {
|
|
||||||
filter.Provider = c.Query("platform")
|
|
||||||
}
|
|
||||||
|
|
||||||
if accountIDStr := c.Query("account_id"); accountIDStr != "" {
|
|
||||||
accountID, err := strconv.ParseInt(accountIDStr, 10, 64)
|
|
||||||
if err != nil || accountID <= 0 {
|
|
||||||
response.BadRequest(c, "Invalid account_id")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filter.AccountID = &accountID
|
|
||||||
}
|
|
||||||
|
|
||||||
out, err := h.opsService.GetErrorLogs(c.Request.Context(), filter)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to get error logs")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
response.Success(c, gin.H{
|
|
||||||
"errors": out.Errors,
|
|
||||||
"total": out.Total,
|
|
||||||
"page": out.Page,
|
|
||||||
"page_size": out.PageSize,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetLatencyHistogram returns the latency distribution histogram.
|
|
||||||
// GET /api/v1/admin/ops/dashboard/latency-histogram
|
|
||||||
func (h *OpsHandler) GetLatencyHistogram(c *gin.Context) {
|
|
||||||
timeRange := c.Query("time_range")
|
|
||||||
if timeRange == "" {
|
|
||||||
timeRange = "1h"
|
|
||||||
}
|
|
||||||
|
|
||||||
buckets, err := h.opsService.GetLatencyHistogram(c.Request.Context(), timeRange)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to get latency histogram")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
totalRequests := int64(0)
|
|
||||||
for _, b := range buckets {
|
|
||||||
totalRequests += b.Count
|
|
||||||
}
|
|
||||||
|
|
||||||
response.Success(c, gin.H{
|
|
||||||
"buckets": buckets,
|
|
||||||
"total_requests": totalRequests,
|
|
||||||
"slow_request_threshold": 1000,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetErrorDistribution returns the error distribution.
|
|
||||||
// GET /api/v1/admin/ops/dashboard/errors/distribution
|
|
||||||
func (h *OpsHandler) GetErrorDistribution(c *gin.Context) {
|
|
||||||
timeRange := c.Query("time_range")
|
|
||||||
if timeRange == "" {
|
|
||||||
timeRange = "1h"
|
|
||||||
}
|
|
||||||
|
|
||||||
items, err := h.opsService.GetErrorDistribution(c.Request.Context(), timeRange)
|
|
||||||
if err != nil {
|
|
||||||
response.Error(c, http.StatusInternalServerError, "Failed to get error distribution")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
response.Success(c, gin.H{
|
|
||||||
"items": items,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,286 +0,0 @@
|
|||||||
package admin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
|
||||||
"github.com/gorilla/websocket"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OpsWSProxyConfig struct {
|
|
||||||
TrustProxy bool
|
|
||||||
TrustedProxies []netip.Prefix
|
|
||||||
OriginPolicy string
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
envOpsWSTrustProxy = "OPS_WS_TRUST_PROXY"
|
|
||||||
envOpsWSTrustedProxies = "OPS_WS_TRUSTED_PROXIES"
|
|
||||||
envOpsWSOriginPolicy = "OPS_WS_ORIGIN_POLICY"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
OriginPolicyStrict = "strict"
|
|
||||||
OriginPolicyPermissive = "permissive"
|
|
||||||
)
|
|
||||||
|
|
||||||
var opsWSProxyConfig = loadOpsWSProxyConfigFromEnv()
|
|
||||||
|
|
||||||
var upgrader = websocket.Upgrader{
|
|
||||||
CheckOrigin: func(r *http.Request) bool {
|
|
||||||
return isAllowedOpsWSOrigin(r)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// QPSWSHandler handles realtime QPS push via WebSocket.
|
|
||||||
// GET /api/v1/admin/ops/ws/qps
|
|
||||||
func (h *OpsHandler) QPSWSHandler(c *gin.Context) {
|
|
||||||
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsWS] upgrade failed: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer func() { _ = conn.Close() }()
|
|
||||||
|
|
||||||
// Set pong handler
|
|
||||||
if err := conn.SetReadDeadline(time.Now().Add(60 * time.Second)); err != nil {
|
|
||||||
log.Printf("[OpsWS] set read deadline failed: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
conn.SetPongHandler(func(string) error {
|
|
||||||
return conn.SetReadDeadline(time.Now().Add(60 * time.Second))
|
|
||||||
})
|
|
||||||
|
|
||||||
// Push QPS data every 2 seconds
|
|
||||||
ticker := time.NewTicker(2 * time.Second)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
// Heartbeat ping every 30 seconds
|
|
||||||
pingTicker := time.NewTicker(30 * time.Second)
|
|
||||||
defer pingTicker.Stop()
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(c.Request.Context())
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
// Fetch 1m window stats for current QPS
|
|
||||||
data, err := h.opsService.GetDashboardOverview(ctx, "5m")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsWS] get overview failed: %v", err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
payload := gin.H{
|
|
||||||
"type": "qps_update",
|
|
||||||
"timestamp": time.Now().Format(time.RFC3339),
|
|
||||||
"data": gin.H{
|
|
||||||
"qps": data.QPS.Current,
|
|
||||||
"tps": data.TPS.Current,
|
|
||||||
"request_count": data.Errors.TotalCount + int64(data.QPS.Avg1h*60), // Rough estimate
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
msg, _ := json.Marshal(payload)
|
|
||||||
if err := conn.WriteMessage(websocket.TextMessage, msg); err != nil {
|
|
||||||
log.Printf("[OpsWS] write failed: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case <-pingTicker.C:
|
|
||||||
if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil {
|
|
||||||
log.Printf("[OpsWS] ping failed: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func isAllowedOpsWSOrigin(r *http.Request) bool {
|
|
||||||
if r == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
origin := strings.TrimSpace(r.Header.Get("Origin"))
|
|
||||||
if origin == "" {
|
|
||||||
switch strings.ToLower(strings.TrimSpace(opsWSProxyConfig.OriginPolicy)) {
|
|
||||||
case OriginPolicyStrict:
|
|
||||||
return false
|
|
||||||
case OriginPolicyPermissive, "":
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
parsed, err := url.Parse(origin)
|
|
||||||
if err != nil || parsed.Hostname() == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
originHost := strings.ToLower(parsed.Hostname())
|
|
||||||
|
|
||||||
trustProxyHeaders := shouldTrustOpsWSProxyHeaders(r)
|
|
||||||
reqHost := hostWithoutPort(r.Host)
|
|
||||||
if trustProxyHeaders {
|
|
||||||
xfHost := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
|
|
||||||
if xfHost != "" {
|
|
||||||
xfHost = strings.TrimSpace(strings.Split(xfHost, ",")[0])
|
|
||||||
if xfHost != "" {
|
|
||||||
reqHost = hostWithoutPort(xfHost)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reqHost = strings.ToLower(reqHost)
|
|
||||||
if reqHost == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return originHost == reqHost
|
|
||||||
}
|
|
||||||
|
|
||||||
func shouldTrustOpsWSProxyHeaders(r *http.Request) bool {
|
|
||||||
if r == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !opsWSProxyConfig.TrustProxy {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
peerIP, ok := requestPeerIP(r)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return isAddrInTrustedProxies(peerIP, opsWSProxyConfig.TrustedProxies)
|
|
||||||
}
|
|
||||||
|
|
||||||
func requestPeerIP(r *http.Request) (netip.Addr, bool) {
|
|
||||||
if r == nil {
|
|
||||||
return netip.Addr{}, false
|
|
||||||
}
|
|
||||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
|
||||||
if err != nil {
|
|
||||||
host = strings.TrimSpace(r.RemoteAddr)
|
|
||||||
}
|
|
||||||
host = strings.TrimPrefix(host, "[")
|
|
||||||
host = strings.TrimSuffix(host, "]")
|
|
||||||
if host == "" {
|
|
||||||
return netip.Addr{}, false
|
|
||||||
}
|
|
||||||
addr, err := netip.ParseAddr(host)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Addr{}, false
|
|
||||||
}
|
|
||||||
return addr.Unmap(), true
|
|
||||||
}
|
|
||||||
|
|
||||||
func isAddrInTrustedProxies(addr netip.Addr, trusted []netip.Prefix) bool {
|
|
||||||
if !addr.IsValid() {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, p := range trusted {
|
|
||||||
if p.Contains(addr) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadOpsWSProxyConfigFromEnv() OpsWSProxyConfig {
|
|
||||||
cfg := OpsWSProxyConfig{
|
|
||||||
TrustProxy: true,
|
|
||||||
TrustedProxies: defaultTrustedProxies(),
|
|
||||||
OriginPolicy: OriginPolicyPermissive,
|
|
||||||
}
|
|
||||||
|
|
||||||
if v := strings.TrimSpace(os.Getenv(envOpsWSTrustProxy)); v != "" {
|
|
||||||
if parsed, err := strconv.ParseBool(v); err == nil {
|
|
||||||
cfg.TrustProxy = parsed
|
|
||||||
} else {
|
|
||||||
log.Printf("[OpsWS] invalid %s=%q (expected bool); using default=%v", envOpsWSTrustProxy, v, cfg.TrustProxy)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw := strings.TrimSpace(os.Getenv(envOpsWSTrustedProxies)); raw != "" {
|
|
||||||
prefixes, invalid := parseTrustedProxyList(raw)
|
|
||||||
if len(invalid) > 0 {
|
|
||||||
log.Printf("[OpsWS] invalid %s entries ignored: %s", envOpsWSTrustedProxies, strings.Join(invalid, ", "))
|
|
||||||
}
|
|
||||||
cfg.TrustedProxies = prefixes
|
|
||||||
}
|
|
||||||
|
|
||||||
if v := strings.TrimSpace(os.Getenv(envOpsWSOriginPolicy)); v != "" {
|
|
||||||
normalized := strings.ToLower(v)
|
|
||||||
switch normalized {
|
|
||||||
case OriginPolicyStrict, OriginPolicyPermissive:
|
|
||||||
cfg.OriginPolicy = normalized
|
|
||||||
default:
|
|
||||||
log.Printf("[OpsWS] invalid %s=%q (expected %q or %q); using default=%q", envOpsWSOriginPolicy, v, OriginPolicyStrict, OriginPolicyPermissive, cfg.OriginPolicy)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultTrustedProxies() []netip.Prefix {
|
|
||||||
prefixes, _ := parseTrustedProxyList("127.0.0.0/8,::1/128")
|
|
||||||
return prefixes
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseTrustedProxyList(raw string) (prefixes []netip.Prefix, invalid []string) {
|
|
||||||
for _, token := range strings.Split(raw, ",") {
|
|
||||||
item := strings.TrimSpace(token)
|
|
||||||
if item == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
p netip.Prefix
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
if strings.Contains(item, "/") {
|
|
||||||
p, err = netip.ParsePrefix(item)
|
|
||||||
} else {
|
|
||||||
var addr netip.Addr
|
|
||||||
addr, err = netip.ParseAddr(item)
|
|
||||||
if err == nil {
|
|
||||||
addr = addr.Unmap()
|
|
||||||
bits := 128
|
|
||||||
if addr.Is4() {
|
|
||||||
bits = 32
|
|
||||||
}
|
|
||||||
p = netip.PrefixFrom(addr, bits)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil || !p.IsValid() {
|
|
||||||
invalid = append(invalid, item)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
prefixes = append(prefixes, p.Masked())
|
|
||||||
}
|
|
||||||
return prefixes, invalid
|
|
||||||
}
|
|
||||||
|
|
||||||
func hostWithoutPort(hostport string) string {
|
|
||||||
hostport = strings.TrimSpace(hostport)
|
|
||||||
if hostport == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
if host, _, err := net.SplitHostPort(hostport); err == nil {
|
|
||||||
return host
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(hostport, "[") && strings.HasSuffix(hostport, "]") {
|
|
||||||
return strings.Trim(hostport, "[]")
|
|
||||||
}
|
|
||||||
parts := strings.Split(hostport, ":")
|
|
||||||
return parts[0]
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package admin
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestIsAllowedOpsWSOrigin_AllowsEmptyOrigin(t *testing.T) {
|
|
||||||
original := opsWSProxyConfig
|
|
||||||
t.Cleanup(func() { opsWSProxyConfig = original })
|
|
||||||
opsWSProxyConfig = OpsWSProxyConfig{OriginPolicy: OriginPolicyPermissive}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, "http://example.test", nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !isAllowedOpsWSOrigin(req) {
|
|
||||||
t.Fatalf("expected empty Origin to be allowed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsAllowedOpsWSOrigin_RejectsEmptyOrigin_WhenStrict(t *testing.T) {
|
|
||||||
original := opsWSProxyConfig
|
|
||||||
t.Cleanup(func() { opsWSProxyConfig = original })
|
|
||||||
opsWSProxyConfig = OpsWSProxyConfig{OriginPolicy: OriginPolicyStrict}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, "http://example.test", nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if isAllowedOpsWSOrigin(req) {
|
|
||||||
t.Fatalf("expected empty Origin to be rejected under strict policy")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsAllowedOpsWSOrigin_UsesXForwardedHostOnlyFromTrustedProxy(t *testing.T) {
|
|
||||||
original := opsWSProxyConfig
|
|
||||||
t.Cleanup(func() { opsWSProxyConfig = original })
|
|
||||||
|
|
||||||
opsWSProxyConfig = OpsWSProxyConfig{
|
|
||||||
TrustProxy: true,
|
|
||||||
TrustedProxies: []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("127.0.0.0/8"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Untrusted peer: ignore X-Forwarded-Host and compare against r.Host.
|
|
||||||
{
|
|
||||||
req, err := http.NewRequest(http.MethodGet, "http://internal.service.local", nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest: %v", err)
|
|
||||||
}
|
|
||||||
req.RemoteAddr = "192.0.2.1:12345"
|
|
||||||
req.Host = "internal.service.local"
|
|
||||||
req.Header.Set("Origin", "https://public.example.com")
|
|
||||||
req.Header.Set("X-Forwarded-Host", "public.example.com")
|
|
||||||
|
|
||||||
if isAllowedOpsWSOrigin(req) {
|
|
||||||
t.Fatalf("expected Origin to be rejected when peer is not a trusted proxy")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Trusted peer: allow X-Forwarded-Host to participate in Origin validation.
|
|
||||||
{
|
|
||||||
req, err := http.NewRequest(http.MethodGet, "http://internal.service.local", nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest: %v", err)
|
|
||||||
}
|
|
||||||
req.RemoteAddr = "127.0.0.1:23456"
|
|
||||||
req.Host = "internal.service.local"
|
|
||||||
req.Header.Set("Origin", "https://public.example.com")
|
|
||||||
req.Header.Set("X-Forwarded-Host", "public.example.com")
|
|
||||||
|
|
||||||
if !isAllowedOpsWSOrigin(req) {
|
|
||||||
t.Fatalf("expected Origin to be accepted when peer is a trusted proxy")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoadOpsWSProxyConfigFromEnv_OriginPolicy(t *testing.T) {
|
|
||||||
t.Setenv(envOpsWSOriginPolicy, "STRICT")
|
|
||||||
cfg := loadOpsWSProxyConfigFromEnv()
|
|
||||||
if cfg.OriginPolicy != OriginPolicyStrict {
|
|
||||||
t.Fatalf("OriginPolicy=%q, want %q", cfg.OriginPolicy, OriginPolicyStrict)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoadOpsWSProxyConfigFromEnv_OriginPolicyInvalidUsesDefault(t *testing.T) {
|
|
||||||
t.Setenv(envOpsWSOriginPolicy, "nope")
|
|
||||||
cfg := loadOpsWSProxyConfigFromEnv()
|
|
||||||
if cfg.OriginPolicy != OriginPolicyPermissive {
|
|
||||||
t.Fatalf("OriginPolicy=%q, want %q", cfg.OriginPolicy, OriginPolicyPermissive)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseTrustedProxyList(t *testing.T) {
|
|
||||||
prefixes, invalid := parseTrustedProxyList("10.0.0.1, 10.0.0.0/8, bad, ::1/128")
|
|
||||||
if len(prefixes) != 3 {
|
|
||||||
t.Fatalf("prefixes=%d, want 3", len(prefixes))
|
|
||||||
}
|
|
||||||
if len(invalid) != 1 || invalid[0] != "bad" {
|
|
||||||
t.Fatalf("invalid=%v, want [bad]", invalid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRequestPeerIP_ParsesIPv6(t *testing.T) {
|
|
||||||
req, err := http.NewRequest(http.MethodGet, "http://example.test", nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRequest: %v", err)
|
|
||||||
}
|
|
||||||
req.RemoteAddr = "[::1]:1234"
|
|
||||||
|
|
||||||
addr, ok := requestPeerIP(req)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected IPv6 peer IP to parse")
|
|
||||||
}
|
|
||||||
if addr != netip.MustParseAddr("::1") {
|
|
||||||
t.Fatalf("addr=%s, want ::1", addr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,166 +0,0 @@
|
|||||||
package handler
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
|
|
||||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
|
||||||
"github.com/gin-gonic/gin"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
opsModelKey = "ops_model"
|
|
||||||
opsStreamKey = "ops_stream"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
opsErrorLogWorkerCount = 10
|
|
||||||
opsErrorLogQueueSize = 256
|
|
||||||
opsErrorLogTimeout = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
type opsErrorLogJob struct {
|
|
||||||
ops *service.OpsService
|
|
||||||
entry *service.OpsErrorLog
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
opsErrorLogOnce sync.Once
|
|
||||||
opsErrorLogQueue chan opsErrorLogJob
|
|
||||||
)
|
|
||||||
|
|
||||||
func startOpsErrorLogWorkers() {
|
|
||||||
opsErrorLogQueue = make(chan opsErrorLogJob, opsErrorLogQueueSize)
|
|
||||||
for i := 0; i < opsErrorLogWorkerCount; i++ {
|
|
||||||
go func() {
|
|
||||||
for job := range opsErrorLogQueue {
|
|
||||||
if job.ops == nil || job.entry == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), opsErrorLogTimeout)
|
|
||||||
_ = job.ops.RecordError(ctx, job.entry)
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func enqueueOpsErrorLog(ops *service.OpsService, entry *service.OpsErrorLog) {
|
|
||||||
if ops == nil || entry == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
opsErrorLogOnce.Do(startOpsErrorLogWorkers)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case opsErrorLogQueue <- opsErrorLogJob{ops: ops, entry: entry}:
|
|
||||||
default:
|
|
||||||
// Queue is full; drop to avoid blocking request handling.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func setOpsRequestContext(c *gin.Context, model string, stream bool) {
|
|
||||||
c.Set(opsModelKey, model)
|
|
||||||
c.Set(opsStreamKey, stream)
|
|
||||||
}
|
|
||||||
|
|
||||||
func recordOpsError(c *gin.Context, ops *service.OpsService, status int, errType, message, fallbackPlatform string) {
|
|
||||||
if ops == nil || c == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
model, _ := c.Get(opsModelKey)
|
|
||||||
stream, _ := c.Get(opsStreamKey)
|
|
||||||
|
|
||||||
var modelName string
|
|
||||||
if m, ok := model.(string); ok {
|
|
||||||
modelName = m
|
|
||||||
}
|
|
||||||
streaming, _ := stream.(bool)
|
|
||||||
|
|
||||||
apiKey, _ := middleware2.GetAPIKeyFromContext(c)
|
|
||||||
|
|
||||||
logEntry := &service.OpsErrorLog{
|
|
||||||
Phase: classifyOpsPhase(errType, message),
|
|
||||||
Type: errType,
|
|
||||||
Severity: classifyOpsSeverity(errType, status),
|
|
||||||
StatusCode: status,
|
|
||||||
Platform: resolveOpsPlatform(apiKey, fallbackPlatform),
|
|
||||||
Model: modelName,
|
|
||||||
RequestID: c.Writer.Header().Get("x-request-id"),
|
|
||||||
Message: message,
|
|
||||||
ClientIP: c.ClientIP(),
|
|
||||||
RequestPath: func() string {
|
|
||||||
if c.Request != nil && c.Request.URL != nil {
|
|
||||||
return c.Request.URL.Path
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}(),
|
|
||||||
Stream: streaming,
|
|
||||||
}
|
|
||||||
|
|
||||||
if apiKey != nil {
|
|
||||||
logEntry.APIKeyID = &apiKey.ID
|
|
||||||
if apiKey.User != nil {
|
|
||||||
logEntry.UserID = &apiKey.User.ID
|
|
||||||
}
|
|
||||||
if apiKey.GroupID != nil {
|
|
||||||
logEntry.GroupID = apiKey.GroupID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
enqueueOpsErrorLog(ops, logEntry)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveOpsPlatform(apiKey *service.APIKey, fallback string) string {
|
|
||||||
if apiKey != nil && apiKey.Group != nil && apiKey.Group.Platform != "" {
|
|
||||||
return apiKey.Group.Platform
|
|
||||||
}
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
|
|
||||||
func classifyOpsPhase(errType, message string) string {
|
|
||||||
msg := strings.ToLower(message)
|
|
||||||
switch errType {
|
|
||||||
case "authentication_error":
|
|
||||||
return "auth"
|
|
||||||
case "billing_error", "subscription_error":
|
|
||||||
return "billing"
|
|
||||||
case "rate_limit_error":
|
|
||||||
if strings.Contains(msg, "concurrency") || strings.Contains(msg, "pending") {
|
|
||||||
return "concurrency"
|
|
||||||
}
|
|
||||||
return "upstream"
|
|
||||||
case "invalid_request_error":
|
|
||||||
return "response"
|
|
||||||
case "upstream_error", "overloaded_error":
|
|
||||||
return "upstream"
|
|
||||||
case "api_error":
|
|
||||||
if strings.Contains(msg, "no available accounts") {
|
|
||||||
return "scheduling"
|
|
||||||
}
|
|
||||||
return "internal"
|
|
||||||
default:
|
|
||||||
return "internal"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func classifyOpsSeverity(errType string, status int) string {
|
|
||||||
switch errType {
|
|
||||||
case "invalid_request_error", "authentication_error", "billing_error", "subscription_error":
|
|
||||||
return "P3"
|
|
||||||
}
|
|
||||||
if status >= 500 {
|
|
||||||
return "P1"
|
|
||||||
}
|
|
||||||
if status == 429 {
|
|
||||||
return "P1"
|
|
||||||
}
|
|
||||||
if status >= 400 {
|
|
||||||
return "P2"
|
|
||||||
}
|
|
||||||
return "P3"
|
|
||||||
}
|
|
||||||
@@ -1,190 +0,0 @@
|
|||||||
package repository
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ListErrorLogs queries ops_error_logs with optional filters and pagination.
|
|
||||||
// It returns the list items and the total count of matching rows.
|
|
||||||
func (r *OpsRepository) ListErrorLogs(ctx context.Context, filter *service.ErrorLogFilter) ([]*service.ErrorLog, int64, error) {
|
|
||||||
page := 1
|
|
||||||
pageSize := 20
|
|
||||||
if filter != nil {
|
|
||||||
if filter.Page > 0 {
|
|
||||||
page = filter.Page
|
|
||||||
}
|
|
||||||
if filter.PageSize > 0 {
|
|
||||||
pageSize = filter.PageSize
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if pageSize > 100 {
|
|
||||||
pageSize = 100
|
|
||||||
}
|
|
||||||
offset := (page - 1) * pageSize
|
|
||||||
|
|
||||||
conditions := make([]string, 0)
|
|
||||||
args := make([]any, 0)
|
|
||||||
|
|
||||||
addCondition := func(condition string, values ...any) {
|
|
||||||
conditions = append(conditions, condition)
|
|
||||||
args = append(args, values...)
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter != nil {
|
|
||||||
// 默认查询最近 24 小时
|
|
||||||
if filter.StartTime == nil && filter.EndTime == nil {
|
|
||||||
defaultStart := time.Now().Add(-24 * time.Hour)
|
|
||||||
filter.StartTime = &defaultStart
|
|
||||||
}
|
|
||||||
|
|
||||||
if filter.StartTime != nil {
|
|
||||||
addCondition(fmt.Sprintf("created_at >= $%d", len(args)+1), *filter.StartTime)
|
|
||||||
}
|
|
||||||
if filter.EndTime != nil {
|
|
||||||
addCondition(fmt.Sprintf("created_at <= $%d", len(args)+1), *filter.EndTime)
|
|
||||||
}
|
|
||||||
if filter.ErrorCode != nil {
|
|
||||||
addCondition(fmt.Sprintf("status_code = $%d", len(args)+1), *filter.ErrorCode)
|
|
||||||
}
|
|
||||||
if provider := strings.TrimSpace(filter.Provider); provider != "" {
|
|
||||||
addCondition(fmt.Sprintf("platform = $%d", len(args)+1), provider)
|
|
||||||
}
|
|
||||||
if filter.AccountID != nil {
|
|
||||||
addCondition(fmt.Sprintf("account_id = $%d", len(args)+1), *filter.AccountID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
where := ""
|
|
||||||
if len(conditions) > 0 {
|
|
||||||
where = "WHERE " + strings.Join(conditions, " AND ")
|
|
||||||
}
|
|
||||||
|
|
||||||
countQuery := fmt.Sprintf(`SELECT COUNT(1) FROM ops_error_logs %s`, where)
|
|
||||||
var total int64
|
|
||||||
if err := scanSingleRow(ctx, r.sql, countQuery, args, &total); err != nil {
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
total = 0
|
|
||||||
} else {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
listQuery := fmt.Sprintf(`
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
created_at,
|
|
||||||
severity,
|
|
||||||
request_id,
|
|
||||||
account_id,
|
|
||||||
request_path,
|
|
||||||
platform,
|
|
||||||
model,
|
|
||||||
status_code,
|
|
||||||
error_message,
|
|
||||||
duration_ms,
|
|
||||||
retry_count,
|
|
||||||
stream
|
|
||||||
FROM ops_error_logs
|
|
||||||
%s
|
|
||||||
ORDER BY created_at DESC
|
|
||||||
LIMIT $%d OFFSET $%d
|
|
||||||
`, where, len(args)+1, len(args)+2)
|
|
||||||
|
|
||||||
listArgs := append(append([]any{}, args...), pageSize, offset)
|
|
||||||
rows, err := r.sql.QueryContext(ctx, listQuery, listArgs...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
results := make([]*service.ErrorLog, 0)
|
|
||||||
for rows.Next() {
|
|
||||||
var (
|
|
||||||
id int64
|
|
||||||
createdAt time.Time
|
|
||||||
severity sql.NullString
|
|
||||||
requestID sql.NullString
|
|
||||||
accountID sql.NullInt64
|
|
||||||
requestURI sql.NullString
|
|
||||||
platform sql.NullString
|
|
||||||
model sql.NullString
|
|
||||||
statusCode sql.NullInt64
|
|
||||||
message sql.NullString
|
|
||||||
durationMs sql.NullInt64
|
|
||||||
retryCount sql.NullInt64
|
|
||||||
stream sql.NullBool
|
|
||||||
)
|
|
||||||
|
|
||||||
if err := rows.Scan(
|
|
||||||
&id,
|
|
||||||
&createdAt,
|
|
||||||
&severity,
|
|
||||||
&requestID,
|
|
||||||
&accountID,
|
|
||||||
&requestURI,
|
|
||||||
&platform,
|
|
||||||
&model,
|
|
||||||
&statusCode,
|
|
||||||
&message,
|
|
||||||
&durationMs,
|
|
||||||
&retryCount,
|
|
||||||
&stream,
|
|
||||||
); err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := &service.ErrorLog{
|
|
||||||
ID: id,
|
|
||||||
Timestamp: createdAt,
|
|
||||||
Level: levelFromSeverity(severity.String),
|
|
||||||
RequestID: requestID.String,
|
|
||||||
APIPath: requestURI.String,
|
|
||||||
Provider: platform.String,
|
|
||||||
Model: model.String,
|
|
||||||
HTTPCode: int(statusCode.Int64),
|
|
||||||
Stream: stream.Bool,
|
|
||||||
}
|
|
||||||
if accountID.Valid {
|
|
||||||
entry.AccountID = strconv.FormatInt(accountID.Int64, 10)
|
|
||||||
}
|
|
||||||
if message.Valid {
|
|
||||||
entry.ErrorMessage = message.String
|
|
||||||
}
|
|
||||||
if durationMs.Valid {
|
|
||||||
v := int(durationMs.Int64)
|
|
||||||
entry.DurationMs = &v
|
|
||||||
}
|
|
||||||
if retryCount.Valid {
|
|
||||||
v := int(retryCount.Int64)
|
|
||||||
entry.RetryCount = &v
|
|
||||||
}
|
|
||||||
|
|
||||||
results = append(results, entry)
|
|
||||||
}
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return results, total, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func levelFromSeverity(severity string) string {
|
|
||||||
sev := strings.ToUpper(strings.TrimSpace(severity))
|
|
||||||
switch sev {
|
|
||||||
case "P0", "P1":
|
|
||||||
return "CRITICAL"
|
|
||||||
case "P2":
|
|
||||||
return "ERROR"
|
|
||||||
case "P3":
|
|
||||||
return "WARN"
|
|
||||||
default:
|
|
||||||
return "ERROR"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
package repository
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
opsLatestMetricsKey = "ops:metrics:latest"
|
|
||||||
|
|
||||||
opsDashboardOverviewKeyPrefix = "ops:dashboard:overview:"
|
|
||||||
|
|
||||||
opsLatestMetricsTTL = 10 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
func (r *OpsRepository) GetCachedLatestSystemMetric(ctx context.Context) (*service.OpsMetrics, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if r == nil || r.rdb == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := r.rdb.Get(ctx, opsLatestMetricsKey).Bytes()
|
|
||||||
if errors.Is(err, redis.Nil) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("redis get cached latest system metric: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var metric service.OpsMetrics
|
|
||||||
if err := json.Unmarshal(data, &metric); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal cached latest system metric: %w", err)
|
|
||||||
}
|
|
||||||
return &metric, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *OpsRepository) SetCachedLatestSystemMetric(ctx context.Context, metric *service.OpsMetrics) error {
|
|
||||||
if metric == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if r == nil || r.rdb == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := json.Marshal(metric)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("marshal cached latest system metric: %w", err)
|
|
||||||
}
|
|
||||||
return r.rdb.Set(ctx, opsLatestMetricsKey, data, opsLatestMetricsTTL).Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *OpsRepository) GetCachedDashboardOverview(ctx context.Context, timeRange string) (*service.DashboardOverviewData, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if r == nil || r.rdb == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
rangeKey := strings.TrimSpace(timeRange)
|
|
||||||
if rangeKey == "" {
|
|
||||||
rangeKey = "1h"
|
|
||||||
}
|
|
||||||
|
|
||||||
key := opsDashboardOverviewKeyPrefix + rangeKey
|
|
||||||
data, err := r.rdb.Get(ctx, key).Bytes()
|
|
||||||
if errors.Is(err, redis.Nil) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("redis get cached dashboard overview: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var overview service.DashboardOverviewData
|
|
||||||
if err := json.Unmarshal(data, &overview); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal cached dashboard overview: %w", err)
|
|
||||||
}
|
|
||||||
return &overview, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *OpsRepository) SetCachedDashboardOverview(ctx context.Context, timeRange string, data *service.DashboardOverviewData, ttl time.Duration) error {
|
|
||||||
if data == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if ttl <= 0 {
|
|
||||||
ttl = 10 * time.Second
|
|
||||||
}
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if r == nil || r.rdb == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
rangeKey := strings.TrimSpace(timeRange)
|
|
||||||
if rangeKey == "" {
|
|
||||||
rangeKey = "1h"
|
|
||||||
}
|
|
||||||
|
|
||||||
payload, err := json.Marshal(data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("marshal cached dashboard overview: %w", err)
|
|
||||||
}
|
|
||||||
key := opsDashboardOverviewKeyPrefix + rangeKey
|
|
||||||
return r.rdb.Set(ctx, key, payload, ttl).Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *OpsRepository) PingRedis(ctx context.Context) error {
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if r == nil || r.rdb == nil {
|
|
||||||
return errors.New("redis client is nil")
|
|
||||||
}
|
|
||||||
return r.rdb.Ping(ctx).Err()
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,55 +0,0 @@
|
|||||||
package middleware
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
opsAuthErrorLogWorkerCount = 10
|
|
||||||
opsAuthErrorLogQueueSize = 256
|
|
||||||
opsAuthErrorLogTimeout = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
type opsAuthErrorLogJob struct {
|
|
||||||
ops *service.OpsService
|
|
||||||
entry *service.OpsErrorLog
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
opsAuthErrorLogOnce sync.Once
|
|
||||||
opsAuthErrorLogQueue chan opsAuthErrorLogJob
|
|
||||||
)
|
|
||||||
|
|
||||||
func startOpsAuthErrorLogWorkers() {
|
|
||||||
opsAuthErrorLogQueue = make(chan opsAuthErrorLogJob, opsAuthErrorLogQueueSize)
|
|
||||||
for i := 0; i < opsAuthErrorLogWorkerCount; i++ {
|
|
||||||
go func() {
|
|
||||||
for job := range opsAuthErrorLogQueue {
|
|
||||||
if job.ops == nil || job.entry == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), opsAuthErrorLogTimeout)
|
|
||||||
_ = job.ops.RecordError(ctx, job.entry)
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func enqueueOpsAuthErrorLog(ops *service.OpsService, entry *service.OpsErrorLog) {
|
|
||||||
if ops == nil || entry == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
opsAuthErrorLogOnce.Do(startOpsAuthErrorLogWorkers)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case opsAuthErrorLogQueue <- opsAuthErrorLogJob{ops: ops, entry: entry}:
|
|
||||||
default:
|
|
||||||
// Queue is full; drop to avoid blocking request handling.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrorLog represents an ops error log item for list queries.
|
|
||||||
//
|
|
||||||
// Field naming matches docs/API-运维监控中心2.0.md (L3 根因追踪 - 错误日志列表).
|
|
||||||
type ErrorLog struct {
|
|
||||||
ID int64 `json:"id"`
|
|
||||||
Timestamp time.Time `json:"timestamp"`
|
|
||||||
|
|
||||||
Level string `json:"level,omitempty"`
|
|
||||||
RequestID string `json:"request_id,omitempty"`
|
|
||||||
AccountID string `json:"account_id,omitempty"`
|
|
||||||
APIPath string `json:"api_path,omitempty"`
|
|
||||||
Provider string `json:"provider,omitempty"`
|
|
||||||
Model string `json:"model,omitempty"`
|
|
||||||
HTTPCode int `json:"http_code,omitempty"`
|
|
||||||
ErrorMessage string `json:"error_message,omitempty"`
|
|
||||||
|
|
||||||
DurationMs *int `json:"duration_ms,omitempty"`
|
|
||||||
RetryCount *int `json:"retry_count,omitempty"`
|
|
||||||
Stream bool `json:"stream,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrorLogFilter describes optional filters and pagination for listing ops error logs.
|
|
||||||
type ErrorLogFilter struct {
|
|
||||||
StartTime *time.Time
|
|
||||||
EndTime *time.Time
|
|
||||||
|
|
||||||
ErrorCode *int
|
|
||||||
Provider string
|
|
||||||
AccountID *int64
|
|
||||||
|
|
||||||
Page int
|
|
||||||
PageSize int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *ErrorLogFilter) normalize() (page, pageSize int) {
|
|
||||||
page = 1
|
|
||||||
pageSize = 20
|
|
||||||
if f == nil {
|
|
||||||
return page, pageSize
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.Page > 0 {
|
|
||||||
page = f.Page
|
|
||||||
}
|
|
||||||
if f.PageSize > 0 {
|
|
||||||
pageSize = f.PageSize
|
|
||||||
}
|
|
||||||
if pageSize > 100 {
|
|
||||||
pageSize = 100
|
|
||||||
}
|
|
||||||
return page, pageSize
|
|
||||||
}
|
|
||||||
|
|
||||||
type ErrorLogListResponse struct {
|
|
||||||
Errors []*ErrorLog `json:"errors"`
|
|
||||||
Total int64 `json:"total"`
|
|
||||||
Page int `json:"page"`
|
|
||||||
PageSize int `json:"page_size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) GetErrorLogs(ctx context.Context, filter *ErrorLogFilter) (*ErrorLogListResponse, error) {
|
|
||||||
if s == nil || s.repo == nil {
|
|
||||||
return &ErrorLogListResponse{
|
|
||||||
Errors: []*ErrorLog{},
|
|
||||||
Total: 0,
|
|
||||||
Page: 1,
|
|
||||||
PageSize: 20,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
page, pageSize := filter.normalize()
|
|
||||||
if filter == nil {
|
|
||||||
filter = &ErrorLogFilter{}
|
|
||||||
}
|
|
||||||
filter.Page = page
|
|
||||||
filter.PageSize = pageSize
|
|
||||||
|
|
||||||
items, total, err := s.repo.ListErrorLogs(ctx, filter)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if items == nil {
|
|
||||||
items = []*ErrorLog{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ErrorLogListResponse{
|
|
||||||
Errors: items,
|
|
||||||
Total: total,
|
|
||||||
Page: page,
|
|
||||||
PageSize: pageSize,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@@ -1,834 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OpsAlertService struct {
|
|
||||||
opsService *OpsService
|
|
||||||
userService *UserService
|
|
||||||
emailService *EmailService
|
|
||||||
httpClient *http.Client
|
|
||||||
|
|
||||||
interval time.Duration
|
|
||||||
|
|
||||||
startOnce sync.Once
|
|
||||||
stopOnce sync.Once
|
|
||||||
stopCtx context.Context
|
|
||||||
stop context.CancelFunc
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
|
||||||
|
|
||||||
// opsAlertEvalInterval defines how often OpsAlertService evaluates alert rules.
|
|
||||||
//
|
|
||||||
// Production uses opsMetricsInterval. Tests may override this variable to keep
|
|
||||||
// integration tests fast without changing production defaults.
|
|
||||||
var opsAlertEvalInterval = opsMetricsInterval
|
|
||||||
|
|
||||||
func NewOpsAlertService(opsService *OpsService, userService *UserService, emailService *EmailService) *OpsAlertService {
|
|
||||||
return &OpsAlertService{
|
|
||||||
opsService: opsService,
|
|
||||||
userService: userService,
|
|
||||||
emailService: emailService,
|
|
||||||
httpClient: &http.Client{Timeout: 10 * time.Second},
|
|
||||||
interval: opsAlertEvalInterval,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start launches the background alert evaluation loop.
|
|
||||||
//
|
|
||||||
// Stop must be called during shutdown to ensure the goroutine exits.
|
|
||||||
func (s *OpsAlertService) Start() {
|
|
||||||
s.StartWithContext(context.Background())
|
|
||||||
}
|
|
||||||
|
|
||||||
// StartWithContext is like Start but allows the caller to provide a parent context.
|
|
||||||
// When the parent context is canceled, the service stops automatically.
|
|
||||||
func (s *OpsAlertService) StartWithContext(ctx context.Context) {
|
|
||||||
if s == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
|
|
||||||
s.startOnce.Do(func() {
|
|
||||||
if s.interval <= 0 {
|
|
||||||
s.interval = opsAlertEvalInterval
|
|
||||||
}
|
|
||||||
|
|
||||||
s.stopCtx, s.stop = context.WithCancel(ctx)
|
|
||||||
s.wg.Add(1)
|
|
||||||
go s.run()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop gracefully stops the background goroutine started by Start/StartWithContext.
|
|
||||||
// It is safe to call Stop multiple times.
|
|
||||||
func (s *OpsAlertService) Stop() {
|
|
||||||
if s == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.stopOnce.Do(func() {
|
|
||||||
if s.stop != nil {
|
|
||||||
s.stop()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
s.wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) run() {
|
|
||||||
defer s.wg.Done()
|
|
||||||
|
|
||||||
ticker := time.NewTicker(s.interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
s.evaluateOnce()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
s.evaluateOnce()
|
|
||||||
case <-s.stopCtx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) evaluateOnce() {
|
|
||||||
ctx, cancel := context.WithTimeout(s.stopCtx, opsAlertEvaluateTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
s.Evaluate(ctx, time.Now())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) Evaluate(ctx context.Context, now time.Time) {
|
|
||||||
if s == nil || s.opsService == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rules, err := s.opsService.ListAlertRules(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to list rules: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(rules) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
maxSustainedByWindow := make(map[int]int)
|
|
||||||
for _, rule := range rules {
|
|
||||||
if !rule.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
window := rule.WindowMinutes
|
|
||||||
if window <= 0 {
|
|
||||||
window = 1
|
|
||||||
}
|
|
||||||
sustained := rule.SustainedMinutes
|
|
||||||
if sustained <= 0 {
|
|
||||||
sustained = 1
|
|
||||||
}
|
|
||||||
if sustained > maxSustainedByWindow[window] {
|
|
||||||
maxSustainedByWindow[window] = sustained
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
metricsByWindow := make(map[int][]OpsMetrics)
|
|
||||||
for window, limit := range maxSustainedByWindow {
|
|
||||||
metrics, err := s.opsService.ListRecentSystemMetrics(ctx, window, limit)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to load metrics window=%dm: %v", window, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
metricsByWindow[window] = metrics
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rule := range rules {
|
|
||||||
if !rule.Enabled {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
window := rule.WindowMinutes
|
|
||||||
if window <= 0 {
|
|
||||||
window = 1
|
|
||||||
}
|
|
||||||
sustained := rule.SustainedMinutes
|
|
||||||
if sustained <= 0 {
|
|
||||||
sustained = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics := metricsByWindow[window]
|
|
||||||
selected, ok := selectContiguousMetrics(metrics, sustained, now)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
breached, latestValue, ok := evaluateRule(rule, selected)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
activeEvent, err := s.opsService.GetActiveAlertEvent(ctx, rule.ID)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to get active event (rule=%d): %v", rule.ID, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if breached {
|
|
||||||
if activeEvent != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
lastEvent, err := s.opsService.GetLatestAlertEvent(ctx, rule.ID)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to get latest event (rule=%d): %v", rule.ID, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if lastEvent != nil && rule.CooldownMinutes > 0 {
|
|
||||||
cooldown := time.Duration(rule.CooldownMinutes) * time.Minute
|
|
||||||
if now.Sub(lastEvent.FiredAt) < cooldown {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
event := &OpsAlertEvent{
|
|
||||||
RuleID: rule.ID,
|
|
||||||
Severity: rule.Severity,
|
|
||||||
Status: OpsAlertStatusFiring,
|
|
||||||
Title: fmt.Sprintf("%s: %s", rule.Severity, rule.Name),
|
|
||||||
Description: buildAlertDescription(rule, latestValue),
|
|
||||||
MetricValue: latestValue,
|
|
||||||
ThresholdValue: rule.Threshold,
|
|
||||||
FiredAt: now,
|
|
||||||
CreatedAt: now,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.opsService.CreateAlertEvent(ctx, event); err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to create event (rule=%d): %v", rule.ID, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
emailSent, webhookSent := s.dispatchNotifications(ctx, rule, event)
|
|
||||||
if emailSent || webhookSent {
|
|
||||||
if err := s.opsService.UpdateAlertEventNotifications(ctx, event.ID, emailSent, webhookSent); err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to update notification flags (event=%d): %v", event.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if activeEvent != nil {
|
|
||||||
resolvedAt := now
|
|
||||||
if err := s.opsService.UpdateAlertEventStatus(ctx, activeEvent.ID, OpsAlertStatusResolved, &resolvedAt); err != nil {
|
|
||||||
log.Printf("[OpsAlert] failed to resolve event (event=%d): %v", activeEvent.ID, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const opsMetricsContinuityTolerance = 20 * time.Second
|
|
||||||
|
|
||||||
// selectContiguousMetrics picks the newest N metrics and verifies they are continuous.
|
|
||||||
//
|
|
||||||
// This prevents a sustained rule from triggering when metrics sampling has gaps
|
|
||||||
// (e.g. collector downtime) and avoids evaluating "stale" data.
|
|
||||||
//
|
|
||||||
// Assumptions:
|
|
||||||
// - Metrics are ordered by UpdatedAt DESC (newest first).
|
|
||||||
// - Metrics are expected to be collected at opsMetricsInterval cadence.
|
|
||||||
func selectContiguousMetrics(metrics []OpsMetrics, needed int, now time.Time) ([]OpsMetrics, bool) {
|
|
||||||
if needed <= 0 {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
if len(metrics) < needed {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
newest := metrics[0].UpdatedAt
|
|
||||||
if newest.IsZero() {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
if now.Sub(newest) > opsMetricsInterval+opsMetricsContinuityTolerance {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
selected := metrics[:needed]
|
|
||||||
for i := 0; i < len(selected)-1; i++ {
|
|
||||||
a := selected[i].UpdatedAt
|
|
||||||
b := selected[i+1].UpdatedAt
|
|
||||||
if a.IsZero() || b.IsZero() {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
gap := a.Sub(b)
|
|
||||||
if gap < opsMetricsInterval-opsMetricsContinuityTolerance || gap > opsMetricsInterval+opsMetricsContinuityTolerance {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return selected, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func evaluateRule(rule OpsAlertRule, metrics []OpsMetrics) (bool, float64, bool) {
|
|
||||||
if len(metrics) == 0 {
|
|
||||||
return false, 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
latestValue, ok := metricValue(metrics[0], rule.MetricType)
|
|
||||||
if !ok {
|
|
||||||
return false, 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, metric := range metrics {
|
|
||||||
value, ok := metricValue(metric, rule.MetricType)
|
|
||||||
if !ok || !compareMetric(value, rule.Operator, rule.Threshold) {
|
|
||||||
return false, latestValue, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true, latestValue, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func metricValue(metric OpsMetrics, metricType string) (float64, bool) {
|
|
||||||
switch metricType {
|
|
||||||
case OpsMetricSuccessRate:
|
|
||||||
if metric.RequestCount == 0 {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
return metric.SuccessRate, true
|
|
||||||
case OpsMetricErrorRate:
|
|
||||||
if metric.RequestCount == 0 {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
return metric.ErrorRate, true
|
|
||||||
case OpsMetricP95LatencyMs:
|
|
||||||
return float64(metric.P95LatencyMs), true
|
|
||||||
case OpsMetricP99LatencyMs:
|
|
||||||
return float64(metric.P99LatencyMs), true
|
|
||||||
case OpsMetricHTTP2Errors:
|
|
||||||
return float64(metric.HTTP2Errors), true
|
|
||||||
case OpsMetricCPUUsagePercent:
|
|
||||||
return metric.CPUUsagePercent, true
|
|
||||||
case OpsMetricMemoryUsagePercent:
|
|
||||||
return metric.MemoryUsagePercent, true
|
|
||||||
case OpsMetricQueueDepth:
|
|
||||||
return float64(metric.ConcurrencyQueueDepth), true
|
|
||||||
default:
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func compareMetric(value float64, operator string, threshold float64) bool {
|
|
||||||
switch operator {
|
|
||||||
case ">":
|
|
||||||
return value > threshold
|
|
||||||
case ">=":
|
|
||||||
return value >= threshold
|
|
||||||
case "<":
|
|
||||||
return value < threshold
|
|
||||||
case "<=":
|
|
||||||
return value <= threshold
|
|
||||||
case "==":
|
|
||||||
return value == threshold
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildAlertDescription(rule OpsAlertRule, value float64) string {
|
|
||||||
window := rule.WindowMinutes
|
|
||||||
if window <= 0 {
|
|
||||||
window = 1
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("Rule %s triggered: %s %s %.2f (current %.2f) over last %dm",
|
|
||||||
rule.Name,
|
|
||||||
rule.MetricType,
|
|
||||||
rule.Operator,
|
|
||||||
rule.Threshold,
|
|
||||||
value,
|
|
||||||
window,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) dispatchNotifications(ctx context.Context, rule OpsAlertRule, event *OpsAlertEvent) (bool, bool) {
|
|
||||||
emailSent := false
|
|
||||||
webhookSent := false
|
|
||||||
|
|
||||||
notifyCtx, cancel := s.notificationContext(ctx)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if rule.NotifyEmail {
|
|
||||||
emailSent = s.sendEmailNotification(notifyCtx, rule, event)
|
|
||||||
}
|
|
||||||
if rule.NotifyWebhook && rule.WebhookURL != "" {
|
|
||||||
webhookSent = s.sendWebhookNotification(notifyCtx, rule, event)
|
|
||||||
}
|
|
||||||
// Fallback channel: if email is enabled but ultimately fails, try webhook even if the
|
|
||||||
// webhook toggle is off (as long as a webhook URL is configured).
|
|
||||||
if rule.NotifyEmail && !emailSent && !rule.NotifyWebhook && rule.WebhookURL != "" {
|
|
||||||
log.Printf("[OpsAlert] email failed; attempting webhook fallback (rule=%d)", rule.ID)
|
|
||||||
webhookSent = s.sendWebhookNotification(notifyCtx, rule, event)
|
|
||||||
}
|
|
||||||
|
|
||||||
return emailSent, webhookSent
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
opsAlertEvaluateTimeout = 45 * time.Second
|
|
||||||
opsAlertNotificationTimeout = 30 * time.Second
|
|
||||||
opsAlertEmailMaxRetries = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
var opsAlertEmailBackoff = []time.Duration{
|
|
||||||
1 * time.Second,
|
|
||||||
2 * time.Second,
|
|
||||||
4 * time.Second,
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) notificationContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
|
||||||
parent := ctx
|
|
||||||
if s != nil && s.stopCtx != nil {
|
|
||||||
parent = s.stopCtx
|
|
||||||
}
|
|
||||||
if parent == nil {
|
|
||||||
parent = context.Background()
|
|
||||||
}
|
|
||||||
return context.WithTimeout(parent, opsAlertNotificationTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
var opsAlertSleep = sleepWithContext
|
|
||||||
|
|
||||||
func sleepWithContext(ctx context.Context, d time.Duration) error {
|
|
||||||
if d <= 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if ctx == nil {
|
|
||||||
time.Sleep(d)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
timer := time.NewTimer(d)
|
|
||||||
defer timer.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
case <-timer.C:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func retryWithBackoff(
|
|
||||||
ctx context.Context,
|
|
||||||
maxRetries int,
|
|
||||||
backoff []time.Duration,
|
|
||||||
fn func() error,
|
|
||||||
onError func(attempt int, total int, nextDelay time.Duration, err error),
|
|
||||||
) error {
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if maxRetries < 0 {
|
|
||||||
maxRetries = 0
|
|
||||||
}
|
|
||||||
totalAttempts := maxRetries + 1
|
|
||||||
|
|
||||||
var lastErr error
|
|
||||||
for attempt := 1; attempt <= totalAttempts; attempt++ {
|
|
||||||
if attempt > 1 {
|
|
||||||
backoffIdx := attempt - 2
|
|
||||||
if backoffIdx < len(backoff) {
|
|
||||||
if err := opsAlertSleep(ctx, backoff[backoffIdx]); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := fn(); err != nil {
|
|
||||||
lastErr = err
|
|
||||||
nextDelay := time.Duration(0)
|
|
||||||
if attempt < totalAttempts {
|
|
||||||
nextIdx := attempt - 1
|
|
||||||
if nextIdx < len(backoff) {
|
|
||||||
nextDelay = backoff[nextIdx]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if onError != nil {
|
|
||||||
onError(attempt, totalAttempts, nextDelay, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return lastErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) sendEmailNotification(ctx context.Context, rule OpsAlertRule, event *OpsAlertEvent) bool {
|
|
||||||
if s.emailService == nil || s.userService == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
|
|
||||||
admin, err := s.userService.GetFirstAdmin(ctx)
|
|
||||||
if err != nil || admin == nil || admin.Email == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
subject := fmt.Sprintf("[Ops Alert][%s] %s", rule.Severity, rule.Name)
|
|
||||||
body := fmt.Sprintf(
|
|
||||||
"Alert triggered: %s\n\nMetric: %s\nThreshold: %.2f\nCurrent: %.2f\nWindow: %dm\nStatus: %s\nTime: %s",
|
|
||||||
rule.Name,
|
|
||||||
rule.MetricType,
|
|
||||||
rule.Threshold,
|
|
||||||
event.MetricValue,
|
|
||||||
rule.WindowMinutes,
|
|
||||||
event.Status,
|
|
||||||
event.FiredAt.Format(time.RFC3339),
|
|
||||||
)
|
|
||||||
|
|
||||||
config, err := s.emailService.GetSMTPConfig(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] email config load failed: %v", err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := retryWithBackoff(
|
|
||||||
ctx,
|
|
||||||
opsAlertEmailMaxRetries,
|
|
||||||
opsAlertEmailBackoff,
|
|
||||||
func() error {
|
|
||||||
return s.emailService.SendEmailWithConfig(config, admin.Email, subject, body)
|
|
||||||
},
|
|
||||||
func(attempt int, total int, nextDelay time.Duration, err error) {
|
|
||||||
if attempt < total {
|
|
||||||
log.Printf("[OpsAlert] email send failed (attempt=%d/%d), retrying in %s: %v", attempt, total, nextDelay, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Printf("[OpsAlert] email send failed (attempt=%d/%d), giving up: %v", attempt, total, err)
|
|
||||||
},
|
|
||||||
); err != nil {
|
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
log.Printf("[OpsAlert] email send canceled: %v", err)
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsAlertService) sendWebhookNotification(ctx context.Context, rule OpsAlertRule, event *OpsAlertEvent) bool {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
webhookTarget, err := validateWebhookURL(ctx, rule.WebhookURL)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] invalid webhook url (rule=%d): %v", rule.ID, err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
payload := map[string]any{
|
|
||||||
"rule_id": rule.ID,
|
|
||||||
"rule_name": rule.Name,
|
|
||||||
"severity": rule.Severity,
|
|
||||||
"status": event.Status,
|
|
||||||
"metric_type": rule.MetricType,
|
|
||||||
"metric_value": event.MetricValue,
|
|
||||||
"threshold_value": rule.Threshold,
|
|
||||||
"window_minutes": rule.WindowMinutes,
|
|
||||||
"fired_at": event.FiredAt.Format(time.RFC3339),
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := json.Marshal(payload)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookTarget.URL.String(), bytes.NewReader(body))
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
resp, err := buildWebhookHTTPClient(s.httpClient, webhookTarget).Do(req)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsAlert] webhook send failed: %v", err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
|
||||||
log.Printf("[OpsAlert] webhook returned status %d", resp.StatusCode)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
const webhookHTTPClientTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
func buildWebhookHTTPClient(base *http.Client, webhookTarget *validatedWebhookTarget) *http.Client {
|
|
||||||
var client http.Client
|
|
||||||
if base != nil {
|
|
||||||
client = *base
|
|
||||||
}
|
|
||||||
if client.Timeout <= 0 {
|
|
||||||
client.Timeout = webhookHTTPClientTimeout
|
|
||||||
}
|
|
||||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
|
||||||
return http.ErrUseLastResponse
|
|
||||||
}
|
|
||||||
if webhookTarget != nil {
|
|
||||||
client.Transport = buildWebhookTransport(client.Transport, webhookTarget)
|
|
||||||
}
|
|
||||||
return &client
|
|
||||||
}
|
|
||||||
|
|
||||||
var disallowedWebhookIPNets = []net.IPNet{
|
|
||||||
// "this host on this network" / unspecified.
|
|
||||||
mustParseCIDR("0.0.0.0/8"),
|
|
||||||
mustParseCIDR("127.0.0.0/8"), // loopback (includes 127.0.0.1)
|
|
||||||
mustParseCIDR("10.0.0.0/8"), // RFC1918
|
|
||||||
mustParseCIDR("192.168.0.0/16"), // RFC1918
|
|
||||||
mustParseCIDR("172.16.0.0/12"), // RFC1918 (172.16.0.0 - 172.31.255.255)
|
|
||||||
mustParseCIDR("100.64.0.0/10"), // RFC6598 (carrier-grade NAT)
|
|
||||||
mustParseCIDR("169.254.0.0/16"), // IPv4 link-local (includes 169.254.169.254 metadata IP on many clouds)
|
|
||||||
mustParseCIDR("198.18.0.0/15"), // RFC2544 benchmark testing
|
|
||||||
mustParseCIDR("224.0.0.0/4"), // IPv4 multicast
|
|
||||||
mustParseCIDR("240.0.0.0/4"), // IPv4 reserved
|
|
||||||
mustParseCIDR("::/128"), // IPv6 unspecified
|
|
||||||
mustParseCIDR("::1/128"), // IPv6 loopback
|
|
||||||
mustParseCIDR("fc00::/7"), // IPv6 unique local
|
|
||||||
mustParseCIDR("fe80::/10"), // IPv6 link-local
|
|
||||||
mustParseCIDR("ff00::/8"), // IPv6 multicast
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustParseCIDR(cidr string) net.IPNet {
|
|
||||||
_, block, err := net.ParseCIDR(cidr)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return *block
|
|
||||||
}
|
|
||||||
|
|
||||||
var lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
return net.DefaultResolver.LookupIPAddr(ctx, host)
|
|
||||||
}
|
|
||||||
|
|
||||||
type validatedWebhookTarget struct {
|
|
||||||
URL *url.URL
|
|
||||||
|
|
||||||
host string
|
|
||||||
port string
|
|
||||||
pinnedIPs []net.IP
|
|
||||||
}
|
|
||||||
|
|
||||||
var webhookBaseDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
dialer := net.Dialer{
|
|
||||||
Timeout: 5 * time.Second,
|
|
||||||
KeepAlive: 30 * time.Second,
|
|
||||||
}
|
|
||||||
return dialer.DialContext(ctx, network, addr)
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildWebhookTransport(base http.RoundTripper, webhookTarget *validatedWebhookTarget) http.RoundTripper {
|
|
||||||
if webhookTarget == nil || webhookTarget.URL == nil {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
|
|
||||||
var transport *http.Transport
|
|
||||||
switch typed := base.(type) {
|
|
||||||
case *http.Transport:
|
|
||||||
if typed != nil {
|
|
||||||
transport = typed.Clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if transport == nil {
|
|
||||||
if defaultTransport, ok := http.DefaultTransport.(*http.Transport); ok && defaultTransport != nil {
|
|
||||||
transport = defaultTransport.Clone()
|
|
||||||
} else {
|
|
||||||
transport = (&http.Transport{}).Clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookHost := webhookTarget.host
|
|
||||||
webhookPort := webhookTarget.port
|
|
||||||
pinnedIPs := append([]net.IP(nil), webhookTarget.pinnedIPs...)
|
|
||||||
|
|
||||||
transport.Proxy = nil
|
|
||||||
transport.DialTLSContext = nil
|
|
||||||
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
host, port, err := net.SplitHostPort(addr)
|
|
||||||
if err != nil || host == "" || port == "" {
|
|
||||||
return nil, fmt.Errorf("webhook dial target is invalid: %q", addr)
|
|
||||||
}
|
|
||||||
|
|
||||||
canonicalHost := strings.TrimSuffix(strings.ToLower(host), ".")
|
|
||||||
if canonicalHost != webhookHost || port != webhookPort {
|
|
||||||
return nil, fmt.Errorf("webhook dial target mismatch: %q", addr)
|
|
||||||
}
|
|
||||||
|
|
||||||
var lastErr error
|
|
||||||
for _, ip := range pinnedIPs {
|
|
||||||
if isDisallowedWebhookIP(ip) {
|
|
||||||
lastErr = fmt.Errorf("webhook target resolves to a disallowed ip")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
dialAddr := net.JoinHostPort(ip.String(), port)
|
|
||||||
conn, err := webhookBaseDialContext(ctx, network, dialAddr)
|
|
||||||
if err == nil {
|
|
||||||
return conn, nil
|
|
||||||
}
|
|
||||||
lastErr = err
|
|
||||||
}
|
|
||||||
if lastErr == nil {
|
|
||||||
lastErr = errors.New("webhook target has no resolved addresses")
|
|
||||||
}
|
|
||||||
return nil, lastErr
|
|
||||||
}
|
|
||||||
|
|
||||||
return transport
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateWebhookURL(ctx context.Context, raw string) (*validatedWebhookTarget, error) {
|
|
||||||
raw = strings.TrimSpace(raw)
|
|
||||||
if raw == "" {
|
|
||||||
return nil, errors.New("webhook url is empty")
|
|
||||||
}
|
|
||||||
// Avoid request smuggling / header injection vectors.
|
|
||||||
if strings.ContainsAny(raw, "\r\n") {
|
|
||||||
return nil, errors.New("webhook url contains invalid characters")
|
|
||||||
}
|
|
||||||
|
|
||||||
parsed, err := url.Parse(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.New("webhook url format is invalid")
|
|
||||||
}
|
|
||||||
if !strings.EqualFold(parsed.Scheme, "https") {
|
|
||||||
return nil, errors.New("webhook url scheme must be https")
|
|
||||||
}
|
|
||||||
parsed.Scheme = "https"
|
|
||||||
if parsed.Host == "" || parsed.Hostname() == "" {
|
|
||||||
return nil, errors.New("webhook url must include host")
|
|
||||||
}
|
|
||||||
if parsed.User != nil {
|
|
||||||
return nil, errors.New("webhook url must not include userinfo")
|
|
||||||
}
|
|
||||||
if parsed.Port() != "" {
|
|
||||||
port, err := strconv.Atoi(parsed.Port())
|
|
||||||
if err != nil || port < 1 || port > 65535 {
|
|
||||||
return nil, errors.New("webhook url port is invalid")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
host := strings.TrimSuffix(strings.ToLower(parsed.Hostname()), ".")
|
|
||||||
if host == "localhost" {
|
|
||||||
return nil, errors.New("webhook url host must not be localhost")
|
|
||||||
}
|
|
||||||
|
|
||||||
if ip := net.ParseIP(host); ip != nil {
|
|
||||||
if isDisallowedWebhookIP(ip) {
|
|
||||||
return nil, errors.New("webhook url host resolves to a disallowed ip")
|
|
||||||
}
|
|
||||||
return &validatedWebhookTarget{
|
|
||||||
URL: parsed,
|
|
||||||
host: host,
|
|
||||||
port: portForScheme(parsed),
|
|
||||||
pinnedIPs: []net.IP{ip},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
ips, err := lookupIPAddrs(ctx, host)
|
|
||||||
if err != nil || len(ips) == 0 {
|
|
||||||
return nil, errors.New("webhook url host cannot be resolved")
|
|
||||||
}
|
|
||||||
pinned := make([]net.IP, 0, len(ips))
|
|
||||||
for _, addr := range ips {
|
|
||||||
if isDisallowedWebhookIP(addr.IP) {
|
|
||||||
return nil, errors.New("webhook url host resolves to a disallowed ip")
|
|
||||||
}
|
|
||||||
if addr.IP != nil {
|
|
||||||
pinned = append(pinned, addr.IP)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(pinned) == 0 {
|
|
||||||
return nil, errors.New("webhook url host cannot be resolved")
|
|
||||||
}
|
|
||||||
|
|
||||||
return &validatedWebhookTarget{
|
|
||||||
URL: parsed,
|
|
||||||
host: host,
|
|
||||||
port: portForScheme(parsed),
|
|
||||||
pinnedIPs: uniqueResolvedIPs(pinned),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isDisallowedWebhookIP(ip net.IP) bool {
|
|
||||||
if ip == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if ip4 := ip.To4(); ip4 != nil {
|
|
||||||
ip = ip4
|
|
||||||
} else if ip16 := ip.To16(); ip16 != nil {
|
|
||||||
ip = ip16
|
|
||||||
} else {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Disallow non-public addresses even if they're not explicitly covered by the CIDR list.
|
|
||||||
// This provides defense-in-depth against SSRF targets such as link-local, multicast, and
|
|
||||||
// unspecified addresses, and ensures any "pinned" IP is still blocked at dial time.
|
|
||||||
if ip.IsUnspecified() ||
|
|
||||||
ip.IsLoopback() ||
|
|
||||||
ip.IsMulticast() ||
|
|
||||||
ip.IsLinkLocalUnicast() ||
|
|
||||||
ip.IsLinkLocalMulticast() ||
|
|
||||||
ip.IsPrivate() {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, block := range disallowedWebhookIPNets {
|
|
||||||
if block.Contains(ip) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func portForScheme(u *url.URL) string {
|
|
||||||
if u != nil && u.Port() != "" {
|
|
||||||
return u.Port()
|
|
||||||
}
|
|
||||||
return "443"
|
|
||||||
}
|
|
||||||
|
|
||||||
func uniqueResolvedIPs(ips []net.IP) []net.IP {
|
|
||||||
seen := make(map[string]struct{}, len(ips))
|
|
||||||
out := make([]net.IP, 0, len(ips))
|
|
||||||
for _, ip := range ips {
|
|
||||||
if ip == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := ip.String()
|
|
||||||
if _, ok := seen[key]; ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[key] = struct{}{}
|
|
||||||
out = append(out, ip)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -1,271 +0,0 @@
|
|||||||
//go:build integration
|
|
||||||
|
|
||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// This integration test protects the DI startup contract for OpsAlertService.
|
|
||||||
//
|
|
||||||
// Background:
|
|
||||||
// - OpsMetricsCollector previously called alertService.Start()/Evaluate() directly.
|
|
||||||
// - Those direct calls were removed, so OpsAlertService must now start via DI
|
|
||||||
// (ProvideOpsAlertService in wire.go) and run its own evaluation ticker.
|
|
||||||
//
|
|
||||||
// What we validate here:
|
|
||||||
// 1. When we construct via the Wire provider functions (ProvideOpsAlertService +
|
|
||||||
// ProvideOpsMetricsCollector), OpsAlertService starts automatically.
|
|
||||||
// 2. Its evaluation loop continues to tick even if OpsMetricsCollector is stopped,
|
|
||||||
// proving the alert evaluator is independent.
|
|
||||||
// 3. The evaluation path can trigger alert logic (CreateAlertEvent called).
|
|
||||||
func TestOpsAlertService_StartedViaWireProviders_RunsIndependentTicker(t *testing.T) {
|
|
||||||
oldInterval := opsAlertEvalInterval
|
|
||||||
opsAlertEvalInterval = 25 * time.Millisecond
|
|
||||||
t.Cleanup(func() { opsAlertEvalInterval = oldInterval })
|
|
||||||
|
|
||||||
repo := newFakeOpsRepository()
|
|
||||||
opsService := NewOpsService(repo, nil)
|
|
||||||
|
|
||||||
// Start via the Wire provider function (the production DI path).
|
|
||||||
alertService := ProvideOpsAlertService(opsService, nil, nil)
|
|
||||||
t.Cleanup(alertService.Stop)
|
|
||||||
|
|
||||||
// Construct via ProvideOpsMetricsCollector (wire.go). Stop immediately to ensure
|
|
||||||
// the alert ticker keeps running without the metrics collector.
|
|
||||||
collector := ProvideOpsMetricsCollector(opsService, NewConcurrencyService(nil))
|
|
||||||
collector.Stop()
|
|
||||||
|
|
||||||
// Wait for at least one evaluation (run() calls evaluateOnce immediately).
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
return repo.listRulesCalls.Load() >= 1
|
|
||||||
}, 1*time.Second, 5*time.Millisecond)
|
|
||||||
|
|
||||||
// Confirm the evaluation loop keeps ticking after the metrics collector is stopped.
|
|
||||||
callsAfterCollectorStop := repo.listRulesCalls.Load()
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
return repo.listRulesCalls.Load() >= callsAfterCollectorStop+2
|
|
||||||
}, 1*time.Second, 5*time.Millisecond)
|
|
||||||
|
|
||||||
// Confirm the evaluation logic actually fires an alert event at least once.
|
|
||||||
select {
|
|
||||||
case <-repo.eventCreatedCh:
|
|
||||||
// ok
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatalf("expected OpsAlertService to create an alert event, but none was created (ListAlertRules calls=%d)", repo.listRulesCalls.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newFakeOpsRepository() *fakeOpsRepository {
|
|
||||||
return &fakeOpsRepository{
|
|
||||||
eventCreatedCh: make(chan struct{}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fakeOpsRepository is a lightweight in-memory stub of OpsRepository for integration tests.
|
|
||||||
// It avoids real DB/Redis usage and provides deterministic responses fast.
|
|
||||||
type fakeOpsRepository struct {
|
|
||||||
listRulesCalls atomic.Int64
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
activeEvent *OpsAlertEvent
|
|
||||||
latestEvent *OpsAlertEvent
|
|
||||||
nextEventID int64
|
|
||||||
eventCreatedCh chan struct{}
|
|
||||||
eventOnce sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) CreateErrorLog(ctx context.Context, log *OpsErrorLog) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) ListErrorLogsLegacy(ctx context.Context, filters OpsErrorLogFilters) ([]OpsErrorLog, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) ListErrorLogs(ctx context.Context, filter *ErrorLogFilter) ([]*ErrorLog, int64, error) {
|
|
||||||
return nil, 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetLatestSystemMetric(ctx context.Context) (*OpsMetrics, error) {
|
|
||||||
return &OpsMetrics{WindowMinutes: 1}, sql.ErrNoRows
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) CreateSystemMetric(ctx context.Context, metric *OpsMetrics) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetWindowStats(ctx context.Context, startTime, endTime time.Time) (*OpsWindowStats, error) {
|
|
||||||
return &OpsWindowStats{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetProviderStats(ctx context.Context, startTime, endTime time.Time) ([]*ProviderStats, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetLatencyHistogram(ctx context.Context, startTime, endTime time.Time) ([]*LatencyHistogramItem, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetErrorDistribution(ctx context.Context, startTime, endTime time.Time) ([]*ErrorDistributionItem, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) ListRecentSystemMetrics(ctx context.Context, windowMinutes, limit int) ([]OpsMetrics, error) {
|
|
||||||
if limit <= 0 {
|
|
||||||
limit = 1
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
metrics := make([]OpsMetrics, 0, limit)
|
|
||||||
for i := 0; i < limit; i++ {
|
|
||||||
metrics = append(metrics, OpsMetrics{
|
|
||||||
WindowMinutes: windowMinutes,
|
|
||||||
CPUUsagePercent: 99,
|
|
||||||
UpdatedAt: now.Add(-time.Duration(i) * opsMetricsInterval),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return metrics, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) ListSystemMetricsRange(ctx context.Context, windowMinutes int, startTime, endTime time.Time, limit int) ([]OpsMetrics, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) ListAlertRules(ctx context.Context) ([]OpsAlertRule, error) {
|
|
||||||
call := r.listRulesCalls.Add(1)
|
|
||||||
// Delay enabling rules slightly so the test can stop OpsMetricsCollector first,
|
|
||||||
// then observe the alert evaluator ticking independently.
|
|
||||||
if call < 5 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return []OpsAlertRule{
|
|
||||||
{
|
|
||||||
ID: 1,
|
|
||||||
Name: "cpu too high (test)",
|
|
||||||
Enabled: true,
|
|
||||||
MetricType: OpsMetricCPUUsagePercent,
|
|
||||||
Operator: ">",
|
|
||||||
Threshold: 0,
|
|
||||||
WindowMinutes: 1,
|
|
||||||
SustainedMinutes: 1,
|
|
||||||
Severity: "P1",
|
|
||||||
NotifyEmail: false,
|
|
||||||
NotifyWebhook: false,
|
|
||||||
CooldownMinutes: 0,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetActiveAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if r.activeEvent == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if r.activeEvent.RuleID != ruleID {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if r.activeEvent.Status != OpsAlertStatusFiring {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
clone := *r.activeEvent
|
|
||||||
return &clone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetLatestAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if r.latestEvent == nil || r.latestEvent.RuleID != ruleID {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
clone := *r.latestEvent
|
|
||||||
return &clone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) CreateAlertEvent(ctx context.Context, event *OpsAlertEvent) error {
|
|
||||||
if event == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
r.nextEventID++
|
|
||||||
event.ID = r.nextEventID
|
|
||||||
|
|
||||||
clone := *event
|
|
||||||
r.latestEvent = &clone
|
|
||||||
if clone.Status == OpsAlertStatusFiring {
|
|
||||||
r.activeEvent = &clone
|
|
||||||
}
|
|
||||||
|
|
||||||
r.eventOnce.Do(func() { close(r.eventCreatedCh) })
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) UpdateAlertEventStatus(ctx context.Context, eventID int64, status string, resolvedAt *time.Time) error {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if r.activeEvent != nil && r.activeEvent.ID == eventID {
|
|
||||||
r.activeEvent.Status = status
|
|
||||||
r.activeEvent.ResolvedAt = resolvedAt
|
|
||||||
}
|
|
||||||
if r.latestEvent != nil && r.latestEvent.ID == eventID {
|
|
||||||
r.latestEvent.Status = status
|
|
||||||
r.latestEvent.ResolvedAt = resolvedAt
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) UpdateAlertEventNotifications(ctx context.Context, eventID int64, emailSent, webhookSent bool) error {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if r.activeEvent != nil && r.activeEvent.ID == eventID {
|
|
||||||
r.activeEvent.EmailSent = emailSent
|
|
||||||
r.activeEvent.WebhookSent = webhookSent
|
|
||||||
}
|
|
||||||
if r.latestEvent != nil && r.latestEvent.ID == eventID {
|
|
||||||
r.latestEvent.EmailSent = emailSent
|
|
||||||
r.latestEvent.WebhookSent = webhookSent
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) CountActiveAlerts(ctx context.Context) (int, error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if r.activeEvent == nil {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
return 1, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetOverviewStats(ctx context.Context, startTime, endTime time.Time) (*OverviewStats, error) {
|
|
||||||
return &OverviewStats{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetCachedLatestSystemMetric(ctx context.Context) (*OpsMetrics, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) SetCachedLatestSystemMetric(ctx context.Context, metric *OpsMetrics) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) GetCachedDashboardOverview(ctx context.Context, timeRange string) (*DashboardOverviewData, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) SetCachedDashboardOverview(ctx context.Context, timeRange string, data *DashboardOverviewData, ttl time.Duration) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *fakeOpsRepository) PingRedis(ctx context.Context) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,315 +0,0 @@
|
|||||||
//go:build unit || opsalert_unit
|
|
||||||
|
|
||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSelectContiguousMetrics_Contiguous(t *testing.T) {
|
|
||||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
||||||
metrics := []OpsMetrics{
|
|
||||||
{UpdatedAt: now},
|
|
||||||
{UpdatedAt: now.Add(-1 * time.Minute)},
|
|
||||||
{UpdatedAt: now.Add(-2 * time.Minute)},
|
|
||||||
}
|
|
||||||
|
|
||||||
selected, ok := selectContiguousMetrics(metrics, 3, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
require.Len(t, selected, 3)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectContiguousMetrics_GapFails(t *testing.T) {
|
|
||||||
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
||||||
metrics := []OpsMetrics{
|
|
||||||
{UpdatedAt: now},
|
|
||||||
// Missing the -1m sample (gap ~=2m).
|
|
||||||
{UpdatedAt: now.Add(-2 * time.Minute)},
|
|
||||||
{UpdatedAt: now.Add(-3 * time.Minute)},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, ok := selectContiguousMetrics(metrics, 3, now)
|
|
||||||
require.False(t, ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectContiguousMetrics_StaleNewestFails(t *testing.T) {
|
|
||||||
now := time.Date(2026, 1, 1, 0, 10, 0, 0, time.UTC)
|
|
||||||
metrics := []OpsMetrics{
|
|
||||||
{UpdatedAt: now.Add(-10 * time.Minute)},
|
|
||||||
{UpdatedAt: now.Add(-11 * time.Minute)},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, ok := selectContiguousMetrics(metrics, 2, now)
|
|
||||||
require.False(t, ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMetricValue_SuccessRate_NoTrafficIsNoData(t *testing.T) {
|
|
||||||
metric := OpsMetrics{
|
|
||||||
RequestCount: 0,
|
|
||||||
SuccessRate: 0,
|
|
||||||
}
|
|
||||||
value, ok := metricValue(metric, OpsMetricSuccessRate)
|
|
||||||
require.False(t, ok)
|
|
||||||
require.Equal(t, 0.0, value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpsAlertService_StopWithoutStart_NoPanic(t *testing.T) {
|
|
||||||
s := NewOpsAlertService(nil, nil, nil)
|
|
||||||
require.NotPanics(t, func() { s.Stop() })
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpsAlertService_StartStop_Graceful(t *testing.T) {
|
|
||||||
s := NewOpsAlertService(nil, nil, nil)
|
|
||||||
s.interval = 5 * time.Millisecond
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
s.StartWithContext(ctx)
|
|
||||||
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
s.Stop()
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
// ok
|
|
||||||
case <-time.After(1 * time.Second):
|
|
||||||
t.Fatal("Stop did not return; background goroutine likely stuck")
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NotPanics(t, func() { s.Stop() })
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildWebhookHTTPClient_DefaultTimeout(t *testing.T) {
|
|
||||||
client := buildWebhookHTTPClient(nil, nil)
|
|
||||||
require.Equal(t, webhookHTTPClientTimeout, client.Timeout)
|
|
||||||
require.NotNil(t, client.CheckRedirect)
|
|
||||||
require.ErrorIs(t, client.CheckRedirect(nil, nil), http.ErrUseLastResponse)
|
|
||||||
|
|
||||||
base := &http.Client{}
|
|
||||||
client = buildWebhookHTTPClient(base, nil)
|
|
||||||
require.Equal(t, webhookHTTPClientTimeout, client.Timeout)
|
|
||||||
require.NotNil(t, client.CheckRedirect)
|
|
||||||
|
|
||||||
base = &http.Client{Timeout: 2 * time.Second}
|
|
||||||
client = buildWebhookHTTPClient(base, nil)
|
|
||||||
require.Equal(t, 2*time.Second, client.Timeout)
|
|
||||||
require.NotNil(t, client.CheckRedirect)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RequiresHTTPS(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := validateWebhookURL(context.Background(), "http://example.com/webhook")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_InvalidFormatRejected(t *testing.T) {
|
|
||||||
_, err := validateWebhookURL(context.Background(), "https://[::1")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RejectsUserinfo(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := validateWebhookURL(context.Background(), "https://user:pass@example.com/webhook")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RejectsLocalhost(t *testing.T) {
|
|
||||||
_, err := validateWebhookURL(context.Background(), "https://localhost/webhook")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RejectsPrivateIPLiteral(t *testing.T) {
|
|
||||||
cases := []string{
|
|
||||||
"https://0.0.0.0/webhook",
|
|
||||||
"https://127.0.0.1/webhook",
|
|
||||||
"https://10.0.0.1/webhook",
|
|
||||||
"https://192.168.1.2/webhook",
|
|
||||||
"https://172.16.0.1/webhook",
|
|
||||||
"https://172.31.255.255/webhook",
|
|
||||||
"https://100.64.0.1/webhook",
|
|
||||||
"https://169.254.169.254/webhook",
|
|
||||||
"https://198.18.0.1/webhook",
|
|
||||||
"https://224.0.0.1/webhook",
|
|
||||||
"https://240.0.0.1/webhook",
|
|
||||||
"https://[::]/webhook",
|
|
||||||
"https://[::1]/webhook",
|
|
||||||
"https://[ff02::1]/webhook",
|
|
||||||
}
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc, func(t *testing.T) {
|
|
||||||
_, err := validateWebhookURL(context.Background(), tc)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RejectsPrivateIPViaDNS(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
require.Equal(t, "internal.example", host)
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("10.0.0.2")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := validateWebhookURL(context.Background(), "https://internal.example/webhook")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RejectsLinkLocalIPViaDNS(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
require.Equal(t, "metadata.example", host)
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("169.254.169.254")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := validateWebhookURL(context.Background(), "https://metadata.example/webhook")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_AllowsPublicHostViaDNS(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
require.Equal(t, "example.com", host)
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
target, err := validateWebhookURL(context.Background(), "https://example.com:443/webhook")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "https", target.URL.Scheme)
|
|
||||||
require.Equal(t, "example.com", target.URL.Hostname())
|
|
||||||
require.Equal(t, "443", target.URL.Port())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateWebhookURL_RejectsInvalidPort(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
t.Cleanup(func() { lookupIPAddrs = oldLookup })
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := validateWebhookURL(context.Background(), "https://example.com:99999/webhook")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookTransport_UsesPinnedIP_NoDNSRebinding(t *testing.T) {
|
|
||||||
oldLookup := lookupIPAddrs
|
|
||||||
oldDial := webhookBaseDialContext
|
|
||||||
t.Cleanup(func() {
|
|
||||||
lookupIPAddrs = oldLookup
|
|
||||||
webhookBaseDialContext = oldDial
|
|
||||||
})
|
|
||||||
|
|
||||||
lookupCalls := 0
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
lookupCalls++
|
|
||||||
require.Equal(t, "example.com", host)
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
target, err := validateWebhookURL(context.Background(), "https://example.com/webhook")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, 1, lookupCalls)
|
|
||||||
|
|
||||||
lookupIPAddrs = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
|
||||||
lookupCalls++
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("10.0.0.1")}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var dialAddrs []string
|
|
||||||
webhookBaseDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
dialAddrs = append(dialAddrs, addr)
|
|
||||||
return nil, errors.New("dial blocked in test")
|
|
||||||
}
|
|
||||||
|
|
||||||
client := buildWebhookHTTPClient(nil, target)
|
|
||||||
transport, ok := client.Transport.(*http.Transport)
|
|
||||||
require.True(t, ok)
|
|
||||||
|
|
||||||
_, err = transport.DialContext(context.Background(), "tcp", "example.com:443")
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Equal(t, []string{"93.184.216.34:443"}, dialAddrs)
|
|
||||||
require.Equal(t, 1, lookupCalls, "dial path must not re-resolve DNS")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRetryWithBackoff_SucceedsAfterRetries(t *testing.T) {
|
|
||||||
oldSleep := opsAlertSleep
|
|
||||||
t.Cleanup(func() { opsAlertSleep = oldSleep })
|
|
||||||
|
|
||||||
var slept []time.Duration
|
|
||||||
opsAlertSleep = func(ctx context.Context, d time.Duration) error {
|
|
||||||
slept = append(slept, d)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
attempts := 0
|
|
||||||
err := retryWithBackoff(
|
|
||||||
context.Background(),
|
|
||||||
3,
|
|
||||||
[]time.Duration{time.Second, 2 * time.Second, 4 * time.Second},
|
|
||||||
func() error {
|
|
||||||
attempts++
|
|
||||||
if attempts <= 3 {
|
|
||||||
return errors.New("send failed")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, 4, attempts)
|
|
||||||
require.Equal(t, []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}, slept)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRetryWithBackoff_ContextCanceledStopsRetries(t *testing.T) {
|
|
||||||
oldSleep := opsAlertSleep
|
|
||||||
t.Cleanup(func() { opsAlertSleep = oldSleep })
|
|
||||||
|
|
||||||
var slept []time.Duration
|
|
||||||
opsAlertSleep = func(ctx context.Context, d time.Duration) error {
|
|
||||||
slept = append(slept, d)
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
attempts := 0
|
|
||||||
err := retryWithBackoff(
|
|
||||||
ctx,
|
|
||||||
3,
|
|
||||||
[]time.Duration{time.Second, 2 * time.Second, 4 * time.Second},
|
|
||||||
func() error {
|
|
||||||
attempts++
|
|
||||||
return errors.New("send failed")
|
|
||||||
},
|
|
||||||
func(attempt int, total int, nextDelay time.Duration, err error) {
|
|
||||||
if attempt == 1 {
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.ErrorIs(t, err, context.Canceled)
|
|
||||||
require.Equal(t, 1, attempts)
|
|
||||||
require.Equal(t, []time.Duration{time.Second}, slept)
|
|
||||||
}
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
OpsAlertStatusFiring = "firing"
|
|
||||||
OpsAlertStatusResolved = "resolved"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
OpsMetricSuccessRate = "success_rate"
|
|
||||||
OpsMetricErrorRate = "error_rate"
|
|
||||||
OpsMetricP95LatencyMs = "p95_latency_ms"
|
|
||||||
OpsMetricP99LatencyMs = "p99_latency_ms"
|
|
||||||
OpsMetricHTTP2Errors = "http2_errors"
|
|
||||||
OpsMetricCPUUsagePercent = "cpu_usage_percent"
|
|
||||||
OpsMetricMemoryUsagePercent = "memory_usage_percent"
|
|
||||||
OpsMetricQueueDepth = "concurrency_queue_depth"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OpsAlertRule struct {
|
|
||||||
ID int64 `json:"id"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
Enabled bool `json:"enabled"`
|
|
||||||
MetricType string `json:"metric_type"`
|
|
||||||
Operator string `json:"operator"`
|
|
||||||
Threshold float64 `json:"threshold"`
|
|
||||||
WindowMinutes int `json:"window_minutes"`
|
|
||||||
SustainedMinutes int `json:"sustained_minutes"`
|
|
||||||
Severity string `json:"severity"`
|
|
||||||
NotifyEmail bool `json:"notify_email"`
|
|
||||||
NotifyWebhook bool `json:"notify_webhook"`
|
|
||||||
WebhookURL string `json:"webhook_url"`
|
|
||||||
CooldownMinutes int `json:"cooldown_minutes"`
|
|
||||||
DimensionFilters map[string]any `json:"dimension_filters,omitempty"`
|
|
||||||
NotifyChannels []string `json:"notify_channels,omitempty"`
|
|
||||||
NotifyConfig map[string]any `json:"notify_config,omitempty"`
|
|
||||||
CreatedAt time.Time `json:"created_at"`
|
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type OpsAlertEvent struct {
|
|
||||||
ID int64 `json:"id"`
|
|
||||||
RuleID int64 `json:"rule_id"`
|
|
||||||
Severity string `json:"severity"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
MetricValue float64 `json:"metric_value"`
|
|
||||||
ThresholdValue float64 `json:"threshold_value"`
|
|
||||||
FiredAt time.Time `json:"fired_at"`
|
|
||||||
ResolvedAt *time.Time `json:"resolved_at"`
|
|
||||||
EmailSent bool `json:"email_sent"`
|
|
||||||
WebhookSent bool `json:"webhook_sent"`
|
|
||||||
CreatedAt time.Time `json:"created_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) ListAlertRules(ctx context.Context) ([]OpsAlertRule, error) {
|
|
||||||
return s.repo.ListAlertRules(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) GetActiveAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
|
||||||
return s.repo.GetActiveAlertEvent(ctx, ruleID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) GetLatestAlertEvent(ctx context.Context, ruleID int64) (*OpsAlertEvent, error) {
|
|
||||||
return s.repo.GetLatestAlertEvent(ctx, ruleID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) CreateAlertEvent(ctx context.Context, event *OpsAlertEvent) error {
|
|
||||||
return s.repo.CreateAlertEvent(ctx, event)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) UpdateAlertEventStatus(ctx context.Context, eventID int64, status string, resolvedAt *time.Time) error {
|
|
||||||
return s.repo.UpdateAlertEventStatus(ctx, eventID, status, resolvedAt)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) UpdateAlertEventNotifications(ctx context.Context, eventID int64, emailSent, webhookSent bool) error {
|
|
||||||
return s.repo.UpdateAlertEventNotifications(ctx, eventID, emailSent, webhookSent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) ListRecentSystemMetrics(ctx context.Context, windowMinutes, limit int) ([]OpsMetrics, error) {
|
|
||||||
return s.repo.ListRecentSystemMetrics(ctx, windowMinutes, limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *OpsService) CountActiveAlerts(ctx context.Context) (int, error) {
|
|
||||||
return s.repo.CountActiveAlerts(ctx)
|
|
||||||
}
|
|
||||||
@@ -1,203 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log"
|
|
||||||
"runtime"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/shirou/gopsutil/v4/cpu"
|
|
||||||
"github.com/shirou/gopsutil/v4/mem"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
opsMetricsInterval = 1 * time.Minute
|
|
||||||
opsMetricsCollectTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
opsMetricsWindowShortMinutes = 1
|
|
||||||
opsMetricsWindowLongMinutes = 5
|
|
||||||
|
|
||||||
bytesPerMB = 1024 * 1024
|
|
||||||
cpuUsageSampleInterval = 0 * time.Second
|
|
||||||
|
|
||||||
percentScale = 100
|
|
||||||
)
|
|
||||||
|
|
||||||
type OpsMetricsCollector struct {
|
|
||||||
opsService *OpsService
|
|
||||||
concurrencyService *ConcurrencyService
|
|
||||||
interval time.Duration
|
|
||||||
lastGCPauseTotal uint64
|
|
||||||
lastGCPauseMu sync.Mutex
|
|
||||||
stopCh chan struct{}
|
|
||||||
startOnce sync.Once
|
|
||||||
stopOnce sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewOpsMetricsCollector(opsService *OpsService, concurrencyService *ConcurrencyService) *OpsMetricsCollector {
|
|
||||||
return &OpsMetricsCollector{
|
|
||||||
opsService: opsService,
|
|
||||||
concurrencyService: concurrencyService,
|
|
||||||
interval: opsMetricsInterval,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) Start() {
|
|
||||||
if c == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c.startOnce.Do(func() {
|
|
||||||
if c.stopCh == nil {
|
|
||||||
c.stopCh = make(chan struct{})
|
|
||||||
}
|
|
||||||
go c.run()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) Stop() {
|
|
||||||
if c == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c.stopOnce.Do(func() {
|
|
||||||
if c.stopCh != nil {
|
|
||||||
close(c.stopCh)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) run() {
|
|
||||||
ticker := time.NewTicker(c.interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
c.collectOnce()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
c.collectOnce()
|
|
||||||
case <-c.stopCh:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) collectOnce() {
|
|
||||||
if c.opsService == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), opsMetricsCollectTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
systemStats := c.collectSystemStats(ctx)
|
|
||||||
queueDepth := c.collectQueueDepth(ctx)
|
|
||||||
activeAlerts := c.collectActiveAlerts(ctx)
|
|
||||||
|
|
||||||
for _, window := range []int{opsMetricsWindowShortMinutes, opsMetricsWindowLongMinutes} {
|
|
||||||
startTime := now.Add(-time.Duration(window) * time.Minute)
|
|
||||||
windowStats, err := c.opsService.GetWindowStats(ctx, startTime, now)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsMetrics] failed to get window stats (%dm): %v", window, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
successRate, errorRate := computeRates(windowStats.SuccessCount, windowStats.ErrorCount)
|
|
||||||
requestCount := windowStats.SuccessCount + windowStats.ErrorCount
|
|
||||||
metric := &OpsMetrics{
|
|
||||||
WindowMinutes: window,
|
|
||||||
RequestCount: requestCount,
|
|
||||||
SuccessCount: windowStats.SuccessCount,
|
|
||||||
ErrorCount: windowStats.ErrorCount,
|
|
||||||
SuccessRate: successRate,
|
|
||||||
ErrorRate: errorRate,
|
|
||||||
P95LatencyMs: windowStats.P95LatencyMs,
|
|
||||||
P99LatencyMs: windowStats.P99LatencyMs,
|
|
||||||
HTTP2Errors: windowStats.HTTP2Errors,
|
|
||||||
ActiveAlerts: activeAlerts,
|
|
||||||
CPUUsagePercent: systemStats.cpuUsage,
|
|
||||||
MemoryUsedMB: systemStats.memoryUsedMB,
|
|
||||||
MemoryTotalMB: systemStats.memoryTotalMB,
|
|
||||||
MemoryUsagePercent: systemStats.memoryUsagePercent,
|
|
||||||
HeapAllocMB: systemStats.heapAllocMB,
|
|
||||||
GCPauseMs: systemStats.gcPauseMs,
|
|
||||||
ConcurrencyQueueDepth: queueDepth,
|
|
||||||
UpdatedAt: now,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := c.opsService.RecordMetrics(ctx, metric); err != nil {
|
|
||||||
log.Printf("[OpsMetrics] failed to record metrics (%dm): %v", window, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func computeRates(successCount, errorCount int64) (float64, float64) {
|
|
||||||
total := successCount + errorCount
|
|
||||||
if total == 0 {
|
|
||||||
// No traffic => no data. Rates are kept at 0 and request_count will be 0.
|
|
||||||
// The UI should render this as N/A instead of "100% success".
|
|
||||||
return 0, 0
|
|
||||||
}
|
|
||||||
successRate := float64(successCount) / float64(total) * percentScale
|
|
||||||
errorRate := float64(errorCount) / float64(total) * percentScale
|
|
||||||
return successRate, errorRate
|
|
||||||
}
|
|
||||||
|
|
||||||
type opsSystemStats struct {
|
|
||||||
cpuUsage float64
|
|
||||||
memoryUsedMB int64
|
|
||||||
memoryTotalMB int64
|
|
||||||
memoryUsagePercent float64
|
|
||||||
heapAllocMB int64
|
|
||||||
gcPauseMs float64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) collectSystemStats(ctx context.Context) opsSystemStats {
|
|
||||||
stats := opsSystemStats{}
|
|
||||||
|
|
||||||
if percents, err := cpu.PercentWithContext(ctx, cpuUsageSampleInterval, false); err == nil && len(percents) > 0 {
|
|
||||||
stats.cpuUsage = percents[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
if vm, err := mem.VirtualMemoryWithContext(ctx); err == nil {
|
|
||||||
stats.memoryUsedMB = int64(vm.Used / bytesPerMB)
|
|
||||||
stats.memoryTotalMB = int64(vm.Total / bytesPerMB)
|
|
||||||
stats.memoryUsagePercent = vm.UsedPercent
|
|
||||||
}
|
|
||||||
|
|
||||||
var memStats runtime.MemStats
|
|
||||||
runtime.ReadMemStats(&memStats)
|
|
||||||
stats.heapAllocMB = int64(memStats.HeapAlloc / bytesPerMB)
|
|
||||||
c.lastGCPauseMu.Lock()
|
|
||||||
if c.lastGCPauseTotal != 0 && memStats.PauseTotalNs >= c.lastGCPauseTotal {
|
|
||||||
stats.gcPauseMs = float64(memStats.PauseTotalNs-c.lastGCPauseTotal) / float64(time.Millisecond)
|
|
||||||
}
|
|
||||||
c.lastGCPauseTotal = memStats.PauseTotalNs
|
|
||||||
c.lastGCPauseMu.Unlock()
|
|
||||||
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) collectQueueDepth(ctx context.Context) int {
|
|
||||||
if c.concurrencyService == nil {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
depth, err := c.concurrencyService.GetTotalWaitCount(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("[OpsMetrics] failed to get queue depth: %v", err)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return depth
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpsMetricsCollector) collectActiveAlerts(ctx context.Context) int {
|
|
||||||
if c.opsService == nil {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
count, err := c.opsService.CountActiveAlerts(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return count
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,48 +0,0 @@
|
|||||||
-- Ops error logs and system metrics
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS ops_error_logs (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
request_id VARCHAR(64),
|
|
||||||
user_id BIGINT,
|
|
||||||
api_key_id BIGINT,
|
|
||||||
account_id BIGINT,
|
|
||||||
group_id BIGINT,
|
|
||||||
client_ip INET,
|
|
||||||
error_phase VARCHAR(32) NOT NULL,
|
|
||||||
error_type VARCHAR(64) NOT NULL,
|
|
||||||
severity VARCHAR(4) NOT NULL,
|
|
||||||
status_code INT,
|
|
||||||
platform VARCHAR(32),
|
|
||||||
model VARCHAR(100),
|
|
||||||
request_path VARCHAR(256),
|
|
||||||
stream BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
error_message TEXT,
|
|
||||||
error_body TEXT,
|
|
||||||
provider_error_code VARCHAR(64),
|
|
||||||
provider_error_type VARCHAR(64),
|
|
||||||
is_retryable BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
is_user_actionable BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
retry_count INT NOT NULL DEFAULT 0,
|
|
||||||
completion_status VARCHAR(16),
|
|
||||||
duration_ms INT,
|
|
||||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_error_logs_created_at ON ops_error_logs (created_at DESC);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_error_logs_phase ON ops_error_logs (error_phase);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_error_logs_platform ON ops_error_logs (platform);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_error_logs_severity ON ops_error_logs (severity);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_error_logs_phase_platform_time ON ops_error_logs (error_phase, platform, created_at DESC);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS ops_system_metrics (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
success_rate DOUBLE PRECISION,
|
|
||||||
error_rate DOUBLE PRECISION,
|
|
||||||
p95_latency_ms INT,
|
|
||||||
p99_latency_ms INT,
|
|
||||||
http2_errors INT,
|
|
||||||
active_alerts INT,
|
|
||||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_system_metrics_created_at ON ops_system_metrics (created_at DESC);
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
-- Extend ops_system_metrics with windowed/system stats
|
|
||||||
|
|
||||||
ALTER TABLE ops_system_metrics
|
|
||||||
ADD COLUMN IF NOT EXISTS window_minutes INT NOT NULL DEFAULT 1,
|
|
||||||
ADD COLUMN IF NOT EXISTS cpu_usage_percent DOUBLE PRECISION,
|
|
||||||
ADD COLUMN IF NOT EXISTS memory_used_mb BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS memory_total_mb BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS memory_usage_percent DOUBLE PRECISION,
|
|
||||||
ADD COLUMN IF NOT EXISTS heap_alloc_mb BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS gc_pause_ms DOUBLE PRECISION,
|
|
||||||
ADD COLUMN IF NOT EXISTS concurrency_queue_depth INT;
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_system_metrics_window_time
|
|
||||||
ON ops_system_metrics (window_minutes, created_at DESC);
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
-- Ops alert rules and events
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS ops_alert_rules (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
name VARCHAR(128) NOT NULL,
|
|
||||||
description TEXT,
|
|
||||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
|
||||||
metric_type VARCHAR(64) NOT NULL,
|
|
||||||
operator VARCHAR(8) NOT NULL,
|
|
||||||
threshold DOUBLE PRECISION NOT NULL,
|
|
||||||
window_minutes INT NOT NULL DEFAULT 1,
|
|
||||||
sustained_minutes INT NOT NULL DEFAULT 1,
|
|
||||||
severity VARCHAR(4) NOT NULL DEFAULT 'P1',
|
|
||||||
notify_email BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
notify_webhook BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
webhook_url TEXT,
|
|
||||||
cooldown_minutes INT NOT NULL DEFAULT 10,
|
|
||||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_alert_rules_enabled ON ops_alert_rules (enabled);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_alert_rules_metric ON ops_alert_rules (metric_type, window_minutes);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS ops_alert_events (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
rule_id BIGINT NOT NULL REFERENCES ops_alert_rules(id) ON DELETE CASCADE,
|
|
||||||
severity VARCHAR(4) NOT NULL,
|
|
||||||
status VARCHAR(16) NOT NULL DEFAULT 'firing',
|
|
||||||
title VARCHAR(200),
|
|
||||||
description TEXT,
|
|
||||||
metric_value DOUBLE PRECISION,
|
|
||||||
threshold_value DOUBLE PRECISION,
|
|
||||||
fired_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
||||||
resolved_at TIMESTAMPTZ,
|
|
||||||
email_sent BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
webhook_sent BOOLEAN NOT NULL DEFAULT FALSE,
|
|
||||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_alert_events_rule_status ON ops_alert_events (rule_id, status);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_alert_events_fired_at ON ops_alert_events (fired_at DESC);
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
-- Seed default ops alert rules (idempotent)
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'Global success rate < 99%',
|
|
||||||
'Trigger when the 1-minute success rate drops below 99% for 2 consecutive minutes.',
|
|
||||||
TRUE,
|
|
||||||
'success_rate',
|
|
||||||
'<',
|
|
||||||
99,
|
|
||||||
1,
|
|
||||||
2,
|
|
||||||
'P1',
|
|
||||||
TRUE,
|
|
||||||
FALSE,
|
|
||||||
NULL,
|
|
||||||
10
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules);
|
|
||||||
@@ -1,205 +0,0 @@
|
|||||||
-- Seed additional ops alert rules (idempotent)
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'Global error rate > 1%',
|
|
||||||
'Trigger when the 1-minute error rate exceeds 1% for 2 consecutive minutes.',
|
|
||||||
TRUE,
|
|
||||||
'error_rate',
|
|
||||||
'>',
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
2,
|
|
||||||
'P1',
|
|
||||||
TRUE,
|
|
||||||
CASE
|
|
||||||
WHEN (SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1) IS NULL THEN FALSE
|
|
||||||
ELSE TRUE
|
|
||||||
END,
|
|
||||||
(SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1),
|
|
||||||
10
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules WHERE name = 'Global error rate > 1%');
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'P99 latency > 2000ms',
|
|
||||||
'Trigger when the 5-minute P99 latency exceeds 2000ms for 2 consecutive samples.',
|
|
||||||
TRUE,
|
|
||||||
'p99_latency_ms',
|
|
||||||
'>',
|
|
||||||
2000,
|
|
||||||
5,
|
|
||||||
2,
|
|
||||||
'P1',
|
|
||||||
TRUE,
|
|
||||||
CASE
|
|
||||||
WHEN (SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1) IS NULL THEN FALSE
|
|
||||||
ELSE TRUE
|
|
||||||
END,
|
|
||||||
(SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1),
|
|
||||||
15
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules WHERE name = 'P99 latency > 2000ms');
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'HTTP/2 errors > 20',
|
|
||||||
'Trigger when HTTP/2 errors exceed 20 in the last minute for 2 consecutive minutes.',
|
|
||||||
TRUE,
|
|
||||||
'http2_errors',
|
|
||||||
'>',
|
|
||||||
20,
|
|
||||||
1,
|
|
||||||
2,
|
|
||||||
'P2',
|
|
||||||
FALSE,
|
|
||||||
CASE
|
|
||||||
WHEN (SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1) IS NULL THEN FALSE
|
|
||||||
ELSE TRUE
|
|
||||||
END,
|
|
||||||
(SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1),
|
|
||||||
10
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules WHERE name = 'HTTP/2 errors > 20');
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'CPU usage > 85%',
|
|
||||||
'Trigger when CPU usage exceeds 85% for 5 consecutive minutes.',
|
|
||||||
TRUE,
|
|
||||||
'cpu_usage_percent',
|
|
||||||
'>',
|
|
||||||
85,
|
|
||||||
1,
|
|
||||||
5,
|
|
||||||
'P2',
|
|
||||||
FALSE,
|
|
||||||
CASE
|
|
||||||
WHEN (SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1) IS NULL THEN FALSE
|
|
||||||
ELSE TRUE
|
|
||||||
END,
|
|
||||||
(SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1),
|
|
||||||
15
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules WHERE name = 'CPU usage > 85%');
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'Memory usage > 90%',
|
|
||||||
'Trigger when memory usage exceeds 90% for 5 consecutive minutes.',
|
|
||||||
TRUE,
|
|
||||||
'memory_usage_percent',
|
|
||||||
'>',
|
|
||||||
90,
|
|
||||||
1,
|
|
||||||
5,
|
|
||||||
'P2',
|
|
||||||
FALSE,
|
|
||||||
CASE
|
|
||||||
WHEN (SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1) IS NULL THEN FALSE
|
|
||||||
ELSE TRUE
|
|
||||||
END,
|
|
||||||
(SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1),
|
|
||||||
15
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules WHERE name = 'Memory usage > 90%');
|
|
||||||
|
|
||||||
INSERT INTO ops_alert_rules (
|
|
||||||
name,
|
|
||||||
description,
|
|
||||||
enabled,
|
|
||||||
metric_type,
|
|
||||||
operator,
|
|
||||||
threshold,
|
|
||||||
window_minutes,
|
|
||||||
sustained_minutes,
|
|
||||||
severity,
|
|
||||||
notify_email,
|
|
||||||
notify_webhook,
|
|
||||||
webhook_url,
|
|
||||||
cooldown_minutes
|
|
||||||
)
|
|
||||||
SELECT
|
|
||||||
'Queue depth > 50',
|
|
||||||
'Trigger when concurrency queue depth exceeds 50 for 2 consecutive minutes.',
|
|
||||||
TRUE,
|
|
||||||
'concurrency_queue_depth',
|
|
||||||
'>',
|
|
||||||
50,
|
|
||||||
1,
|
|
||||||
2,
|
|
||||||
'P2',
|
|
||||||
FALSE,
|
|
||||||
CASE
|
|
||||||
WHEN (SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1) IS NULL THEN FALSE
|
|
||||||
ELSE TRUE
|
|
||||||
END,
|
|
||||||
(SELECT webhook_url FROM ops_alert_rules WHERE webhook_url IS NOT NULL AND webhook_url <> '' LIMIT 1),
|
|
||||||
10
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM ops_alert_rules WHERE name = 'Queue depth > 50');
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
-- Enable webhook notifications for rules with webhook_url configured
|
|
||||||
|
|
||||||
UPDATE ops_alert_rules
|
|
||||||
SET notify_webhook = TRUE
|
|
||||||
WHERE webhook_url IS NOT NULL
|
|
||||||
AND webhook_url <> ''
|
|
||||||
AND notify_webhook IS DISTINCT FROM TRUE;
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
-- Add request counts to ops_system_metrics so the UI/alerts can distinguish "no traffic" from "healthy".
|
|
||||||
|
|
||||||
ALTER TABLE ops_system_metrics
|
|
||||||
ADD COLUMN IF NOT EXISTS request_count BIGINT NOT NULL DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS success_count BIGINT NOT NULL DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS error_count BIGINT NOT NULL DEFAULT 0;
|
|
||||||
@@ -1,272 +0,0 @@
|
|||||||
-- 运维监控中心 2.0 - 数据库 Schema 增强
|
|
||||||
-- 创建时间: 2026-01-02
|
|
||||||
-- 说明: 扩展监控指标,支持多维度分析和告警管理
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 1. 扩展 ops_system_metrics 表
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
-- 添加 RED 指标列
|
|
||||||
ALTER TABLE ops_system_metrics
|
|
||||||
ADD COLUMN IF NOT EXISTS qps DECIMAL(10,2) DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS tps DECIMAL(10,2) DEFAULT 0,
|
|
||||||
|
|
||||||
-- 错误分类
|
|
||||||
ADD COLUMN IF NOT EXISTS error_4xx_count BIGINT DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS error_5xx_count BIGINT DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS error_timeout_count BIGINT DEFAULT 0,
|
|
||||||
|
|
||||||
-- 延迟指标扩展
|
|
||||||
ADD COLUMN IF NOT EXISTS latency_p50 DECIMAL(10,2),
|
|
||||||
ADD COLUMN IF NOT EXISTS latency_p999 DECIMAL(10,2),
|
|
||||||
ADD COLUMN IF NOT EXISTS latency_avg DECIMAL(10,2),
|
|
||||||
ADD COLUMN IF NOT EXISTS latency_max DECIMAL(10,2),
|
|
||||||
|
|
||||||
-- 上游延迟
|
|
||||||
ADD COLUMN IF NOT EXISTS upstream_latency_avg DECIMAL(10,2),
|
|
||||||
|
|
||||||
-- 资源指标
|
|
||||||
ADD COLUMN IF NOT EXISTS disk_used BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS disk_total BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS disk_iops BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS network_in_bytes BIGINT,
|
|
||||||
ADD COLUMN IF NOT EXISTS network_out_bytes BIGINT,
|
|
||||||
|
|
||||||
-- 饱和度指标
|
|
||||||
ADD COLUMN IF NOT EXISTS goroutine_count INT,
|
|
||||||
ADD COLUMN IF NOT EXISTS db_conn_active INT,
|
|
||||||
ADD COLUMN IF NOT EXISTS db_conn_idle INT,
|
|
||||||
ADD COLUMN IF NOT EXISTS db_conn_waiting INT,
|
|
||||||
|
|
||||||
-- 业务指标
|
|
||||||
ADD COLUMN IF NOT EXISTS token_consumed BIGINT DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS token_rate DECIMAL(10,2) DEFAULT 0,
|
|
||||||
ADD COLUMN IF NOT EXISTS active_subscriptions INT DEFAULT 0,
|
|
||||||
|
|
||||||
-- 维度标签 (支持多维度分析)
|
|
||||||
ADD COLUMN IF NOT EXISTS tags JSONB;
|
|
||||||
|
|
||||||
-- 添加 JSONB 索引以加速标签查询
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_metrics_tags ON ops_system_metrics USING GIN(tags);
|
|
||||||
|
|
||||||
-- 添加注释
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.qps IS '每秒查询数 (Queries Per Second)';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.tps IS '每秒事务数 (Transactions Per Second)';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.error_4xx_count IS '客户端错误数量 (4xx)';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.error_5xx_count IS '服务端错误数量 (5xx)';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.error_timeout_count IS '超时错误数量';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.upstream_latency_avg IS '上游 API 平均延迟 (ms)';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.goroutine_count IS 'Goroutine 数量 (检测泄露)';
|
|
||||||
COMMENT ON COLUMN ops_system_metrics.tags IS '维度标签 (JSON), 如: {"account_id": "123", "api_path": "/v1/chat"}';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 2. 创建维度统计表
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS ops_dimension_stats (
|
|
||||||
id BIGSERIAL PRIMARY KEY,
|
|
||||||
timestamp TIMESTAMPTZ NOT NULL,
|
|
||||||
|
|
||||||
-- 维度类型: account, api_path, provider, region
|
|
||||||
dimension_type VARCHAR(50) NOT NULL,
|
|
||||||
dimension_value VARCHAR(255) NOT NULL,
|
|
||||||
|
|
||||||
-- 统计指标
|
|
||||||
request_count BIGINT DEFAULT 0,
|
|
||||||
success_count BIGINT DEFAULT 0,
|
|
||||||
error_count BIGINT DEFAULT 0,
|
|
||||||
success_rate DECIMAL(5,2),
|
|
||||||
error_rate DECIMAL(5,2),
|
|
||||||
|
|
||||||
-- 性能指标
|
|
||||||
latency_p50 DECIMAL(10,2),
|
|
||||||
latency_p95 DECIMAL(10,2),
|
|
||||||
latency_p99 DECIMAL(10,2),
|
|
||||||
|
|
||||||
-- 业务指标
|
|
||||||
token_consumed BIGINT DEFAULT 0,
|
|
||||||
cost_usd DECIMAL(10,4) DEFAULT 0,
|
|
||||||
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
-- 创建复合索引以加速维度查询
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_dim_type_value_time
|
|
||||||
ON ops_dimension_stats(dimension_type, dimension_value, timestamp DESC);
|
|
||||||
|
|
||||||
-- 创建单独的时间索引用于范围查询
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_ops_dim_timestamp
|
|
||||||
ON ops_dimension_stats(timestamp DESC);
|
|
||||||
|
|
||||||
-- 添加注释
|
|
||||||
COMMENT ON TABLE ops_dimension_stats IS '多维度统计表,支持按账户/API/Provider等维度下钻分析';
|
|
||||||
COMMENT ON COLUMN ops_dimension_stats.dimension_type IS '维度类型: account(账户), api_path(接口), provider(上游), region(地域)';
|
|
||||||
COMMENT ON COLUMN ops_dimension_stats.dimension_value IS '维度值,如: 账户ID, /v1/chat, openai, us-east-1';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 3. 创建告警规则表
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
ALTER TABLE ops_alert_rules
|
|
||||||
ADD COLUMN IF NOT EXISTS dimension_filters JSONB,
|
|
||||||
ADD COLUMN IF NOT EXISTS notify_channels JSONB,
|
|
||||||
ADD COLUMN IF NOT EXISTS notify_config JSONB,
|
|
||||||
ADD COLUMN IF NOT EXISTS created_by VARCHAR(100),
|
|
||||||
ADD COLUMN IF NOT EXISTS last_triggered_at TIMESTAMPTZ;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 4. 告警历史表 (使用现有的 ops_alert_events)
|
|
||||||
-- ============================================
|
|
||||||
-- 注意: 后端代码使用 ops_alert_events 表,不创建新表
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 5. 创建数据清理配置表
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS ops_data_retention_config (
|
|
||||||
id SERIAL PRIMARY KEY,
|
|
||||||
table_name VARCHAR(100) NOT NULL UNIQUE,
|
|
||||||
retention_days INT NOT NULL, -- 保留天数
|
|
||||||
enabled BOOLEAN DEFAULT true,
|
|
||||||
last_cleanup_at TIMESTAMPTZ,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
-- 插入默认配置
|
|
||||||
INSERT INTO ops_data_retention_config (table_name, retention_days) VALUES
|
|
||||||
('ops_system_metrics', 30), -- 系统指标保留 30 天
|
|
||||||
('ops_dimension_stats', 30), -- 维度统计保留 30 天
|
|
||||||
('ops_error_logs', 30), -- 错误日志保留 30 天
|
|
||||||
('ops_alert_events', 90), -- 告警事件保留 90 天
|
|
||||||
('usage_logs', 90) -- 使用日志保留 90 天
|
|
||||||
ON CONFLICT (table_name) DO NOTHING;
|
|
||||||
|
|
||||||
COMMENT ON TABLE ops_data_retention_config IS '数据保留策略配置表';
|
|
||||||
COMMENT ON COLUMN ops_data_retention_config.retention_days IS '数据保留天数,超过此天数的数据将被自动清理';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 6. 创建辅助函数
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
-- 函数: 计算健康度评分
|
|
||||||
-- 权重: SLA(40%) + 错误率(30%) + 延迟(20%) + 资源(10%)
|
|
||||||
CREATE OR REPLACE FUNCTION calculate_health_score(
|
|
||||||
p_success_rate DECIMAL,
|
|
||||||
p_error_rate DECIMAL,
|
|
||||||
p_latency_p99 DECIMAL,
|
|
||||||
p_cpu_usage DECIMAL
|
|
||||||
) RETURNS INT AS $$
|
|
||||||
DECLARE
|
|
||||||
sla_score INT;
|
|
||||||
error_score INT;
|
|
||||||
latency_score INT;
|
|
||||||
resource_score INT;
|
|
||||||
BEGIN
|
|
||||||
-- SLA 评分 (40分)
|
|
||||||
sla_score := CASE
|
|
||||||
WHEN p_success_rate >= 99.9 THEN 40
|
|
||||||
WHEN p_success_rate >= 99.5 THEN 35
|
|
||||||
WHEN p_success_rate >= 99.0 THEN 30
|
|
||||||
WHEN p_success_rate >= 95.0 THEN 20
|
|
||||||
ELSE 10
|
|
||||||
END;
|
|
||||||
|
|
||||||
-- 错误率评分 (30分)
|
|
||||||
error_score := CASE
|
|
||||||
WHEN p_error_rate <= 0.1 THEN 30
|
|
||||||
WHEN p_error_rate <= 0.5 THEN 25
|
|
||||||
WHEN p_error_rate <= 1.0 THEN 20
|
|
||||||
WHEN p_error_rate <= 5.0 THEN 10
|
|
||||||
ELSE 5
|
|
||||||
END;
|
|
||||||
|
|
||||||
-- 延迟评分 (20分)
|
|
||||||
latency_score := CASE
|
|
||||||
WHEN p_latency_p99 <= 500 THEN 20
|
|
||||||
WHEN p_latency_p99 <= 1000 THEN 15
|
|
||||||
WHEN p_latency_p99 <= 3000 THEN 10
|
|
||||||
WHEN p_latency_p99 <= 5000 THEN 5
|
|
||||||
ELSE 0
|
|
||||||
END;
|
|
||||||
|
|
||||||
-- 资源评分 (10分)
|
|
||||||
resource_score := CASE
|
|
||||||
WHEN p_cpu_usage <= 50 THEN 10
|
|
||||||
WHEN p_cpu_usage <= 70 THEN 7
|
|
||||||
WHEN p_cpu_usage <= 85 THEN 5
|
|
||||||
ELSE 2
|
|
||||||
END;
|
|
||||||
|
|
||||||
RETURN sla_score + error_score + latency_score + resource_score;
|
|
||||||
END;
|
|
||||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
|
||||||
|
|
||||||
COMMENT ON FUNCTION calculate_health_score IS '计算系统健康度评分 (0-100),权重: SLA 40% + 错误率 30% + 延迟 20% + 资源 10%';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 7. 创建视图: 最新指标快照
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
CREATE OR REPLACE VIEW ops_latest_metrics AS
|
|
||||||
SELECT
|
|
||||||
m.*,
|
|
||||||
calculate_health_score(
|
|
||||||
m.success_rate::DECIMAL,
|
|
||||||
m.error_rate::DECIMAL,
|
|
||||||
m.p99_latency_ms::DECIMAL,
|
|
||||||
m.cpu_usage_percent::DECIMAL
|
|
||||||
) AS health_score
|
|
||||||
FROM ops_system_metrics m
|
|
||||||
WHERE m.window_minutes = 1
|
|
||||||
AND m.created_at = (SELECT MAX(created_at) FROM ops_system_metrics WHERE window_minutes = 1)
|
|
||||||
LIMIT 1;
|
|
||||||
|
|
||||||
COMMENT ON VIEW ops_latest_metrics IS '最新的系统指标快照,包含健康度评分';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 8. 创建视图: 活跃告警列表
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
CREATE OR REPLACE VIEW ops_active_alerts AS
|
|
||||||
SELECT
|
|
||||||
e.id,
|
|
||||||
e.rule_id,
|
|
||||||
r.name AS rule_name,
|
|
||||||
r.metric_type,
|
|
||||||
e.fired_at,
|
|
||||||
e.metric_value,
|
|
||||||
e.threshold_value,
|
|
||||||
r.severity,
|
|
||||||
EXTRACT(EPOCH FROM (NOW() - e.fired_at))::INT AS duration_seconds
|
|
||||||
FROM ops_alert_events e
|
|
||||||
JOIN ops_alert_rules r ON e.rule_id = r.id
|
|
||||||
WHERE e.status = 'firing'
|
|
||||||
ORDER BY e.fired_at DESC;
|
|
||||||
|
|
||||||
COMMENT ON VIEW ops_active_alerts IS '当前活跃的告警列表';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 9. 权限设置 (可选)
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
-- 如果有专门的 ops 用户,可以授权
|
|
||||||
-- GRANT SELECT, INSERT, UPDATE ON ops_system_metrics TO ops_user;
|
|
||||||
-- GRANT SELECT, INSERT ON ops_dimension_stats TO ops_user;
|
|
||||||
-- GRANT ALL ON ops_alert_rules TO ops_user;
|
|
||||||
-- GRANT ALL ON ops_alert_events TO ops_user;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 10. 数据完整性检查
|
|
||||||
-- ============================================
|
|
||||||
|
|
||||||
-- 确保现有数据的兼容性
|
|
||||||
UPDATE ops_system_metrics
|
|
||||||
SET
|
|
||||||
qps = COALESCE(request_count / (window_minutes * 60.0), 0),
|
|
||||||
error_rate = COALESCE((error_count::DECIMAL / NULLIF(request_count, 0)) * 100, 0)
|
|
||||||
WHERE qps = 0 AND request_count > 0;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- 完成
|
|
||||||
-- ============================================
|
|
||||||
@@ -1,324 +0,0 @@
|
|||||||
/**
|
|
||||||
* Admin Ops API endpoints
|
|
||||||
* Provides stability metrics and error logs for ops dashboard
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { apiClient } from '../client'
|
|
||||||
|
|
||||||
export type OpsSeverity = 'P0' | 'P1' | 'P2' | 'P3'
|
|
||||||
export type OpsPhase =
|
|
||||||
| 'auth'
|
|
||||||
| 'concurrency'
|
|
||||||
| 'billing'
|
|
||||||
| 'scheduling'
|
|
||||||
| 'network'
|
|
||||||
| 'upstream'
|
|
||||||
| 'response'
|
|
||||||
| 'internal'
|
|
||||||
export type OpsPlatform = 'gemini' | 'openai' | 'anthropic' | 'antigravity'
|
|
||||||
|
|
||||||
export interface OpsMetrics {
|
|
||||||
window_minutes: number
|
|
||||||
request_count: number
|
|
||||||
success_count: number
|
|
||||||
error_count: number
|
|
||||||
success_rate: number
|
|
||||||
error_rate: number
|
|
||||||
p95_latency_ms: number
|
|
||||||
p99_latency_ms: number
|
|
||||||
http2_errors: number
|
|
||||||
active_alerts: number
|
|
||||||
cpu_usage_percent?: number
|
|
||||||
memory_used_mb?: number
|
|
||||||
memory_total_mb?: number
|
|
||||||
memory_usage_percent?: number
|
|
||||||
heap_alloc_mb?: number
|
|
||||||
gc_pause_ms?: number
|
|
||||||
concurrency_queue_depth?: number
|
|
||||||
updated_at?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OpsErrorLog {
|
|
||||||
id: number
|
|
||||||
created_at: string
|
|
||||||
phase: OpsPhase
|
|
||||||
type: string
|
|
||||||
severity: OpsSeverity
|
|
||||||
status_code: number
|
|
||||||
platform: OpsPlatform
|
|
||||||
model: string
|
|
||||||
latency_ms: number | null
|
|
||||||
request_id: string
|
|
||||||
message: string
|
|
||||||
user_id?: number | null
|
|
||||||
api_key_id?: number | null
|
|
||||||
account_id?: number | null
|
|
||||||
group_id?: number | null
|
|
||||||
client_ip?: string
|
|
||||||
request_path?: string
|
|
||||||
stream?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OpsErrorListParams {
|
|
||||||
start_time?: string
|
|
||||||
end_time?: string
|
|
||||||
platform?: OpsPlatform
|
|
||||||
phase?: OpsPhase
|
|
||||||
severity?: OpsSeverity
|
|
||||||
q?: string
|
|
||||||
/**
|
|
||||||
* Max 500 (legacy endpoint uses a hard cap); use paginated /admin/ops/errors for larger result sets.
|
|
||||||
*/
|
|
||||||
limit?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OpsErrorListResponse {
|
|
||||||
items: OpsErrorLog[]
|
|
||||||
total?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OpsMetricsHistoryParams {
|
|
||||||
window_minutes?: number
|
|
||||||
minutes?: number
|
|
||||||
start_time?: string
|
|
||||||
end_time?: string
|
|
||||||
limit?: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OpsMetricsHistoryResponse {
|
|
||||||
items: OpsMetrics[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get latest ops metrics snapshot
|
|
||||||
*/
|
|
||||||
export async function getMetrics(): Promise<OpsMetrics> {
|
|
||||||
const { data } = await apiClient.get<OpsMetrics>('/admin/ops/metrics')
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* List metrics history for charts
|
|
||||||
*/
|
|
||||||
export async function listMetricsHistory(params?: OpsMetricsHistoryParams): Promise<OpsMetricsHistoryResponse> {
|
|
||||||
const { data } = await apiClient.get<OpsMetricsHistoryResponse>('/admin/ops/metrics/history', { params })
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* List recent error logs with optional filters
|
|
||||||
*/
|
|
||||||
export async function listErrors(params?: OpsErrorListParams): Promise<OpsErrorListResponse> {
|
|
||||||
const { data } = await apiClient.get<OpsErrorListResponse>('/admin/ops/error-logs', { params })
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface OpsDashboardOverview {
|
|
||||||
timestamp: string
|
|
||||||
health_score: number
|
|
||||||
sla: {
|
|
||||||
current: number
|
|
||||||
threshold: number
|
|
||||||
status: string
|
|
||||||
trend: string
|
|
||||||
change_24h: number
|
|
||||||
}
|
|
||||||
qps: {
|
|
||||||
current: number
|
|
||||||
peak_1h: number
|
|
||||||
avg_1h: number
|
|
||||||
change_vs_yesterday: number
|
|
||||||
}
|
|
||||||
tps: {
|
|
||||||
current: number
|
|
||||||
peak_1h: number
|
|
||||||
avg_1h: number
|
|
||||||
}
|
|
||||||
latency: {
|
|
||||||
p50: number
|
|
||||||
p95: number
|
|
||||||
p99: number
|
|
||||||
p999: number
|
|
||||||
avg: number
|
|
||||||
max: number
|
|
||||||
threshold_p99: number
|
|
||||||
status: string
|
|
||||||
}
|
|
||||||
errors: {
|
|
||||||
total_count: number
|
|
||||||
error_rate: number
|
|
||||||
'4xx_count': number
|
|
||||||
'5xx_count': number
|
|
||||||
timeout_count: number
|
|
||||||
top_error?: {
|
|
||||||
code: string
|
|
||||||
message: string
|
|
||||||
count: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resources: {
|
|
||||||
cpu_usage: number
|
|
||||||
memory_usage: number
|
|
||||||
disk_usage: number
|
|
||||||
goroutines: number
|
|
||||||
db_connections: {
|
|
||||||
active: number
|
|
||||||
idle: number
|
|
||||||
waiting: number
|
|
||||||
max: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
system_status: {
|
|
||||||
redis: string
|
|
||||||
database: string
|
|
||||||
background_jobs: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ProviderHealthData {
|
|
||||||
name: string
|
|
||||||
request_count: number
|
|
||||||
success_rate: number
|
|
||||||
error_rate: number
|
|
||||||
latency_avg: number
|
|
||||||
latency_p99: number
|
|
||||||
status: string
|
|
||||||
errors_by_type: {
|
|
||||||
'4xx': number
|
|
||||||
'5xx': number
|
|
||||||
timeout: number
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ProviderHealthResponse {
|
|
||||||
providers: ProviderHealthData[]
|
|
||||||
summary: {
|
|
||||||
total_requests: number
|
|
||||||
avg_success_rate: number
|
|
||||||
best_provider: string
|
|
||||||
worst_provider: string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface LatencyHistogramResponse {
|
|
||||||
buckets: {
|
|
||||||
range: string
|
|
||||||
count: number
|
|
||||||
percentage: number
|
|
||||||
}[]
|
|
||||||
total_requests: number
|
|
||||||
slow_request_threshold: number
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ErrorDistributionResponse {
|
|
||||||
items: {
|
|
||||||
code: string
|
|
||||||
message: string
|
|
||||||
count: number
|
|
||||||
percentage: number
|
|
||||||
}[]
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get realtime ops dashboard overview
|
|
||||||
*/
|
|
||||||
export async function getDashboardOverview(timeRange = '1h'): Promise<OpsDashboardOverview> {
|
|
||||||
const { data } = await apiClient.get<OpsDashboardOverview>('/admin/ops/dashboard/overview', {
|
|
||||||
params: { time_range: timeRange }
|
|
||||||
})
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get provider health comparison
|
|
||||||
*/
|
|
||||||
export async function getProviderHealth(timeRange = '1h'): Promise<ProviderHealthResponse> {
|
|
||||||
const { data } = await apiClient.get<ProviderHealthResponse>('/admin/ops/dashboard/providers', {
|
|
||||||
params: { time_range: timeRange }
|
|
||||||
})
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get latency histogram
|
|
||||||
*/
|
|
||||||
export async function getLatencyHistogram(timeRange = '1h'): Promise<LatencyHistogramResponse> {
|
|
||||||
const { data } = await apiClient.get<LatencyHistogramResponse>('/admin/ops/dashboard/latency-histogram', {
|
|
||||||
params: { time_range: timeRange }
|
|
||||||
})
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get error distribution
|
|
||||||
*/
|
|
||||||
export async function getErrorDistribution(timeRange = '1h'): Promise<ErrorDistributionResponse> {
|
|
||||||
const { data } = await apiClient.get<ErrorDistributionResponse>('/admin/ops/dashboard/errors/distribution', {
|
|
||||||
params: { time_range: timeRange }
|
|
||||||
})
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Subscribe to realtime QPS updates via WebSocket
|
|
||||||
*/
|
|
||||||
export function subscribeQPS(onMessage: (data: any) => void): () => void {
|
|
||||||
let ws: WebSocket | null = null
|
|
||||||
let reconnectAttempts = 0
|
|
||||||
const maxReconnectAttempts = 5
|
|
||||||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null
|
|
||||||
let shouldReconnect = true
|
|
||||||
|
|
||||||
const connect = () => {
|
|
||||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
|
|
||||||
const host = window.location.host
|
|
||||||
ws = new WebSocket(`${protocol}//${host}/api/v1/admin/ops/ws/qps`)
|
|
||||||
|
|
||||||
ws.onopen = () => {
|
|
||||||
console.log('[OpsWS] Connected')
|
|
||||||
reconnectAttempts = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.onmessage = (e) => {
|
|
||||||
const data = JSON.parse(e.data)
|
|
||||||
onMessage(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.onerror = (error) => {
|
|
||||||
console.error('[OpsWS] Connection error:', error)
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.onclose = () => {
|
|
||||||
console.log('[OpsWS] Connection closed')
|
|
||||||
if (shouldReconnect && reconnectAttempts < maxReconnectAttempts) {
|
|
||||||
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30000)
|
|
||||||
console.log(`[OpsWS] Reconnecting in ${delay}ms...`)
|
|
||||||
reconnectTimer = setTimeout(() => {
|
|
||||||
reconnectAttempts++
|
|
||||||
connect()
|
|
||||||
}, delay)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
connect()
|
|
||||||
|
|
||||||
return () => {
|
|
||||||
shouldReconnect = false
|
|
||||||
if (reconnectTimer) clearTimeout(reconnectTimer)
|
|
||||||
if (ws) ws.close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const opsAPI = {
|
|
||||||
getMetrics,
|
|
||||||
listMetricsHistory,
|
|
||||||
listErrors,
|
|
||||||
getDashboardOverview,
|
|
||||||
getProviderHealth,
|
|
||||||
getLatencyHistogram,
|
|
||||||
getErrorDistribution,
|
|
||||||
subscribeQPS
|
|
||||||
}
|
|
||||||
|
|
||||||
export default opsAPI
|
|
||||||
@@ -1,417 +0,0 @@
|
|||||||
<script setup lang="ts">
|
|
||||||
import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
|
|
||||||
import { useI18n } from 'vue-i18n'
|
|
||||||
import { Bar, Doughnut } from 'vue-chartjs'
|
|
||||||
import {
|
|
||||||
Chart as ChartJS,
|
|
||||||
Title,
|
|
||||||
Tooltip,
|
|
||||||
Legend,
|
|
||||||
LineElement,
|
|
||||||
LinearScale,
|
|
||||||
PointElement,
|
|
||||||
CategoryScale,
|
|
||||||
BarElement,
|
|
||||||
ArcElement
|
|
||||||
} from 'chart.js'
|
|
||||||
import { useIntervalFn } from '@vueuse/core'
|
|
||||||
import AppLayout from '@/components/layout/AppLayout.vue'
|
|
||||||
import { opsAPI, type OpsDashboardOverview, type ProviderHealthData, type LatencyHistogramResponse, type ErrorDistributionResponse } from '@/api/admin/ops'
|
|
||||||
import { useAuthStore } from '@/stores/auth'
|
|
||||||
|
|
||||||
ChartJS.register(
|
|
||||||
Title,
|
|
||||||
Tooltip,
|
|
||||||
Legend,
|
|
||||||
LineElement,
|
|
||||||
LinearScale,
|
|
||||||
PointElement,
|
|
||||||
CategoryScale,
|
|
||||||
BarElement,
|
|
||||||
ArcElement
|
|
||||||
)
|
|
||||||
|
|
||||||
const { t } = useI18n()
|
|
||||||
const authStore = useAuthStore()
|
|
||||||
const loading = ref(false)
|
|
||||||
const errorMessage = ref('')
|
|
||||||
const timeRange = ref('1h')
|
|
||||||
const lastUpdated = ref(new Date())
|
|
||||||
|
|
||||||
const overview = ref<OpsDashboardOverview | null>(null)
|
|
||||||
const providers = ref<ProviderHealthData[]>([])
|
|
||||||
const latencyData = ref<LatencyHistogramResponse | null>(null)
|
|
||||||
const errorDistribution = ref<ErrorDistributionResponse | null>(null)
|
|
||||||
|
|
||||||
// WebSocket for real-time QPS
|
|
||||||
const realTimeQPS = ref(0)
|
|
||||||
const realTimeTPS = ref(0)
|
|
||||||
const wsConnected = ref(false)
|
|
||||||
let ws: WebSocket | null = null
|
|
||||||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null
|
|
||||||
|
|
||||||
const connectWS = () => {
|
|
||||||
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
|
|
||||||
const wsBaseUrl = import.meta.env.VITE_WS_BASE_URL || window.location.host
|
|
||||||
const wsURL = new URL(`${protocol}//${wsBaseUrl}/api/v1/admin/ops/ws/qps`)
|
|
||||||
const token = authStore.token || localStorage.getItem('auth_token')
|
|
||||||
if (token) {
|
|
||||||
wsURL.searchParams.set('token', token)
|
|
||||||
}
|
|
||||||
ws = new WebSocket(wsURL.toString())
|
|
||||||
|
|
||||||
ws.onopen = () => {
|
|
||||||
wsConnected.value = true
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.onmessage = (event) => {
|
|
||||||
try {
|
|
||||||
const payload = JSON.parse(event.data)
|
|
||||||
if (payload && typeof payload === 'object' && payload.type === 'qps_update' && payload.data) {
|
|
||||||
realTimeQPS.value = payload.data.qps || 0
|
|
||||||
realTimeTPS.value = payload.data.tps || 0
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
console.error('WS parse error', e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.onclose = () => {
|
|
||||||
wsConnected.value = false
|
|
||||||
if (reconnectTimer) clearTimeout(reconnectTimer)
|
|
||||||
reconnectTimer = setTimeout(connectWS, 5000)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const fetchData = async () => {
|
|
||||||
loading.value = true
|
|
||||||
errorMessage.value = ''
|
|
||||||
try {
|
|
||||||
const [ov, pr, lt, er] = await Promise.all([
|
|
||||||
opsAPI.getDashboardOverview(timeRange.value),
|
|
||||||
opsAPI.getProviderHealth(timeRange.value),
|
|
||||||
opsAPI.getLatencyHistogram(timeRange.value),
|
|
||||||
opsAPI.getErrorDistribution(timeRange.value)
|
|
||||||
])
|
|
||||||
overview.value = ov
|
|
||||||
providers.value = pr.providers
|
|
||||||
latencyData.value = lt
|
|
||||||
errorDistribution.value = er
|
|
||||||
lastUpdated.value = new Date()
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to fetch ops data', err)
|
|
||||||
errorMessage.value = '数据加载失败，请稍后重试'
|
|
||||||
} finally {
|
|
||||||
loading.value = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Refresh data every 30 seconds (fallback for L2/L3)
|
|
||||||
useIntervalFn(fetchData, 30000)
|
|
||||||
|
|
||||||
onMounted(() => {
|
|
||||||
fetchData()
|
|
||||||
connectWS()
|
|
||||||
})
|
|
||||||
|
|
||||||
onUnmounted(() => {
|
|
||||||
if (ws) ws.close()
|
|
||||||
if (reconnectTimer) clearTimeout(reconnectTimer)
|
|
||||||
})
|
|
||||||
|
|
||||||
watch(timeRange, () => {
|
|
||||||
fetchData()
|
|
||||||
})
|
|
||||||
|
|
||||||
// Chart Data: Latency Distribution
|
|
||||||
const latencyChartData = computed(() => {
|
|
||||||
if (!latencyData.value) return null
|
|
||||||
return {
|
|
||||||
labels: latencyData.value.buckets.map(b => b.range),
|
|
||||||
datasets: [
|
|
||||||
{
|
|
||||||
label: t('admin.ops.charts.requestCount'),
|
|
||||||
data: latencyData.value.buckets.map(b => b.count),
|
|
||||||
backgroundColor: '#3b82f6',
|
|
||||||
borderRadius: 4
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Chart Data: Error Distribution
|
|
||||||
const errorChartData = computed(() => {
|
|
||||||
if (!errorDistribution.value) return null
|
|
||||||
return {
|
|
||||||
labels: errorDistribution.value.items.map(i => i.code),
|
|
||||||
datasets: [
|
|
||||||
{
|
|
||||||
data: errorDistribution.value.items.map(i => i.count),
|
|
||||||
backgroundColor: [
|
|
||||||
'#ef4444', '#f59e0b', '#3b82f6', '#10b981', '#8b5cf6', '#ec4899'
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Chart Data: Provider SLA
|
|
||||||
const providerChartData = computed(() => {
|
|
||||||
if (!providers.value.length) return null
|
|
||||||
return {
|
|
||||||
labels: providers.value.map(p => p.name),
|
|
||||||
datasets: [
|
|
||||||
{
|
|
||||||
label: 'SLA (%)',
|
|
||||||
data: providers.value.map(p => p.success_rate),
|
|
||||||
backgroundColor: providers.value.map(p => p.success_rate > 99.5 ? '#10b981' : p.success_rate > 98 ? '#f59e0b' : '#ef4444'),
|
|
||||||
borderRadius: 4
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const chartOptions = {
|
|
||||||
responsive: true,
|
|
||||||
maintainAspectRatio: false,
|
|
||||||
plugins: {
|
|
||||||
legend: {
|
|
||||||
display: false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
scales: {
|
|
||||||
y: {
|
|
||||||
beginAtZero: true,
|
|
||||||
grid: {
|
|
||||||
display: false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
x: {
|
|
||||||
grid: {
|
|
||||||
display: false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const healthScoreClass = computed(() => {
|
|
||||||
const score = overview.value?.health_score || 0
|
|
||||||
if (score >= 90) return 'text-green-500 border-green-500'
|
|
||||||
if (score >= 70) return 'text-yellow-500 border-yellow-500'
|
|
||||||
return 'text-red-500 border-red-500'
|
|
||||||
})
|
|
||||||
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<template>
|
|
||||||
<AppLayout>
|
|
||||||
<div class="space-y-6 pb-12">
|
|
||||||
<!-- Error Message -->
|
|
||||||
<div v-if="errorMessage" class="rounded-2xl bg-red-50 p-4 text-sm text-red-600 dark:bg-red-900/20 dark:text-red-400">
|
|
||||||
{{ errorMessage }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- L1: Header & Realtime Stats -->
|
|
||||||
<div class="flex flex-wrap items-center justify-between gap-4 rounded-2xl bg-white p-6 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="flex items-center gap-6">
|
|
||||||
<!-- Health Score Gauge -->
|
|
||||||
<div class="flex h-20 w-20 flex-col items-center justify-center rounded-full border-4 bg-gray-50 dark:bg-dark-900" :class="healthScoreClass">
|
|
||||||
<span class="text-2xl font-black">{{ overview?.health_score || '--' }}</span>
|
|
||||||
<span class="text-[10px] font-bold opacity-60">HEALTH</span>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<h1 class="text-xl font-black text-gray-900 dark:text-white">运维监控中心 2.0</h1>
|
|
||||||
<div class="mt-1 flex items-center gap-3">
|
|
||||||
<span class="flex items-center gap-1.5">
|
|
||||||
<span class="h-2 w-2 rounded-full bg-green-500 animate-pulse" v-if="wsConnected"></span>
|
|
||||||
<span class="h-2 w-2 rounded-full bg-red-500" v-else></span>
|
|
||||||
<span class="text-xs font-medium text-gray-500">{{ wsConnected ? '实时连接中' : '连接已断开' }}</span>
|
|
||||||
</span>
|
|
||||||
<span class="text-xs text-gray-400">最后更新: {{ lastUpdated.toLocaleTimeString() }}</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="flex items-center gap-4">
|
|
||||||
<div class="hidden items-center gap-6 border-r border-gray-100 pr-6 dark:border-dark-700 lg:flex">
|
|
||||||
<div class="text-center">
|
|
||||||
<div class="text-sm font-black text-gray-900 dark:text-white">{{ realTimeQPS.toFixed(1) }}</div>
|
|
||||||
<div class="text-[10px] font-bold text-gray-400 uppercase">实时 QPS</div>
|
|
||||||
</div>
|
|
||||||
<div class="text-center">
|
|
||||||
<div class="text-sm font-black text-gray-900 dark:text-white">{{ (realTimeTPS / 1000).toFixed(1) }}K</div>
|
|
||||||
<div class="text-[10px] font-bold text-gray-400 uppercase">实时 TPS</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<select v-model="timeRange" class="rounded-lg border-gray-200 bg-gray-50 py-1.5 pl-3 pr-8 text-sm font-medium text-gray-700 focus:border-blue-500 focus:ring-blue-500 dark:border-dark-700 dark:bg-dark-900 dark:text-gray-300">
|
|
||||||
<option value="5m">5 分钟</option>
|
|
||||||
<option value="30m">30 分钟</option>
|
|
||||||
<option value="1h">1 小时</option>
|
|
||||||
<option value="24h">24 小时</option>
|
|
||||||
</select>
|
|
||||||
|
|
||||||
<button @click="fetchData" :disabled="loading" class="flex h-9 w-9 items-center justify-center rounded-lg bg-gray-100 text-gray-500 hover:bg-gray-200 dark:bg-dark-700 dark:text-gray-400">
|
|
||||||
<svg class="h-5 w-5" :class="{ 'animate-spin': loading }" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
|
||||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
|
|
||||||
</svg>
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- L1: Core Metrics Grid -->
|
|
||||||
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4">
|
|
||||||
<div class="rounded-2xl bg-white p-5 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-xs font-bold text-gray-400 uppercase tracking-wider">服务可用率 (SLA)</span>
|
|
||||||
<span class="rounded-full bg-green-50 px-2 py-0.5 text-[10px] font-bold text-green-600 dark:bg-green-900/30">{{ overview?.sla.status }}</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-2 flex items-baseline gap-2">
|
|
||||||
<span class="text-2xl font-black text-gray-900 dark:text-white">{{ overview?.sla.current.toFixed(2) }}%</span>
|
|
||||||
<span class="text-xs font-bold" :class="overview?.sla.change_24h && overview.sla.change_24h >= 0 ? 'text-green-500' : 'text-red-500'">
|
|
||||||
{{ overview?.sla.change_24h && overview.sla.change_24h >= 0 ? '+' : '' }}{{ overview?.sla.change_24h }}%
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-3 h-1 w-full overflow-hidden rounded-full bg-gray-100 dark:bg-dark-700">
|
|
||||||
<div class="h-full bg-green-500" :style="{ width: `${overview?.sla.current}%` }"></div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="rounded-2xl bg-white p-5 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-xs font-bold text-gray-400 uppercase tracking-wider">P99 响应延迟</span>
|
|
||||||
<span class="rounded-full bg-blue-50 px-2 py-0.5 text-[10px] font-bold text-blue-600 dark:bg-blue-900/30">Target 1s</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-2 flex items-baseline gap-2">
|
|
||||||
<span class="text-2xl font-black text-gray-900 dark:text-white">{{ overview?.latency.p99 }}ms</span>
|
|
||||||
<span class="text-xs font-bold text-gray-400">Avg: {{ overview?.latency.avg }}ms</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-3 flex gap-1">
|
|
||||||
<div v-for="i in 10" :key="i" class="h-1 flex-1 rounded-full" :class="i <= (overview?.latency.p99 || 0) / 200 ? 'bg-blue-500' : 'bg-gray-100 dark:bg-dark-700'"></div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="rounded-2xl bg-white p-5 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-xs font-bold text-gray-400 uppercase tracking-wider">周期请求总数</span>
|
|
||||||
<svg class="h-4 w-4 text-gray-300" fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 7h8m0 0v8m0-8l-8 8-4-4-6 6" /></svg>
|
|
||||||
</div>
|
|
||||||
<div class="mt-2 flex items-baseline gap-2">
|
|
||||||
<span class="text-2xl font-black text-gray-900 dark:text-white">{{ overview?.qps.avg_1h.toFixed(1) }}</span>
|
|
||||||
<span class="text-xs font-bold text-gray-400">req/s</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-1 text-[10px] font-bold text-gray-400 uppercase">对比昨日: {{ overview?.qps.change_vs_yesterday }}%</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="rounded-2xl bg-white p-5 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-xs font-bold text-gray-400 uppercase tracking-wider">周期错误数</span>
|
|
||||||
<span class="rounded-full bg-red-50 px-2 py-0.5 text-[10px] font-bold text-red-600 dark:bg-red-900/30">{{ overview?.errors.error_rate.toFixed(2) }}%</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-2 flex items-baseline gap-2">
|
|
||||||
<span class="text-2xl font-black text-gray-900 dark:text-white">{{ overview?.errors.total_count }}</span>
|
|
||||||
<span class="text-xs font-bold text-red-500">5xx: {{ overview?.errors['5xx_count'] }}</span>
|
|
||||||
</div>
|
|
||||||
<div class="mt-1 text-[10px] font-bold text-gray-400 uppercase">主要错误码: {{ overview?.errors.top_error?.code || 'N/A' }}</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- L2: Visual Analysis -->
|
|
||||||
<div class="grid grid-cols-1 gap-6 lg:grid-cols-2">
|
|
||||||
<!-- Latency Distribution -->
|
|
||||||
<div class="rounded-2xl bg-white p-6 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="mb-6 flex items-center justify-between">
|
|
||||||
<h3 class="text-sm font-black text-gray-900 dark:text-white uppercase tracking-wider">请求延迟分布</h3>
|
|
||||||
</div>
|
|
||||||
<div class="h-64">
|
|
||||||
<Bar v-if="latencyChartData" :data="latencyChartData" :options="chartOptions" />
|
|
||||||
<div v-else class="flex h-full items-center justify-center text-gray-400">加载中...</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Provider Health -->
|
|
||||||
<div class="rounded-2xl bg-white p-6 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="mb-6 flex items-center justify-between">
|
|
||||||
<h3 class="text-sm font-black text-gray-900 dark:text-white uppercase tracking-wider">上游供应商健康度 (SLA)</h3>
|
|
||||||
</div>
|
|
||||||
<div class="h-64">
|
|
||||||
<Bar v-if="providerChartData" :data="providerChartData" :options="chartOptions" />
|
|
||||||
<div v-else class="flex h-full items-center justify-center text-gray-400">加载中...</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Error Distribution -->
|
|
||||||
<div class="rounded-2xl bg-white p-6 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="mb-6 flex items-center justify-between">
|
|
||||||
<h3 class="text-sm font-black text-gray-900 dark:text-white uppercase tracking-wider">错误类型分布</h3>
|
|
||||||
</div>
|
|
||||||
<div class="flex h-64 gap-6">
|
|
||||||
<div class="relative w-1/2">
|
|
||||||
<Doughnut v-if="errorChartData" :data="errorChartData" :options="{ ...chartOptions, cutout: '70%' }" />
|
|
||||||
</div>
|
|
||||||
<div class="flex flex-1 flex-col justify-center space-y-3">
|
|
||||||
<div v-for="(item, idx) in errorDistribution?.items.slice(0, 5)" :key="item.code" class="flex items-center justify-between">
|
|
||||||
<div class="flex items-center gap-2">
|
|
||||||
<div class="h-2 w-2 rounded-full" :style="{ backgroundColor: ['#ef4444', '#f59e0b', '#3b82f6', '#10b981', '#8b5cf6'][idx] }"></div>
|
|
||||||
<span class="text-xs font-bold text-gray-700 dark:text-gray-300">{{ item.code }}</span>
|
|
||||||
</div>
|
|
||||||
<span class="text-xs font-black text-gray-900 dark:text-white">{{ item.percentage }}%</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- System Resources -->
|
|
||||||
<div class="rounded-2xl bg-white p-6 shadow-sm ring-1 ring-gray-900/5 dark:bg-dark-800 dark:ring-dark-700">
|
|
||||||
<div class="mb-6 flex items-center justify-between">
|
|
||||||
<h3 class="text-sm font-black text-gray-900 dark:text-white uppercase tracking-wider">系统运行状态</h3>
|
|
||||||
</div>
|
|
||||||
<div class="grid grid-cols-2 gap-6">
|
|
||||||
<div class="space-y-4">
|
|
||||||
<div>
|
|
||||||
<div class="mb-1 flex justify-between text-[10px] font-bold text-gray-400 uppercase">CPU 使用率</div>
|
|
||||||
<div class="h-2 w-full rounded-full bg-gray-100 dark:bg-dark-700">
|
|
||||||
<div class="h-full rounded-full bg-purple-500" :style="{ width: `${overview?.resources.cpu_usage}%` }"></div>
|
|
||||||
</div>
|
|
||||||
<div class="mt-1 text-right text-xs font-bold text-gray-900 dark:text-white">{{ overview?.resources.cpu_usage }}%</div>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<div class="mb-1 flex justify-between text-[10px] font-bold text-gray-400 uppercase">内存使用率</div>
|
|
||||||
<div class="h-2 w-full rounded-full bg-gray-100 dark:bg-dark-700">
|
|
||||||
<div class="h-full rounded-full bg-indigo-500" :style="{ width: `${overview?.resources.memory_usage}%` }"></div>
|
|
||||||
</div>
|
|
||||||
<div class="mt-1 text-right text-xs font-bold text-gray-900 dark:text-white">{{ overview?.resources.memory_usage }}%</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="flex flex-col justify-center space-y-4 rounded-xl bg-gray-50 p-4 dark:bg-dark-900">
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-[10px] font-bold text-gray-400 uppercase">Redis 状态</span>
|
|
||||||
<span class="text-xs font-bold text-green-500 uppercase">{{ overview?.system_status.redis }}</span>
|
|
||||||
</div>
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-[10px] font-bold text-gray-400 uppercase">DB 连接</span>
|
|
||||||
<span class="text-xs font-bold text-gray-900 dark:text-white">{{ overview?.resources.db_connections.active }} / {{ overview?.resources.db_connections.max }}</span>
|
|
||||||
</div>
|
|
||||||
<div class="flex items-center justify-between">
|
|
||||||
<span class="text-[10px] font-bold text-gray-400 uppercase">Goroutines</span>
|
|
||||||
<span class="text-xs font-bold text-gray-900 dark:text-white">{{ overview?.resources.goroutines }}</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</AppLayout>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
/* Custom select styling */
|
|
||||||
select {
|
|
||||||
appearance: none;
|
|
||||||
background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%236b7280' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e");
|
|
||||||
background-repeat: no-repeat;
|
|
||||||
background-position: right 0.5rem center;
|
|
||||||
background-size: 1.5em 1.5em;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
Reference in New Issue
Block a user