fix moderation key handling and key UI

backend/internal/service/content_moderation.go

@@ -29,6 +29,9 @@ const (
 	ContentModerationModeObserve  = "observe"
 	ContentModerationModePreBlock = "pre_block"
 
+	contentModerationAPIKeysModeAppend  = "append"
+	contentModerationAPIKeysModeReplace = "replace"
+
 	ContentModerationActionAllow     = "allow"
 	ContentModerationActionBlock     = "block"
 	ContentModerationActionHashBlock = "hash_block"
@@ -61,9 +64,11 @@ const (
 	defaultContentModerationNonHitRetentionDays  = 3
 	maxContentModerationRetentionDays            = 3650
 	maxContentModerationNonHitRetentionDays      = 3
-	contentModerationKeyFailureFreezeThreshold   = 3
-	contentModerationKeyFreezeDuration           = time.Minute
-	maxContentModerationTestImages               = 4
+	contentModerationKeyRateLimitFreezeDuration  = time.Minute
+	contentModerationKeyAuthFreezeDuration       = 10 * time.Minute
+	contentModerationKeyHTTPErrorFreezeDuration  = 10 * time.Second
+	maxContentModerationInputImages              = 1
+	maxContentModerationTestImages               = maxContentModerationInputImages
 	maxContentModerationTestImageBytes           = 8 * 1024 * 1024
 	maxContentModerationTestImageDataURLBytes    = 12 * 1024 * 1024
 
@@ -215,6 +220,8 @@ type UpdateContentModerationConfigInput struct {
 	Model                *string   `json:"model"`
 	APIKey               *string   `json:"api_key"`
 	APIKeys              *[]string `json:"api_keys"`
+	APIKeysMode          string    `json:"api_keys_mode"`
+	DeleteAPIKeyHashes   *[]string `json:"delete_api_key_hashes"`
 	ClearAPIKey          bool      `json:"clear_api_key"`
 	TimeoutMS            *int      `json:"timeout_ms"`
 	SampleRate           *int      `json:"sample_rate"`
@@ -260,7 +267,7 @@ func (in *ContentModerationInput) Normalize() {
 		return
 	}
 	in.Text = trimRunes(normalizeContentModerationText(in.Text), maxModerationInputRunes)
-	in.Images = normalizeModerationImages(in.Images)
+	in.Images = limitContentModerationImages(normalizeModerationImages(in.Images))
 }
 
 func (in ContentModerationInput) IsEmpty() bool {
@@ -268,14 +275,15 @@ func (in ContentModerationInput) IsEmpty() bool {
 }
 
 func (in ContentModerationInput) ModerationInput() any {
-	if len(in.Images) == 0 {
+	images := limitContentModerationImages(in.Images)
+	if len(images) == 0 {
 		return in.Text
 	}
-	parts := make([]moderationAPIInputPart, 0, len(in.Images)+1)
+	parts := make([]moderationAPIInputPart, 0, len(images)+1)
 	if strings.TrimSpace(in.Text) != "" {
 		parts = append(parts, moderationAPIInputPart{Type: "text", Text: in.Text})
 	}
-	for _, image := range in.Images {
+	for _, image := range images {
 		parts = append(parts, moderationAPIInputPart{
 			Type:     "image_url",
 			ImageURL: &moderationAPIImageURLRef{URL: image},
@@ -565,8 +573,17 @@ func (s *ContentModerationService) UpdateConfig(ctx context.Context, input Updat
 		cfg.APIKey = ""
 		cfg.APIKeys = []string{}
 	} else {
+		apiKeysMode := normalizeContentModerationAPIKeysMode(input.APIKeysMode)
+		if input.DeleteAPIKeyHashes != nil && apiKeysMode != contentModerationAPIKeysModeReplace {
+			cfg.APIKeys = deleteModerationAPIKeysByHash(cfg.apiKeys(), *input.DeleteAPIKeyHashes)
+			cfg.APIKey = ""
+		}
 		if input.APIKeys != nil {
-			cfg.APIKeys = normalizeModerationAPIKeys(*input.APIKeys)
+			if apiKeysMode == contentModerationAPIKeysModeReplace {
+				cfg.APIKeys = normalizeModerationAPIKeys(*input.APIKeys)
+			} else {
+				cfg.APIKeys = normalizeModerationAPIKeys(append(cfg.apiKeys(), *input.APIKeys...))
+			}
 			cfg.APIKey = ""
 		}
 		if input.APIKey != nil && strings.TrimSpace(*input.APIKey) != "" {
@@ -636,7 +653,7 @@ func (s *ContentModerationService) TestAPIKeys(ctx context.Context, input TestCo
 		latency := int(time.Since(start).Milliseconds())
 		keyHash := moderationAPIKeyHash(key)
 		if err != nil {
-			s.markAPIKeyFailure(key, err.Error(), latency, httpStatus)
+			s.markAPIKeyError(key, err.Error(), latency, httpStatus)
 		} else {
 			s.markAPIKeySuccess(key, latency, httpStatus)
 			if auditResult == nil {
@@ -1227,8 +1244,11 @@ func (s *ContentModerationService) callModeration(ctx context.Context, cfg *Cont
 			s.markAPIKeySuccess(key, latency, httpStatus)
 			return result, nil
 		}
-		s.markAPIKeyFailure(key, err.Error(), latency, httpStatus)
+		s.markAPIKeyError(key, err.Error(), latency, httpStatus)
 		lastErr = err
+		if httpStatus == http.StatusBadRequest {
+			break
+		}
 		if attempt == attempts-1 {
 			break
 		}
@@ -1599,7 +1619,7 @@ func (s *ContentModerationService) markAPIKeySuccess(key string, latencyMS int,
 	state.LastTested = true
 }
 
-func (s *ContentModerationService) markAPIKeyFailure(key string, errText string, latencyMS int, httpStatus int) {
+func (s *ContentModerationService) markAPIKeyError(key string, errText string, latencyMS int, httpStatus int) {
 	hash := moderationAPIKeyHash(key)
 	if hash == "" || s == nil {
 		return
@@ -1607,14 +1627,29 @@ func (s *ContentModerationService) markAPIKeyFailure(key string, errText string,
 	s.keyHealthMu.Lock()
 	defer s.keyHealthMu.Unlock()
 	state := s.ensureAPIKeyHealthLocked(hash, maskSecretTail(key))
-	state.FailureCount++
+	if contentModerationFreezeDurationForHTTPStatus(httpStatus) > 0 {
+		state.FailureCount++
+	}
 	state.LastError = trimRunes(errText, 180)
 	state.LastCheckedAt = time.Now()
 	state.LastLatencyMS = latencyMS
 	state.LastHTTPStatus = httpStatus
 	state.LastTested = true
-	if state.FailureCount >= contentModerationKeyFailureFreezeThreshold {
-		state.FrozenUntil = time.Now().Add(contentModerationKeyFreezeDuration)
+	if freezeDuration := contentModerationFreezeDurationForHTTPStatus(httpStatus); freezeDuration > 0 {
+		state.FrozenUntil = time.Now().Add(freezeDuration)
+	}
+}
+
+func contentModerationFreezeDurationForHTTPStatus(httpStatus int) time.Duration {
+	switch httpStatus {
+	case 0, http.StatusBadRequest:
+		return 0
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return contentModerationKeyAuthFreezeDuration
+	case http.StatusTooManyRequests, 529:
+		return contentModerationKeyRateLimitFreezeDuration
+	default:
+		return contentModerationKeyHTTPErrorFreezeDuration
 	}
 }
 
@@ -1929,6 +1964,37 @@ func normalizeModerationAPIKeys(keys []string) []string {
 	return out
 }
 
+func deleteModerationAPIKeysByHash(keys []string, hashes []string) []string {
+	keys = normalizeModerationAPIKeys(keys)
+	deleteHashes := make(map[string]struct{}, len(hashes))
+	for _, hash := range hashes {
+		hash = normalizeContentModerationHash(hash)
+		if hash != "" {
+			deleteHashes[hash] = struct{}{}
+		}
+	}
+	if len(deleteHashes) == 0 {
+		return keys
+	}
+	out := make([]string, 0, len(keys))
+	for _, key := range keys {
+		if _, ok := deleteHashes[moderationAPIKeyHash(key)]; ok {
+			continue
+		}
+		out = append(out, key)
+	}
+	return out
+}
+
+func normalizeContentModerationAPIKeysMode(mode string) string {
+	switch strings.ToLower(strings.TrimSpace(mode)) {
+	case contentModerationAPIKeysModeReplace:
+		return contentModerationAPIKeysModeReplace
+	default:
+		return contentModerationAPIKeysModeAppend
+	}
+}
+
 func normalizeContentModerationHash(inputHash string) string {
 	inputHash = strings.ToLower(strings.TrimSpace(inputHash))
 	if len(inputHash) != sha256.Size*2 {