feat(idempotency): 为关键写接口接入幂等并完善并发容错

2026-02-23 12:45:37 +08:00
parent 3b6584cc8d
commit 5fa45f3b8c
40 changed files with 4383 additions and 223 deletions
--- a/backend/internal/handler/admin/account_data.go
+++ b/backend/internal/handler/admin/account_data.go
@@ -175,22 +175,28 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 		return
 	}

-	dataPayload := req.Data
-	if err := validateDataHeader(dataPayload); err != nil {
+	if err := validateDataHeader(req.Data); err != nil {
 		response.BadRequest(c, err.Error())
 		return
 	}

+	executeAdminIdempotentJSON(c, "admin.accounts.import_data", req, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		return h.importData(ctx, req)
+	})
+}
+
+func (h *AccountHandler) importData(ctx context.Context, req DataImportRequest) (DataImportResult, error) {
 	skipDefaultGroupBind := true
 	if req.SkipDefaultGroupBind != nil {
 		skipDefaultGroupBind = *req.SkipDefaultGroupBind
 	}

+	dataPayload := req.Data
 	result := DataImportResult{}
-	existingProxies, err := h.listAllProxies(c.Request.Context())
+
+	existingProxies, err := h.listAllProxies(ctx)
 	if err != nil {
-		response.ErrorFrom(c, err)
-		return
+		return result, err
 	}

 	proxyKeyToID := make(map[string]int64, len(existingProxies))
@@ -221,8 +227,8 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 			proxyKeyToID[key] = existingID
 			result.ProxyReused++
 			if normalizedStatus != "" {
-				if proxy, err := h.adminService.GetProxy(c.Request.Context(), existingID); err == nil && proxy != nil && proxy.Status != normalizedStatus {
-					_, _ = h.adminService.UpdateProxy(c.Request.Context(), existingID, &service.UpdateProxyInput{
+				if proxy, getErr := h.adminService.GetProxy(ctx, existingID); getErr == nil && proxy != nil && proxy.Status != normalizedStatus {
+					_, _ = h.adminService.UpdateProxy(ctx, existingID, &service.UpdateProxyInput{
 						Status: normalizedStatus,
 					})
 				}
@@ -230,7 +236,7 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 			continue
 		}

-		created, err := h.adminService.CreateProxy(c.Request.Context(), &service.CreateProxyInput{
+		created, createErr := h.adminService.CreateProxy(ctx, &service.CreateProxyInput{
 			Name:     defaultProxyName(item.Name),
 			Protocol: item.Protocol,
 			Host:     item.Host,
@@ -238,13 +244,13 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 			Username: item.Username,
 			Password: item.Password,
 		})
-		if err != nil {
+		if createErr != nil {
 			result.ProxyFailed++
 			result.Errors = append(result.Errors, DataImportError{
 				Kind:     "proxy",
 				Name:     item.Name,
 				ProxyKey: key,
-				Message:  err.Error(),
+				Message:  createErr.Error(),
 			})
 			continue
 		}
@@ -252,7 +258,7 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 		result.ProxyCreated++

 		if normalizedStatus != "" && normalizedStatus != created.Status {
-			_, _ = h.adminService.UpdateProxy(c.Request.Context(), created.ID, &service.UpdateProxyInput{
+			_, _ = h.adminService.UpdateProxy(ctx, created.ID, &service.UpdateProxyInput{
 				Status: normalizedStatus,
 			})
 		}
@@ -303,7 +309,7 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 			SkipDefaultGroupBind: skipDefaultGroupBind,
 		}

-		if _, err := h.adminService.CreateAccount(c.Request.Context(), accountInput); err != nil {
+		if _, err := h.adminService.CreateAccount(ctx, accountInput); err != nil {
 			result.AccountFailed++
 			result.Errors = append(result.Errors, DataImportError{
 				Kind:    "account",
@@ -315,7 +321,7 @@ func (h *AccountHandler) ImportData(c *gin.Context) {
 		result.AccountCreated++
 	}

-	response.Success(c, result)
+	return result, nil
 }

 func (h *AccountHandler) listAllProxies(ctx context.Context) ([]service.Proxy, error) {
--- a/backend/internal/handler/admin/account_handler.go
+++ b/backend/internal/handler/admin/account_handler.go
@@ -405,21 +405,27 @@ func (h *AccountHandler) Create(c *gin.Context) {
 	// 确定是否跳过混合渠道检查
 	skipCheck := req.ConfirmMixedChannelRisk != nil && *req.ConfirmMixedChannelRisk

-	account, err := h.adminService.CreateAccount(c.Request.Context(), &service.CreateAccountInput{
-		Name:                  req.Name,
-		Notes:                 req.Notes,
-		Platform:              req.Platform,
-		Type:                  req.Type,
-		Credentials:           req.Credentials,
-		Extra:                 req.Extra,
-		ProxyID:               req.ProxyID,
-		Concurrency:           req.Concurrency,
-		Priority:              req.Priority,
-		RateMultiplier:        req.RateMultiplier,
-		GroupIDs:              req.GroupIDs,
-		ExpiresAt:             req.ExpiresAt,
-		AutoPauseOnExpired:    req.AutoPauseOnExpired,
-		SkipMixedChannelCheck: skipCheck,
+	result, err := executeAdminIdempotent(c, "admin.accounts.create", req, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		account, execErr := h.adminService.CreateAccount(ctx, &service.CreateAccountInput{
+			Name:                  req.Name,
+			Notes:                 req.Notes,
+			Platform:              req.Platform,
+			Type:                  req.Type,
+			Credentials:           req.Credentials,
+			Extra:                 req.Extra,
+			ProxyID:               req.ProxyID,
+			Concurrency:           req.Concurrency,
+			Priority:              req.Priority,
+			RateMultiplier:        req.RateMultiplier,
+			GroupIDs:              req.GroupIDs,
+			ExpiresAt:             req.ExpiresAt,
+			AutoPauseOnExpired:    req.AutoPauseOnExpired,
+			SkipMixedChannelCheck: skipCheck,
+		})
+		if execErr != nil {
+			return nil, execErr
+		}
+		return h.buildAccountResponseWithRuntime(ctx, account), nil
 	})
 	if err != nil {
 		// 检查是否为混合渠道错误
@@ -440,11 +446,17 @@ func (h *AccountHandler) Create(c *gin.Context) {
 			return
 		}

+		if retryAfter := service.RetryAfterSecondsFromError(err); retryAfter > 0 {
+			c.Header("Retry-After", strconv.Itoa(retryAfter))
+		}
 		response.ErrorFrom(c, err)
 		return
 	}

-	response.Success(c, h.buildAccountResponseWithRuntime(c.Request.Context(), account))
+	if result != nil && result.Replayed {
+		c.Header("X-Idempotency-Replayed", "true")
+	}
+	response.Success(c, result.Data)
 }

 // Update handles updating an account
@@ -838,61 +850,62 @@ func (h *AccountHandler) BatchCreate(c *gin.Context) {
 		return
 	}

-	ctx := c.Request.Context()
-	success := 0
-	failed := 0
-	results := make([]gin.H, 0, len(req.Accounts))
+	executeAdminIdempotentJSON(c, "admin.accounts.batch_create", req, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		success := 0
+		failed := 0
+		results := make([]gin.H, 0, len(req.Accounts))

-	for _, item := range req.Accounts {
-		if item.RateMultiplier != nil && *item.RateMultiplier < 0 {
-			failed++
+		for _, item := range req.Accounts {
+			if item.RateMultiplier != nil && *item.RateMultiplier < 0 {
+				failed++
+				results = append(results, gin.H{
+					"name":    item.Name,
+					"success": false,
+					"error":   "rate_multiplier must be >= 0",
+				})
+				continue
+			}
+
+			skipCheck := item.ConfirmMixedChannelRisk != nil && *item.ConfirmMixedChannelRisk
+
+			account, err := h.adminService.CreateAccount(ctx, &service.CreateAccountInput{
+				Name:                  item.Name,
+				Notes:                 item.Notes,
+				Platform:              item.Platform,
+				Type:                  item.Type,
+				Credentials:           item.Credentials,
+				Extra:                 item.Extra,
+				ProxyID:               item.ProxyID,
+				Concurrency:           item.Concurrency,
+				Priority:              item.Priority,
+				RateMultiplier:        item.RateMultiplier,
+				GroupIDs:              item.GroupIDs,
+				ExpiresAt:             item.ExpiresAt,
+				AutoPauseOnExpired:    item.AutoPauseOnExpired,
+				SkipMixedChannelCheck: skipCheck,
+			})
+			if err != nil {
+				failed++
+				results = append(results, gin.H{
+					"name":    item.Name,
+					"success": false,
+					"error":   err.Error(),
+				})
+				continue
+			}
+			success++
 			results = append(results, gin.H{
 				"name":    item.Name,
-				"success": false,
-				"error":   "rate_multiplier must be >= 0",
+				"id":      account.ID,
+				"success": true,
 			})
-			continue
 		}

-		skipCheck := item.ConfirmMixedChannelRisk != nil && *item.ConfirmMixedChannelRisk
-
-		account, err := h.adminService.CreateAccount(ctx, &service.CreateAccountInput{
-			Name:                  item.Name,
-			Notes:                 item.Notes,
-			Platform:              item.Platform,
-			Type:                  item.Type,
-			Credentials:           item.Credentials,
-			Extra:                 item.Extra,
-			ProxyID:               item.ProxyID,
-			Concurrency:           item.Concurrency,
-			Priority:              item.Priority,
-			RateMultiplier:        item.RateMultiplier,
-			GroupIDs:              item.GroupIDs,
-			ExpiresAt:             item.ExpiresAt,
-			AutoPauseOnExpired:    item.AutoPauseOnExpired,
-			SkipMixedChannelCheck: skipCheck,
-		})
-		if err != nil {
-			failed++
-			results = append(results, gin.H{
-				"name":    item.Name,
-				"success": false,
-				"error":   err.Error(),
-			})
-			continue
-		}
-		success++
-		results = append(results, gin.H{
-			"name":    item.Name,
-			"id":      account.ID,
-			"success": true,
-		})
-	}
-
-	response.Success(c, gin.H{
-		"success": success,
-		"failed":  failed,
-		"results": results,
+		return gin.H{
+			"success": success,
+			"failed":  failed,
+			"results": results,
+		}, nil
 	})
 }

--- a/backend/internal/handler/admin/idempotency_helper.go
+++ b/backend/internal/handler/admin/idempotency_helper.go
@@ -0,0 +1,115 @@
+package admin
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
+	"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
+	"github.com/Wei-Shaw/sub2api/internal/pkg/response"
+	middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
+	"github.com/Wei-Shaw/sub2api/internal/service"
+
+	"github.com/gin-gonic/gin"
+)
+
+type idempotencyStoreUnavailableMode int
+
+const (
+	idempotencyStoreUnavailableFailClose idempotencyStoreUnavailableMode = iota
+	idempotencyStoreUnavailableFailOpen
+)
+
+func executeAdminIdempotent(
+	c *gin.Context,
+	scope string,
+	payload any,
+	ttl time.Duration,
+	execute func(context.Context) (any, error),
+) (*service.IdempotencyExecuteResult, error) {
+	coordinator := service.DefaultIdempotencyCoordinator()
+	if coordinator == nil {
+		data, err := execute(c.Request.Context())
+		if err != nil {
+			return nil, err
+		}
+		return &service.IdempotencyExecuteResult{Data: data}, nil
+	}
+
+	actorScope := "admin:0"
+	if subject, ok := middleware2.GetAuthSubjectFromContext(c); ok {
+		actorScope = "admin:" + strconv.FormatInt(subject.UserID, 10)
+	}
+
+	return coordinator.Execute(c.Request.Context(), service.IdempotencyExecuteOptions{
+		Scope:          scope,
+		ActorScope:     actorScope,
+		Method:         c.Request.Method,
+		Route:          c.FullPath(),
+		IdempotencyKey: c.GetHeader("Idempotency-Key"),
+		Payload:        payload,
+		RequireKey:     true,
+		TTL:            ttl,
+	}, execute)
+}
+
+func executeAdminIdempotentJSON(
+	c *gin.Context,
+	scope string,
+	payload any,
+	ttl time.Duration,
+	execute func(context.Context) (any, error),
+) {
+	executeAdminIdempotentJSONWithMode(c, scope, payload, ttl, idempotencyStoreUnavailableFailClose, execute)
+}
+
+func executeAdminIdempotentJSONFailOpenOnStoreUnavailable(
+	c *gin.Context,
+	scope string,
+	payload any,
+	ttl time.Duration,
+	execute func(context.Context) (any, error),
+) {
+	executeAdminIdempotentJSONWithMode(c, scope, payload, ttl, idempotencyStoreUnavailableFailOpen, execute)
+}
+
+func executeAdminIdempotentJSONWithMode(
+	c *gin.Context,
+	scope string,
+	payload any,
+	ttl time.Duration,
+	mode idempotencyStoreUnavailableMode,
+	execute func(context.Context) (any, error),
+) {
+	result, err := executeAdminIdempotent(c, scope, payload, ttl, execute)
+	if err != nil {
+		if infraerrors.Code(err) == infraerrors.Code(service.ErrIdempotencyStoreUnavail) {
+			strategy := "fail_close"
+			if mode == idempotencyStoreUnavailableFailOpen {
+				strategy = "fail_open"
+			}
+			service.RecordIdempotencyStoreUnavailable(c.FullPath(), scope, "handler_"+strategy)
+			logger.LegacyPrintf("handler.idempotency", "[Idempotency] store unavailable: method=%s route=%s scope=%s strategy=%s", c.Request.Method, c.FullPath(), scope, strategy)
+			if mode == idempotencyStoreUnavailableFailOpen {
+				data, fallbackErr := execute(c.Request.Context())
+				if fallbackErr != nil {
+					response.ErrorFrom(c, fallbackErr)
+					return
+				}
+				c.Header("X-Idempotency-Degraded", "store-unavailable")
+				response.Success(c, data)
+				return
+			}
+		}
+		if retryAfter := service.RetryAfterSecondsFromError(err); retryAfter > 0 {
+			c.Header("Retry-After", strconv.Itoa(retryAfter))
+		}
+		response.ErrorFrom(c, err)
+		return
+	}
+	if result != nil && result.Replayed {
+		c.Header("X-Idempotency-Replayed", "true")
+	}
+	response.Success(c, result.Data)
+}
--- a/backend/internal/handler/admin/idempotency_helper_test.go
+++ b/backend/internal/handler/admin/idempotency_helper_test.go
@@ -0,0 +1,285 @@
+package admin
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/Wei-Shaw/sub2api/internal/service"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+)
+
+type storeUnavailableRepoStub struct{}
+
+func (storeUnavailableRepoStub) CreateProcessing(context.Context, *service.IdempotencyRecord) (bool, error) {
+	return false, errors.New("store unavailable")
+}
+func (storeUnavailableRepoStub) GetByScopeAndKeyHash(context.Context, string, string) (*service.IdempotencyRecord, error) {
+	return nil, errors.New("store unavailable")
+}
+func (storeUnavailableRepoStub) TryReclaim(context.Context, int64, string, time.Time, time.Time, time.Time) (bool, error) {
+	return false, errors.New("store unavailable")
+}
+func (storeUnavailableRepoStub) ExtendProcessingLock(context.Context, int64, string, time.Time, time.Time) (bool, error) {
+	return false, errors.New("store unavailable")
+}
+func (storeUnavailableRepoStub) MarkSucceeded(context.Context, int64, int, string, time.Time) error {
+	return errors.New("store unavailable")
+}
+func (storeUnavailableRepoStub) MarkFailedRetryable(context.Context, int64, string, time.Time, time.Time) error {
+	return errors.New("store unavailable")
+}
+func (storeUnavailableRepoStub) DeleteExpired(context.Context, time.Time, int) (int64, error) {
+	return 0, errors.New("store unavailable")
+}
+
+func TestExecuteAdminIdempotentJSONFailCloseOnStoreUnavailable(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(storeUnavailableRepoStub{}, service.DefaultIdempotencyConfig()))
+	t.Cleanup(func() {
+		service.SetDefaultIdempotencyCoordinator(nil)
+	})
+
+	var executed int
+	router := gin.New()
+	router.POST("/idempotent", func(c *gin.Context) {
+		executeAdminIdempotentJSON(c, "admin.test.high", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
+			executed++
+			return gin.H{"ok": true}, nil
+		})
+	})
+
+	req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Idempotency-Key", "test-key-1")
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusServiceUnavailable, rec.Code)
+	require.Equal(t, 0, executed, "fail-close should block business execution when idempotency store is unavailable")
+}
+
+func TestExecuteAdminIdempotentJSONFailOpenOnStoreUnavailable(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(storeUnavailableRepoStub{}, service.DefaultIdempotencyConfig()))
+	t.Cleanup(func() {
+		service.SetDefaultIdempotencyCoordinator(nil)
+	})
+
+	var executed int
+	router := gin.New()
+	router.POST("/idempotent", func(c *gin.Context) {
+		executeAdminIdempotentJSONFailOpenOnStoreUnavailable(c, "admin.test.medium", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
+			executed++
+			return gin.H{"ok": true}, nil
+		})
+	})
+
+	req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Idempotency-Key", "test-key-2")
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusOK, rec.Code)
+	require.Equal(t, "store-unavailable", rec.Header().Get("X-Idempotency-Degraded"))
+	require.Equal(t, 1, executed, "fail-open strategy should allow semantic idempotent path to continue")
+}
+
+type memoryIdempotencyRepoStub struct {
+	mu     sync.Mutex
+	nextID int64
+	data   map[string]*service.IdempotencyRecord
+}
+
+func newMemoryIdempotencyRepoStub() *memoryIdempotencyRepoStub {
+	return &memoryIdempotencyRepoStub{
+		nextID: 1,
+		data:   make(map[string]*service.IdempotencyRecord),
+	}
+}
+
+func (r *memoryIdempotencyRepoStub) key(scope, keyHash string) string {
+	return scope + "|" + keyHash
+}
+
+func (r *memoryIdempotencyRepoStub) clone(in *service.IdempotencyRecord) *service.IdempotencyRecord {
+	if in == nil {
+		return nil
+	}
+	out := *in
+	if in.LockedUntil != nil {
+		v := *in.LockedUntil
+		out.LockedUntil = &v
+	}
+	if in.ResponseBody != nil {
+		v := *in.ResponseBody
+		out.ResponseBody = &v
+	}
+	if in.ResponseStatus != nil {
+		v := *in.ResponseStatus
+		out.ResponseStatus = &v
+	}
+	if in.ErrorReason != nil {
+		v := *in.ErrorReason
+		out.ErrorReason = &v
+	}
+	return &out
+}
+
+func (r *memoryIdempotencyRepoStub) CreateProcessing(_ context.Context, record *service.IdempotencyRecord) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	k := r.key(record.Scope, record.IdempotencyKeyHash)
+	if _, ok := r.data[k]; ok {
+		return false, nil
+	}
+	cp := r.clone(record)
+	cp.ID = r.nextID
+	r.nextID++
+	r.data[k] = cp
+	record.ID = cp.ID
+	return true, nil
+}
+
+func (r *memoryIdempotencyRepoStub) GetByScopeAndKeyHash(_ context.Context, scope, keyHash string) (*service.IdempotencyRecord, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.clone(r.data[r.key(scope, keyHash)]), nil
+}
+
+func (r *memoryIdempotencyRepoStub) TryReclaim(_ context.Context, id int64, fromStatus string, now, newLockedUntil, newExpiresAt time.Time) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		if rec.Status != fromStatus {
+			return false, nil
+		}
+		if rec.LockedUntil != nil && rec.LockedUntil.After(now) {
+			return false, nil
+		}
+		rec.Status = service.IdempotencyStatusProcessing
+		rec.LockedUntil = &newLockedUntil
+		rec.ExpiresAt = newExpiresAt
+		rec.ErrorReason = nil
+		return true, nil
+	}
+	return false, nil
+}
+
+func (r *memoryIdempotencyRepoStub) ExtendProcessingLock(_ context.Context, id int64, requestFingerprint string, newLockedUntil, newExpiresAt time.Time) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		if rec.Status != service.IdempotencyStatusProcessing || rec.RequestFingerprint != requestFingerprint {
+			return false, nil
+		}
+		rec.LockedUntil = &newLockedUntil
+		rec.ExpiresAt = newExpiresAt
+		return true, nil
+	}
+	return false, nil
+}
+
+func (r *memoryIdempotencyRepoStub) MarkSucceeded(_ context.Context, id int64, responseStatus int, responseBody string, expiresAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		rec.Status = service.IdempotencyStatusSucceeded
+		rec.LockedUntil = nil
+		rec.ExpiresAt = expiresAt
+		rec.ResponseStatus = &responseStatus
+		rec.ResponseBody = &responseBody
+		rec.ErrorReason = nil
+		return nil
+	}
+	return nil
+}
+
+func (r *memoryIdempotencyRepoStub) MarkFailedRetryable(_ context.Context, id int64, errorReason string, lockedUntil, expiresAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		rec.Status = service.IdempotencyStatusFailedRetryable
+		rec.LockedUntil = &lockedUntil
+		rec.ExpiresAt = expiresAt
+		rec.ErrorReason = &errorReason
+		return nil
+	}
+	return nil
+}
+
+func (r *memoryIdempotencyRepoStub) DeleteExpired(_ context.Context, _ time.Time, _ int) (int64, error) {
+	return 0, nil
+}
+
+func TestExecuteAdminIdempotentJSONConcurrentRetryOnlyOneSideEffect(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	repo := newMemoryIdempotencyRepoStub()
+	cfg := service.DefaultIdempotencyConfig()
+	cfg.ProcessingTimeout = 2 * time.Second
+	service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(repo, cfg))
+	t.Cleanup(func() {
+		service.SetDefaultIdempotencyCoordinator(nil)
+	})
+
+	var executed atomic.Int32
+	router := gin.New()
+	router.POST("/idempotent", func(c *gin.Context) {
+		executeAdminIdempotentJSON(c, "admin.test.concurrent", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
+			executed.Add(1)
+			time.Sleep(120 * time.Millisecond)
+			return gin.H{"ok": true}, nil
+		})
+	})
+
+	call := func() (int, http.Header) {
+		req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Idempotency-Key", "same-key")
+		rec := httptest.NewRecorder()
+		router.ServeHTTP(rec, req)
+		return rec.Code, rec.Header()
+	}
+
+	var status1, status2 int
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		status1, _ = call()
+	}()
+	go func() {
+		defer wg.Done()
+		status2, _ = call()
+	}()
+	wg.Wait()
+
+	require.Contains(t, []int{http.StatusOK, http.StatusConflict}, status1)
+	require.Contains(t, []int{http.StatusOK, http.StatusConflict}, status2)
+	require.Equal(t, int32(1), executed.Load(), "same idempotency key should execute side-effect only once")
+
+	status3, headers3 := call()
+	require.Equal(t, http.StatusOK, status3)
+	require.Equal(t, "true", headers3.Get("X-Idempotency-Replayed"))
+	require.Equal(t, int32(1), executed.Load())
+}
--- a/backend/internal/handler/admin/proxy_handler.go
+++ b/backend/internal/handler/admin/proxy_handler.go
@@ -1,6 +1,7 @@
 package admin

 import (
+	"context"
 	"strconv"
 	"strings"

@@ -130,20 +131,20 @@ func (h *ProxyHandler) Create(c *gin.Context) {
 		return
 	}

-	proxy, err := h.adminService.CreateProxy(c.Request.Context(), &service.CreateProxyInput{
-		Name:     strings.TrimSpace(req.Name),
-		Protocol: strings.TrimSpace(req.Protocol),
-		Host:     strings.TrimSpace(req.Host),
-		Port:     req.Port,
-		Username: strings.TrimSpace(req.Username),
-		Password: strings.TrimSpace(req.Password),
+	executeAdminIdempotentJSON(c, "admin.proxies.create", req, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		proxy, err := h.adminService.CreateProxy(ctx, &service.CreateProxyInput{
+			Name:     strings.TrimSpace(req.Name),
+			Protocol: strings.TrimSpace(req.Protocol),
+			Host:     strings.TrimSpace(req.Host),
+			Port:     req.Port,
+			Username: strings.TrimSpace(req.Username),
+			Password: strings.TrimSpace(req.Password),
+		})
+		if err != nil {
+			return nil, err
+		}
+		return dto.ProxyFromService(proxy), nil
 	})
-	if err != nil {
-		response.ErrorFrom(c, err)
-		return
-	}
-
-	response.Success(c, dto.ProxyFromService(proxy))
 }

 // Update handles updating a proxy
--- a/backend/internal/handler/admin/redeem_handler.go
+++ b/backend/internal/handler/admin/redeem_handler.go
@@ -2,6 +2,7 @@ package admin

 import (
 	"bytes"
+	"context"
 	"encoding/csv"
 	"fmt"
 	"strconv"
@@ -88,23 +89,24 @@ func (h *RedeemHandler) Generate(c *gin.Context) {
 		return
 	}

-	codes, err := h.adminService.GenerateRedeemCodes(c.Request.Context(), &service.GenerateRedeemCodesInput{
-		Count:        req.Count,
-		Type:         req.Type,
-		Value:        req.Value,
-		GroupID:      req.GroupID,
-		ValidityDays: req.ValidityDays,
-	})
-	if err != nil {
-		response.ErrorFrom(c, err)
-		return
-	}
+	executeAdminIdempotentJSON(c, "admin.redeem_codes.generate", req, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		codes, execErr := h.adminService.GenerateRedeemCodes(ctx, &service.GenerateRedeemCodesInput{
+			Count:        req.Count,
+			Type:         req.Type,
+			Value:        req.Value,
+			GroupID:      req.GroupID,
+			ValidityDays: req.ValidityDays,
+		})
+		if execErr != nil {
+			return nil, execErr
+		}

-	out := make([]dto.AdminRedeemCode, 0, len(codes))
-	for i := range codes {
-		out = append(out, *dto.RedeemCodeFromServiceAdmin(&codes[i]))
-	}
-	response.Success(c, out)
+		out := make([]dto.AdminRedeemCode, 0, len(codes))
+		for i := range codes {
+			out = append(out, *dto.RedeemCodeFromServiceAdmin(&codes[i]))
+		}
+		return out, nil
+	})
 }

 // Delete handles deleting a redeem code
--- a/backend/internal/handler/admin/subscription_handler.go
+++ b/backend/internal/handler/admin/subscription_handler.go
@@ -1,6 +1,7 @@
 package admin

 import (
+	"context"
 	"strconv"

 	"github.com/Wei-Shaw/sub2api/internal/handler/dto"
@@ -199,13 +200,20 @@ func (h *SubscriptionHandler) Extend(c *gin.Context) {
 		return
 	}

-	subscription, err := h.subscriptionService.ExtendSubscription(c.Request.Context(), subscriptionID, req.Days)
-	if err != nil {
-		response.ErrorFrom(c, err)
-		return
+	idempotencyPayload := struct {
+		SubscriptionID int64                     `json:"subscription_id"`
+		Body           AdjustSubscriptionRequest `json:"body"`
+	}{
+		SubscriptionID: subscriptionID,
+		Body:           req,
 	}
-
-	response.Success(c, dto.UserSubscriptionFromServiceAdmin(subscription))
+	executeAdminIdempotentJSON(c, "admin.subscriptions.extend", idempotencyPayload, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		subscription, execErr := h.subscriptionService.ExtendSubscription(ctx, subscriptionID, req.Days)
+		if execErr != nil {
+			return nil, execErr
+		}
+		return dto.UserSubscriptionFromServiceAdmin(subscription), nil
+	})
 }

 // Revoke handles revoking a subscription
--- a/backend/internal/handler/admin/system_handler.go
+++ b/backend/internal/handler/admin/system_handler.go
@@ -1,11 +1,15 @@
 package admin

 import (
+	"context"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"

 	"github.com/Wei-Shaw/sub2api/internal/pkg/response"
 	"github.com/Wei-Shaw/sub2api/internal/pkg/sysutil"
+	middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
 	"github.com/Wei-Shaw/sub2api/internal/service"

 	"github.com/gin-gonic/gin"
@@ -14,12 +18,14 @@ import (
 // SystemHandler handles system-related operations
 type SystemHandler struct {
 	updateSvc *service.UpdateService
+	lockSvc   *service.SystemOperationLockService
 }

 // NewSystemHandler creates a new SystemHandler
-func NewSystemHandler(updateSvc *service.UpdateService) *SystemHandler {
+func NewSystemHandler(updateSvc *service.UpdateService, lockSvc *service.SystemOperationLockService) *SystemHandler {
 	return &SystemHandler{
 		updateSvc: updateSvc,
+		lockSvc:   lockSvc,
 	}
 }

@@ -47,41 +53,125 @@ func (h *SystemHandler) CheckUpdates(c *gin.Context) {
 // PerformUpdate downloads and applies the update
 // POST /api/v1/admin/system/update
 func (h *SystemHandler) PerformUpdate(c *gin.Context) {
-	if err := h.updateSvc.PerformUpdate(c.Request.Context()); err != nil {
-		response.Error(c, http.StatusInternalServerError, err.Error())
-		return
-	}
-	response.Success(c, gin.H{
-		"message":      "Update completed. Please restart the service.",
-		"need_restart": true,
+	operationID := buildSystemOperationID(c, "update")
+	payload := gin.H{"operation_id": operationID}
+	executeAdminIdempotentJSON(c, "admin.system.update", payload, service.DefaultSystemOperationIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		lock, release, err := h.acquireSystemLock(ctx, operationID)
+		if err != nil {
+			return nil, err
+		}
+		var releaseReason string
+		succeeded := false
+		defer func() {
+			release(releaseReason, succeeded)
+		}()
+
+		if err := h.updateSvc.PerformUpdate(ctx); err != nil {
+			releaseReason = "SYSTEM_UPDATE_FAILED"
+			return nil, err
+		}
+		succeeded = true
+
+		return gin.H{
+			"message":      "Update completed. Please restart the service.",
+			"need_restart": true,
+			"operation_id": lock.OperationID(),
+		}, nil
 	})
 }

 // Rollback restores the previous version
 // POST /api/v1/admin/system/rollback
 func (h *SystemHandler) Rollback(c *gin.Context) {
-	if err := h.updateSvc.Rollback(); err != nil {
-		response.Error(c, http.StatusInternalServerError, err.Error())
-		return
-	}
-	response.Success(c, gin.H{
-		"message":      "Rollback completed. Please restart the service.",
-		"need_restart": true,
+	operationID := buildSystemOperationID(c, "rollback")
+	payload := gin.H{"operation_id": operationID}
+	executeAdminIdempotentJSON(c, "admin.system.rollback", payload, service.DefaultSystemOperationIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		lock, release, err := h.acquireSystemLock(ctx, operationID)
+		if err != nil {
+			return nil, err
+		}
+		var releaseReason string
+		succeeded := false
+		defer func() {
+			release(releaseReason, succeeded)
+		}()
+
+		if err := h.updateSvc.Rollback(); err != nil {
+			releaseReason = "SYSTEM_ROLLBACK_FAILED"
+			return nil, err
+		}
+		succeeded = true
+
+		return gin.H{
+			"message":      "Rollback completed. Please restart the service.",
+			"need_restart": true,
+			"operation_id": lock.OperationID(),
+		}, nil
 	})
 }

 // RestartService restarts the systemd service
 // POST /api/v1/admin/system/restart
 func (h *SystemHandler) RestartService(c *gin.Context) {
-	// Schedule service restart in background after sending response
-	// This ensures the client receives the success response before the service restarts
-	go func() {
-		// Wait a moment to ensure the response is sent
-		time.Sleep(500 * time.Millisecond)
-		sysutil.RestartServiceAsync()
-	}()
+	operationID := buildSystemOperationID(c, "restart")
+	payload := gin.H{"operation_id": operationID}
+	executeAdminIdempotentJSON(c, "admin.system.restart", payload, service.DefaultSystemOperationIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		lock, release, err := h.acquireSystemLock(ctx, operationID)
+		if err != nil {
+			return nil, err
+		}
+		succeeded := false
+		defer func() {
+			release("", succeeded)
+		}()

-	response.Success(c, gin.H{
-		"message": "Service restart initiated",
+		// Schedule service restart in background after sending response
+		// This ensures the client receives the success response before the service restarts
+		go func() {
+			// Wait a moment to ensure the response is sent
+			time.Sleep(500 * time.Millisecond)
+			sysutil.RestartServiceAsync()
+		}()
+		succeeded = true
+		return gin.H{
+			"message":      "Service restart initiated",
+			"operation_id": lock.OperationID(),
+		}, nil
 	})
 }
+
+func (h *SystemHandler) acquireSystemLock(
+	ctx context.Context,
+	operationID string,
+) (*service.SystemOperationLock, func(string, bool), error) {
+	if h.lockSvc == nil {
+		return nil, nil, service.ErrIdempotencyStoreUnavail
+	}
+	lock, err := h.lockSvc.Acquire(ctx, operationID)
+	if err != nil {
+		return nil, nil, err
+	}
+	release := func(reason string, succeeded bool) {
+		releaseCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		_ = h.lockSvc.Release(releaseCtx, lock, succeeded, reason)
+	}
+	return lock, release, nil
+}
+
+func buildSystemOperationID(c *gin.Context, operation string) string {
+	key := strings.TrimSpace(c.GetHeader("Idempotency-Key"))
+	if key == "" {
+		return "sysop-" + operation + "-" + strconv.FormatInt(time.Now().UnixNano(), 36)
+	}
+	actorScope := "admin:0"
+	if subject, ok := middleware2.GetAuthSubjectFromContext(c); ok {
+		actorScope = "admin:" + strconv.FormatInt(subject.UserID, 10)
+	}
+	seed := operation + "|" + actorScope + "|" + c.FullPath() + "|" + key
+	hash := service.HashIdempotencyKey(seed)
+	if len(hash) > 24 {
+		hash = hash[:24]
+	}
+	return "sysop-" + hash
+}
--- a/backend/internal/handler/admin/usage_handler.go
+++ b/backend/internal/handler/admin/usage_handler.go
@@ -1,6 +1,7 @@
 package admin

 import (
+	"context"
 	"net/http"
 	"strconv"
 	"strings"
@@ -472,29 +473,36 @@ func (h *UsageHandler) CreateCleanupTask(c *gin.Context) {
 		billingType = *filters.BillingType
 	}

-	logger.LegacyPrintf("handler.admin.usage", "[UsageCleanup] 请求创建清理任务: operator=%d start=%s end=%s user_id=%v api_key_id=%v account_id=%v group_id=%v model=%v stream=%v billing_type=%v tz=%q",
-		subject.UserID,
-		filters.StartTime.Format(time.RFC3339),
-		filters.EndTime.Format(time.RFC3339),
-		userID,
-		apiKeyID,
-		accountID,
-		groupID,
-		model,
-		stream,
-		billingType,
-		req.Timezone,
-	)
-
-	task, err := h.cleanupService.CreateTask(c.Request.Context(), filters, subject.UserID)
-	if err != nil {
-		logger.LegacyPrintf("handler.admin.usage", "[UsageCleanup] 创建清理任务失败: operator=%d err=%v", subject.UserID, err)
-		response.ErrorFrom(c, err)
-		return
+	idempotencyPayload := struct {
+		OperatorID int64                         `json:"operator_id"`
+		Body       CreateUsageCleanupTaskRequest `json:"body"`
+	}{
+		OperatorID: subject.UserID,
+		Body:       req,
 	}
+	executeAdminIdempotentJSON(c, "admin.usage.cleanup_tasks.create", idempotencyPayload, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		logger.LegacyPrintf("handler.admin.usage", "[UsageCleanup] 请求创建清理任务: operator=%d start=%s end=%s user_id=%v api_key_id=%v account_id=%v group_id=%v model=%v stream=%v billing_type=%v tz=%q",
+			subject.UserID,
+			filters.StartTime.Format(time.RFC3339),
+			filters.EndTime.Format(time.RFC3339),
+			userID,
+			apiKeyID,
+			accountID,
+			groupID,
+			model,
+			stream,
+			billingType,
+			req.Timezone,
+		)

-	logger.LegacyPrintf("handler.admin.usage", "[UsageCleanup] 清理任务已创建: task=%d operator=%d status=%s", task.ID, subject.UserID, task.Status)
-	response.Success(c, dto.UsageCleanupTaskFromService(task))
+		task, err := h.cleanupService.CreateTask(ctx, filters, subject.UserID)
+		if err != nil {
+			logger.LegacyPrintf("handler.admin.usage", "[UsageCleanup] 创建清理任务失败: operator=%d err=%v", subject.UserID, err)
+			return nil, err
+		}
+		logger.LegacyPrintf("handler.admin.usage", "[UsageCleanup] 清理任务已创建: task=%d operator=%d status=%s", task.ID, subject.UserID, task.Status)
+		return dto.UsageCleanupTaskFromService(task), nil
+	})
 }

 // CancelCleanupTask handles canceling a usage cleanup task
--- a/backend/internal/handler/admin/user_handler.go
+++ b/backend/internal/handler/admin/user_handler.go
@@ -1,6 +1,7 @@
 package admin

 import (
+	"context"
 	"strconv"
 	"strings"

@@ -257,13 +258,20 @@ func (h *UserHandler) UpdateBalance(c *gin.Context) {
 		return
 	}

-	user, err := h.adminService.UpdateUserBalance(c.Request.Context(), userID, req.Balance, req.Operation, req.Notes)
-	if err != nil {
-		response.ErrorFrom(c, err)
-		return
+	idempotencyPayload := struct {
+		UserID int64                `json:"user_id"`
+		Body   UpdateBalanceRequest `json:"body"`
+	}{
+		UserID: userID,
+		Body:   req,
 	}
-
-	response.Success(c, dto.UserFromServiceAdmin(user))
+	executeAdminIdempotentJSON(c, "admin.users.balance.update", idempotencyPayload, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		user, execErr := h.adminService.UpdateUserBalance(ctx, userID, req.Balance, req.Operation, req.Notes)
+		if execErr != nil {
+			return nil, execErr
+		}
+		return dto.UserFromServiceAdmin(user), nil
+	})
 }

 // GetUserAPIKeys handles getting user's API keys
--- a/backend/internal/handler/api_key_handler.go
+++ b/backend/internal/handler/api_key_handler.go
@@ -2,6 +2,7 @@
 package handler

 import (
+	"context"
 	"strconv"
 	"time"

@@ -130,13 +131,14 @@ func (h *APIKeyHandler) Create(c *gin.Context) {
 	if req.Quota != nil {
 		svcReq.Quota = *req.Quota
 	}
-	key, err := h.apiKeyService.Create(c.Request.Context(), subject.UserID, svcReq)
-	if err != nil {
-		response.ErrorFrom(c, err)
-		return
-	}

-	response.Success(c, dto.APIKeyFromService(key))
+	executeUserIdempotentJSON(c, "user.api_keys.create", req, service.DefaultWriteIdempotencyTTL(), func(ctx context.Context) (any, error) {
+		key, err := h.apiKeyService.Create(ctx, subject.UserID, svcReq)
+		if err != nil {
+			return nil, err
+		}
+		return dto.APIKeyFromService(key), nil
+	})
 }

 // Update handles updating an API key
--- a/backend/internal/handler/dto/mappers.go
+++ b/backend/internal/handler/dto/mappers.go
@@ -2,6 +2,7 @@
 package dto

 import (
+	"strconv"
 	"time"

 	"github.com/Wei-Shaw/sub2api/internal/service"
@@ -542,11 +543,18 @@ func BulkAssignResultFromService(r *service.BulkAssignResult) *BulkAssignResult
 	for i := range r.Subscriptions {
 		subs = append(subs, *UserSubscriptionFromServiceAdmin(&r.Subscriptions[i]))
 	}
+	statuses := make(map[string]string, len(r.Statuses))
+	for userID, status := range r.Statuses {
+		statuses[strconv.FormatInt(userID, 10)] = status
+	}
 	return &BulkAssignResult{
 		SuccessCount:  r.SuccessCount,
+		CreatedCount:  r.CreatedCount,
+		ReusedCount:   r.ReusedCount,
 		FailedCount:   r.FailedCount,
 		Subscriptions: subs,
 		Errors:        r.Errors,
+		Statuses:      statuses,
 	}
 }

--- a/backend/internal/handler/dto/types.go
+++ b/backend/internal/handler/dto/types.go
@@ -395,9 +395,12 @@ type AdminUserSubscription struct {

 type BulkAssignResult struct {
 	SuccessCount  int                     `json:"success_count"`
+	CreatedCount  int                     `json:"created_count"`
+	ReusedCount   int                     `json:"reused_count"`
 	FailedCount   int                     `json:"failed_count"`
 	Subscriptions []AdminUserSubscription `json:"subscriptions"`
 	Errors        []string                `json:"errors"`
+	Statuses      map[string]string       `json:"statuses,omitempty"`
 }

 // PromoCode 注册优惠码
--- a/backend/internal/handler/idempotency_helper.go
+++ b/backend/internal/handler/idempotency_helper.go
@@ -0,0 +1,65 @@
+package handler
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
+	"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
+	"github.com/Wei-Shaw/sub2api/internal/pkg/response"
+	middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
+	"github.com/Wei-Shaw/sub2api/internal/service"
+
+	"github.com/gin-gonic/gin"
+)
+
+func executeUserIdempotentJSON(
+	c *gin.Context,
+	scope string,
+	payload any,
+	ttl time.Duration,
+	execute func(context.Context) (any, error),
+) {
+	coordinator := service.DefaultIdempotencyCoordinator()
+	if coordinator == nil {
+		data, err := execute(c.Request.Context())
+		if err != nil {
+			response.ErrorFrom(c, err)
+			return
+		}
+		response.Success(c, data)
+		return
+	}
+
+	actorScope := "user:0"
+	if subject, ok := middleware2.GetAuthSubjectFromContext(c); ok {
+		actorScope = "user:" + strconv.FormatInt(subject.UserID, 10)
+	}
+
+	result, err := coordinator.Execute(c.Request.Context(), service.IdempotencyExecuteOptions{
+		Scope:          scope,
+		ActorScope:     actorScope,
+		Method:         c.Request.Method,
+		Route:          c.FullPath(),
+		IdempotencyKey: c.GetHeader("Idempotency-Key"),
+		Payload:        payload,
+		RequireKey:     true,
+		TTL:            ttl,
+	}, execute)
+	if err != nil {
+		if infraerrors.Code(err) == infraerrors.Code(service.ErrIdempotencyStoreUnavail) {
+			service.RecordIdempotencyStoreUnavailable(c.FullPath(), scope, "handler_fail_close")
+			logger.LegacyPrintf("handler.idempotency", "[Idempotency] store unavailable: method=%s route=%s scope=%s strategy=fail_close", c.Request.Method, c.FullPath(), scope)
+		}
+		if retryAfter := service.RetryAfterSecondsFromError(err); retryAfter > 0 {
+			c.Header("Retry-After", strconv.Itoa(retryAfter))
+		}
+		response.ErrorFrom(c, err)
+		return
+	}
+	if result != nil && result.Replayed {
+		c.Header("X-Idempotency-Replayed", "true")
+	}
+	response.Success(c, result.Data)
+}
--- a/backend/internal/handler/idempotency_helper_test.go
+++ b/backend/internal/handler/idempotency_helper_test.go
@@ -0,0 +1,285 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/Wei-Shaw/sub2api/internal/server/middleware"
+	"github.com/Wei-Shaw/sub2api/internal/service"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+)
+
+type userStoreUnavailableRepoStub struct{}
+
+func (userStoreUnavailableRepoStub) CreateProcessing(context.Context, *service.IdempotencyRecord) (bool, error) {
+	return false, errors.New("store unavailable")
+}
+func (userStoreUnavailableRepoStub) GetByScopeAndKeyHash(context.Context, string, string) (*service.IdempotencyRecord, error) {
+	return nil, errors.New("store unavailable")
+}
+func (userStoreUnavailableRepoStub) TryReclaim(context.Context, int64, string, time.Time, time.Time, time.Time) (bool, error) {
+	return false, errors.New("store unavailable")
+}
+func (userStoreUnavailableRepoStub) ExtendProcessingLock(context.Context, int64, string, time.Time, time.Time) (bool, error) {
+	return false, errors.New("store unavailable")
+}
+func (userStoreUnavailableRepoStub) MarkSucceeded(context.Context, int64, int, string, time.Time) error {
+	return errors.New("store unavailable")
+}
+func (userStoreUnavailableRepoStub) MarkFailedRetryable(context.Context, int64, string, time.Time, time.Time) error {
+	return errors.New("store unavailable")
+}
+func (userStoreUnavailableRepoStub) DeleteExpired(context.Context, time.Time, int) (int64, error) {
+	return 0, errors.New("store unavailable")
+}
+
+type userMemoryIdempotencyRepoStub struct {
+	mu     sync.Mutex
+	nextID int64
+	data   map[string]*service.IdempotencyRecord
+}
+
+func newUserMemoryIdempotencyRepoStub() *userMemoryIdempotencyRepoStub {
+	return &userMemoryIdempotencyRepoStub{
+		nextID: 1,
+		data:   make(map[string]*service.IdempotencyRecord),
+	}
+}
+
+func (r *userMemoryIdempotencyRepoStub) key(scope, keyHash string) string {
+	return scope + "|" + keyHash
+}
+
+func (r *userMemoryIdempotencyRepoStub) clone(in *service.IdempotencyRecord) *service.IdempotencyRecord {
+	if in == nil {
+		return nil
+	}
+	out := *in
+	if in.LockedUntil != nil {
+		v := *in.LockedUntil
+		out.LockedUntil = &v
+	}
+	if in.ResponseBody != nil {
+		v := *in.ResponseBody
+		out.ResponseBody = &v
+	}
+	if in.ResponseStatus != nil {
+		v := *in.ResponseStatus
+		out.ResponseStatus = &v
+	}
+	if in.ErrorReason != nil {
+		v := *in.ErrorReason
+		out.ErrorReason = &v
+	}
+	return &out
+}
+
+func (r *userMemoryIdempotencyRepoStub) CreateProcessing(_ context.Context, record *service.IdempotencyRecord) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	k := r.key(record.Scope, record.IdempotencyKeyHash)
+	if _, ok := r.data[k]; ok {
+		return false, nil
+	}
+	cp := r.clone(record)
+	cp.ID = r.nextID
+	r.nextID++
+	r.data[k] = cp
+	record.ID = cp.ID
+	return true, nil
+}
+
+func (r *userMemoryIdempotencyRepoStub) GetByScopeAndKeyHash(_ context.Context, scope, keyHash string) (*service.IdempotencyRecord, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.clone(r.data[r.key(scope, keyHash)]), nil
+}
+
+func (r *userMemoryIdempotencyRepoStub) TryReclaim(_ context.Context, id int64, fromStatus string, now, newLockedUntil, newExpiresAt time.Time) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		if rec.Status != fromStatus {
+			return false, nil
+		}
+		if rec.LockedUntil != nil && rec.LockedUntil.After(now) {
+			return false, nil
+		}
+		rec.Status = service.IdempotencyStatusProcessing
+		rec.LockedUntil = &newLockedUntil
+		rec.ExpiresAt = newExpiresAt
+		rec.ErrorReason = nil
+		return true, nil
+	}
+	return false, nil
+}
+
+func (r *userMemoryIdempotencyRepoStub) ExtendProcessingLock(_ context.Context, id int64, requestFingerprint string, newLockedUntil, newExpiresAt time.Time) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		if rec.Status != service.IdempotencyStatusProcessing || rec.RequestFingerprint != requestFingerprint {
+			return false, nil
+		}
+		rec.LockedUntil = &newLockedUntil
+		rec.ExpiresAt = newExpiresAt
+		return true, nil
+	}
+	return false, nil
+}
+
+func (r *userMemoryIdempotencyRepoStub) MarkSucceeded(_ context.Context, id int64, responseStatus int, responseBody string, expiresAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		rec.Status = service.IdempotencyStatusSucceeded
+		rec.LockedUntil = nil
+		rec.ExpiresAt = expiresAt
+		rec.ResponseStatus = &responseStatus
+		rec.ResponseBody = &responseBody
+		rec.ErrorReason = nil
+		return nil
+	}
+	return nil
+}
+
+func (r *userMemoryIdempotencyRepoStub) MarkFailedRetryable(_ context.Context, id int64, errorReason string, lockedUntil, expiresAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, rec := range r.data {
+		if rec.ID != id {
+			continue
+		}
+		rec.Status = service.IdempotencyStatusFailedRetryable
+		rec.LockedUntil = &lockedUntil
+		rec.ExpiresAt = expiresAt
+		rec.ErrorReason = &errorReason
+		return nil
+	}
+	return nil
+}
+
+func (r *userMemoryIdempotencyRepoStub) DeleteExpired(_ context.Context, _ time.Time, _ int) (int64, error) {
+	return 0, nil
+}
+
+func withUserSubject(userID int64) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: userID})
+		c.Next()
+	}
+}
+
+func TestExecuteUserIdempotentJSONFallbackWithoutCoordinator(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	service.SetDefaultIdempotencyCoordinator(nil)
+
+	var executed int
+	router := gin.New()
+	router.Use(withUserSubject(1))
+	router.POST("/idempotent", func(c *gin.Context) {
+		executeUserIdempotentJSON(c, "user.test.scope", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
+			executed++
+			return gin.H{"ok": true}, nil
+		})
+	})
+
+	req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
+	req.Header.Set("Content-Type", "application/json")
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusOK, rec.Code)
+	require.Equal(t, 1, executed)
+}
+
+func TestExecuteUserIdempotentJSONFailCloseOnStoreUnavailable(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(userStoreUnavailableRepoStub{}, service.DefaultIdempotencyConfig()))
+	t.Cleanup(func() {
+		service.SetDefaultIdempotencyCoordinator(nil)
+	})
+
+	var executed int
+	router := gin.New()
+	router.Use(withUserSubject(2))
+	router.POST("/idempotent", func(c *gin.Context) {
+		executeUserIdempotentJSON(c, "user.test.scope", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
+			executed++
+			return gin.H{"ok": true}, nil
+		})
+	})
+
+	req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Idempotency-Key", "k1")
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+
+	require.Equal(t, http.StatusServiceUnavailable, rec.Code)
+	require.Equal(t, 0, executed)
+}
+
+func TestExecuteUserIdempotentJSONConcurrentRetrySingleSideEffectAndReplay(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	repo := newUserMemoryIdempotencyRepoStub()
+	cfg := service.DefaultIdempotencyConfig()
+	cfg.ProcessingTimeout = 2 * time.Second
+	service.SetDefaultIdempotencyCoordinator(service.NewIdempotencyCoordinator(repo, cfg))
+	t.Cleanup(func() {
+		service.SetDefaultIdempotencyCoordinator(nil)
+	})
+
+	var executed atomic.Int32
+	router := gin.New()
+	router.Use(withUserSubject(3))
+	router.POST("/idempotent", func(c *gin.Context) {
+		executeUserIdempotentJSON(c, "user.test.scope", map[string]any{"a": 1}, time.Minute, func(ctx context.Context) (any, error) {
+			executed.Add(1)
+			time.Sleep(80 * time.Millisecond)
+			return gin.H{"ok": true}, nil
+		})
+	})
+
+	call := func() (int, http.Header) {
+		req := httptest.NewRequest(http.MethodPost, "/idempotent", bytes.NewBufferString(`{"a":1}`))
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Idempotency-Key", "same-user-key")
+		rec := httptest.NewRecorder()
+		router.ServeHTTP(rec, req)
+		return rec.Code, rec.Header()
+	}
+
+	var status1, status2 int
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() { defer wg.Done(); status1, _ = call() }()
+	go func() { defer wg.Done(); status2, _ = call() }()
+	wg.Wait()
+
+	require.Contains(t, []int{http.StatusOK, http.StatusConflict}, status1)
+	require.Contains(t, []int{http.StatusOK, http.StatusConflict}, status2)
+	require.Equal(t, int32(1), executed.Load())
+
+	status3, headers3 := call()
+	require.Equal(t, http.StatusOK, status3)
+	require.Equal(t, "true", headers3.Get("X-Idempotency-Replayed"))
+	require.Equal(t, int32(1), executed.Load())
+}
--- a/backend/internal/handler/wire.go
+++ b/backend/internal/handler/wire.go
@@ -53,8 +53,8 @@ func ProvideAdminHandlers(
 }

 // ProvideSystemHandler creates admin.SystemHandler with UpdateService
-func ProvideSystemHandler(updateService *service.UpdateService) *admin.SystemHandler {
-	return admin.NewSystemHandler(updateService)
+func ProvideSystemHandler(updateService *service.UpdateService, lockService *service.SystemOperationLockService) *admin.SystemHandler {
+	return admin.NewSystemHandler(updateService, lockService)
 }

 // ProvideSettingHandler creates SettingHandler with version from BuildInfo
@@ -77,6 +77,8 @@ func ProvideHandlers(
 	soraGatewayHandler *SoraGatewayHandler,
 	settingHandler *SettingHandler,
 	totpHandler *TotpHandler,
+	_ *service.IdempotencyCoordinator,
+	_ *service.IdempotencyCleanupService,
 ) *Handlers {
 	return &Handlers{
 		Auth:          authHandler,