超时重试与补偿机制

// OkHttp 配置
OkHttpClient client = new OkHttpClient.Builder()
    .connectTimeout(2, TimeUnit.SECONDS)  // 连接超时 2 秒
    .readTimeout(5, TimeUnit.SECONDS)     // 读取超时 5 秒
    .writeTimeout(5, TimeUnit.SECONDS)    // 写入超时 5 秒
    .build();

// HttpClient 配置
RequestConfig config = RequestConfig.custom()
    .setConnectTimeout(2000)              // 连接超时 2 秒
    .setSocketTimeout(5000)               // Socket 超时 5 秒
    .setConnectionRequestTimeout(1000)    // 从连接池获取连接超时 1 秒
    .build();

// 数据库连接池配置(HikariCP)
HikariConfig config = new HikariConfig();
config.setConnectionTimeout(3000);        // 等待连接超时 3 秒
config.setMaximumPoolSize(20);            // 最大连接数
config.setMinimumIdle(5);                 // 最小空闲连接
config.setIdleTimeout(600000);            // 空闲连接超时 10 分钟
config.setMaxLifetime(1800000);           // 连接最大生命周期 30 分钟

// HTTP 连接池配置
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(100);                      // 最大连接数
cm.setDefaultMaxPerRoute(20);             // 每个 route 最大连接数

// 方式 1: Future.get() 超时
ExecutorService executor = Executors.newFixedThreadPool(10);
Future<String> future = executor.submit(() -> callRemoteService());
try {
    String result = future.get(5, TimeUnit.SECONDS);  // 超时 5 秒
    return result;
} catch (TimeoutException e) {
    future.cancel(true);  // 中断任务执行
    throw new ServiceTimeoutException("任务执行超时");
}

// 方式 2: Spring @Timeout 注解
@Timeout(value = 5, unit = TimeUnit.SECONDS)
public void asyncTask() {
    // 异步任务逻辑
}

// 方式 3: CompletableFuture 超时控制
CompletableFuture<String> future = CompletableFuture.supplyAsync(() -> {
    return callRemoteService();
});

// 超时后返回默认值
CompletableFuture<String> result = future
    .completeOnTimeout("default", 5, TimeUnit.SECONDS);

// 或者超时后执行补偿
CompletableFuture<String> result = future
    .orTimeout(5, TimeUnit.SECONDS)
    .exceptionally(ex -> {
        log.warn("任务超时", ex);
        return "fallback";
    });

超时时间 = min(系统承载极限 × 0.8, 业务容忍上限)

应用层超时(最外层)
    ↓
框架层超时(HTTP Client/数据库连接池)
    ↓
网络层超时(TCP KeepAlive)
    ↓
业务层超时(单次操作最大时间)

// 业务场景:调用第三方支付接口
// 业务容忍上限:10 秒
// 第三方 SLA:99% 请求在 5 秒内完成
// 建议配置:
connectTimeout = 2s     // 连接建立
readTimeout = 8s        // 等待响应(留 2s 余量给其他操作)
totalTimeout = 10s      // 整体超时保护

public class AdaptiveTimeout {
    private final SlidingWindow responseTimeWindow = new SlidingWindow(100);
    
    public int calculateTimeout() {
        // 基于最近 100 次请求的 P99 响应时间
        long p99 = responseTimeWindow.getP99();
        
        // 设置超时为 P99 的 1.5 倍,至少 1 秒
        int timeout = (int) Math.max(1000, p99 * 1.5);
        
        // 上限不超过业务容忍极限
        return Math.min(timeout, 10000);
    }
    
    public void recordResponseTime(long responseTime) {
        responseTimeWindow.add(responseTime);
    }
}

超时监控指标:
  - timeout_count: 超时次数
  - timeout_rate: 超时率(超时次数/总请求数)
  - avg_response_time: 平均响应时间
  - p99_response_time: P99 响应时间
  - active_thread_count: 活跃线程数
  
告警规则:
  - 超时率 > 1%: 警告
  - 超时率 > 5%: 严重告警
  - 连续超时次数 > 10: 触发熔断

@Component
public class TimeoutMonitor {
    private final MeterRegistry meterRegistry;
    
    public void recordTimeout(String serviceName, long timeout) {
        Timer.builder("service.timeout")
            .tag("service", serviceName)
            .register(meterRegistry)
            .record(timeout, TimeUnit.MILLISECONDS);
        
        Counter.builder("service.timeout.count")
            .tag("service", serviceName)
            .register(meterRegistry)
            .increment();
    }
    
    public void checkAndAlert(String serviceName) {
        double timeoutRate = getTimeoutRate(serviceName);
        
        if (timeoutRate > 0.05) {
            alertService.sendAlert(
                AlertLevel.CRITICAL,
                serviceName + " 超时率过高: " + (timeoutRate * 100) + "%"
            );
        } else if (timeoutRate > 0.01) {
            alertService.sendAlert(
                AlertLevel.WARNING,
                serviceName + " 超时率升高: " + (timeoutRate * 100) + "%"
            );
        }
    }
}

public class RetryConfig {
    // 读操作配置
    public static final int READ_RETRY_COUNT = 3;
    
    // 写操作配置
    public static final int WRITE_RETRY_COUNT = 2;
    
    // 关键业务配置
    public static final int CRITICAL_RETRY_COUNT = 5;
}

public class FixedRetryPolicy implements RetryPolicy {
    private final long intervalMillis;
    
    @Override
    public long getNextWaitTime(int attemptCount) {
        return intervalMillis;  // 每次固定等待
    }
}

// 示例:每次间隔 1 秒
FixedRetryPolicy policy = new FixedRetryPolicy(1000);

public class ExponentialBackoffPolicy implements RetryPolicy {
    private final long initialInterval;
    private final double multiplier;
    private final long maxInterval;
    
    @Override
    public long getNextWaitTime(int attemptCount) {
        long waitTime = (long) (initialInterval * Math.pow(multiplier, attemptCount - 1));
        return Math.min(waitTime, maxInterval);
    }
}

// 示例:初始 100ms,每次翻倍,最大 10s
ExponentialBackoffPolicy policy = new ExponentialBackoffPolicy(
    100,    // 初始间隔 100ms
    2.0,    // 乘数
    10000   // 最大间隔 10s
);
// 第1次重试:100ms
// 第2次重试:200ms
// 第3次重试:400ms
// 第4次重试:800ms

public class JitterRetryPolicy implements RetryPolicy {
    private final RetryPolicy basePolicy;
    private final double jitterFactor;  // 抖动因子 0-1
    
    @Override
    public long getNextWaitTime(int attemptCount) {
        long baseWait = basePolicy.getNextWaitTime(attemptCount);
        long jitter = (long) (baseWait * jitterFactor * Math.random());
        return baseWait + jitter;
    }
}

// 示例:在指数退避基础上增加 20% 随机抖动
RetryPolicy policy = new JitterRetryPolicy(
    new ExponentialBackoffPolicy(100, 2.0, 10000),
    0.2
);

public class SmartRetryTemplate {
    private final int maxRetries;
    private final RetryPolicy retryPolicy;
    private final Predicate<Exception> retryPredicate;
    
    public <T> T execute(Callable<T> task) {
        Exception lastException = null;
        
        for (int attempt = 1; attempt <= maxRetries; attempt++) {
            try {
                return task.call();
            } catch (Exception e) {
                lastException = e;
                
                // 判断是否应该重试
                if (!shouldRetry(e, attempt)) {
                    break;
                }
                
                // 等待下次重试
                if (attempt < maxRetries) {
                    long waitTime = retryPolicy.getNextWaitTime(attempt);
                    sleep(waitTime);
                }
            }
        }
        
        throw new RetryException("重试失败", lastException);
    }
    
    private boolean shouldRetry(Exception e, int attempt) {
        // 已达最大重试次数
        if (attempt >= maxRetries) {
            return false;
        }
        
        // 判断异常类型是否可重试
        return retryPredicate.test(e);
    }
    
    private void sleep(long millis) {
        try {
            Thread.sleep(millis);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
}

public class RetryableExceptionPredicate implements Predicate<Exception> {
    // 可重试的异常类型
    private static final Set<Class<? extends Exception>> RETRYABLE_EXCEPTIONS = Set.of(
        SocketTimeoutException.class,
        ConnectException.class,
        SocketException.class,
        UnknownHostException.class
    );
    
    // 可重试的 HTTP 状态码
    private static final Set<Integer> RETRYABLE_STATUS_CODES = Set.of(
        408,  // Request Timeout
        429,  // Too Many Requests
        500,  // Internal Server Error
        502,  // Bad Gateway
        503,  // Service Unavailable
        504   // Gateway Timeout
    );
    
    @Override
    public boolean test(Exception e) {
        // 检查异常类型
        if (RETRYABLE_EXCEPTIONS.contains(e.getClass())) {
            return true;
        }
        
        // 检查 HTTP 状态码
        if (e instanceof HttpClientException) {
            HttpClientException httpEx = (HttpClientException) e;
            return RETRYABLE_STATUS_CODES.contains(httpEx.getStatusCode());
        }
        
        // 检查是否为业务可重试异常
        if (e instanceof RetryableException) {
            return true;
        }
        
        return false;
    }
}

public class RetryContext {
    private final String requestId;
    private int attemptCount = 0;
    private long totalWaitTime = 0;
    private long startTime;
    private List<Exception> exceptions = new ArrayList<>();
    
    public RetryContext(String requestId) {
        this.requestId = requestId;
        this.startTime = System.currentTimeMillis();
    }
    
    public void recordAttempt(Exception exception, long waitTime) {
        this.attemptCount++;
        this.totalWaitTime += waitTime;
        this.exceptions.add(exception);
    }
    
    public long getElapsedTime() {
        return System.currentTimeMillis() - startTime;
    }
    
    public boolean shouldAbort(long maxElapsedTime) {
        return getElapsedTime() + totalWaitTime > maxElapsedTime;
    }
    
    // Getters
    public String getRequestId() { return requestId; }
    public int getAttemptCount() { return attemptCount; }
    public List<Exception> getExceptions() { return exceptions; }
}

@Configuration
@EnableRetry
public class RetryConfig {
    
    @Bean
    public RetryTemplate retryTemplate() {
        RetryTemplate template = new RetryTemplate();
        
        // 设置重试策略
        SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy();
        retryPolicy.setMaxAttempts(3);
        template.setRetryPolicy(retryPolicy);
        
        // 设置退避策略
        ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
        backOffPolicy.setInitialInterval(100);
        backOffPolicy.setMultiplier(2.0);
        backOffPolicy.setMaxInterval(5000);
        template.setBackOffPolicy(backOffPolicy);
        
        return template;
    }
}

@Service
public class PaymentService {
    
    @Retryable(
        value = {SocketTimeoutException.class, ConnectException.class},
        maxAttempts = 3,
        backoff = @Backoff(delay = 100, multiplier = 2)
    )
    public PaymentResult processPayment(PaymentRequest request) {
        // 调用第三方支付接口
        return thirdPartyPaymentClient.pay(request);
    }
    
    @Recover
    public PaymentResult recover(SocketTimeoutException e, PaymentRequest request) {
        // 重试失败后的兜底逻辑
        log.error("支付超时,订单号: {}", request.getOrderNo(), e);
        return PaymentResult.timeout(request.getOrderNo());
    }
    
    @Recover
    public PaymentResult recover(ConnectException e, PaymentRequest request) {
        // 连接失败的兜底逻辑
        log.error("支付服务不可用,订单号: {}", request.getOrderNo(), e);
        return PaymentResult.serviceUnavailable(request.getOrderNo());
    }
}

// 配置
RetryConfig config = RetryConfig.custom()
    .maxAttempts(3)
    .waitDuration(Duration.ofMillis(100))
    .intervalFunction(IntervalFunction.ofExponentialBackoff(100, 2.0))
    .retryOnException(e -> e instanceof SocketTimeoutException)
    .retryOnResult(result -> {
        // 根据结果判断是否重试
        if (result instanceof Response) {
            return ((Response) result).getStatus() == 503;
        }
        return false;
    })
    .build();

// 创建 Retry 实例
Retry retry = Retry.of("paymentService", config);

// 使用
Supplier<PaymentResult> supplier = Retry.decorateSupplier(retry, () -> {
    return paymentClient.processPayment(request);
});

Try<PaymentResult> result = Try.ofSupplier(supplier);

@Component
public class RetryMetrics {
    private final MeterRegistry meterRegistry;
    
    public void recordRetryAttempt(String serviceName, int attemptCount, boolean success) {
        Counter.builder("retry.attempt")
            .tag("service", serviceName)
            .tag("attempt", String.valueOf(attemptCount))
            .tag("success", String.valueOf(success))
            .register(meterRegistry)
            .increment();
    }
    
    public void recordRetryExhausted(String serviceName) {
        Counter.builder("retry.exhausted")
            .tag("service", serviceName)
            .register(meterRegistry)
            .increment();
    }
    
    public void recordRetrySuccess(String serviceName, int attemptCount) {
        Counter.builder("retry.success")
            .tag("service", serviceName)
            .tag("attempts", String.valueOf(attemptCount))
            .register(meterRegistry)
            .increment();
    }
}

-- 创建唯一索引
CREATE UNIQUE INDEX uk_order_no ON payment_records(order_no);

-- 插入时利用唯一约束防重
INSERT INTO payment_records (order_no, amount, status)
VALUES ('ORDER_123', 100.00, 'SUCCESS')
ON DUPLICATE KEY UPDATE status = status;  -- 已存在则忽略

@Service
public class PaymentService {
    
    @Transactional
    public PaymentResult pay(PaymentRequest request) {
        try {
            // 插入支付记录(利用唯一约束)
            PaymentRecord record = new PaymentRecord();
            record.setOrderNo(request.getOrderNo());
            record.setAmount(request.getAmount());
            record.setStatus("PROCESSING");
            paymentRecordMapper.insert(record);
            
            // 调用支付渠道
            ThirdPartyResult result = paymentChannel.pay(request);
            
            // 更新状态
            record.setStatus(result.isSuccess() ? "SUCCESS" : "FAILED");
            paymentRecordMapper.updateById(record);
            
            return PaymentResult.success(record);
            
        } catch (DuplicateKeyException e) {
            // 订单已处理,查询返回
            PaymentRecord existing = paymentRecordMapper
                .selectByOrderNo(request.getOrderNo());
            return PaymentResult.success(existing);
        }
    }
}

@Service
public class IdempotentService {
    
    // Redis 存储 token
    @Autowired
    private RedisTemplate<String, String> redisTemplate;
    
    /**
     * 生成幂等 token
     */
    public String generateToken(String businessType, String businessId) {
        String token = UUID.randomUUID().toString();
        String key = buildTokenKey(businessType, businessId);
        
        // token 有效期 1 小时
        redisTemplate.opsForValue().set(key, token, 1, TimeUnit.HOURS);
        
        return token;
    }
    
    /**
     * 校验并消费 token
     */
    public boolean validateAndConsumeToken(String businessType, 
                                           String businessId, 
                                           String token) {
        String key = buildTokenKey(businessType, businessId);
        
        // Lua 脚本保证原子性
        String script = 
            "if redis.call('GET', KEYS[1]) == ARGV[1] then " +
            "    redis.call('DEL', KEYS[1]) " +
            "    return 1 " +
            "else " +
            "    return 0 " +
            "end";
        
        RedisScript<Long> redisScript = RedisScript.of(script, Long.class);
        Long result = redisTemplate.execute(redisScript, 
                                           Collections.singletonList(key), 
                                           token);
        
        return result != null && result == 1;
    }
    
    private String buildTokenKey(String businessType, String businessId) {
        return String.format("idempotent:token:%s:%s", businessType, businessId);
    }
}

// 使用示例
@RestController
public class OrderController {
    
    @GetMapping("/order/token")
    public String generateOrderToken(String userId) {
        return idempotentService.generateToken("CREATE_ORDER", userId);
    }
    
    @PostMapping("/order/create")
    public OrderResult createOrder(@RequestBody CreateOrderRequest request,
                                   @RequestHeader("Idempotent-Token") String token) {
        // 校验 token
        if (!idempotentService.validateAndConsumeToken("CREATE_ORDER", 
                                                       request.getUserId(), 
                                                       token)) {
            throw new BusinessException("token 无效或已过期");
        }
        
        // 执行业务逻辑
        return orderService.createOrder(request);
    }
}

CREATE TABLE idempotent_log (
    id BIGINT PRIMARY KEY AUTO_INCREMENT,
    request_id VARCHAR(64) NOT NULL UNIQUE COMMENT '请求唯一标识',
    business_type VARCHAR(32) NOT NULL COMMENT '业务类型',
    business_key VARCHAR(128) NOT NULL COMMENT '业务键',
    request_data TEXT COMMENT '请求数据',
    response_data TEXT COMMENT '响应数据',
    status VARCHAR(16) NOT NULL COMMENT '状态:PROCESSING/SUCCESS/FAILED',
    retry_count INT DEFAULT 0 COMMENT '重试次数',
    created_at DATETIME NOT NULL,
    updated_at DATETIME NOT NULL,
    INDEX idx_business (business_type, business_key)
);

@Service
public class IdempotentLogService {
    
    @Autowired
    private IdempotentLogMapper idempotentLogMapper;
    
    /**
     * 开始处理请求
     */
    @Transactional
    public IdempotentLog startProcess(String requestId, 
                                      String businessType,
                                      String businessKey, 
                                      String requestData) {
        // 插入处理记录
        IdempotentLog log = new IdempotentLog();
        log.setRequestId(requestId);
        log.setBusinessType(businessType);
        log.setBusinessKey(businessKey);
        log.setRequestData(requestData);
        log.setStatus("PROCESSING");
        log.setCreatedAt(LocalDateTime.now());
        log.setUpdatedAt(LocalDateTime.now());
        
        try {
            idempotentLogMapper.insert(log);
            return log;
        } catch (DuplicateKeyException e) {
            // 已有记录,返回现有记录
            IdempotentLog existing = idempotentLogMapper
                .selectByRequestId(requestId);
            
            // 如果已在处理中,抛出异常
            if ("PROCESSING".equals(existing.getStatus())) {
                throw new ConcurrentProcessingException("请求正在处理中");
            }
            
            return existing;
        }
    }
    
    /**
     * 标记处理成功
     */
    public void markSuccess(String requestId, String responseData) {
        idempotentLogMapper.updateStatus(requestId, "SUCCESS", responseData);
    }
    
    /**
     * 标记处理失败
     */
    public void markFailed(String requestId) {
        idempotentLogMapper.updateStatus(requestId, "FAILED", null);
    }
    
    /**
     * 查询处理结果
     */
    public IdempotentLog queryResult(String requestId) {
        return idempotentLogMapper.selectByRequestId(requestId);
    }
}

// 使用示例
@Service
public class PaymentService {
    
    public PaymentResult pay(PaymentRequest request) {
        String requestId = request.getRequestId();
        
        // 1. 检查是否已处理
        IdempotentLog log = idempotentLogService.queryResult(requestId);
        if (log != null) {
            if ("SUCCESS".equals(log.getStatus())) {
                // 返回缓存结果
                return JsonUtils.fromJson(log.getResponseData(), PaymentResult.class);
            } else if ("PROCESSING".equals(log.getStatus())) {
                throw new ConcurrentProcessingException("请求正在处理中");
            }
        }
        
        // 2. 开始处理
        log = idempotentLogService.startProcess(
            requestId,
            "PAYMENT",
            request.getOrderNo(),
            JsonUtils.toJson(request)
        );
        
        try {
            // 3. 执行支付逻辑
            PaymentResult result = doPayment(request);
            
            // 4. 标记成功
            idempotentLogService.markSuccess(requestId, JsonUtils.toJson(result));
            
            return result;
            
        } catch (Exception e) {
            // 标记失败
            idempotentLogService.markFailed(requestId);
            throw e;
        }
    }
    
    private PaymentResult doPayment(PaymentRequest request) {
        // 实际支付逻辑
    }
}

@Service
public class OrderService {
    
    @Autowired
    private OrderMapper orderMapper;
    
    @Transactional
    public Order cancelOrder(String orderNo, String requestId) {
        Order order = orderMapper.selectByOrderNo(orderNo);
        
        if (order == null) {
            throw new BusinessException("订单不存在");
        }
        
        // 状态机判断
        if (!order.canCancel()) {
            throw new BusinessException("订单状态不允许取消");
        }
        
        // 幂等性判断:已取消的订单直接返回
        if ("CANCELLED".equals(order.getStatus())) {
            return order;
        }
        
        // 执行取消逻辑
        order.setStatus("CANCELLED");
        order.setCancelTime(LocalDateTime.now());
        order.setCancelReason("用户取消");
        
        int updated = orderMapper.updateByIdAndStatus(
            order.getId(), 
            "PAID",  // 原状态
            order    // 新状态
        );
        
        if (updated == 0) {
            // 乐观锁失败,可能是并发修改
            throw new OptimisticLockException("订单状态已变更");
        }
        
        return order;
    }
}

// 订单实体
public class Order {
    private String status;
    
    public boolean canCancel() {
        // 只有已支付和待支付状态可以取消
        return "PAID".equals(status) || "PENDING".equals(status);
    }
}

// 好的幂等键设计
requestId = UUID.randomUUID().toString();  // 全局唯一

// 业务键 + 时间戳(防止重放)
String businessKey = orderNo + "_" + DateUtil.format(new Date(), "yyyyMMdd");

// 多维度幂等
String idempotentKey = String.format("%s:%s:%s", 
    businessType,    // 业务类型
    businessId,      // 业务 ID
    operationType    // 操作类型
);

@Service
public class InventoryService {
    
    @Retryable(
        value = {SocketTimeoutException.class},
        maxAttempts = 3,
        backoff = @Backoff(delay = 100, multiplier = 2)
    )
    public DeductResult deductInventory(DeductRequest request) {
        // 1. 幂等性检查
        String requestId = request.getRequestId();
        DeductLog existingLog = deductLogMapper.selectByRequestId(requestId);
        
        if (existingLog != null) {
            if ("SUCCESS".equals(existingLog.getStatus())) {
                // 已成功,返回缓存结果
                return JsonUtils.fromJson(existingLog.getResult(), DeductResult.class);
            } else if ("PROCESSING".equals(existingLog.getStatus())) {
                // 正在处理中,等待或重试
                throw new RetryableException("处理中,稍后重试");
            }
        }
        
        // 2. 开始处理
        DeductLog log = new DeductLog();
        log.setRequestId(requestId);
        log.setSkuId(request.getSkuId());
        log.setQuantity(request.getQuantity());
        log.setStatus("PROCESSING");
        deductLogMapper.insert(log);
        
        try {
            // 3. 扣减库存
            int updated = inventoryMapper.deduct(
                request.getSkuId(), 
                request.getQuantity()
            );
            
            if (updated == 0) {
                throw new InsufficientInventoryException("库存不足");
            }
            
            // 4. 标记成功
            DeductResult result = DeductResult.success(request);
            log.setStatus("SUCCESS");
            log.setResult(JsonUtils.toJson(result));
            deductLogMapper.updateById(log);
            
            return result;
            
        } catch (Exception e) {
            // 标记失败
            log.setStatus("FAILED");
            log.setResult(e.getMessage());
            deductLogMapper.updateById(log);
            throw e;
        }
    }
}

// 补偿任务状态定义
public enum CompensationStatus {
    PENDING("待处理"),
    PROCESSING("处理中"),
    SUCCESS("成功"),
    FAILED("失败"),
    MANUAL_REQUIRED("需人工介入"),
    TIMEOUT("超时");
    
    private String desc;
    
    CompensationStatus(String desc) {
        this.desc = desc;
    }
}

CREATE TABLE compensation_task (
    id BIGINT PRIMARY KEY AUTO_INCREMENT,
    task_id VARCHAR(64) NOT NULL UNIQUE COMMENT '任务唯一标识',
    business_type VARCHAR(32) NOT NULL COMMENT '业务类型',
    business_id VARCHAR(128) NOT NULL COMMENT '业务 ID',
    compensation_type VARCHAR(32) NOT NULL COMMENT '补偿类型',
    original_data TEXT COMMENT '原始数据',
    compensation_data TEXT COMMENT '补偿数据',
    status VARCHAR(16) NOT NULL COMMENT '状态',
    retry_count INT DEFAULT 0 COMMENT '重试次数',
    max_retry_count INT DEFAULT 5 COMMENT '最大重试次数',
    next_retry_time DATETIME COMMENT '下次重试时间',
    created_at DATETIME NOT NULL,
    updated_at DATETIME NOT NULL,
    INDEX idx_status_time (status, next_retry_time),
    INDEX idx_business (business_type, business_id)
);

public interface CompensationExecutor {
    
    /**
     * 获取支持的补偿类型
     */
    String getCompensationType();
    
    /**
     * 执行补偿
     */
    CompensationResult execute(CompensationTask task);
    
    /**
     * 判断是否需要人工介入
     */
    default boolean needManualIntervention(CompensationTask task) {
        return task.getRetryCount() >= task.getMaxRetryCount();
    }
}

@Service
public class CompensationScheduler {
    
    @Autowired
    private CompensationTaskMapper taskMapper;
    
    @Autowired
    private Map<String, CompensationExecutor> executorMap;
    
    @Scheduled(fixedDelay = 5000)  // 每 5 秒执行一次
    public void processCompensationTasks() {
        // 1. 查询待处理的补偿任务
        List<CompensationTask> tasks = taskMapper.selectPendingTasks(
            LocalDateTime.now(),
            100  // 批量处理 100 条
        );
        
        if (tasks.isEmpty()) {
            return;
        }
        
        // 2. 并行处理
        tasks.parallelStream().forEach(task -> {
            try {
                processTask(task);
            } catch (Exception e) {
                log.error("补偿任务处理失败: {}", task.getTaskId(), e);
            }
        });
    }
    
    private void processTask(CompensationTask task) {
        // 获取对应的执行器
        CompensationExecutor executor = executorMap.get(task.getCompensationType());
        
        if (executor == null) {
            log.error("未找到补偿执行器: {}", task.getCompensationType());
            return;
        }
        
        try {
            // 执行补偿
            CompensationResult result = executor.execute(task);
            
            if (result.isSuccess()) {
                // 标记成功
                task.setStatus(CompensationStatus.SUCCESS.name());
                task.setUpdatedAt(LocalDateTime.now());
                taskMapper.updateById(task);
                
                log.info("补偿成功: {}", task.getTaskId());
                
            } else {
                handleFailure(task, result.getErrorMessage());
            }
            
        } catch (Exception e) {
            handleFailure(task, e.getMessage());
        }
    }
    
    private void handleFailure(CompensationTask task, String errorMessage) {
        // 增加重试次数
        task.setRetryCount(task.getRetryCount() + 1);
        task.setUpdatedAt(LocalDateTime.now());
        
        // 判断是否需要人工介入
        if (task.getRetryCount() >= task.getMaxRetryCount()) {
            task.setStatus(CompensationStatus.MANUAL_REQUIRED.name());
            
            // 发送告警
            alertService.sendAlert(
                AlertLevel.HIGH,
                String.format("补偿任务需人工介入: %s, 原因: %s", 
                    task.getTaskId(), errorMessage)
            );
        } else {
            task.setStatus(CompensationStatus.PENDING.name());
            
            // 设置下次重试时间(指数退避)
            long nextDelay = calculateNextDelay(task.getRetryCount());
            task.setNextRetryTime(LocalDateTime.now().plusSeconds(nextDelay));
        }
        
        taskMapper.updateById(task);
    }
    
    private long calculateNextDelay(int retryCount) {
        // 指数退避:1分钟、2分钟、4分钟、8分钟、16分钟
        return (long) (Math.pow(2, retryCount - 1) * 60);
    }
}

@Component
public class PaymentTimeoutCompensationExecutor implements CompensationExecutor {
    
    @Autowired
    private PaymentRecordMapper paymentMapper;
    
    @Autowired
    private OrderService orderService;
    
    @Override
    public String getCompensationType() {
        return "PAYMENT_TIMEOUT";
    }
    
    @Override
    public CompensationResult execute(CompensationTask task) {
        // 解析原始数据
        PaymentData paymentData = JsonUtils.fromJson(
            task.getOriginalData(), 
            PaymentData.class
        );
        
        String orderNo = paymentData.getOrderNo();
        
        // 1. 查询支付状态
        PaymentRecord payment = paymentMapper.selectByOrderNo(orderNo);
        
        if (payment == null) {
            // 没有支付记录,直接取消订单
            orderService.cancelOrder(orderNo, "支付超时");
            return CompensationResult.success();
        }
        
        // 2. 根据支付状态处理
        switch (payment.getStatus()) {
            case "SUCCESS":
                // 已支付成功,完成订单
                orderService.completeOrder(orderNo);
                return CompensationResult.success();
                
            case "PROCESSING":
                // 支付处理中,查询第三方状态
                ThirdPartyPayStatus status = queryThirdPartyStatus(payment.getPayId());
                
                if (status.isSuccess()) {
                    // 支付成功,更新状态
                    payment.setStatus("SUCCESS");
                    paymentMapper.updateById(payment);
                    orderService.completeOrder(orderNo);
                    return CompensationResult.success();
                    
                } else if (status.isFailed()) {
                    // 支付失败,取消订单
                    payment.setStatus("FAILED");
                    paymentMapper.updateById(payment);
                    orderService.cancelOrder(orderNo, "支付失败");
                    return CompensationResult.success();
                    
                } else {
                    // 仍然未知,返回失败等待下次重试
                    return CompensationResult.failure("支付状态未知");
                }
                
            case "FAILED":
                // 已失败,取消订单
                orderService.cancelOrder(orderNo, "支付失败");
                return CompensationResult.success();
                
            default:
                return CompensationResult.failure("未知支付状态: " + payment.getStatus());
        }
    }
    
    private ThirdPartyPayStatus queryThirdPartyStatus(String payId) {
        // 查询第三方支付状态
        return thirdPartyPaymentClient.queryStatus(payId);
    }
}

@Component
public class InventoryDeductCompensationExecutor implements CompensationExecutor {
    
    @Autowired
    private InventoryMapper inventoryMapper;
    
    @Autowired
    private DeductLogMapper deductLogMapper;
    
    @Override
    public String getCompensationType() {
        return "INVENTORY_DEDUCT";
    }
    
    @Override
    public CompensationResult execute(CompensationTask task) {
        DeductData deductData = JsonUtils.fromJson(
            task.getOriginalData(), 
            DeductData.class
        );
        
        String requestId = deductData.getRequestId();
        Long skuId = deductData.getSkuId();
        Integer quantity = deductData.getQuantity();
        
        // 1. 查询扣减日志
        DeductLog log = deductLogMapper.selectByRequestId(requestId);
        
        if (log == null) {
            // 没有扣减记录,说明扣减失败,无需补偿
            return CompensationResult.success();
        }
        
        // 2. 根据状态处理
        if ("SUCCESS".equals(log.getStatus())) {
            // 扣减成功,检查订单状态
            Order order = orderMapper.selectByOrderNo(deductData.getOrderNo());
            
            if ("CANCELLED".equals(order.getStatus())) {
                // 订单已取消,回退库存
                inventoryMapper.restore(skuId, quantity);
                log.setStatus("COMPENSATED");
                deductLogMapper.updateById(log);
            }
            
            return CompensationResult.success();
            
        } else if ("PROCESSING".equals(log.getStatus())) {
            // 处理中,需要重新检查
            // 这里可以选择等待或者标记为失败
            return CompensationResult.failure("扣减仍在处理中");
        }
        
        return CompensationResult.success();
    }
}

@Component
public class MessageSendCompensationExecutor implements CompensationExecutor {
    
    @Autowired
    private MessageQueue messageQueue;
    
    @Autowired
    private MessageLogMapper messageLogMapper;
    
    @Override
    public String getCompensationType() {
        return "MESSAGE_SEND";
    }
    
    @Override
    public CompensationResult execute(CompensationTask task) {
        MessageData messageData = JsonUtils.fromJson(
            task.getOriginalData(), 
            MessageData.class
        );
        
        String messageId = messageData.getMessageId();
        
        // 1. 查询消息发送记录
        MessageLog log = messageLogMapper.selectByMessageId(messageId);
        
        if (log == null) {
            // 没有记录,尝试发送
            return sendMessage(messageData);
        }
        
        // 2. 根据状态处理
        switch (log.getStatus()) {
            case "SUCCESS":
                // 已发送成功
                return CompensationResult.success();
                
            case "FAILED":
                // 发送失败,重试发送
                return sendMessage(messageData);
                
            case "SENDING":
                // 发送中,检查是否超时
                if (isTimeout(log.getCreatedAt())) {
                    // 超时,重新发送
                    return sendMessage(messageData);
                }
                return CompensationResult.failure("消息发送中");
                
            default:
                return CompensationResult.failure("未知状态: " + log.getStatus());
        }
    }
    
    private CompensationResult sendMessage(MessageData messageData) {
        try {
            // 发送消息
            messageQueue.send(
                messageData.getTopic(),
                messageData.getTag(),
                messageData.getKey(),
                messageData.getBody()
            );
            
            // 更新状态
            MessageLog log = messageLogMapper.selectByMessageId(messageData.getMessageId());
            if (log != null) {
                log.setStatus("SUCCESS");
                log.setUpdatedAt(LocalDateTime.now());
                messageLogMapper.updateById(log);
            } else {
                log = new MessageLog();
                log.setMessageId(messageData.getMessageId());
                log.setStatus("SUCCESS");
                log.setCreatedAt(LocalDateTime.now());
                messageLogMapper.insert(log);
            }
            
            return CompensationResult.success();
            
        } catch (Exception e) {
            log.error("消息发送失败", e);
            return CompensationResult.failure(e.getMessage());
        }
    }
    
    private boolean isTimeout(LocalDateTime createdAt) {
        // 超过 5 分钟视为超时
        return createdAt.plusMinutes(5).isBefore(LocalDateTime.now());
    }
}

// 好的实践:在业务操作开始时就创建补偿任务
@Service
public class OrderService {
    
    @Transactional
    public OrderResult createOrder(CreateOrderRequest request) {
        String orderNo = generateOrderNo();
        
        // 1. 创建补偿任务(前置)
        CompensationTask compensationTask = createCompensationTask(orderNo, request);
        compensationTaskMapper.insert(compensationTask);
        
        try {
            // 2. 执行业务逻辑
            Order order = doCreateOrder(orderNo, request);
            
            // 3. 业务成功,删除补偿任务
            compensationTaskMapper.deleteById(compensationTask.getId());
            
            return OrderResult.success(order);
            
        } catch (Exception e) {
            // 4. 业务失败,补偿任务会被调度器处理
            log.error("订单创建失败: {}", orderNo, e);
            throw e;
        }
    }
}

@Component
public class OrderCancelCompensationExecutor implements CompensationExecutor {
    
    @Override
    public CompensationResult execute(CompensationTask task) {
        String orderNo = JsonUtils.fromJson(
            task.getOriginalData(), 
            OrderData.class
        ).getOrderNo();
        
        // 查询订单状态
        Order order = orderMapper.selectByOrderNo(orderNo);
        
        // 幂等性判断:已取消的订单不再处理
        if ("CANCELLED".equals(order.getStatus())) {
            return CompensationResult.success();
        }
        
        // 执行取消
        orderService.cancelOrder(orderNo, "补偿取消");
        
        return CompensationResult.success();
    }
}

@Service
public class CompensationMonitor {
    
    @Scheduled(cron = "0 0 * * * ?")  // 每小时执行一次
    public void checkCompensationTasks() {
        // 1. 统计需人工介入的任务
        long manualRequiredCount = compensationTaskMapper
            .countByStatus(CompensationStatus.MANUAL_REQUIRED);
        
        if (manualRequiredCount > 0) {
            alertService.sendAlert(
                AlertLevel.HIGH,
                String.format("有 %d 个补偿任务需要人工介入", manualRequiredCount)
            );
        }
        
        // 2. 统计长时间未处理的任务
        List<CompensationTask> timeoutTasks = compensationTaskMapper
            .selectTimeoutTasks(LocalDateTime.now().minusHours(24));
        
        if (!timeoutTasks.isEmpty()) {
            alertService.sendAlert(
                AlertLevel.HIGH,
                String.format("有 %d 个补偿任务超过 24 小时未处理", timeoutTasks.size())
            );
        }
        
        // 3. 统计补偿成功率
        CompensationStatistics stats = compensationTaskMapper.getStatistics();
        if (stats.getSuccessRate() < 0.95) {
            alertService.sendAlert(
                AlertLevel.WARNING,
                String.format("补偿成功率: %.2f%%, 低于 95%%", stats.getSuccessRate() * 100)
            );
        }
    }
}

         失败率 > 阈值
    ┌────────────────────┐
    │                    │
    ▼                    │
┌────────┐  半开状态成功  ┌──────────┐
│  关闭  │◄──────────────│   半开   │
│ Closed │               │Half-Open│
└────────┘  半开状态失败  └──────────┘
    │                    ▲          │
    │ 失败率正常          │          │
    └────────────────────┘          │
         ▲                          │
         │                          │
         │        超时/重试         │
         │                          │
    ┌────────┐                     │
    │  打开  │─────────────────────┘
    │  Open │  超时后进入半开
    └────────┘

@Configuration
public class CircuitBreakerConfig {
    
    @Bean
    public CircuitBreakerRegistry circuitBreakerRegistry() {
        // 熔断器配置
        io.github.resilience4j.circuitbreaker.CircuitBreakerConfig config = 
            io.github.resilience4j.circuitbreaker.CircuitBreakerConfig.custom()
                // 故障率阈值:50%
                .failureRateThreshold(50)
                
                // 慢调用率阈值:50%
                .slowCallRateThreshold(50)
                
                // 慢调用时间阈值:2 秒
                .slowCallDurationThreshold(Duration.ofSeconds(2))
                
                // 最小调用次数:100(至少调用 100 次才开始计算故障率)
                .minimumNumberOfCalls(100)
                
                // 滑动窗口类型:基于数量
                .slidingWindowType(
                    io.github.resilience4j.circuitbreaker.CircuitBreakerConfig.SlidingWindowType.COUNT_BASED
                )
                
                // 滑动窗口大小:最近 100 次调用
                .slidingWindowSize(100)
                
                // 半开状态允许的调用次数:10
                .permittedNumberOfCallsInHalfOpenState(10)
                
                // 熔断器打开持续时间:30 秒
                .waitDurationInOpenState(Duration.ofSeconds(30))
                
                // 自动从打开转换到半开
                .automaticTransitionFromOpenToHalfOpenEnabled(true)
                
                // 记录哪些异常为失败
                .recordExceptions(IOException.class, TimeoutException.class)
                
                // 忽略哪些异常(不计入失败)
                .ignoreExceptions(BusinessException.class)
                
                .build();
        
        return CircuitBreakerRegistry.of(config);
    }
}

@Service
public class PaymentService {
    
    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;
    
    private final PaymentClient paymentClient;
    
    public PaymentResult pay(PaymentRequest request) {
        // 获取或创建熔断器
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitBreaker("paymentService");
        
        // 使用熔断器包装调用
        Supplier<PaymentResult> supplier = CircuitBreaker
            .decorateSupplier(circuitBreaker, () -> {
                return paymentClient.processPayment(request);
            });
        
        // 执行(可结合降级)
        Try<PaymentResult> result = Try.ofSupplier(supplier)
            .recover(throwable -> {
                log.warn("支付服务熔断,返回降级结果", throwable);
                return PaymentResult.fallback(request);
            });
        
        return result.get();
    }
}

@Service
public class OrderService {
    
    @Autowired
    private InventoryClient inventoryClient;
    
    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;
    
    /**
     * 查询库存 - 完整的熔断降级示例
     */
    public InventoryInfo getInventory(Long skuId) {
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitBreaker("inventoryService");
        
        // 定义主逻辑
        Supplier<InventoryInfo> supplier = CircuitBreaker
            .decorateSupplier(circuitBreaker, () -> {
                return inventoryClient.getInventory(skuId);
            });
        
        // 定义降级逻辑
        Supplier<InventoryInfo> recoverySupplier = () -> {
            // 降级策略 1: 从缓存读取
            InventoryInfo cached = getFromCache(skuId);
            if (cached != null) {
                return cached;
            }
            
            // 降级策略 2: 返回默认值
            return InventoryInfo.defaultInventory(skuId);
        };
        
        // 执行
        return Try.ofSupplier(supplier)
            .recoverWith(throwable -> {
                log.warn("库存服务不可用,使用降级策略: {}", throwable.getMessage());
                return Try.ofSupplier(recoverySupplier);
            })
            .get();
    }
    
    private InventoryInfo getFromCache(Long skuId) {
        String key = "inventory:" + skuId;
        String cached = redisTemplate.opsForValue().get(key);
        
        if (cached != null) {
            return JsonUtils.fromJson(cached, InventoryInfo.class);
        }
        
        return null;
    }
}

@RestController
@RequestMapping("/api/circuit-breaker")
public class CircuitBreakerController {
    
    @Autowired
    private CircuitBreakerRegistry registry;
    
    /**
     * 动态调整熔断器参数
     */
    @PostMapping("/{name}/config")
    public String updateConfig(
        @PathVariable String name,
        @RequestBody CircuitBreakerUpdateRequest request
    ) {
        CircuitBreaker circuitBreaker = registry.circuitBreaker(name);
        
        // 更新配置
        CircuitBreakerConfig newConfig = CircuitBreakerConfig.custom()
            .failureRateThreshold(request.getFailureRateThreshold())
            .slowCallRateThreshold(request.getSlowCallRateThreshold())
            .slowCallDurationThreshold(Duration.ofMillis(request.getSlowCallDuration()))
            .minimumNumberOfCalls(request.getMinimumNumberOfCalls())
            .slidingWindowSize(request.getSlidingWindowSize())
            .waitDurationInOpenState(Duration.ofMillis(request.getWaitDuration()))
            .build();
        
        circuitBreaker.changeConfig(newConfig);
        
        return "配置已更新";
    }
    
    /**
     * 强制打开熔断器
     */
    @PostMapping("/{name}/force-open")
    public String forceOpen(@PathVariable String name) {
        CircuitBreaker circuitBreaker = registry.circuitBreaker(name);
        circuitBreaker.transitionToOpenState();
        return "熔断器已强制打开";
    }
    
    /**
     * 强制关闭熔断器
     */
    @PostMapping("/{name}/force-close")
    public String forceClose(@PathVariable String name) {
        CircuitBreaker circuitBreaker = registry.circuitBreaker(name);
        circuitBreaker.transitionToClosedState();
        return "熔断器已强制关闭";
    }
    
    /**
     * 查询熔断器状态
     */
    @GetMapping("/{name}/metrics")
    public CircuitBreakerMetrics getMetrics(@PathVariable String name) {
        CircuitBreaker circuitBreaker = registry.circuitBreaker(name);
        CircuitBreaker.Metrics metrics = circuitBreaker.getMetrics();
        
        return CircuitBreakerMetrics.builder()
            .state(circuitBreaker.getState().name())
            .failureRate(metrics.getFailureRate())
            .slowCallRate(metrics.getSlowCallRate())
            .numberOfCalls(metrics.getNumberOfCalls())
            .numberOfFailedCalls(metrics.getNumberOfFailedCalls())
            .numberOfSlowCalls(metrics.getNumberOfSlowCalls())
            .build();
    }
}

public InventoryInfo getDefaultInventory(Long skuId) {
    // 返回安全默认值
    return InventoryInfo.builder()
        .skuId(skuId)
        .stock(0)  // 默认无库存
        .available(false)
        .build();
}

public InventoryInfo getCachedInventory(Long skuId) {
    // 从本地缓存或 Redis 获取
    String key = "inventory:" + skuId;
    String cached = redisTemplate.opsForValue().get(key);
    
    if (cached != null) {
        return JsonUtils.fromJson(cached, InventoryInfo.class);
    }
    
    // 缓存也没有,返回默认值
    return getDefaultInventory(skuId);
}

public PaymentResult fallbackPayment(PaymentRequest request) {
    // 尝试备用支付渠道
    try {
        return backupPaymentClient.processPayment(request);
    } catch (Exception e) {
        log.error("备用支付渠道也失败", e);
        return PaymentResult.serviceUnavailable();
    }
}

public class RateLimitFallback {
    
    private final RateLimiter rateLimiter = RateLimiter.create(100);  // 100 QPS
    
    public InventoryInfo getInventoryWithRateLimit(Long skuId) {
        if (rateLimiter.tryAcquire()) {
            // 正常调用
            return inventoryClient.getInventory(skuId);
        } else {
            // 限流降级
            log.warn("库存服务限流");
            return getCachedInventory(skuId);
        }
    }
}

@RestController
@RequestMapping("/api/circuit-breaker/dashboard")
public class CircuitBreakerDashboardController {
    
    @Autowired
    private CircuitBreakerRegistry registry;
    
    /**
     * 获取所有熔断器状态
     */
    @GetMapping("/overview")
    public List<CircuitBreakerStatus> getOverview() {
        return registry.getAllCircuitBreakers()
            .stream()
            .map(this::toStatus)
            .collect(Collectors.toList());
    }
    
    private CircuitBreakerStatus toStatus(CircuitBreaker cb) {
        CircuitBreaker.Metrics metrics = cb.getMetrics();
        
        return CircuitBreakerStatus.builder()
            .name(cb.getName())
            .state(cb.getState().name())
            .failureRate(metrics.getFailureRate())
            .slowCallRate(metrics.getSlowCallRate())
            .numberOfCalls(metrics.getNumberOfCalls())
            .numberOfFailedCalls(metrics.getNumberOfFailedCalls())
            .numberOfSlowCalls(metrics.getNumberOfSlowCalls())
            .build();
    }
}

@Service
public class OrderService {
    
    @Autowired
    private InventoryClient inventoryClient;
    
    @Autowired
    private DeductLogMapper deductLogMapper;
    
    @Autowired
    private CompensationTaskMapper compensationTaskMapper;
    
    @Retryable(
        value = {SocketTimeoutException.class, ConnectException.class},
        maxAttempts = 3,
        backoff = @Backoff(delay = 100, multiplier = 2)
    )
    @Transactional
    public OrderResult createOrder(CreateOrderRequest request) {
        String orderNo = generateOrderNo();
        String requestId = request.getRequestId();
        
        // 1. 幂等性检查
        Order existingOrder = orderMapper.selectByRequestId(requestId);
        if (existingOrder != null) {
            return OrderResult.success(existingOrder);
        }
        
        // 2. 创建订单(初始状态)
        Order order = new Order();
        order.setOrderNo(orderNo);
        order.setUserId(request.getUserId());
        order.setStatus("CREATED");
        order.setRequestId(requestId);
        orderMapper.insert(order);
        
        // 3. 扣减库存
        try {
            DeductInventoryRequest deductRequest = new DeductInventoryRequest();
            deductRequest.setRequestId(requestId);
            deductRequest.setOrderNo(orderNo);
            deductRequest.setItems(request.getItems());
            
            DeductResult deductResult = inventoryClient.deduct(deductRequest);
            
            if (!deductResult.isSuccess()) {
                // 库存扣减失败
                order.setStatus("CANCELLED");
                order.setCancelReason("库存不足");
                orderMapper.updateById(order);
                return OrderResult.failure("库存不足");
            }
            
            // 4. 记录扣减日志
            DeductLog deductLog = new DeductLog();
            deductLog.setRequestId(requestId);
            deductLog.setOrderNo(orderNo);
            deductLog.setStatus("SUCCESS");
            deductLog.setResult(JsonUtils.toJson(deductResult));
            deductLogMapper.insert(deductLog);
            
            // 5. 订单状态更新
            order.setStatus("PAID");
            orderMapper.updateById(order);
            
            return OrderResult.success(order);
            
        } catch (SocketTimeoutException e) {
            // 超时异常,创建补偿任务
            createCompensationTask(orderNo, requestId, "INVENTORY_TIMEOUT");
            
            // 返回处理中状态
            return OrderResult.processing(orderNo, "订单处理中,请稍后查询结果");
        }
    }
    
    private void createCompensationTask(String orderNo, 
                                        String requestId, 
                                        String compensationType) {
        CompensationTask task = new CompensationTask();
        task.setTaskId(UUID.randomUUID().toString());
        task.setBusinessType("ORDER");
        task.setBusinessId(orderNo);
        task.setCompensationType(compensationType);
        task.setOriginalData(JsonUtils.toJson(Map.of(
            "orderNo", orderNo,
            "requestId", requestId
        )));
        task.setStatus("PENDING");
        task.setRetryCount(0);
        task.setMaxRetryCount(5);
        task.setNextRetryTime(LocalDateTime.now().plusMinutes(1));
        task.setCreatedAt(LocalDateTime.now());
        
        compensationTaskMapper.insert(task);
    }
}

// 库存客户端(带熔断)
@Service
public class InventoryClient {
    
    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;
    
    public DeductResult deduct(DeductInventoryRequest request) {
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitBreaker("inventoryService");
        
        Supplier<DeductResult> supplier = CircuitBreaker
            .decorateSupplier(circuitBreaker, () -> {
                return doDeduct(request);
            });
        
        return Try.ofSupplier(supplier)
            .recover(throwable -> {
                log.warn("库存服务不可用", throwable);
                return DeductResult.serviceUnavailable();
            })
            .get();
    }
    
    private DeductResult doDeduct(DeductInventoryRequest request) {
        // HTTP 调用
        // ...
    }
}

@Component
@RocketMQMessageListener(
    topic = "PAYMENT_SUCCESS",
    consumerGroup = "ORDER_CONSUMER_GROUP"
)
public class PaymentSuccessConsumer implements RocketMQListener<MessageExt> {
    
    @Autowired
    private OrderService orderService;
    
    @Autowired
    private MessageConsumeLogMapper consumeLogMapper;
    
    @Override
    public void onMessage(MessageExt message) {
        String messageId = message.getMsgId();
        String messageKey = message.getKeys();
        
        try {
            // 1. 幂等性检查
            MessageConsumeLog log = consumeLogMapper.selectByMessageId(messageId);
            if (log != null && "SUCCESS".equals(log.getStatus())) {
                // 已消费成功,跳过
                return;
            }
            
            // 2. 记录消费日志
            if (log == null) {
                log = new MessageConsumeLog();
                log.setMessageId(messageId);
                log.setMessageKey(messageKey);
                log.setTopic(message.getTopic());
                log.setBody(new String(message.getBody()));
                log.setConsumeCount(1);
                log.setStatus("PROCESSING");
                log.setCreatedAt(LocalDateTime.now());
                consumeLogMapper.insert(log);
            } else {
                // 更新消费次数
                log.setConsumeCount(log.getConsumeCount() + 1);
                log.setUpdatedAt(LocalDateTime.now());
                consumeLogMapper.updateById(log);
            }
            
            // 3. 消费逻辑
            PaymentSuccessMessage msg = JsonUtils.fromJson(
                new String(message.getBody()),
                PaymentSuccessMessage.class
            );
            
            orderService.completeOrder(msg.getOrderNo(), msg.getPayNo());
            
            // 4. 标记成功
            log.setStatus("SUCCESS");
            log.setUpdatedAt(LocalDateTime.now());
            consumeLogMapper.updateById(log);
            
        } catch (Exception e) {
            log.error("消息消费失败: {}", messageId, e);
            
            // 5. 判断是否达到最大重试次数
            MessageConsumeLog log = consumeLogMapper.selectByMessageId(messageId);
            
            if (log.getConsumeCount() >= 16) {  // RocketMQ 默认最大重试 16 次
                // 转入补偿表
                createCompensationTask(message, log);
                
                // 标记为需人工处理
                log.setStatus("MANUAL_REQUIRED");
                consumeLogMapper.updateById(log);
                
            } else {
                // 标记失败,等待下次重试
                log.setStatus("FAILED");
                log.setErrorMessage(e.getMessage());
                consumeLogMapper.updateById(log);
                
                // 抛出异常,触发消息重试
                throw new RuntimeException("消息消费失败,等待重试", e);
            }
        }
    }
    
    private void createCompensationTask(MessageExt message, MessageConsumeLog log) {
        CompensationTask task = new CompensationTask();
        task.setTaskId(UUID.randomUUID().toString());
        task.setBusinessType("MESSAGE_CONSUME");
        task.setBusinessId(message.getMsgId());
        task.setCompensationType("PAYMENT_SUCCESS_MESSAGE");
        task.setOriginalData(JsonUtils.toJson(Map.of(
            "messageId", message.getMsgId(),
            "messageKey", message.getKeys(),
            "body", new String(message.getBody()),
            "consumeCount", log.getConsumeCount()
        )));
        task.setStatus("PENDING");
        task.setRetryCount(0);
        task.setMaxRetryCount(5);
        task.setCreatedAt(LocalDateTime.now());
        
        compensationTaskMapper.insert(task);
    }
}

@Service
public class ThirdPartyPaymentService {
    
    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;
    
    @Autowired
    private PaymentRecordMapper paymentMapper;
    
    @Autowired
    private CompensationTaskMapper compensationTaskMapper;
    
    private final OkHttpClient httpClient = new OkHttpClient.Builder()
        .connectTimeout(2, TimeUnit.SECONDS)
        .readTimeout(5, TimeUnit.SECONDS)
        .writeTimeout(5, TimeUnit.SECONDS)
        .build();
    
    /**
     * 支付请求 - 带熔断、降级、补偿
     */
    public PaymentResult pay(PaymentRequest request) {
        // 1. 生成请求 ID
        String requestId = UUID.randomUUID().toString();
        request.setRequestId(requestId);
        
        // 2. 幂等性检查
        PaymentRecord existing = paymentMapper.selectByRequestId(requestId);
        if (existing != null) {
            return PaymentResult.fromRecord(existing);
        }
        
        // 3. 创建支付记录
        PaymentRecord record = createPaymentRecord(request);
        paymentMapper.insert(record);
        
        // 4. 执行支付(带熔断)
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitBreaker("thirdPartyPayment");
        
        Supplier<PaymentResult> supplier = CircuitBreaker
            .decorateSupplier(circuitBreaker, () -> {
                return doPayment(request, record);
            });
        
        PaymentResult result;
        
        try {
            // 尝试支付
            result = Try.ofSupplier(supplier)
                .recover(throwable -> {
                    log.warn("第三方支付失败", throwable);
                    return handlePaymentFailure(request, record, throwable);
                })
                .get();
                
        } catch (Exception e) {
            log.error("支付异常", e);
            result = handlePaymentFailure(request, record, e);
        }
        
        return result;
    }
    
    private PaymentResult doPayment(PaymentRequest request, PaymentRecord record) {
        try {
            // 构建请求
            ThirdPartyPayRequest thirdPartyRequest = buildThirdPartyRequest(request);
            
            // 调用第三方接口
            String response = callThirdParty(thirdPartyRequest);
            
            // 解析响应
            ThirdPartyPayResponse thirdPartyResponse = parseResponse(response);
            
            // 更新支付记录
            record.setThirdPartyPayId(thirdPartyResponse.getPayId());
            record.setStatus(thirdPartyResponse.isSuccess() ? "SUCCESS" : "FAILED");
            record.setResponse(response);
            record.setUpdatedAt(LocalDateTime.now());
            paymentMapper.updateById(record);
            
            if (thirdPartyResponse.isSuccess()) {
                return PaymentResult.success(record);
            } else {
                return PaymentResult.failure(record, thirdPartyResponse.getErrorMessage());
            }
            
        } catch (SocketTimeoutException e) {
            // 超时,状态未知,需要补偿
            record.setStatus("TIMEOUT");
            record.setErrorMessage("支付超时");
            paymentMapper.updateById(record);
            
            throw new PaymentTimeoutException("支付超时", e);
            
        } catch (IOException e) {
            // 网络异常
            record.setStatus("NETWORK_ERROR");
            record.setErrorMessage(e.getMessage());
            paymentMapper.updateById(record);
            
            throw new PaymentNetworkException("网络异常", e);
        }
    }
    
    private PaymentResult handlePaymentFailure(
        PaymentRequest request, 
        PaymentRecord record, 
        Throwable throwable
    ) {
        // 判断异常类型
        if (throwable instanceof PaymentTimeoutException) {
            // 超时,创建补偿任务
            createCompensationTask(record, "PAYMENT_TIMEOUT");
            
            return PaymentResult.processing(record, "支付处理中,请稍后查询结果");
            
        } else if (throwable instanceof PaymentNetworkException) {
            // 网络异常,可重试
            createCompensationTask(record, "PAYMENT_NETWORK_ERROR");
            
            return PaymentResult.processing(record, "支付处理中,请稍后查询结果");
            
        } else {
            // 其他异常
            return PaymentResult.failure(record, "支付失败: " + throwable.getMessage());
        }
    }
    
    private void createCompensationTask(PaymentRecord record, String compensationType) {
        CompensationTask task = new CompensationTask();
        task.setTaskId(UUID.randomUUID().toString());
        task.setBusinessType("PAYMENT");
        task.setBusinessId(record.getOrderNo());
        task.setCompensationType(compensationType);
        task.setOriginalData(JsonUtils.toJson(record));
        task.setStatus("PENDING");
        task.setRetryCount(0);
        task.setMaxRetryCount(5);
        task.setNextRetryTime(LocalDateTime.now().plusMinutes(1));
        task.setCreatedAt(LocalDateTime.now());
        
        compensationTaskMapper.insert(task);
    }
    
    private String callThirdParty(ThirdPartyPayRequest request) throws IOException {
        Request httpRequest = new Request.Builder()
            .url("https://api.thirdparty.com/pay")
            .post(RequestBody.create(
                MediaType.parse("application/json"),
                JsonUtils.toJson(request)
            ))
            .build();
        
        try (Response response = httpClient.newCall(httpRequest).execute()) {
            if (!response.isSuccessful()) {
                throw new IOException("第三方支付失败: " + response.code());
            }
            return response.body().string();
        }
    }
}

// 熔断器配置
@Configuration
public class PaymentCircuitBreakerConfig {
    
    @Bean
    public CircuitBreaker thirdPartyPaymentCircuitBreaker() {
        CircuitBreakerConfig config = CircuitBreakerConfig.custom()
            .failureRateThreshold(30)              // 故障率 30% 触发熔断
            .slowCallRateThreshold(50)             // 慢调用率 50%
            .slowCallDurationThreshold(Duration.ofSeconds(3))
            .minimumNumberOfCalls(20)              // 最少调用 20 次
            .slidingWindowSize(50)                 // 滑动窗口 50 次
            .waitDurationInOpenState(Duration.ofSeconds(30))  // 熔断 30 秒
            .permittedNumberOfCallsInHalfOpenState(5)
            .recordExceptions(
                SocketTimeoutException.class,
                IOException.class,
                PaymentTimeoutException.class,
                PaymentNetworkException.class
            )
            .build();
        
        return CircuitBreaker.of("thirdPartyPayment", config);
    }
}

// 1. 监控超时指标
@Component
public class TimeoutMonitor {
    
    private final MeterRegistry meterRegistry;
    
    public void recordTimeout(String serviceName, String timeoutType, long duration) {
        Timer.builder("service.timeout")
            .tag("service", serviceName)
            .tag("type", timeoutType)  // connect/read/write
            .register(meterRegistry)
            .record(duration, TimeUnit.MILLISECONDS);
    }
    
    public TimeoutStatistics getStatistics(String serviceName) {
        // 获取超时统计
        return TimeoutStatistics.builder()
            .connectTimeoutCount(getTimeoutCount(serviceName, "connect"))
            .readTimeoutCount(getTimeoutCount(serviceName, "read"))
            .writeTimeoutCount(getTimeoutCount(serviceName, "write"))
            .avgTimeoutDuration(getAvgTimeoutDuration(serviceName))
            .build();
    }
}

// 2. 线程池监控
@Component
public class ThreadPoolMonitor {
    
    @Scheduled(fixedRate = 5000)
    public void monitorThreadPools() {
        // 监控各线程池状态
        Map<String, ThreadPoolExecutor> pools = getThreadPools();
        
        pools.forEach((name, pool) -> {
            // 记录指标
            Gauge.builder("thread.pool.active", pool, ThreadPoolExecutor::getActiveCount)
                .tag("pool", name)
                .register(meterRegistry);
            
            Gauge.builder("thread.pool.queue.size", pool, p -> p.getQueue().size())
                .tag("pool", name)
                .register(meterRegistry);
            
            // 判断是否告警
            if (pool.getActiveCount() >= pool.getMaximumPoolSize() * 0.8) {
                alertService.sendAlert(
                    AlertLevel.WARNING,
                    String.format("线程池 %s 接近满载: %d/%d", 
                        name, pool.getActiveCount(), pool.getMaximumPoolSize())
                );
            }
        });
    }
}

// 3. 连接池监控
@Component
public class ConnectionPoolMonitor {
    
    @Scheduled(fixedRate = 5000)
    public void monitorConnectionPools() {
        // 监控数据库连接池
        HikariDataSource dataSource = getHikariDataSource();
        HikariPoolMXBean pool = dataSource.getHikariPoolMXBean();
        
        if (pool != null) {
            // 记录指标
            Gauge.builder("db.pool.active", pool, HikariPoolMXBean::getActiveConnections)
                .register(meterRegistry);
            
            Gauge.builder("db.pool.idle", pool, HikariPoolMXBean::getIdleConnections)
                .register(meterRegistry);
            
            Gauge.builder("db.pool.waiting", pool, HikariPoolMXBean::getThreadsAwaitingConnection)
                .register(meterRegistry);
            
            // 判断是否告警
            if (pool.getThreadsAwaitingConnection() > 10) {
                alertService.sendAlert(
                    AlertLevel.WARNING,
                    String.format("数据库连接池等待线程数: %d", 
                        pool.getThreadsAwaitingConnection())
                );
            }
        }
    }
}

// 1. 重试链路追踪
@Component
@Aspect
public class RetryTraceAspect {
    
    @Around("@annotation(retryable)")
    public Object traceRetry(ProceedingJoinPoint pjp, Retryable retryable) {
        String traceId = MDC.get("traceId");
        int retryCount = getRetryCount(traceId);
        
        // 记录重试次数
        MDC.put("retryCount", String.valueOf(retryCount));
        
        log.info("重试调用: {} 第 {} 次", 
            pjp.getSignature().toShortString(), retryCount);
        
        try {
            Object result = pjp.proceed();
            
            // 记录重试成功
            retryMetrics.recordSuccess(
                pjp.getTarget().getClass().getSimpleName(),
                retryCount
            );
            
            return result;
            
        } catch (Throwable e) {
            // 记录重试失败
            retryMetrics.recordFailure(
                pjp.getTarget().getClass().getSimpleName(),
                retryCount
            );
            
            throw e;
            
        } finally {
            MDC.remove("retryCount");
        }
    }
    
    private int getRetryCount(String traceId) {
        // 从 Redis 获取该 trace 的重试次数
        String key = "retry:count:" + traceId;
        Long count = redisTemplate.opsForValue().increment(key);
        redisTemplate.expire(key, 1, TimeUnit.HOURS);
        return count.intValue();
    }
}

// 2. 重试风暴检测
@Component
public class RetryStormDetector {
    
    private final LoadingCache<String, AtomicLong> retryCountCache = CacheBuilder.newBuilder()
        .expireAfterWrite(1, TimeUnit.MINUTES)
        .build(new CacheLoader<String, AtomicLong>() {
            @Override
            public AtomicLong load(String key) {
                return new AtomicLong(0);
            }
        });
    
    public void checkRetryStorm(String serviceName) {
        String key = "retry:" + serviceName + ":" + System.currentTimeMillis() / 60000;
        
        long count = retryCountCache.getUnchecked(key).incrementAndGet();
        
        // 每分钟超过 1000 次重试视为风暴
        if (count > 1000) {
            alertService.sendAlert(
                AlertLevel.CRITICAL,
                String.format("检测到重试风暴: %s 在 1 分钟内重试 %d 次", 
                    serviceName, count)
            );
        }
    }
}

// 1. 补偿任务健康检查
@Service
public class CompensationHealthChecker {
    
    @Scheduled(cron = "0 0 * * * ?")  // 每小时执行
    public void checkCompensationHealth() {
        // 统计各状态任务数
        Map<String, Long> statusCounts = compensationTaskMapper.countByStatus();
        
        // 生成健康报告
        CompensationHealthReport report = CompensationHealthReport.builder()
            .pendingCount(statusCounts.getOrDefault("PENDING", 0L))
            .processingCount(statusCounts.getOrDefault("PROCESSING", 0L))
            .successCount(statusCounts.getOrDefault("SUCCESS", 0L))
            .failedCount(statusCounts.getOrDefault("FAILED", 0L))
            .manualRequiredCount(statusCounts.getOrDefault("MANUAL_REQUIRED", 0L))
            .build();
        
        // 判断是否健康
        if (report.getManualRequiredCount() > 10) {
            alertService.sendAlert(
                AlertLevel.HIGH,
                String.format("有 %d 个补偿任务需要人工处理", report.getManualRequiredCount())
            );
        }
        
        if (report.getPendingCount() > 1000) {
            alertService.sendAlert(
                AlertLevel.WARNING,
                String.format("有 %d 个补偿任务待处理,可能存在积压", report.getPendingCount())
            );
        }
        
        // 记录指标
        meterRegistry.gauge("compensation.pending", report.getPendingCount());
        meterRegistry.gauge("compensation.manual", report.getManualRequiredCount());
    }
}

// 2. 补偿任务清理
@Service
public class CompensationTaskCleaner {
    
    @Scheduled(cron = "0 0 2 * * ?")  // 每天凌晨 2 点执行
    public void cleanOldTasks() {
        // 删除 30 天前已成功的任务
        int deleted = compensationTaskMapper.deleteOldSuccessTasks(
            LocalDateTime.now().minusDays(30)
        );
        
        log.info("清理了 {} 个已完成的补偿任务", deleted);
    }
}

// 3. 补偿任务大盘
@RestController
@RequestMapping("/api/compensation/dashboard")
public class CompensationDashboardController {
    
    @GetMapping("/overview")
    public CompensationDashboard getOverview() {
        return CompensationDashboard.builder()
            .statistics(compensationTaskMapper.getStatistics())
            .recentTasks(compensationTaskMapper.selectRecentTasks(10))
            .topFailures(compensationTaskMapper.selectTopFailures(10))
            .build();
    }
    
    @GetMapping("/task/{taskId}")
    public CompensationTaskDetail getTaskDetail(@PathVariable String taskId) {
        CompensationTask task = compensationTaskMapper.selectByTaskId(taskId);
        
        return CompensationTaskDetail.builder()
            .task(task)
            .retryHistory(compensationTaskMapper.selectRetryHistory(taskId))
            .relatedBusinessData(getRelatedBusinessData(task))
            .build();
    }
    
    @PostMapping("/task/{taskId}/manual-process")
    public String manualProcess(@PathVariable String taskId, 
                               @RequestBody ManualProcessRequest request) {
        CompensationTask task = compensationTaskMapper.selectByTaskId(taskId);
        
        // 执行人工处理逻辑
        manualProcessService.process(task, request);
        
        return "处理完成";
    }
}

// 1. 统计请求耗时分布
@Component
public class ResponseTimeAnalyzer {
    
    public void analyzeResponseTime(String serviceName) {
        // 获取最近 1 小时的响应时间
        List<Long> responseTimes = getRecentResponseTimes(serviceName, 1, TimeUnit.HOURS);
        
        // 计算分位数
        long p50 = calculatePercentile(responseTimes, 50);
        long p90 = calculatePercentile(responseTimes, 90);
        long p95 = calculatePercentile(responseTimes, 95);
        long p99 = calculatePercentile(responseTimes, 99);
        long max = Collections.max(responseTimes);
        
        // 建议超时时间
        long suggestedTimeout = (long) (p99 * 1.5);
        
        log.info("{} 响应时间统计: P50={}, P90={}, P95={}, P99={}, Max={}", 
            serviceName, p50, p90, p95, p99, max);
        log.info("建议超时时间: {}ms", suggestedTimeout);
    }
}

// 动态调整超时时间
@Configuration
public class DynamicTimeoutConfig {
    
    @Bean
    public DynamicTimeoutService dynamicTimeoutService() {
        return new DynamicTimeoutService();
    }
}

@Service
public class DynamicTimeoutService {
    
    private final LoadingCache<String, AdaptiveTimeout> timeoutCache = CacheBuilder.newBuilder()
        .refreshAfterWrite(5, TimeUnit.MINUTES)
        .build(new CacheLoader<String, AdaptiveTimeout>() {
            @Override
            public AdaptiveTimeout load(String serviceName) {
                return new AdaptiveTimeout(serviceName);
            }
        });
    
    public int getTimeout(String serviceName) {
        return timeoutCache.getUnchecked(serviceName).getTimeout();
    }
}

public class AdaptiveTimeout {
    private final String serviceName;
    private final SlidingWindow responseTimeWindow = new SlidingWindow(1000);
    
    public AdaptiveTimeout(String serviceName) {
        this.serviceName = serviceName;
    }
    
    public int getTimeout() {
        long p99 = responseTimeWindow.getP99();
        
        // P99 的 1.5 倍,最少 1 秒,最多 30 秒
        int timeout = (int) Math.max(1000, Math.min(p99 * 1.5, 30000));
        
        return timeout;
    }
    
    public void recordResponseTime(long responseTime) {
        responseTimeWindow.add(responseTime);
    }
}

// 1. 检测重试风暴
@Component
public class RetryStormMonitor {
    
    private final Map<String, AtomicLong> retryCounter = new ConcurrentHashMap<>();
    
    @Scheduled(fixedRate = 1000)
    public void checkRetryStorm() {
        long threshold = 1000;  // 每秒 1000 次
        
        retryCounter.forEach((serviceName, counter) -> {
            long count = counter.getAndSet(0);
            
            if (count > threshold) {
                // 触发告警
                alertService.sendAlert(
                    AlertLevel.CRITICAL,
                    String.format("检测到重试风暴: %s 每秒 %d 次重试", 
                        serviceName, count)
                );
                
                // 自动熔断
                circuitBreakerRegistry.circuitBreaker(serviceName)
                    .transitionToOpenState();
            }
        });
    }
    
    public void recordRetry(String serviceName) {
        retryCounter.computeIfAbsent(serviceName, k -> new AtomicLong(0))
            .incrementAndGet();
    }
}

// 2. 防止重试风暴
@Service
public class SafeRetryService {
    
    public <T> T executeWithRetry(String serviceName, Callable<T> task) {
        // 全局重试限流
        if (!globalRetryLimiter.tryAcquire()) {
            throw new RetryLimitExceededException("全局重试限流");
        }
        
        // 服务级重试限流
        RateLimiter serviceLimiter = getServiceRetryLimiter(serviceName);
        if (!serviceLimiter.tryAcquire()) {
            throw new RetryLimitExceededException("服务重试限流: " + serviceName);
        }
        
        // 执行重试
        return retryTemplate.execute(context -> {
            retryStormMonitor.recordRetry(serviceName);
            return task.call();
        });
    }
}

// 1. 补偿任务优先级调度
@Service
public class PriorityCompensationScheduler {
    
    @Scheduled(fixedDelay = 5000)
    public void processCompensationTasks() {
        // 按优先级查询任务
        List<CompensationTask> highPriorityTasks = taskMapper.selectHighPriorityTasks(50);
        List<CompensationTask> normalTasks = taskMapper.selectNormalTasks(30);
        List<CompensationTask> lowPriorityTasks = taskMapper.selectLowPriorityTasks(20);
        
        // 合并并按优先级处理
        List<CompensationTask> allTasks = new ArrayList<>();
        allTasks.addAll(highPriorityTasks);
        allTasks.addAll(normalTasks);
        allTasks.addAll(lowPriorityTasks);
        
        // 并行处理
        allTasks.parallelStream().forEach(this::processTask);
    }
}

// 2. 补偿任务批量处理
@Service
public class BatchCompensationService {
    
    public void batchProcess(List<CompensationTask> tasks) {
        // 按 businessType 分组
        Map<String, List<CompensationTask>> groupedTasks = tasks.stream()
            .collect(Collectors.groupingBy(CompensationTask::getBusinessType));
        
        // 并行处理每组
        groupedTasks.entrySet().parallelStream().forEach(entry -> {
            String businessType = entry.getKey();
            List<CompensationTask> batchTasks = entry.getValue();
            
            CompensationExecutor executor = executorMap.get(businessType);
            
            if (executor != null && executor.supportsBatch()) {
                // 批量处理
                List<CompensationResult> results = executor.batchExecute(batchTasks);
                
                // 更新状态
                for (int i = 0; i < batchTasks.size(); i++) {
                    CompensationTask task = batchTasks.get(i);
                    CompensationResult result = results.get(i);
                    
                    if (result.isSuccess()) {
                        task.setStatus("SUCCESS");
                    } else {
                        handleFailure(task, result.getErrorMessage());
                    }
                    
                    taskMapper.updateById(task);
                }
            } else {
                // 逐个处理
                batchTasks.forEach(this::processTask);
            }
        });
    }
}

// 1. 分析熔断触发原因
@Service
public class CircuitBreakerAnalyzer {
    
    public void analyzeCircuitBreaker(String serviceName) {
        CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker(serviceName);
        CircuitBreaker.Metrics metrics = cb.getMetrics();
        
        // 分析失败原因
        Map<Class<? extends Throwable>, Long> failureDistribution = 
            getFailureDistribution(serviceName);
        
        log.info("熔断器 {} 状态: {}", serviceName, cb.getState());
        log.info("故障率: {}%, 慢调用率: {}%", 
            metrics.getFailureRate(), metrics.getSlowCallRate());
        log.info("失败分布: {}", failureDistribution);
        
        // 判断是否为误触发
        if (metrics.getFailureRate() < 10 && cb.getState() == CircuitBreaker.State.OPEN) {
            log.warn("熔断器可能误触发,故障率 {}% 但熔断器已打开", metrics.getFailureRate());
            
            // 建议:降低故障率阈值或增加最小调用次数
        }
    }
}

// 2. 调整熔断器配置
@Configuration
public class AdaptiveCircuitBreakerConfig {
    
    @Bean
    public CircuitBreaker adaptiveCircuitBreaker() {
        CircuitBreakerConfig config = CircuitBreakerConfig.custom()
            .failureRateThreshold(50)              // 故障率 50%
            .slowCallRateThreshold(80)             // 慢调用率 80%
            .minimumNumberOfCalls(50)              // 最少调用 50 次
            .slidingWindowSize(100)                // 滑动窗口 100 次
            .waitDurationInOpenState(Duration.ofSeconds(20))  // 熔断 20 秒
            .permittedNumberOfCallsInHalfOpenState(10)
            .automaticTransitionFromOpenToHalfOpenEnabled(true)
            .recordExceptions(
                SocketTimeoutException.class,
                ConnectException.class,
                IOException.class
            )
            .ignoreExceptions(
                BusinessException.class,
                ValidationException.class
            )
            .build();
        
        return CircuitBreaker.of("adaptive", config);
    }
}

for (int attempt = 1; attempt <= 3; attempt++) {
    try {
        inventoryClient.deduct(request);
        return;
    } catch (TransientException ex) {
        Thread.sleep(attempt * 200L);
    }
}

compensationService.markPending(request.getOrderNo());

@Service
public class OrderService {
    
    @Autowired
    private InventoryClient inventoryClient;
    
    @Autowired
    private PaymentClient paymentClient;
    
    @Autowired
    private OrderMapper orderMapper;
    
    @Autowired
    private CompensationTaskMapper compensationTaskMapper;
    
    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;
    
    /**
     * 创建订单 - 完整的容错示例
     */
    @Transactional
    public OrderResult createOrder(CreateOrderRequest request) {
        String orderNo = generateOrderNo();
        String requestId = request.getRequestId();
        
        // 1. 幂等性检查
        Order existingOrder = orderMapper.selectByRequestId(requestId);
        if (existingOrder != null) {
            return OrderResult.success(existingOrder);
        }
        
        // 2. 创建订单(初始状态)
        Order order = new Order();
        order.setOrderNo(orderNo);
        order.setUserId(request.getUserId());
        order.setStatus("CREATED");
        order.setRequestId(requestId);
        orderMapper.insert(order);
        
        try {
            // 3. 扣减库存(带熔断和重试)
            deductInventoryWithRetry(order, request);
            
            // 4. 创建支付(带熔断和降级)
            createPaymentWithFallback(order, request);
            
            // 5. 订单状态更新
            order.setStatus("PAID");
            orderMapper.updateById(order);
            
            return OrderResult.success(order);
            
        } catch (InsufficientInventoryException e) {
            // 库存不足,取消订单
            cancelOrder(order, "库存不足");
            return OrderResult.failure("库存不足");
            
        } catch (PaymentException e) {
            // 支付失败,回滚库存并取消订单
            rollbackInventory(order);
            cancelOrder(order, "支付失败");
            return OrderResult.failure("支付失败");
            
        } catch (Exception e) {
            // 其他异常,创建补偿任务
            createCompensationTask(order, "ORDER_CREATE_FAILED");
            return OrderResult.processing(orderNo, "订单处理中,请稍后查询结果");
        }
    }
    
    /**
     * 扣减库存 - 带重试
     */
    @Retryable(
        value = {SocketTimeoutException.class, ConnectException.class},
        maxAttempts = 3,
        backoff = @Backoff(delay = 100, multiplier = 2)
    )
    private void deductInventoryWithRetry(Order order, CreateOrderRequest request) {
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitBreaker("inventoryService");
        
        Supplier<DeductResult> supplier = CircuitBreaker
            .decorateSupplier(circuitBreaker, () -> {
                DeductInventoryRequest deductRequest = new DeductInventoryRequest();
                deductRequest.setRequestId(request.getRequestId());
                deductRequest.setOrderNo(order.getOrderNo());
                deductRequest.setItems(request.getItems());
                
                return inventoryClient.deduct(deductRequest);
            });
        
        DeductResult result = Try.ofSupplier(supplier)
            .recover(throwable -> {
                log.warn("库存服务不可用", throwable);
                throw new ServiceUnavailableException("库存服务不可用");
            })
            .get();
        
        if (!result.isSuccess()) {
            throw new InsufficientInventoryException("库存不足");
        }
        
        // 记录扣减日志
        DeductLog log = new DeductLog();
        log.setRequestId(request.getRequestId());
        log.setOrderNo(order.getOrderNo());
        log.setStatus("SUCCESS");
        deductLogMapper.insert(log);
    }
    
    /**
     * 创建支付 - 带熔断降级
     */
    private void createPaymentWithFallback(Order order, CreateOrderRequest request) {
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitBreaker("paymentService");
        
        Supplier<PaymentResult> supplier = CircuitBreaker
            .decorateSupplier(circuitBreaker, () -> {
                PaymentRequest paymentRequest = new PaymentRequest();
                paymentRequest.setRequestId(request.getRequestId());
                paymentRequest.setOrderNo(order.getOrderNo());
                paymentRequest.setAmount(request.getTotalAmount());
                
                return paymentClient.pay(paymentRequest);
            });
        
        PaymentResult result = Try.ofSupplier(supplier)
            .recover(throwable -> {
                log.warn("支付服务不可用,使用降级策略", throwable);
                
                // 降级策略:创建补偿任务,异步处理
                createCompensationTask(order, "PAYMENT_TIMEOUT");
                
                return PaymentResult.processing();
            })
            .get();
        
        if (result.isFailed()) {
            throw new PaymentException("支付失败");
        }
    }
    
    /**
     * 回滚库存
     */
    private void rollbackInventory(Order order) {
        DeductLog log = deductLogMapper.selectByOrderNo(order.getOrderNo());
        
        if (log != null && "SUCCESS".equals(log.getStatus())) {
            // 恢复库存
            inventoryClient.restore(order.getOrderNo());
            
            log.setStatus("ROLLBACK");
            deductLogMapper.updateById(log);
        }
    }
    
    /**
     * 取消订单
     */
    private void cancelOrder(Order order, String reason) {
        order.setStatus("CANCELLED");
        order.setCancelReason(reason);
        order.setCancelTime(LocalDateTime.now());
        orderMapper.updateById(order);
    }
    
    /**
     * 创建补偿任务
     */
    private void createCompensationTask(Order order, String compensationType) {
        CompensationTask task = new CompensationTask();
        task.setTaskId(UUID.randomUUID().toString());
        task.setBusinessType("ORDER");
        task.setBusinessId(order.getOrderNo());
        task.setCompensationType(compensationType);
        task.setOriginalData(JsonUtils.toJson(order));
        task.setStatus("PENDING");
        task.setRetryCount(0);
        task.setMaxRetryCount(5);
        task.setNextRetryTime(LocalDateTime.now().plusMinutes(1));
        task.setCreatedAt(LocalDateTime.now());
        
        compensationTaskMapper.insert(task);
    }
}

// 错误:不设超时
OkHttpClient client = new OkHttpClient.Builder().build();

// 正确:设置合理的超时
OkHttpClient client = new OkHttpClient.Builder()
    .connectTimeout(2, TimeUnit.SECONDS)
    .readTimeout(5, TimeUnit.SECONDS)
    .writeTimeout(5, TimeUnit.SECONDS)
    .build();

// 错误:所有异常都重试
@Retryable(maxAttempts = 3)
public void process(Request request) {
    // ...
}

// 正确:只重试可恢复异常
@Retryable(
    value = {SocketTimeoutException.class, ConnectException.class},
    maxAttempts = 3
)
public void process(Request request) {
    // ...
}

// 只在离调用最近的一层做重试
// 其他层快速失败

// API 网关:不重试
// Spring Cloud:不重试
ribbon:
  MaxAutoRetries: 0
  MaxAutoRetriesNextServer: 0

// HTTP 客户端:重试
@Retryable(maxAttempts = 3)
public Response call(Request request) {
    return httpClient.execute(request);
}

// 定义清晰的状态
public enum OrderStatus {
    CREATED("已创建"),
    INVENTORY_DEDUCTED("已扣减库存"),
    PAYMENT_PROCESSING("支付处理中"),
    PAID("已支付"),
    COMPLETED("已完成"),
    CANCELLED("已取消"),
    COMPENSATING("补偿中")
}

// 记录每个步骤
public class OrderProcessLog {
    private Long orderId;
    private String step;      // INVENTORY_DEDUCT, PAYMENT_CREATE 等
    private String status;    // SUCCESS, FAILED, PROCESSING
    private String data;      // 步骤数据
    private LocalDateTime createdAt;
}

// 补偿失败后转人工
private void handleCompensationFailure(CompensationTask task, Exception e) {
    task.setRetryCount(task.getRetryCount() + 1);
    
    if (task.getRetryCount() >= task.getMaxRetryCount()) {
        // 标记需人工处理
        task.setStatus("MANUAL_REQUIRED");
        
        // 发送告警
        alertService.sendAlert(
            AlertLevel.HIGH,
            String.format("补偿任务 %s 需人工处理,原因: %s", 
                task.getTaskId(), e.getMessage())
        );
        
        // 创建工单
        ticketService.createTicket(
            "COMPENSATION_MANUAL",
            task.getTaskId(),
            String.format("补偿任务失败,需人工处理: %s", e.getMessage())
        );
    }
}

// 假设系统有 200 个线程,请求正常响应时间 100ms
// 如果不设超时,某个下游服务 RT 飙升到 30 秒
// 结果:200 个线程很快被占满,系统完全不可用

// 正确做法:设置 5 秒超时
// 结果:最多 200 × 5s / 0.1s = 10000 个请求被处理,超时的快速失败

// 方案 1:数据库唯一约束
INSERT INTO payment (order_no, ...)
VALUES (?, ...)
ON DUPLICATE KEY UPDATE ...

// 方案 2:Token 机制
if (!tokenService.validateAndConsume(token)) {
    throw new DuplicateRequestException();
}

// 方案 3:去重表
if (idempotentLogMapper.exists(requestId)) {
    return cachedResult;
}

// 回滚:数据库事务
@Transactional
public void transfer(Long fromId, Long toId, BigDecimal amount) {
    accountDao.debit(fromId, amount);
    accountDao.credit(toId, amount);
    // 失败时数据库自动回滚
}

// 补偿:分布式场景
public void transfer(Long fromId, Long toId, BigDecimal amount) {
    // 1. 扣减 A 账户(成功)
    accountService.debit(fromId, amount);
    
    // 2. 增加 B 账户(失败)
    try {
        accountService.credit(toId, amount);
    } catch (Exception e) {
        // 3. 补偿:给 A 账户加回金额
        compensationService.credit(fromId, amount);
    }
}

// 短超时:2 秒连接,5 秒读取
OkHttpClient client = new OkHttpClient.Builder()
    .connectTimeout(2, TimeUnit.SECONDS)
    .readTimeout(5, TimeUnit.SECONDS)
    .build();

// 限次重试:最多 3 次
@Retryable(maxAttempts = 3, backoff = @Backoff(delay = 100))
public ThirdPartyResult callThirdParty(Request request) {
    // ...
}

// 补偿:失败后异步处理
@Recover
public void recover(Exception e, Request request) {
    compensationTaskMapper.insert(createTask(request));
}

public class ReliableRetryTemplate {
    
    private int maxRetries = 3;
    private RetryPolicy retryPolicy = new ExponentialBackoffPolicy(100, 2.0, 10000);
    private Predicate<Exception> retryPredicate = new RetryableExceptionPredicate();
    
    public <T> T execute(Callable<T> task) {
        Exception lastException = null;
        RetryContext context = new RetryContext();
        
        for (int attempt = 1; attempt <= maxRetries; attempt++) {
            try {
                // 执行任务
                T result = task.call();
                
                // 记录成功
                retryMetrics.recordSuccess(attempt);
                
                return result;
                
            } catch (Exception e) {
                lastException = e;
                context.recordException(e);
                
                // 判断是否应该重试
                if (!shouldRetry(e, attempt, context)) {
                    break;
                }
                
                // 等待下次重试
                if (attempt < maxRetries) {
                    long waitTime = retryPolicy.getNextWaitTime(attempt);
                    sleep(waitTime);
                    context.addWaitTime(waitTime);
                }
            }
        }
        
        // 重试失败
        retryMetrics.recordFailure(maxRetries);
        throw new RetryExhaustedException("重试失败", lastException);
    }
    
    private boolean shouldRetry(Exception e, int attempt, RetryContext context) {
        // 已达最大重试次数
        if (attempt >= maxRetries) {
            return false;
        }
        
        // 超过最大等待时间
        if (context.getTotalWaitTime() > 60000) {
            return false;
        }
        
        // 判断异常类型
        return retryPredicate.test(e);
    }
}

@Service
public class IdempotentService {
    
    @Transactional
    public PaymentResult pay(PaymentRequest request) {
        String requestId = request.getRequestId();
        
        // 1. 查询历史记录
        IdempotentLog log = logMapper.selectByRequestId(requestId);
        
        if (log != null) {
            // 2. 状态机判断
            switch (log.getStatus()) {
                case "SUCCESS":
                    // 已成功,返回缓存结果
                    return JsonUtils.fromJson(log.getResponse(), PaymentResult.class);
                    
                case "PROCESSING":
                    // 处理中,拒绝重复请求
                    throw new ConcurrentProcessingException("请求处理中");
                    
                case "FAILED":
                    // 之前失败,可以重试
                    log.setStatus("PROCESSING");
                    logMapper.updateById(log);
                    break;
            }
        } else {
            // 3. 创建新记录
            log = new IdempotentLog();
            log.setRequestId(requestId);
            log.setBusinessType("PAYMENT");
            log.setBusinessKey(request.getOrderNo());
            log.setRequest(JsonUtils.toJson(request));
            log.setStatus("PROCESSING");
            logMapper.insert(log);
        }
        
        try {
            // 4. 执行业务逻辑
            PaymentResult result = doPayment(request);
            
            // 5. 标记成功
            log.setStatus("SUCCESS");
            log.setResponse(JsonUtils.toJson(result));
            logMapper.updateById(log);
            
            return result;
            
        } catch (Exception e) {
            // 6. 标记失败
            log.setStatus("FAILED");
            log.setErrorMessage(e.getMessage());
            logMapper.updateById(log);
            throw e;
        }
    }
}

┌─────────────────┐
│  业务服务       │
│  创建补偿任务   │
└────────┬────────┘
         │
         ▼
┌─────────────────┐
│  补偿任务表     │
│  持久化存储     │
└────────┬────────┘
         │
         ▼
┌─────────────────┐
│  补偿调度器     │
│  定时扫描       │
└────────┬────────┘
         │
         ▼
┌─────────────────┐
│  补偿执行器     │
│  执行具体逻辑   │
└────────┬────────┘
         │
         ▼
┌─────────────────┐
│  监控告警       │
│  人工兜底       │
└─────────────────┘

// 1. 补偿任务定义
public class CompensationTask {
    private String taskId;
    private String businessType;
    private String businessId;
    private String compensationType;
    private String originalData;
    private String status;
    private int retryCount;
    private int maxRetryCount;
    private LocalDateTime nextRetryTime;
}

// 2. 补偿执行器接口
public interface CompensationExecutor {
    String getCompensationType();
    CompensationResult execute(CompensationTask task);
    default boolean supportsBatch() { return false; }
    default List<CompensationResult> batchExecute(List<CompensationTask> tasks) {
        return tasks.stream()
            .map(this::execute)
            .collect(Collectors.toList());
    }
}

// 3. 补偿调度器
@Service
public class CompensationScheduler {
    
    @Scheduled(fixedDelay = 5000)
    public void processTasks() {
        // 查询待处理任务
        List<CompensationTask> tasks = taskMapper.selectPendingTasks(
            LocalDateTime.now(),
            100
        );
        
        // 按类型分组
        Map<String, List<CompensationTask>> groupedTasks = tasks.stream()
            .collect(Collectors.groupingBy(CompensationTask::getCompensationType));
        
        // 并行处理
        groupedTasks.entrySet().parallelStream().forEach(entry -> {
            CompensationExecutor executor = executorMap.get(entry.getKey());
            if (executor != null) {
                entry.getValue().forEach(task -> {
                    try {
                        CompensationResult result = executor.execute(task);
                        handleResult(task, result);
                    } catch (Exception e) {
                        handleFailure(task, e);
                    }
                });
            }
        });
    }
    
    private void handleResult(CompensationTask task, CompensationResult result) {
        if (result.isSuccess()) {
            task.setStatus("SUCCESS");
            taskMapper.updateById(task);
        } else {
            handleFailure(task, new Exception(result.getErrorMessage()));
        }
    }
    
    private void handleFailure(CompensationTask task, Exception e) {
        task.setRetryCount(task.getRetryCount() + 1);
        
        if (task.getRetryCount() >= task.getMaxRetryCount()) {
            task.setStatus("MANUAL_REQUIRED");
            alertService.sendAlert(AlertLevel.HIGH,
                "补偿任务需人工处理: " + task.getTaskId());
        } else {
            long nextDelay = calculateBackoff(task.getRetryCount());
            task.setNextRetryTime(LocalDateTime.now().plusSeconds(nextDelay));
            task.setStatus("PENDING");
        }
        
        taskMapper.updateById(task);
    }
}

public class AdaptiveCircuitBreakerConfig {
    
    public CircuitBreakerConfig createConfig(ServiceMetrics metrics) {
        // 基于历史数据动态计算
        double avgFailureRate = metrics.getAvgFailureRate();
        double avgResponseTime = metrics.getAvgResponseTime();
        
        // 故障率阈值:历史平均故障率 × 2
        float failureRateThreshold = (float) Math.min(avgFailureRate * 2, 50);
        
        // 慢调用时间阈值:历史 P99 × 1.5
        long slowCallDuration = (long) (metrics.getP99ResponseTime() * 1.5);
        
        // 滑动窗口:基于 QPS 调整
        int slidingWindowSize = (int) Math.max(100, metrics.getQPS() * 10);
        
        return CircuitBreakerConfig.custom()
            .failureRateThreshold(failureRateThreshold)
            .slowCallDurationThreshold(Duration.ofMillis(slowCallDuration))
            .minimumNumberOfCalls(slidingWindowSize / 2)
            .slidingWindowSize(slidingWindowSize)
            .waitDurationInOpenState(Duration.ofSeconds(30))
            .build();
    }
}

// 方案 1:查询确认
public void processWithConfirm(Request request) {
    try {
        Result result = service.call(request);
        handleSuccess(result);
    } catch (TimeoutException e) {
        // 超时后,先查询状态
        Status status = service.queryStatus(request.getId());
        
        if (status.isSuccess()) {
            // 已成功,直接处理
            handleSuccess(status.getResult());
        } else if (status.isFailed()) {
            // 已失败,可以重试
            retry(request);
        } else {
            // 仍未知,创建补偿任务
            createCompensationTask(request);
        }
    }
}

// 方案 2:幂等重试
public void processWithIdempotentRetry(Request request) {
    // 确保请求幂等
    request.setRequestId(generateRequestId());
    
    for (int i = 0; i < maxRetries; i++) {
        try {
            Result result = service.call(request);
            handleSuccess(result);
            return;
        } catch (TimeoutException e) {
            if (i == maxRetries - 1) {
                // 最后一次重试失败,进入补偿
                createCompensationTask(request);
            }
        }
    }
}

// 方案 3:TCC 模式
public void processWithTCC(Request request) {
    String txId = generateTxId();
    
    try {
        // 1. Try 阶段
        service.tryAction(txId, request);
        
        // 2. Confirm 阶段
        service.confirm(txId);
        
    } catch (TimeoutException e) {
        // 3. Cancel 阶段
        service.cancel(txId);
        
        // 记录补偿
        createCompensationTask(txId, request);
    }
}

// 方案 1:单层重试
// 只在离调用最近的一层做重试,其他层快速失败

// 网关层:不重试
zuul:
  retryable: false

// 服务层:不重试
ribbon:
  MaxAutoRetries: 0

// 客户端:重试
@Retryable(maxAttempts = 3)
public Response call(Request request) {
    return httpClient.execute(request);
}

// 方案 2:全局限流
@Service
public class GlobalRetryLimiter {
    
    private final RateLimiter globalLimiter = RateLimiter.create(1000);  // 全局 1000 QPS
    private final Map<String, RateLimiter> serviceLimiters = new ConcurrentHashMap<>();
    
    public boolean tryAcquire(String serviceName) {
        // 全局限流
        if (!globalLimiter.tryAcquire()) {
            return false;
        }
        
        // 服务级限流
        RateLimiter serviceLimiter = serviceLimiters.computeIfAbsent(
            serviceName,
            k -> RateLimiter.create(100)  // 每个服务 100 QPS
        );
        
        return serviceLimiter.tryAcquire();
    }
}

// 方案 3:指数退避 + 抖动
@Retryable(
    maxAttempts = 3,
    backoff = @Backoff(
        delay = 100,
        multiplier = 2,
        random = true  // 随机抖动
    )
)
public Response call(Request request) {
    return httpClient.execute(request);
}

// 方案 4:熔断保护
public Response callWithCircuitBreaker(Request request) {
    CircuitBreaker cb = circuitBreakerRegistry.circuitBreaker("service");
    
    if (cb.getState() == CircuitBreaker.State.OPEN) {
        // 熔断状态,快速失败
        throw new CircuitBreakerOpenException();
    }
    
    return cb.executeSupplier(() -> httpClient.execute(request));
}

// 方案 5:重试监控和告警
@Component
public class RetryStormMonitor {
    
    private final Map<String, AtomicLong> retryCounter = new ConcurrentHashMap<>();
    
    @Scheduled(fixedRate = 1000)
    public void checkRetryStorm() {
        retryCounter.forEach((service, counter) -> {
            long count = counter.getAndSet(0);
            
            if (count > 1000) {  // 每秒超过 1000 次
                alertService.sendAlert(
                    AlertLevel.CRITICAL,
                    "检测到重试风暴: " + service
                );
                
                // 自动熔断
                circuitBreakerRegistry.circuitBreaker(service)
                    .transitionToOpenState();
            }
        });
    }
}