服务端出现故障时该如何应对

// Sentinel 1.8+ 流控规则
FlowRule flowRule = new FlowRule()
    .setResource("orderService.createOrder")
    .setGrade(RuleConstant.FLOW_GRADE_QPS)   // QPS 限流
    .setCount(1000)                           // 阈值 1000 QPS
    .setControlBehavior(RuleConstant.CONTROL_BEHAVIOR_WARM_UP)  // 预热模式
    .setWarmUpPeriodSec(10);                  // 预热时长 10s
 
// 线程数限流 —— 更适合微服务场景
FlowRule threadRule = new FlowRule("orderService.createOrder")
    .setGrade(RuleConstant.FLOW_GRADE_THREAD) // 线程数限流
    .setCount(200);                           // 最大并发线程数 200
 
// 热点参数限流 —— 针对特定参数值限流
ParamFlowRule hotRule = new ParamFlowRule("orderService.queryOrder")
    .setParamIdx(0).setCount(50)
    .setGrade(RuleConstant.FLOW_GRADE_QPS);
hotRule.setParamFlowItemList(Collections.singletonList(
    new ParamFlowItem().setObject("12345").setClassType(int.class.getName()).setCount(500)
));

// Sentinel 自适应系统保护
SystemRule systemRule = new SystemRule();
systemRule.setHighestCpuUsage(0.8);   // CPU 使用率阈值 80%
systemRule.setAvgRt(500);             // 平均响应时间阈值 500ms
systemRule.setMaxThread(800);         // 最大并发线程数
systemRule.setQps(5000);              // 系统总 QPS 阈值
SystemRuleManager.loadRules(Collections.singletonList(systemRule));

-- Redis Lua 令牌桶限流脚本
local key = KEYS[1]
local capacity = tonumber(ARGV[1])   -- 桶容量
local rate = tonumber(ARGV[2])       -- 令牌填充速率
local now = tonumber(ARGV[3])        -- 当前时间戳（毫秒）
local requested = tonumber(ARGV[4])  -- 请求令牌数
 
local tokens_key = key .. ":tokens"
local timestamp_key = key .. ":ts"
 
local last_tokens = tonumber(redis.call("get", tokens_key)) or capacity
local last_refreshed = tonumber(redis.call("get", timestamp_key)) or now
 
local delta = math.max(0, now - last_refreshed)
local filled_tokens = math.min(capacity, last_tokens + (delta * rate / 1000))
 
local allowed = filled_tokens >= requested
if allowed then filled_tokens = filled_tokens - requested end
 
redis.call("set", tokens_key, filled_tokens)
redis.call("set", timestamp_key, now)
return { allowed and 1 or 0, filled_tokens }

// Sentinel 1.8+ 熔断规则
 
// 策略一：慢调用比例
DegradeRule slowCallRule = new DegradeRule("orderService")
    .setGrade(CircuitBreakerStrategy.SLOW_REQUEST_RATIO.getType())
    .setCount(500)           // 慢调用临界 RT：500ms
    .setSlowRatioThreshold(0.6)  // 慢调用比例阈值：60%
    .setTimeWindow(30)       // 熔断持续时间：30s
    .setMinRequestAmount(10) // 最小请求数：10
    .setStatIntervalMs(10000); // 统计时长：10s
 
// 策略二：异常比例
DegradeRule errorRatioRule = new DegradeRule("orderService")
    .setGrade(CircuitBreakerStrategy.ERROR_RATIO.getType())
    .setCount(0.5)           // 异常比例阈值：50%
    .setTimeWindow(20)
    .setMinRequestAmount(5)
    .setStatIntervalMs(5000);
 
// 策略三：异常数
DegradeRule errorCountRule = new DegradeRule("orderService")
    .setGrade(CircuitBreakerStrategy.ERROR_COUNT.getType())
    .setCount(10)            // 异常数阈值：10
    .setTimeWindow(60)
    .setMinRequestAmount(5)
    .setStatIntervalMs(10000);
 
DegradeRuleManager.loadRules(
    Arrays.asList(slowCallRule, errorRatioRule, errorCountRule));

# Resilience4j 2.x 熔断器配置
resilience4j:
  circuitbreaker:
    configs:
      default:
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 100
        minimumNumberOfCalls: 10
        failureRateThreshold: 50
        slowCallRateThreshold: 80
        slowCallDurationThreshold: 3s
        waitDurationInOpenState: 30s
        permittedNumberOfCallsInHalfOpenState: 5
        automaticTransitionFromOpenToHalfOpenEnabled: true
        recordExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
    instances:
      orderService:
        baseConfig: default
        failureRateThreshold: 60

// Resilience4j 2.x 编程式熔断
CircuitBreakerConfig config = CircuitBreakerConfig.custom()
    .slidingWindowSize(100)
    .failureRateThreshold(50.0f)
    .slowCallDurationThreshold(Duration.ofSeconds(3))
    .waitDurationInOpenState(Duration.ofSeconds(30))
    .permittedNumberOfCallsInHalfOpenState(5)
    .build();
 
CircuitBreaker circuitBreaker = CircuitBreaker.of("orderService", config);
 
// 使用熔断器包装调用，结合 Fallback
Supplier<Order> supplier = SupplierUtils.recoverWithSupplier(
    CircuitBreaker.decorateSupplier(circuitBreaker,
        () -> orderService.getOrder(orderId)),
    throwable -> Order.defaultOrder());

# Istio OutlierDetection 熔断配置
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: order-service
spec:
  host: order-service.default.svc.cluster.local
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100          # 最大连接数
      http:
        h2UpgradePolicy: DEFAULT
        http1MaxPendingRequests: 50  # 最大挂起请求数
        http2MaxRequests: 200        # 最大请求数
        maxRequestsPerConnection: 10 # 每连接最大请求数
    outlierDetection:
      consecutive5xxErrors: 5        # 连续 5xx 错误数
      interval: 30s                  # 检测间隔
      baseEjectionTime: 60s          # 基础驱逐时间
      maxEjectionPercent: 50         # 最大驱逐比例
      minHealthPercent: 25           # 最小健康实例比例
      consecutiveGatewayErrors: 3    # 连续网关错误数（Istio 1.22+）

// 基于 Nacos 配置中心的动态开关降级
@Configuration
@RefreshScope
public class DegradationConfig {
    @Value("${degradation.recommend.enabled:false}")
    private boolean recommendEnabled;
 
    @Value("${degradation.comment.enabled:false}")
    private boolean commentEnabled;
}
 
@Service
public class RecommendService {
    @Autowired private DegradationConfig config;
    @Autowired private RecommendClient client;
 
    public List<Product> getRecommendations(String userId) {
        if (config.isRecommendEnabled()) {
            return getDefaultHotProducts();  // 降级：返回默认热门商品
        }
        return client.getPersonalizedRecommendations(userId);
    }
}

// Sentinel Fallback 降级
@SentinelResource(value = "getOrder",
    fallback = "getOrderFallback",          // 异常降级
    blockHandler = "getOrderBlockHandler")  // 限流/熔断降级
public Order getOrder(String orderId) {
    return orderClient.getOrder(orderId);
}
 
public Order getOrderFallback(String orderId, Throwable t) {
    return Order.cachedOrder(orderId);  // 返回缓存数据
}
 
public Order getOrderBlockHandler(String orderId, BlockException ex) {
    return Order.defaultOrder();  // 返回默认值
}

// Resilience4j 2.x Fallback
@CircuitBreaker(name = "orderService", fallbackMethod = "getOrderFallback")
@Bulkhead(name = "orderService", fallbackMethod = "getOrderFallback")
public Order getOrder(String orderId) {
    return orderClient.getOrder(orderId);
}
 
private Order getOrderFallback(String orderId, Throwable t) {
    return orderCacheService.getCachedOrder(orderId)
        .orElse(Order.defaultOrder());
}

// Sentinel 线程数限流 —— 实现信号量隔离
FlowRule isolationRule = new FlowRule("orderService")
    .setGrade(RuleConstant.FLOW_GRADE_THREAD)  // 按并发线程数
    .setCount(50);                             // 最大并发线程数 50
 
// 为不同依赖设置不同的线程数上限
FlowRule dependencyA = new FlowRule("inventoryService")
    .setGrade(RuleConstant.FLOW_GRADE_THREAD)
    .setCount(30);
 
FlowRule dependencyB = new FlowRule("paymentService")
    .setGrade(RuleConstant.FLOW_GRADE_THREAD)
    .setCount(20);

# Resilience4j 2.x Bulkhead 配置
resilience4j:
  bulkhead:
    instances:
      orderService:
        maxConcurrentCalls: 30       # 最大并发调用数
        maxWaitDuration: 50ms        # 最大等待时间
  thread-pool-bulkhead:
    instances:
      inventoryService:
        maxThreadPoolSize: 15        # 最大线程数
        coreThreadPoolSize: 8        # 核心线程数
        queueCapacity: 30            # 队列容量

// 组合使用：Bulkhead + CircuitBreaker + RateLimiter
Supplier<Order> supplier = SupplierUtils.recoverWithSupplier(
    Bulkhead.decorateSupplier(bulkhead,
        CircuitBreaker.decorateSupplier(circuitBreaker,
            RateLimiter.decorateSupplier(rateLimiter,
                () -> orderService.getOrder(orderId)))),
    throwable -> Order.defaultOrder());

# Istio 多区域流量调度 + Locality 故障转移
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: order-service
spec:
  host: order-service
  trafficPolicy:
    loadBalancer:
      localityLbSetting:
        enabled: true
        failover:
          - { from: cn-beijing-a, to: cn-beijing-b }
          - { from: cn-beijing-b, to: cn-beijing-a }
    outlierDetection:
      consecutive5xxErrors: 3
      interval: 10s
      baseEjectionTime: 30s
      maxEjectionPercent: 100
  subsets:
    - name: az-a
      labels: { topology.kubernetes.io/zone: cn-beijing-a }
    - name: az-b
      labels: { topology.kubernetes.io/zone: cn-beijing-b }

# Kubernetes Liveness / Readiness Probe
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  replicas: 10
  template:
    spec:
      containers:
        - name: order-service
          image: order-service:2.0.0
          livenessProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8080
            initialDelaySeconds: 60
            periodSeconds: 10
            failureThreshold: 3           # 连续 3 次失败则重启
          readinessProbe:
            httpGet:
              path: /actuator/health/readiness
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 5
            failureThreshold: 3           # 连续 3 次失败则摘除流量

# Chaos Mesh 网络延迟注入
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: order-service-delay
  namespace: chaos-testing
spec:
  action: delay
  mode: one
  selector:
    namespaces: [production]
    labelSelectors: { app: order-service }
  delay:
    latency: "500ms"
    jitter: "100ms"
  duration: "5m"
---
# Chaos Mesh Pod 故障注入
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: order-service-pod-failure
  namespace: chaos-testing
spec:
  action: pod-failure
  mode: fixed-percent
  value: "10"               # 影响 10% 的 Pod
  selector:
    namespaces: [production]
    labelSelectors: { app: order-service }
  duration: "3m"

# Litmus ChaosEngine 配置
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: order-service-chaos
  namespace: chaos-testing
spec:
  appinfo:
    appns: production
    applabel: "app=order-service"
  chaosServiceAccount: litmus-admin
  experiments:
    - name: pod-network-latency
      spec:
        components:
          env:
            - { name: TOTAL_CHAOS_DURATION, value: "60" }
            - { name: LATENCY, value: "500" }
    - name: pod-delete
      spec:
        components:
          env:
            - { name: TOTAL_CHAOS_DURATION, value: "30" }
            - { name: CHAOS_INTERVAL, value: "10" }