混沌工程与故障注入

# 稳态假设示例
steady_state_hypothesis:
  # 核心业务指标
  - metric: "订单创建成功率"
    query: 'success_rate{endpoint="/api/orders"}'
    threshold: "> 99.9%"
    source: "Prometheus"

  - metric: "支付 P99 延迟"
    query: 'histogram_quantile(0.99, rate(payment_duration_seconds_bucket[5m]))'
    threshold: "< 500ms"
    source: "Prometheus"

  # 基础设施指标
  - metric: "CPU 使用率"
    query: 'avg(cpu_usage{service="order-service"})'
    threshold: "< 80%"
    source: "Prometheus"

  # 自动终止条件（任一指标破坏则立即终止实验）
  abort_on_breach: true
  abort_conditions:
    - "订单创建成功率 < 99%"
    - "支付 P99 延迟 > 2000ms"
    - "错误率 > 1%"

# 实验 1: 网络延迟注入
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: order-service-network-delay
  namespace: chaos-testing
spec:
  action: delay
  mode: all
  selector:
    namespaces:
      - production
    labelSelectors:
      app: order-service
  delay:
    latency: "500ms"
    correlation: "0"
    jitter: "50ms"
  direction: to      # 出站延迟
  target:
    selector:
      namespaces:
        - production
      labelSelectors:
        app: payment-service  # 订单 → 支付的通信延迟
    mode: all
  duration: "5m"
  scheduler:
    cron: "@every 1h"  # 每小时执行一次
---
# 实验 2: Pod 故障注入
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: payment-service-pod-kill
  namespace: chaos-testing
spec:
  action: pod-kill   # 杀死 Pod
  mode: fixed-percent
  value: "30"         # 杀死 30% 的 Pod
  selector:
    namespaces:
      - production
    labelSelectors:
      app: payment-service
  duration: "0"       # 立即生效
  scheduler:
    cron: "0 10 * * 2"  # 每周二 10:00
---
# 实验 3: 磁盘 IO 压力
apiVersion: chaos-mesh.org/v1alpha1
kind: IOChaos
metadata:
  name: database-disk-io-latency
  namespace: chaos-testing
spec:
  action: latency
  mode: all
  selector:
    labelSelectors:
      app: postgres
  volumePath: "/var/lib/postgresql/data"
  path: "/var/lib/postgresql/data/**/*"
  delay: "100ms"      # 磁盘 IO 延迟 100ms
  percent: 50          # 50% 的 IO 操作被延迟
  duration: "10m"

# 多故障组合实验：模拟"网络延迟 + 节点宕机"组合故障
apiVersion: chaos-mesh.org/v1alpha1
kind: Workflow
metadata:
  name: combined-failure-experiment
  namespace: chaos-testing
spec:
  entry: main
  templates:
    - name: main
      templateType: Serial
      children:
        - baseline-check
        - network-delay
        - node-drain
        - combined-failure
        - recovery-check

    # 步骤1: 基线检查
    - name: baseline-check
      templateType: Task
      task:
        container:
          image: stefanprodan/podinfo:latest
          command: ["sh", "-c"]
          args:
            - |
              # 检查基线指标
              CURRENT_RATE=$(curl -s prometheus:9090/api/v1/query \
                --data-urlencode 'query=success_rate{endpoint="/api/orders"}' | jq '.data.result[0].value[1]')
              echo "当前成功率: $CURRENT_RATE"
              if [ $(echo "$CURRENT_RATE < 0.999" | bc -l) -eq 1 ]; then
                echo "基线不达标，终止实验"
                exit 1
              fi

    # 步骤2: 网络延迟
    - name: network-delay
      templateType: NetworkChaos
      networkChaos:
        action: delay
        mode: all
        selector:
          labelSelectors:
            app: order-service
        delay:
          latency: "300ms"
        duration: "5m"

    # 步骤3: 节点宕机（与网络延迟串行）
    - name: node-drain
      templateType: PodChaos
      podChaos:
        action: pod-kill
        mode: fixed-percent
        value: "20"
        selector:
          labelSelectors:
            app: order-service
        duration: "0"

    # 步骤4: 组合故障（同时执行）
    - name: combined-failure
      templateType: Parallel
      children:
        - network-delay-2
        - pod-kill-2

    # 步骤5: 恢复验证
    - name: recovery-check
      templateType: Task
      task:
        container:
          image: curlimages/curl
          command: ["sh", "-c"]
          args:
            - |
              # 等待 5 分钟让系统恢复
              sleep 300
              # 验证系统是否回到稳态
              curl -s prometheus:9090/api/v1/query \
                --data-urlencode 'query=success_rate{endpoint="/api/orders"}'