混沌工程与破坏性测试

# 实验一：随机 Kill Pod，验证服务自愈能力
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: pod-kill-experiment
  namespace: chaos-testing
spec:
  action: pod-kill
  mode: one
  selector:
    namespaces:
      - production
    labelSelectors:
      app: orders-service
  scheduler:
    cron: "@every 300s"    # 每 5 分钟执行一次
  duration: "30s"

# 实验二：网络延迟注入，验证服务降级策略
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: network-delay-experiment
  namespace: chaos-testing
spec:
  action: delay
  mode: all
  selector:
    namespaces:
      - production
    labelSelectors:
      app: orders-service
  delay:
    latency: "500ms"       # 注入 500ms 延迟
    correlation: "25"      # 25% 的相关性
    jitter: "100ms"        # ±100ms 抖动
  direction: to
  target:
    selector:
      namespaces:
        - production
      labelSelectors:
        app: payment-service
  duration: "5m"

# 实验三：IO 故障注入，验证存储层容错
apiVersion: chaos-mesh.org/v1alpha1
kind: IOChaos
metadata:
  name: io-fault-experiment
  namespace: chaos-testing
spec:
  action: fault
  mode: one
  selector:
    namespaces:
      - production
    labelSelectors:
      app: orders-service
  volumePath: /data
  path: "/data/**"          # 影响所有数据文件操作
  errno: 5                  # I/O error
  percent: 50               # 50% 的 IO 操作失败
  duration: "2m"

name: Chaos Engineering Validation
 
on:
  schedule:
    - cron: '0 2 * * *'    # 每天凌晨 2:00 执行
  workflow_dispatch:
 
jobs:
  chaos-experiment:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
 
      - name: Setup kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: 'v1.29.0'
 
      - name: Connect to cluster
        run: |
          echo "${{ secrets.KUBE_CONFIG }}" | base64 -d > kubeconfig
          export KUBECONFIG=kubeconfig
 
      - name: Record baseline metrics
        id: baseline
        run: |
          # 记录实验前的关键指标基线
          ERROR_RATE=$(kubectl exec -n monitoring prometheus-0 -- \
            curl -s 'http://localhost:9090/api/v1/query?query=rate(http_requests_total{status=~"5.."}[5m])/rate(http_requests_total[5m])' | \
            jq -r '.data.result[0].value[1]')
          echo "error_rate=$ERROR_RATE" >> $GITHUB_OUTPUT
          echo "Baseline error rate: $ERROR_RATE"
 
      - name: Inject Pod Kill Chaos
        run: |
          kubectl apply -f chaos-experiments/pod-kill.yaml
          echo "Chaos experiment started, waiting for observation period..."
          sleep 120    # 观察 2 分钟
 
      - name: Observe system behavior
        id: observe
        run: |
          # 采集实验期间的关键指标
          PEAK_ERROR_RATE=$(kubectl exec -n monitoring prometheus-0 -- \
            curl -s 'http://localhost:9090/api/v1/query?query=rate(http_requests_total{status=~"5.."}[5m])/rate(http_requests_total[5m])' | \
            jq -r '.data.result[0].value[1]')
          echo "peak_error_rate=$PEAK_ERROR_RATE" >> $GITHUB_OUTPUT
          echo "Peak error rate during experiment: $PEAK_ERROR_RATE"
 
      - name: Cleanup experiment
        if: always()
        run: |
          kubectl delete -f chaos-experiments/pod-kill.yaml --ignore-not-found
          echo "Chaos experiment cleaned up"
 
      - name: Evaluate results
        run: |
          BASELINE=${{ steps.baseline.outputs.error_rate }}
          PEAK=${{ steps.observe.outputs.peak_error_rate }}
          THRESHOLD=0.05    # 5% 错误率阈值
 
          echo "Baseline: $BASELINE"
          echo "Peak during chaos: $PEAK"
          echo "Threshold: $THRESHOLD"
 
          if (( $(echo "$PEAK > $THRESHOLD" | bc -l) )); then
            echo "❌ RESILIENCE CHECK FAILED: Error rate exceeded threshold"
            echo "Action required: Investigate service degradation under Pod failure"
            exit 1
          else
            echo "✅ RESILIENCE CHECK PASSED: System maintained stability"
          fi

## Game Day 检查清单
 
### 准备阶段
- [ ] 确定演练目标和范围
- [ ] 获得管理层批准
- [ ] 通知所有相关团队（提前 48 小时）
- [ ] 确认监控和告警系统正常
- [ ] 准备回滚预案
- [ ] 指定演练指挥官（Commander）和观察员
- [ ] 建立沟通渠道（Slack 频道 / 电话会议）
 
### 执行阶段
- [ ] 记录基线指标
- [ ] 按计划注入故障
- [ ] 观察系统行为和团队响应
- [ ] 记录所有观察到的异常
- [ ] 如超出安全阈值，立即执行回滚
 
### 恢复阶段
- [ ] 停止所有故障注入
- [ ] 确认系统恢复至稳态
- [ ] 记录恢复时间和恢复步骤
 
### 复盘阶段
- [ ] 24 小时内完成复盘会议
- [ ] 记录发现的问题和脆弱点
- [ ] 制定改进计划并分配责任人
- [ ] 更新混沌实验库
- [ ] 分享学习成果