告警策略与On-Call管理

# SLO 燃尽率告警规则（由 Sloth 生成）
groups:
  # 快速检测：1h 窗口 + 6h 窗口，燃烧率 > 14.4x
  - name: orders-service-availability-slo-burn-rate-fast
    rules:
      - alert: OrdersServiceAvailabilitySLOBurnRateFast
        expr: |
          (
            sum(rate(http_requests_total{job="orders-service",code=~"5.."}[1h]))
            /
            sum(rate(http_requests_total{job="orders-service"}[1h]))
          ) > (14.4 * (1 - 0.999))
          and
          (
            sum(rate(http_requests_total{job="orders-service",code=~"5.."}[6h]))
            /
            sum(rate(http_requests_total{job="orders-service"}[6h]))
          ) > (14.4 * (1 - 0.999))
        for: 2m
        labels:
          severity: critical
          team: team-a
          slo: availability
        annotations:
          summary: "Orders service availability SLO is burning very fast"
          description: "The 1h and 6h error rates are 14.4x above the SLO target. Error budget will be exhausted in ~2h."
          runbook: "https://runbooks.example.com/orders-availability"

  # 慢速检测：6h 窗口 + 3d 窗口，燃烧率 > 6x
  - name: orders-service-availability-slo-burn-rate-slow
    rules:
      - alert: OrdersServiceAvailabilitySLOBurnRateSlow
        expr: |
          (
            sum(rate(http_requests_total{job="orders-service",code=~"5.."}[6h]))
            /
            sum(rate(http_requests_total{job="orders-service"}[6h]))
          ) > (6 * (1 - 0.999))
          and
          (
            sum(rate(http_requests_total{job="orders-service",code=~"5.."}[3d]))
            /
            sum(rate(http_requests_total{job="orders-service"}[3d]))
          ) > (6 * (1 - 0.999))
        for: 15m
        labels:
          severity: warning
          team: team-a
          slo: availability
        annotations:
          summary: "Orders service availability SLO is burning slowly"
          description: "The 6h and 3d error rates are 6x above the SLO target. Error budget will be exhausted in ~5d."

# Alertmanager 配置：去重、分组、路由、抑制
global:
  resolve_timeout: 5m
  slack_api_url: "https://hooks.slack.com/services/xxx"

route:
  receiver: "slack-default"
  group_by: ["alertname", "team", "service"]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

  routes:
    # P1/P2 告警路由到 PagerDuty
    - matchers:
        - severity="critical"
      receiver: pagerduty-critical
      group_wait: 10s
      repeat_interval: 1h

    - matchers:
        - severity="warning"
      receiver: slack-warning
      group_wait: 30s
      repeat_interval: 4h

    # 按团队路由（team-a 的告警分流到团队专属接收器）
    - matchers:
        - team="team-a"
      routes:
        - matchers:
            - severity="critical"
          receiver: pagerduty-team-a
        - matchers:
            - severity="warning"
          receiver: slack-team-a

inhibit_rules:
  # 当 critical 告警存在时，抑制同一服务的 warning 告警
  - source_matchers:
      - severity="critical"
    target_matchers:
      - severity="warning"
    equal: ["alertname", "team", "service"]

receivers:
  - name: slack-default
    slack_configs:
      - channel: "#alerts-general"
        title: "[{{ .Status }}] {{ .CommonLabels.alertname }}"
        text: "{{ range .Alerts }}{{ .Annotations.description }}{{ end }}"

  - name: pagerduty-critical
    pagerduty_configs:
      - service_key: "${PAGERDUTY_SERVICE_KEY}"
        severity: critical

  - name: pagerduty-team-a
    pagerduty_configs:
      - service_key: "${PAGERDUTY_TEAM_A_KEY}"

  - name: slack-warning
    slack_configs:
      - channel: "#team-a-alerts"

# Grafana OnCall 轮值配置（Terraform 资源 grafana_oncall_schedule 的字段示意）
resource: grafana_oncall_schedule
name: team-a-primary
type: rolling
time_zone: Asia/Shanghai
shifts:
  - name: "Weekday Primary"
    type: rolling_users
    rolling_users:
      - user_id: user-1
      - user_id: user-2
      - user_id: user-3
    rotation:
      start: "2024-01-01T09:00:00+08:00"
      duration: 24h        # 每人值班 24 小时
      handoff: "09:00"     # 每天 9:00 交接

  - name: "Weekend Primary"
    type: rolling_users
    rolling_users:
      - user_id: user-4
      - user_id: user-5
    rotation:
      start: "2024-01-06T09:00:00+08:00"
      duration: 168h       # 每人值班一周

## 事故复盘报告

### 基本信息
- **事故时间**: YYYY-MM-DD HH:MM - HH:MM (持续 XX 分钟)
- **影响范围**: 受影响的服务/用户
- **严重级别**: P1/P2/P3
- **On-Call 响应**: 响应时间、诊断时间、恢复时间
- **错误预算消耗**: 本次事故消耗了多少错误预算

### 时间线
| 时间 | 事件 |
|------|------|
| HH:MM | 告警触发 |
| HH:MM | On-Call 确认并开始诊断 |
| HH:MM | 定位根因 |
| HH:MM | 执行修复 |
| HH:MM | 服务恢复 |

### 根因分析
- **直接原因**: 什么触发了故障
- **贡献因素**: 哪些条件使故障影响扩大
- **系统性原因**: 流程或系统中哪些缺陷允许故障发生

### 行动项
| 优先级 | 行动项 | 负责人 | 完成日期 |
|--------|--------|--------|---------|
| P0 | 修复 XX 问题 | @engineer-1 | YYYY-MM-DD |
| P1 | 增加告警规则 | @engineer-2 | YYYY-MM-DD |
| P2 | 更新 Runbook | @engineer-3 | YYYY-MM-DD |

### 经验教训
- **什么有效**: 哪些机制在事故中发挥了正面作用
- **什么无效**: 哪些机制需要改进
- **如果重来会怎么做**: 改进建议

### 无责声明
本复盘的目的是改进系统，而非追究个人责任。所有参与者应坦诚分享观察和想法。