SLO/SLI/SLA实践

总请求时间窗口 = 30 × 24 × 60 × 60 = 2,592,000 秒
允许的不可用时间 = 2,592,000 × 0.1% = 2,592 秒 ≈ 43.2 分钟

错误预算消耗趋势（30天窗口）
 
100% |████████████████████████████████| 月初：预算满额
 80% |████████████████████████████    | 正常消耗
 60% |████████████████████████        | 正常消耗
 40% |████████████████                | 一次故障消耗 20%
 20% |████████                        | ⚠️ 警告：预算紧张
  5% |██                              | 🔴 紧急：预算即将耗尽
  0% |                                 | 🚫 停止发布，全力修复

# Sloth SLO 定义文件
version: "prometheus/v1"
service: "orders-service"
slos:
  # SLO 1：请求成功率
  - name: "availability-success-rate"
    description: "订单服务的请求成功率 SLO"
    target: 0.999     # 99.9%
    window: 30d       # 30 天滚动窗口
    sli:
      events:
        error_query:
          expr: sum(rate(http_requests_total{job="orders-service",code=~"5.."}[{{.window}}]))
        total_query:
          expr: sum(rate(http_requests_total{job="orders-service"}[{{.window}}]))
    alerting:
      name: OrdersServiceAvailabilitySLOBurnRate
      labels:
        team: team-a
        service: orders-service
        slo: availability
      annotations:
        summary: "Orders service availability SLO is burning too fast"
        runbook_url: "https://runbooks.example.com/orders-availability"
      page_alert:
        labels:
          severity: critical
      ticket_alert:
        labels:
          severity: warning
 
  # SLO 2：请求延迟
  - name: "latency-p99"
    description: "订单服务 P99 延迟 SLO"
    target: 0.99     # 99% 的请求延迟 < 200ms
    window: 30d
    sli:
      events:
        error_query:
          expr: sum(rate(http_request_duration_seconds_bucket{job="orders-service",le="0.2"}[{{.window}}]))
        total_query:
          expr: sum(rate(http_request_duration_seconds_count{job="orders-service"}[{{.window}}]))
    alerting:
      name: OrdersServiceLatencySLOBurnRate
      labels:
        team: team-a
        service: orders-service
        slo: latency
      annotations:
        summary: "Orders service latency SLO is burning too fast"
      page_alert:
        labels:
          severity: critical
      ticket_alert:
        labels:
          severity: warning

apiVersion: pyrra.dev/v1alpha1
kind: ServiceLevelObjective
metadata:
  name: orders-service-availability
  namespace: monitoring
  labels:
    team: team-a
    service: orders-service
spec:
  target: "99.9"
  window: 30d
  serviceLevelIndicator:
    prometheus:
      errorQuery: sum(rate(http_requests_total{job="orders-service",code=~"5.."}[{{window}}]))
      totalQuery: sum(rate(http_requests_total{job="orders-service"}[{{window}}]))
  alerting:
    name: OrdersServiceAvailabilityBurnRate

name: Error Budget Check
 
on:
  workflow_call:
    inputs:
      service:
        required: true
        type: string
 
jobs:
  check-budget:
    runs-on: ubuntu-latest
    outputs:
      budget_remaining: ${{ steps.check.outputs.budget_remaining }}
      can_deploy: ${{ steps.check.outputs.can_deploy }}
    steps:
      - name: Query error budget
        id: check
        run: |
          # 从 Prometheus 查询错误预算剩余
          BUDGET=$(curl -s "${{ secrets.PROMETHEUS_URL }}/api/v1/query" \
            --data-urlencode "query=1 - (sum(rate(http_requests_total{job=\"${{ inputs.service }}\",code=~\"5..\"}[30d])) / sum(rate(http_requests_total{job=\"${{ inputs.service }}\"}[30d])))" \
            | jq -r '.data.result[0].value[1]')
 
          echo "budget_remaining=$BUDGET" >> $GITHUB_OUTPUT
 
          # 错误预算 > 0 才允许发布
          if (( $(echo "$BUDGET > 0" | bc -l) )); then
            echo "can_deploy=true" >> $GITHUB_OUTPUT
            echo "✅ Error budget: ${BUDGET}% remaining — deployment allowed"
          else
            echo "can_deploy=false" >> $GITHUB_OUTPUT
            echo "🚫 Error budget exhausted — deployment blocked"
          fi
 
  deploy:
    needs: check-budget
    if: needs.check-budget.outputs.can_deploy == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Deploy
        run: echo "Deploying ${SERVICE} with ${BUDGET}% error budget remaining"
        env:
          SERVICE: ${{ inputs.service }}
          BUDGET: ${{ needs.check-budget.outputs.budget_remaining }}