如何做好微服务容量规划

# k6 1.x — 持续负载测试脚本示例 (k6 v1.x, JavaScript ES6+)
# 文件: capacity-test.js
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend } from 'k6/metrics';

// 自定义指标：慢速比（响应时间 > 1s 的比例）
const slowRate = new Rate('slow_request_rate');
const feedLatency = new Trend('feed_latency');

export const options = {
  scenarios: {
    // 阶梯式加压场景
    ramping: {
      executor: 'ramping-arrival-rate',
      startRate: 100,
      timeUnit: '1s',
      preAllocatedVUs: 500,
      maxVUs: 5000,
      stages: [
        { duration: '5m', target: 500 },   // 缓慢加压到 500 RPS
        { duration: '10m', target: 1000 },  // 继续加压到 1000 RPS
        { duration: '5m', target: 2000 },   // 加压到 2000 RPS 观察拐点
        { duration: '5m', target: 0 },      // 减压恢复
      ],
    },
  },
  thresholds: {
    // SLO-based 终止条件：慢速比 > 1% 则测试失败
    slow_request_rate: ['rate<0.01'],
    feed_latency: ['p99<1000'],
    http_req_failed: ['rate<0.001'],
  },
};

export default function () {
  const res = http.get('https://api.example.com/feed', {
    tags: { endpoint: 'feed' },
  });

  const isSlow = res.timings.duration > 1000;
  slowRate.add(isSlow);
  feedLatency.add(res.timings.duration);

  check(res, {
    'status is 200': (r) => r.status === 200,
    'latency < 500ms': (r) => r.timings.duration < 500,
  });

  sleep(1);
}

// 基于 OpenTelemetry Histogram 的区间加权容量计算
// Go 1.22+, OTel SDK v1.28+
package capacity

import (
	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	"math"
)

// 区间权重定义 — 指数递增（键为耗时区间上界，单位秒，与 OTel Histogram 的 Bounds 一致）
var bucketWeights = map[float64]float64{
	0.01:        1,  // 0-10ms
	0.05:        2,  // 10-50ms
	0.1:         4,  // 50-100ms
	0.2:         8,  // 100-200ms
	0.5:         16, // 200-500ms
	1.0:         32, // 500ms-1s
	math.Inf(1): 64, // >1s
}

// CalculateWeightedCapacity 从 OTel Histogram 数据计算区间加权容量
func CalculateWeightedCapacity(histogram metricdata.Histogram[float64]) float64 {
	var capacity float64
	for _, dp := range histogram.DataPoints {
		prevBound := 0.0
		for i, count := range dp.BucketCounts {
			if i >= len(dp.Bounds) {
				break
			}
			upperBound := dp.Bounds[i]
			weight := getWeightForRange(prevBound, upperBound)
			capacity += weight * float64(count)
			prevBound = upperBound
		}
	}
	return capacity
}

func getWeightForRange(lower, upper float64) float64 {
	// 选取区间上界对应的权重
	for bound, weight := range bucketWeights {
		if upper <= bound {
			return weight
		}
	}
	return 64 // 默认最高权重
}

# Prometheus 自定义 Recording Rule — 自动计算集群水位线
# Prometheus Rule v1, Kubernetes 1.30+
groups:
  - name: capacity_waterline
    interval: 60s
    rules:
      # 计算集群运行负荷（区间加权；Prometheus Histogram 为累计桶，需先做相邻桶差分）
      - record: cluster:capacity:load_weighted
        expr: |
          sum by (cluster, service) (
            rate(http_request_duration_seconds_bucket{le="0.01"}[5m]) * 1 +
            (rate(http_request_duration_seconds_bucket{le="0.05"}[5m]) - rate(http_request_duration_seconds_bucket{le="0.01"}[5m])) * 2 +
            (rate(http_request_duration_seconds_bucket{le="0.1"}[5m]) - rate(http_request_duration_seconds_bucket{le="0.05"}[5m])) * 4 +
            (rate(http_request_duration_seconds_bucket{le="0.2"}[5m]) - rate(http_request_duration_seconds_bucket{le="0.1"}[5m])) * 8 +
            (rate(http_request_duration_seconds_bucket{le="0.5"}[5m]) - rate(http_request_duration_seconds_bucket{le="0.2"}[5m])) * 16 +
            (rate(http_request_duration_seconds_bucket{le="1.0"}[5m]) - rate(http_request_duration_seconds_bucket{le="0.5"}[5m])) * 32 +
            (rate(http_request_duration_seconds_bucket{le="+Inf"}[5m]) - rate(http_request_duration_seconds_bucket{le="1.0"}[5m])) * 64
          )

      # 计算集群水位线 = 最大容量 / 运行负荷
      - record: cluster:capacity:waterline
        expr: |
          cluster:capacity:max_weighted /
          cluster:capacity:load_weighted

# HPA 配置示例 — 基于水位线指标的水位线感知伸缩
# Kubernetes 1.30+, autoscaling/v2
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: feed-service-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: feed-service
  minReplicas: 10
  maxReplicas: 200

  metrics:
    # 基于自定义水位线指标
    - type: Pods
      pods:
        metric:
          name: capacity_waterline
        target:
          type: AverageValue
          averageValue: "1.5"  # 水位线目标值 1.5（负荷为容量的 2/3）

    # 兜底：基于 CPU 使用率
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70

  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60    # 扩容稳定窗口 1 分钟
      policies:
        - type: Percent
          value: 30                      # 每次最多扩容 30%
          periodSeconds: 60
        - type: Pods
          value: 5                       # 或每次最多扩 5 个 Pod
          periodSeconds: 60
      selectPolicy: Max                  # 取两种策略的较大值
    scaleDown:
      stabilizationWindowSeconds: 300    # 缩容稳定窗口 5 分钟
      policies:
        - type: Percent
          value: 10                      # 先缩 10%
          periodSeconds: 300
        - type: Percent
          value: 30                      # 逐步可缩 30%
          periodSeconds: 600
      selectPolicy: Min                  # 缩容取较小值，保守缩容

# KEDA 2.15+ ScaledObject — 基于 Kafka 消费者延迟伸缩
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: feed-consumer-scaler
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: feed-consumer
  pollingInterval: 30        # 每 30s 检查一次事件源
  cooldownPeriod: 300        # 缩容冷却期 5 分钟
  minReplicaCount: 2         # 最小副本数
  maxReplicaCount: 50        # 最大副本数
  fallback:                  # 指标源不可用时的降级策略
    failureThreshold: 3
    replicas: 10
  triggers:
    - type: kafka
      metadata:
        bootstrapServers: kafka-cluster.production.svc:9092
        consumerGroup: feed-consumer-group
        topic: feed-events
        lagThreshold: "1000"       # 每个 Partition 积压 1000 条消息时触发伸缩
        activationLagThreshold: "100"  # 积压低于 100 条时不伸缩
    - type: prometheus
      metadata:
        serverAddress: http://prometheus.monitoring:9090
        metricName: capacity_waterline
        threshold: "1.5"
        query: >-
          cluster:capacity:waterline{service="feed-consumer"}
  advanced:
    horizontalPodAutoscalerConfig:
      behavior:                  # 继承 HPA Behavior 配置
        scaleUp:
          stabilizationWindowSeconds: 60
          policies:
            - type: Percent
              value: 30
              periodSeconds: 60
        scaleDown:
          stabilizationWindowSeconds: 300
          policies:
            - type: Percent
              value: 10
              periodSeconds: 300

# 基于 Prophet 的流量预测与预扩容决策
# Python 3.12+, prophet 1.1+
import pandas as pd
from prophet import Prophet
from kubernetes import client, config

class CapacityForecaster:
    """基于 Prophet 的容量预测器"""

    def __init__(self, lookback_days: int = 90, forecast_hours: int = 6):
        self.lookback_days = lookback_days
        self.forecast_hours = forecast_hours
        config.load_incluster_config()
        self.k8s_apps = client.AppsV1Api()

    def predict_qps(self, history_df: pd.DataFrame) -> pd.DataFrame:
        """预测未来 N 小时的 QPS"""
        model = Prophet(
            changepoint_prior_scale=0.05,   # 较低灵敏度，避免过拟合
            seasonality_prior_scale=10,      # 较强季节性
            yearly_seasonality=True,
            weekly_seasonality=True,
            daily_seasonality=True,
        )
        # 添加大促等特殊日期
        model.add_country_holidays(country_name='CN')
        model.fit(history_df)

        future = model.make_future_dataframe(
            periods=self.forecast_hours,
            freq='h'
        )
        forecast = model.predict(future)
        return forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']]

    def calculate_pre_scale_replicas(
        self,
        predicted_qps: float,
        capacity_per_replica: float,
        safety_margin: float = 1.2,
    ) -> int:
        """根据预测 QPS 计算预扩容副本数"""
        required = (predicted_qps * safety_margin) / capacity_per_replica
        return max(1, int(required))

# Cluster Autoscaler 配置示例
# Kubernetes 1.30+, cluster-autoscaler v1.30+
apiVersion: v1
kind: ConfigMap
metadata:
  name: cluster-autoscaler-status
  namespace: kube-system
data:
  autoscaler-config: |
    # 扩容策略
    scale-down-unneeded-time: 10m       # 节点空闲 10 分钟后可缩
    scale-down-delay-after-add: 10m     # 扩容后 10 分钟内不缩
    scale-down-delay-after-delete: 10s  # 缩容后 10 秒内不继续缩
    scale-down-delay-after-failure: 3m  # 缩容失败后 3 分钟重试
    max-graceful-termination-sec: 600   # 优雅终止最长等待时间

    # 扩容优化
    expander: priority                  # 按优先级选择节点组扩容
    balance-similar-node-groups: true   # 相似节点组间均衡
    skip-nodes-with-system-pods: true   # 跳过含系统 Pod 的节点

    # 节点组自动发现
    node-group-auto-discovery: label=cloud.google.com/gke-nodepool

# 容量画像 CronJob — 定期执行 k6 压测更新容量基线
# Kubernetes 1.30+, k6 Operator v0.0.9+
apiVersion: k6.io/v1alpha1
kind: K6
metadata:
  name: capacity-benchmark
  namespace: capacity-planning
spec:
  parallelism: 4                        # 4 个 Worker 并行压测
  script:
    configMap:
      name: k6-capacity-test
      file: capacity-test.js
  arguments: --out experimental-prometheus-rw  # 直接写入 Prometheus
  runner:
    env:
      - name: K6_PROMETHEUS_RW_SERVER_URL
        value: "http://prometheus.monitoring:9090/api/v1/write"
    resources:
      requests:
        cpu: "1"
        memory: "1Gi"
      limits:
        cpu: "2"
        memory: "2Gi"

# 中小规模简化配置
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: service-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-service
  minReplicas: 3
  maxReplicas: 20
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300