微服务容器化运维：容器调度和服务编排

# Kubernetes 1.30+
apiVersion: v1
kind: Pod
metadata:
  name: feed-service
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:  # 硬约束
        nodeSelectorTerms:
        - matchExpressions:
          - key: node-pool
            operator: In
            values: ["compute", "general"]
          - key: zone
            operator: In
            values: ["cn-east-1a", "cn-east-1b"]
      preferredDuringSchedulingIgnoredDuringExecution:  # 软约束（打分）
      - weight: 80
        preference:
          matchExpressions:
          - key: node-type
            operator: In
            values: ["compute"]
  containers:
  - name: feed
    image: registry.example.com/feed:v2.1.0

# 给 GPU 节点打污点
kubectl taint nodes gpu-node-01 hardware=gpu:NoSchedule

# Pod 声明容忍才能调度到 GPU 节点
apiVersion: v1
kind: Pod
metadata:
  name: model-inference
spec:
  tolerations:
  - key: "hardware"
    operator: "Equal"
    value: "gpu"
    effect: "NoSchedule"
  containers:
  - name: inference
    image: registry.example.com/inference:v1.5.0
    resources:
      limits:
        nvidia.com/gpu: "1"

# kube-scheduler 配置 (Kubernetes 1.30+)
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: default-scheduler
  pluginConfig:
  - name: NodeResourcesFit
    args:
      scoringStrategy:
        type: RequestedToCapacityRatio  # 自定义评分曲线
        resources:
        - name: cpu
          weight: 1
        - name: memory
          weight: 1
        requestedToCapacityRatio:
          shape:
          - utilization: 0
            score: 0
          - utilization: 100
            score: 10

# Kubernetes 1.30+
apiVersion: v1
kind: Pod
metadata:
  name: feed-service
  labels:
    app: feed
spec:
  affinity:
    podAffinity:
      # Feed 服务倾向于和 User 服务部署在同一可用区
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchLabels:
              app: user
          topologyKey: topology.kubernetes.io/zone
    podAntiAffinity:
      # Feed 服务的不同副本尽量不在同一节点（反亲和）
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchLabels:
            app: feed
        topologyKey: kubernetes.io/hostname
  containers:
  - name: feed
    image: registry.example.com/feed:v2.1.0

# Kubernetes 1.30+
apiVersion: v1
kind: Pod
metadata:
  name: feed-service
  labels:
    app: feed
spec:
  topologySpreadConstraints:
  - maxSkew: 1                    # 最大偏差：各域之间 Pod 数差值不超过 1
    topologyKey: topology.kubernetes.io/zone  # 按可用区分布
    whenUnsatisfiable: DoNotSchedule  # 硬约束：不满足则不调度
    labelSelector:
      matchLabels:
        app: feed
  - maxSkew: 1
    topologyKey: kubernetes.io/hostname    # 按节点分布
    whenUnsatisfiable: ScheduleAnyway      # 软约束：尽量满足
    labelSelector:
      matchLabels:
        app: feed
  containers:
  - name: feed
    image: registry.example.com/feed:v2.1.0

# Kubernetes 1.30+
apiVersion: v1
kind: Pod
metadata:
  name: feed-service
spec:
  initContainers:
  - name: wait-for-config
    image: busybox:1.36
    command: ['sh', '-c', 'until nslookup config-service.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for config; sleep 2; done']
  - name: wait-for-user
    image: busybox:1.36
    command: ['sh', '-c', 'until nslookup user-service.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for user; sleep 2; done']
  containers:
  - name: feed
    image: registry.example.com/feed:v2.1.0

# Helm 3.15+ - templates/pre-install-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: db-migration
  annotations:
    "helm.sh/hook": pre-install,pre-upgrade      # 安装/升级前执行
    "helm.sh/hook-weight": "-5"                    # 权重控制执行顺序
    "helm.sh/hook-delete-policy": hook-succeeded   # 成功后自动清理
spec:
  template:
    spec:
      containers:
      - name: migrate
        image: registry.example.com/feed-migrate:v2.1.0
        command: ["python", "manage.py", "migrate"]
      restartPolicy: Never

# Kubernetes 1.30+
apiVersion: v1
kind: Service
metadata:
  name: feed-service
  labels:
    app: feed
spec:
  type: ClusterIP
  selector:
    app: feed
  ports:
  - port: 8080
    targetPort: 8080
  topologyKeys:                    # 拓扑感知路由（1.22+ 稳定）
  - "kubernetes.io/hostname"
  - "topology.kubernetes.io/zone"
  - "*"

# Kubernetes 1.30+
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: feed-service
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: feed-service
  minReplicas: 3
  maxReplicas: 50
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 60
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 70
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1000"
  behavior:                        # 扩缩容行为控制（1.18+ 稳定）
    scaleDown:
      stabilizationWindowSeconds: 300  # 缩容冷却期 5 分钟
      policies:
      - type: Percent
        value: 10                  # 每分钟最多缩容 10%
        periodSeconds: 60
    scaleUp:
      stabilizationWindowSeconds: 0
      policies:
      - type: Percent
        value: 100                 # 每分钟最多扩容 100%
        periodSeconds: 60
      - type: Pods
        value: 4
        periodSeconds: 60
      selectPolicy: Max

# KEDA 2.15+
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: feed-scaler
spec:
  scaleTargetRef:
    name: feed-service
  minReplicaCount: 0           # 支持缩容到零
  maxReplicaCount: 50
  cooldownPeriod: 300
  triggers:
  - type: kafka                # 基于 Kafka 消息堆积量
    metadata:
      bootstrapServers: kafka:9092
      consumerGroup: feed-group
      topic: feed-events
      lagThreshold: "100"      # 每个分区堆积超过 100 条时扩容
  - type: prometheus           # 基于 Prometheus 指标
    metadata:
      serverAddress: http://prometheus:9090
      metricName: http_requests_total
      threshold: "1000"
      query: "sum(rate(http_requests_total{app='feed'}[1m]))"

# Helm 3.15+ - OCI 注册表操作
helm registry login registry.example.com -u admin -p $HARBOR_TOKEN
helm push feed-service-2.1.0.tgz oci://registry.example.com/charts
helm install feed-service oci://registry.example.com/charts/feed-service --version 2.1.0

feed-service/
├── Chart.yaml              # Chart 元数据
├── values.yaml             # 默认配置
├── values-production.yaml  # 生产环境覆盖
├── values-staging.yaml     # 预发环境覆盖
├── schemas/
│   └── values.json         # JSON Schema 校验
├── templates/
│   ├── deployment.yaml
│   ├── service.yaml
│   ├── hpa.yaml
│   ├── configmap.yaml
│   ├── secret.yaml
│   ├── _helpers.tpl        # 模板辅助函数
│   └── tests/
│       └── test-connection.yaml
└── .helmignore

base/
├── kustomization.yaml
├── deployment.yaml
└── service.yaml

overlays/
├── staging/
│   ├── kustomization.yaml
│   └── replica-patch.yaml
└── production/
    ├── kustomization.yaml
    ├── replica-patch.yaml
    └── hpa-patch.yaml

# overlays/production/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base
patches:
- path: replica-patch.yaml
  target:
    kind: Deployment
    name: feed-service
replicas:
- name: feed-service
  count: 10
namespace: production
commonLabels:
  env: production
configMapGenerator:
- name: feed-config
  literals:
  - LOG_LEVEL=warn
  - CACHE_TTL=300s

// api/v1alpha1/feedautoscaler_types.go - CRD 定义
// Kubebuilder 4.x / Kubernetes 1.30+
package v1alpha1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// FeedAutoscalerSpec 定义期望状态
type FeedAutoscalerSpec struct {
	//+kubebuilder:validation:Minimum=1
	MinReplicas int32 `json:"minReplicas"`

	//+kubebuilder:validation:Maximum=100
	MaxReplicas int32 `json:"maxReplicas"`

	//+kubebuilder:validation:Minimum=1
	//+kubebuilder:default=60
	TargetCPUUtilization int32 `json:"targetCPUUtilization,omitempty"`

	// 依赖服务的扩缩容比例
	Dependencies []DependencyScaleRatio `json:"dependencies,omitempty"`

	// 定时扩缩容规则
	ScheduleRules []ScheduleRule `json:"scheduleRules,omitempty"`
}

type DependencyScaleRatio struct {
	Name  string `json:"name"`
	Ratio float64 `json:"ratio"`  // 每扩容 10 台 Feed，扩容 Ratio*10 台依赖服务
}

type ScheduleRule struct {
	//+kubebuilder:validation:Pattern="^\\d{2}:\\d{2}$"
	At       string `json:"at"`       // 如 "12:30"
	Replicas int32  `json:"replicas"` // 目标副本数
}

// FeedAutoscalerStatus 定义观测状态
type FeedAutoscalerStatus struct {
	CurrentReplicas int32 `json:"currentReplicas"`
	DesiredReplicas int32 `json:"desiredReplicas"`
	//+kubebuilder:validation:Enum={"Scaling","Stable"}
	Phase string `json:"phase,omitempty"`
}

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:subresource:scale:specpath=.spec.minReplicas,statuspath=.status.desiredReplicas
//+kubebuilder:printcolumn:name="Desired",type="integer",JSONPath=".status.desiredReplicas"
//+kubebuilder:printcolumn:name="Current",type="integer",JSONPath=".status.currentReplicas"
//+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase"

type FeedAutoscaler struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   FeedAutoscalerSpec   `json:"spec,omitempty"`
	Status FeedAutoscalerStatus `json:"status,omitempty"`
}

# 使用 CRD
apiVersion: ops.weibo.com/v1alpha1
kind: FeedAutoscaler
metadata:
  name: feed-scaler
spec:
  minReplicas: 3
  maxReplicas: 50
  targetCPUUtilization: 60
  dependencies:
  - name: user-service
    ratio: 0.4    # 每 10 台 Feed 扩 4 台 User
  - name: card-service
    ratio: 0.3    # 每 10 台 Feed 扩 3 台 Card
  scheduleRules:
  - at: "12:30"   # 午高峰
    replicas: 30
  - at: "22:30"   # 晚高峰
    replicas: 40
  - at: "03:00"   # 凌晨低谷
    replicas: 3

# OAM / KubeVela 1.9+
apiVersion: core.oam.dev/v1beta1
kind: Application
metadata:
  name: feed-app
spec:
  components:
  - name: feed-service
    type: webservice          # 组件类型
    properties:
      image: registry.example.com/feed:v2.1.0
      port: 8080
      cpu: "2"
      memory: "4Gi"
    traits:
    - type: scaler            # 伸缩策略
      properties:
        min: 3
        max: 50
    - type: gateway           # 网关暴露
      properties:
        domain: feed.example.com
        path: /api/feed
    - type: keda-autoscaler   # 事件驱动伸缩
      properties:
        triggers:
        - type: kafka
          metadata:
            topic: feed-events
            lagThreshold: "100"
  policies:
  - name: multi-env
    type: topology
    properties:
      clusters:
      - name: staging
        namespace: staging
      - name: production
        namespace: production
  workflow:
    steps:
    - name: deploy-staging
      type: deploy
      properties:
        policies: ["multi-env"]
        env: staging
    - name: approval
      type: suspend           # 人工审批
    - name: deploy-production
      type: deploy
      properties:
        policies: ["multi-env"]
        env: production

# Crossplane 1.15+
apiVersion: aws.upbound.io/v1beta1
kind: ProviderConfig
metadata:
  name: aws-config
spec:
  credentials:
    source: Secret
    secretRef:
      name: aws-creds
      namespace: crossplane-system
---
# 声明式创建 RDS 实例
apiVersion: rds.aws.upbound.io/v1beta1
kind: Instance
metadata:
  name: feed-db
  annotations:
    crossplane.io/external-name: feed-db-production
spec:
  forProvider:
    engine: mysql
    engineVersion: "8.0"
    instanceClass: db.r6g.large
    allocatedStorage: 100
    region: cn-north-1
    vpcSecurityGroupIds:
    - sg-xxx
  writeConnectionSecretToRef:
    name: feed-db-conn
    namespace: default
---
# Composition：将多个资源组合为可复用的抽象
apiVersion: apiextensions.crossplane.io/v1
kind: CompositeResourceDefinition
metadata:
  name: xmicroservices.ops.example.com
spec:
  group: ops.example.com
  names:
    kind: XMicroService
    plural: xmicroservices
  versions:
  - name: v1alpha1
    served: true
    referenceable: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          spec:
            type: object
            properties:
              image:
                type: string
              replicas:
                type: integer
            required:
            - image