微服务容器化运维：从DCP到云原生容器运维平台

# Dragonfly 2.x Scheduler 配置示例 (v2.1+)
# dragonfly-scheduler-config.yaml
scheduler:
  algorithm: default # 支持 default / backfill / rainbow
  backSourceCount: 3 # P2P 失败时回源并发数
  retry:
    maxAttempts: 5
    retryDelay: 200ms
 
seedPeer:
  enable: true
  concurrent:
    limit: 200 # 种子节点并发下载限制
 
manager:
  security:
    autoIssueCert: true # 自动签发 mTLS 证书

# Harbor OPA 策略示例：强制镜像签名与漏洞门禁
# harbor-policy.yaml (Harbor 2.10+)
policy:
  - name: "require-cosign-signature"
    action: "reject"
    condition:
      type: "signature"
      validator: "cosign"
      required: true
 
  - name: "block-critical-cves"
    action: "reject"
    condition:
      type: "vulnerability"
      severity: ["Critical"]
      maxCount: 0
 
  - name: "require-sbom"
    action: "warn"
    condition:
      type: "artifact"
      hasSBOM: true

# Crossplane: 声明式创建阿里云 ECS (Crossplane 1.14+)
# crossplane-ecs.yaml
apiVersion: ecs.alibabacloud.crossplane.io/v1alpha1
kind: Instance
metadata:
  name: weibo-app-server
  annotations:
    crossplane.io/external-name: i-bp1a2b3c4d5e
spec:
  forProvider:
    imageId: aliyun_3_x64_20G_alibase_
    instanceType: ecs.c7.2xlarge
    region: cn-beijing
    zoneId: cn-beijing-h     # 多可用区由 Crossplane 自动 failover
    securityGroupId: sg-2ze...
    vSwitchId: vsw-2ze...
    systemDiskSize: 100
    systemDiskCategory: cloud_essd
    tags:
      env: production
      team: weibo-infra
  providerConfigRef:
    name: alibabacloud-config
  writeConnectionSecretToRef:
    name: weibo-app-server-secret
    namespace: crossplane-system

# 利用 ResourceClass 和 PriorityClass 实现错峰调度
# priority-class.yaml (Kubernetes 1.30+)
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: online-batch-low-priority
value: 100
preemptionPolicy: PreemptLowerPriority
globalDefault: false
description: "离线批处理任务，可被在线业务抢占"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: online-service-high-priority
value: 1000
preemptionPolicy: PreemptLowerPriority
globalDefault: false
description: "在线业务，可抢占低优先级资源"

# KEDA ScaledObject: 基于 Cron 的弹性扩缩 (KEDA 2.14+)
# keda-cron-scaler.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: batch-job-scaler
  namespace: offline-compute
spec:
  scaleTargetRef:
    name: spark-history-server
  minReplicaCount: 0
  maxReplicaCount: 50
  triggers:
    - type: cron
      metadata:
        timezone: Asia/Shanghai
        start: "00 0 * * *"    # 凌晨0点扩容（离线计算高峰）
        end: "00 8 * * *"      # 上午8点缩容（在线业务高峰）
        desiredReplicas: "50"
    - type: prometheus          # 同时基于指标
      metadata:
        serverAddress: http://prometheus:9090
        metricName: batch_job_queue_depth
        threshold: "10"
        query: batch_job_queue_depth{namespace="offline-compute"}

# Kubecost 成本注解示例
apiVersion: apps/v1
kind: Deployment
metadata:
  name: weibo-feed-service
  annotations:
    kubecost.io/product: "feed-service"
    kubecost.io/team: "weibo-feed"
    kubecost.io/environment: "production"
spec:
  template:
    metadata:
      annotations:
        kubecost.io/product: "feed-service"

// Kubebuilder 生成的 Operator Reconcile 逻辑骨架 (Kubebuilder 4.x)
// internal/controller/clusterinit_controller.go
package controller
 
import (
    "context"
    "fmt"
 
    appsv1 "k8s.io/api/apps/v1"
    corev1 "k8s.io/api/core/v1"
    "k8s.io/apimachinery/pkg/api/errors"
    "k8s.io/apimachinery/pkg/runtime"
    ctrl "sigs.k8s.io/controller-runtime"
    "sigs.k8s.io/controller-runtime/pkg/client"
    "sigs.k8s.io/controller-runtime/pkg/log"
 
    platformv1 "my-platform/api/v1"
)
 
// ClusterInitReconciler 协调集群初始化资源
type ClusterInitReconciler struct {
    client.Client
    Scheme *runtime.Scheme
}
 
func (r *ClusterInitReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
    logger := log.FromContext(ctx)
 
    var clusterInit platformv1.ClusterInit
    if err := r.Get(ctx, req.NamespacedName, &clusterInit); err != nil {
        if errors.IsNotFound(err) {
            return ctrl.Result{}, nil
        }
        return ctrl.Result{}, err
    }
 
    // 1. 确保 DaemonSet 安装基础组件（替代 Ansible 批量下发）
    if err := r.ensureBaseComponents(ctx, &clusterInit); err != nil {
        logger.Error(err, "failed to ensure base components")
        return ctrl.Result{}, err
    }
 
    // 2. 确保 ConfigMap 分发节点配置
    if err := r.ensureNodeConfig(ctx, &clusterInit); err != nil {
        logger.Error(err, "failed to ensure node config")
        return ctrl.Result{}, err
    }
 
    // 3. 更新初始化状态（替代 DB 写入，使用 CRD Status）
    clusterInit.Status.Phase = platformv1.PhaseInitialized
    if err := r.Status().Update(ctx, &clusterInit); err != nil {
        return ctrl.Result{}, err
    }
 
    logger.Info("cluster initialization completed",
        "cluster", req.Name, "phase", clusterInit.Status.Phase)
    return ctrl.Result{}, nil
}

# PropagationPolicy: 定义工作负载的分发策略 (Karmada 1.9+)
# propagation-policy.yaml
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: weibo-feed-propagation
  namespace: production
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: weibo-feed
  placement:
    clusterAffinity:
      clusterNames:
        - bj-private   # 北京自建机房
        - sh-private   # 上海自建机房
        - ali-cloud    # 阿里云
    clusterTolerations:
      - key: "cluster.karmada.io/not-ready"
        operator: "Exists"
        effect: "NoExecute"
        tolerationSeconds: 300  # 集群不健康时 5 分钟后迁移
    spreadConstraints:
      - maxGroups: 3
        minGroups: 2
        spreadByField: cluster  # 至少分布在 2 个集群
    replicaScheduling:
      replicaSchedulingType: Divided
      replicaDivisionPreference: Weighted
      weightPreference:
        staticWeightList:
          - targetCluster:
              clusterNames: ["bj-private"]
            weight: 5
          - targetCluster:
              clusterNames: ["sh-private"]
            weight: 3
          - targetCluster:
              clusterNames: ["ali-cloud"]
            weight: 2

# OverridePolicy: 公有云集群差异化配置 (Karmada 1.9+)
# override-policy.yaml
apiVersion: policy.karmada.io/v1alpha1
kind: OverridePolicy
metadata:
  name: weibo-feed-ali-cloud-override
  namespace: production
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: weibo-feed
  overrideRules:
    - targetCluster:
        clusterNames: ["ali-cloud"]
      overriders:
        plaintext:
          - path: "/spec/template/spec/nodeSelector"
            operator: add
            value:
              topology.kubernetes.io/zone: cn-beijing-h
          - path: "/spec/template/spec/tolerations"
            operator: add
            value:
              - key: "cloud-provider"
                operator: "Equal"
                value: "alibabacloud"
                effect: "NoSchedule"

# 启用应用级故障转移 (Karmada 1.9+)
# 需在 Karmada 配置中开启 failover 功能
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: weibo-feed-with-failover
spec:
  placement:
    clusterAffinity:
      clusterNames: ["bj-private", "sh-private", "ali-cloud"]
  failover:
    application:
      decisionConditions:
        tolerationSeconds: 120  # 2 分钟后触发故障转移
      purgeMode: Immediately    # 立即从故障集群移除
      gracePeriodSeconds: 60

# ArgoCD ApplicationSet: 编排多服务依赖部署 (ArgoCD 2.12+)
# appset-weibo.yaml
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: weibo-feed-stack
  namespace: argocd
spec:
  goTemplate: true
  generators:
    - list:
        elements:
          - name: weibo-feed-db
            path: charts/feed-db
            order: "1"        # 部署顺序
          - name: weibo-feed-cache
            path: charts/feed-cache
            order: "2"
          - name: weibo-feed-rpc
            path: charts/feed-rpc
            order: "3"
          - name: weibo-feed-gateway
            path: charts/feed-gateway
            order: "4"
  template:
    metadata:
      name: '{{.name}}'
      annotations:
        argocd.argoproj.io/sync-wave: '{{.order}}'  # 同步波次控制依赖顺序
    spec:
      project: weibo-production
      source:
        repoURL: https://git.internal.weibo.com/platform/manifests
        targetRevision: main
        path: '{{.path}}'
      destination:
        server: https://kubernetes.default.svc
        namespace: production
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

# Karmada MultiClusterService: 跨集群服务发现 (Karmada 1.9+)
# mcs-weibo-feed.yaml
apiVersion: networking.karmada.io/v1alpha1
kind: MultiClusterService
metadata:
  name: weibo-feed
  namespace: production
spec:
  types:
    - CrossCluster
  ports:
    - name: http
      port: 8080
      targetPort: 8080
  providerClusters:
    - name: bj-private
    - name: sh-private
  consumerClusters:
    - name: ali-cloud      # 阿里云集群可访问私有云的服务

# KEDA ScaledObject: 基于消息队列深度的弹性扩缩 (KEDA 2.14+)
# keda-queue-scaler.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: weibo-feed-scaler
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: weibo-feed
  minReplicaCount: 10
  maxReplicaCount: 500
  cooldownPeriod: 60
  triggers:
    # 触发器 1：RocketMQ 消息堆积
    - type: rocketmq
      metadata:
        namesrvAddr: rocketmq:9876
        topic: feed-update
        consumerGroup: feed-consumer
        lagThreshold: "1000"       # 堆积超过 1000 条触发扩容
    # 触发器 2：Prometheus 自定义指标
    - type: prometheus
      metadata:
        serverAddress: http://mimir:9009
        query: >
          sum(rate(http_requests_total{app="weibo-feed",code=~"5.."}[2m]))
          / sum(rate(http_requests_total{app="weibo-feed"}[2m]))
        threshold: "0.05"          # 5xx 比率超过 5% 触发扩容
    # 触发器 3：Cron 定时预热
    - type: cron
      metadata:
        timezone: Asia/Shanghai
        start: "50 18 * * *"       # 每晚 18:50 预热（春晚/热点事件高发时段）
        end: "00 23 * * *"
        desiredReplicas: "200"

# Cluster Autoscaler 配置 - 多可用区节点池 (ACK 1.30+)
# cluster-autoscaler-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: cluster-autoscaler-status
  namespace: kube-system
data:
  autoscaling-groups: |
    - name: bj-private-on-demand
      minSize: 10
      maxSize: 200
      priority: 1               # 优先使用自有资源
    - name: ali-cloud-spot      # Spot 实例池
      minSize: 0
      maxSize: 500
      priority: 2               # 次优先使用竞价实例
      instance-types:           # 多规格实例，避免库存限制
        - ecs.c7.2xlarge
        - ecs.c7a.2xlarge
        - ecs.c6.2xlarge
    - name: ali-cloud-on-demand # 按量付费池
      minSize: 0
      maxSize: 100
      priority: 3               # 最后使用按量实例

# Prometheus 远程写入配置 - 启用 Exemplar (Mimir 2.14+)
# prometheus-agent.yaml
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: prometheus-agent
  namespace: observability
spec:
  remoteWrite:
    - url: http://mimir:9009/api/v1/push
      sendExemplars: true       # 启用 Exemplar 关联
  exemplarStorage:
    maxExemplars: 100000        # 内存中保留的 Exemplar 数量

# Backstage Software Template: 创建新微服务 (Backstage 1.x)
# template.yaml
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
  name: weibo-microservice
  title: Weibo Microservice Template
  description: 创建微博新微服务，含 Kubernetes 清单、CI/CD 流水线、监控模板
spec:
  owner: platform-team
  type: service
  parameters:
    - title: 服务基本信息
      required:
        - name
        - owner
        - language
      properties:
        name:
          title: 服务名称
          type: string
          pattern: "^[a-z]([a-z0-9-]*[a-z0-9])?$"
        owner:
          title: 负责团队
          type: string
          ui:field: OwnerPicker
        language:
          title: 开发语言
          type: string
          enum: ["java", "golang", "python"]
          default: "java"
    - title: 部署配置
      required:
        - replicas
        - namespace
      properties:
        replicas:
          title: 初始副本数
          type: number
          default: 3
        namespace:
          title: 命名空间
          type: string
          default: production
        enableKEDA:
          title: 启用 KEDA 弹性伸缩
          type: boolean
          default: true
 
  steps:
    - id: template
      name: 生成服务脚手架
      action: fetch:template
      input:
        url: ./skeleton
        values:
          name: ${{ parameters.name }}
          owner: ${{ parameters.owner }}
          language: ${{ parameters.language }}
          replicas: ${{ parameters.replicas }}
          namespace: ${{ parameters.namespace }}
 
    - id: publish
      name: 推送到 Git 仓库
      action: publish:github
      input:
        repoUrl: git.internal.weibo.com/${{ parameters.owner }}/${{ parameters.name }}
 
    - id: register
      name: 注册到 Backstage Catalog
      action: catalog:register
      input:
        catalogInfoUrl: https://git.internal.weibo.com/${{ parameters.owner }}/${{ parameters.name }}/blob/main/catalog-info.yaml
 
  output:
    links:
      - title: 仓库地址
        url: ${{ steps.publish.output.remoteUrl }}
      - title: ArgoCD 部署
        url: https://argocd.internal.weibo.com/applications/${{ parameters.name }}