构建资源弹性伸缩：云原生构建系统

# 构建任务的资源规格示例
resources:
  requests:
    cpu: "2"        # 调度时保证分配 2 核
    memory: "4Gi"   # 调度时保证分配 4GB 内存
  limits:
    cpu: "4"        # 最多使用 4 核
    memory: "8Gi"   # 最多使用 8GB 内存

# tekton/tasks/npm-build.yaml
apiVersion: tekton.dev/v1
kind: Task
metadata:
  name: npm-build
  labels:
    app.kubernetes.io/version: "0.1"
  annotations:
    tekton.dev/pipelines.minVersion: "0.44"
    tekton.dev/tags: build, npm
spec:
  description: >-
    Build a Node.js application using npm,
    with configurable resource requests.
  params:
    - name: node-version
      description: "Node.js version to use"
      default: "20"
    - name: build-command
      description: "Build command to execute"
      default: "npm run build"
    - name: context-path
      description: "Path to the build context"
      default: "."
    - name: cpu-request
      description: "CPU request for the build pod"
      default: "2"
    - name: memory-request
      description: "Memory request for the build pod"
      default: "4Gi"
  workspaces:
    - name: source
      description: "Workspace with source code"
  results:
    - name: build-log
      description: "Build output log path"
  steps:
    - name: install
      image: node:$(params.node-version)-alpine
      workingDir: $(workspaces.source.path)/$(params.context-path)
      script: |
        #!/bin/sh
        set -e
        echo "Installing dependencies..."
        npm ci --prefer-offline
      computeResources:
        requests:
          cpu: $(params.cpu-request)
          memory: $(params.memory-request)
        limits:
          cpu: "4"
          memory: "8Gi"
 
    - name: build
      image: node:$(params.node-version)-alpine
      workingDir: $(workspaces.source.path)/$(params.context-path)
      script: |
        #!/bin/sh
        set -e
        echo "Running build..."
        $(params.build-command)
        echo "Build completed successfully"
      computeResources:
        requests:
          cpu: $(params.cpu-request)
          memory: $(params.memory-request)
        limits:
          cpu: "4"
          memory: "8Gi"
 
    - name: upload-artifact
      image: amazon/aws-cli:latest
      workingDir: $(workspaces.source.path)/$(params.context-path)
      script: |
        #!/bin/sh
        set -e
        aws s3 sync dist/ s3://$(params.context-path)-artifacts/$(context.taskRun.uid)/

# tekton/pipelines/ci-pipeline.yaml
apiVersion: tekton.dev/v1
kind: Pipeline
metadata:
  name: elastic-ci-pipeline
spec:
  description: >-
    Elastic CI pipeline with resource-aware task scheduling.
  params:
    - name: repo-url
      type: string
    - name: revision
      type: string
    - name: image-name
      type: string
  workspaces:
    - name: shared-workspace
    - name: docker-config
  results:
    - name: image-digest
      value: $(tasks.build-image.results.image-digest)
 
  tasks:
    # Step 1: 获取源代码
    - name: fetch-source
      taskRef:
        name: git-clone
        kind: ClusterTask
      workspaces:
        - name: output
          workspace: shared-workspace
      params:
        - name: url
          value: $(params.repo-url)
        - name: revision
          value: $(params.revision)
 
    # Step 2: 并行执行 Lint 和测试（DAG 依赖）
    - name: lint
      runAfter:
        - fetch-source
      taskRef:
        name: npm-build
      workspaces:
        - name: source
          workspace: shared-workspace
      params:
        - name: build-command
          value: "npm run lint"
        - name: cpu-request
          value: "1"
        - name: memory-request
          value: "2Gi"
 
    - name: unit-test
      runAfter:
        - fetch-source
      taskRef:
        name: npm-build
      workspaces:
        - name: source
          workspace: shared-workspace
      params:
        - name: build-command
          value: "npm run test:ci"
        - name: cpu-request
          value: "2"
        - name: memory-request
          value: "4Gi"
 
    # Step 3: 构建应用（依赖 Lint 和测试通过）
    - name: build-app
      runAfter:
        - lint
        - unit-test
      taskRef:
        name: npm-build
      workspaces:
        - name: source
          workspace: shared-workspace
      params:
        - name: build-command
          value: "npm run build:prod"
        - name: cpu-request
          value: "4"
        - name: memory-request
          value: "8Gi"
 
    # Step 4: 构建容器镜像
    - name: build-image
      runAfter:
        - build-app
      taskRef:
        name: kaniko
        kind: ClusterTask
      workspaces:
        - name: source
          workspace: shared-workspace
        - name: dockerconfig
          workspace: docker-config
      params:
        - name: IMAGE
          value: $(params.image-name):$(params.revision)
        - name: BUILDER_IMAGE
          value: gcr.io/kaniko-project/executor:latest
        - name: EXTRA_ARGS
          value:
            - --cache=true
            - --cache-repo=$(params.image-name)/cache
            - --cache-ttl=72h
 
    # Step 5: 安全扫描
    - name: security-scan
      runAfter:
        - build-image
      taskRef:
        name: trivy-scan
      params:
        - name: image-ref
          value: $(params.image-name):$(params.revision)
 
  # 最终清理与通知
  finally:
    - name: notify
      taskRef:
        name: send-notification
      params:
        - name: status
          value: $(tasks.status)
        - name: pipeline-name
          value: $(context.pipeline.name)

# tekton/triggers/ci-trigger.yaml
apiVersion: triggers.tekton.dev/v1beta1
kind: TriggerTemplate
metadata:
  name: ci-trigger-template
spec:
  params:
    - name: git-repo-url
    - name: git-revision
    - name: git-repo-name
  resourcetemplates:
    - apiVersion: tekton.dev/v1
      kind: PipelineRun
      metadata:
        generateName: ci-$(tt.params.git-repo-name)-
        labels:
          tekton.dev/pipeline: elastic-ci-pipeline
          app: $(tt.params.git-repo-name)
          # 优先级标签：用于 K8s 优先级调度
          priority: "medium"
      spec:
        pipelineRef:
          name: elastic-ci-pipeline
          # 超时控制
        timeouts:
          pipeline: "1h0m0s"
          tasks: "45m0s"
        workspaces:
          - name: shared-workspace
            volumeClaimTemplate:
              spec:
                accessModes:
                  - ReadWriteOnce
                resources:
                  requests:
                    storage: 5Gi
                storageClassName: fast-ssd
          - name: docker-config
            secret:
              secretName: registry-credentials
        params:
          - name: repo-url
            value: $(tt.params.git-repo-url)
          - name: revision
            value: $(tt.params.git-revision)
          - name: image-name
            value: ghcr.io/my-org/$(tt.params.git-repo-name)
---
# TriggerBinding - 解析 Webhook 载荷
apiVersion: triggers.tekton.dev/v1beta1
kind: TriggerBinding
metadata:
  name: github-binding
spec:
  params:
    - name: git-repo-url
      value: $(body.repository.clone_url)
    - name: git-revision
      value: $(body.after)
    - name: git-repo-name
      value: $(body.repository.name)
---
# EventListener - 接收 Webhook
apiVersion: triggers.tekton.dev/v1beta1
kind: EventListener
metadata:
  name: github-listener
spec:
  serviceAccountName: tekton-triggers-sa
  triggers:
    - name: github-push
      bindings:
        - ref: github-binding
      template:
        ref: ci-trigger-template
      interceptors:
        - ref:
            name: github
          params:
            - name: eventTypes
              value: ["push", "pull_request"]
            - name: secretRef
              value:
                secretName: github-webhook-secret
                secretKey: token

#!/bin/bash
# install-arc.sh - 安装 Actions Runner Controller
 
# 1. 添加 Helm 仓库
helm repo add actions-runner-controller \
  https://actions-runner-controller.github.io/actions-runner-controller
helm repo update
 
# 2. 安装 ARC
helm install arc \
  actions-runner-controller/actions-runner-controller \
  --namespace arc-system \
  --create-namespace \
  --set githubToken=${GITHUB_TOKEN} \
  --set githubWebhookServer.enabled=true \
  --set githubWebhookServer.secret.create=true \
  --set replicaCount=2
 
# 3. 等待 Controller 就绪
kubectl rollout status deployment/arc-actions-runner-controller \
  --namespace=arc-system --timeout=120s

# arc/runners/ci-runners.yaml
apiVersion: actions-runner-controller.github.com/v1alpha1
kind: RunnerDeployment
metadata:
  name: ci-runners
  namespace: arc-system
spec:
  replicas: 1  # 最小副本数
  template:
    spec:
      organization: my-org
      # 使用 GitHub App 认证（推荐）
      githubAPICredentialsFrom:
        secretRef:
          name: github-app-credentials
 
      # Runner 镜像
      image: summerwind/actions-runner:ubuntu-22.04
      imagePullPolicy: IfNotPresent
 
      # Runner 标签（用于 Workflow 匹配）
      labels:
        - "self-hosted"
        - "linux"
        - "x64"
        - "ci"
 
      # 资源配置
      resources:
        requests:
          cpu: "2"
          memory: "4Gi"
        limits:
          cpu: "4"
          memory: "8Gi"
 
      # 工作目录挂载（构建缓存）
      volumeMounts:
        - name: runner-work
          mountPath: /home/runner/_work
        - name: npm-cache
          mountPath: /home/runner/.npm
      volumes:
        - name: runner-work
          emptyDir: {}
        - name: npm-cache
          persistentVolumeClaim:
            claimName: npm-cache-pvc
 
      # 环境变量
      env:
        - name: RUNNER_FEATURE_FLAG_EPHEMERAL
          value: "true"  # 一次性 Runner，任务完成后自动销毁
---
# HorizontalRunnerAutoscaler - 弹性伸缩配置
apiVersion: actions-runner-controller.github.com/v1alpha1
kind: HorizontalRunnerAutoscaler
metadata:
  name: ci-runners-autoscaler
  namespace: arc-system
spec:
  scaleTargetRef:
    name: ci-runners
  minReplicas: 1
  maxReplicas: 20
 
  # 基于 GitHub Actions 队列长度伸缩
  metrics:
    - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
      repositoryNames:
        - my-org/my-app
        - my-org/my-api
        - my-org/shared-libs
 
  # 伸缩行为
  scaleUpThreshold: 0.75    # 75% 的 Runner 繁忙时开始扩容
  scaleDownDelaySeconds: 300 # 缩容延迟 5 分钟
---
# 大规格 Runner（用于前端构建等内存密集型任务）
apiVersion: actions-runner-controller.github.com/v1alpha1
kind: RunnerDeployment
metadata:
  name: ci-large-runners
  namespace: arc-system
spec:
  replicas: 0  # 默认 0，按需扩容
  template:
    spec:
      organization: my-org
      githubAPICredentialsFrom:
        secretRef:
          name: github-app-credentials
      image: summerwind/actions-runner:ubuntu-22.04
      labels:
        - "self-hosted"
        - "linux"
        - "x64"
        - "ci-large"
      resources:
        requests:
          cpu: "8"
          memory: "32Gi"
        limits:
          cpu: "16"
          memory: "64Gi"
      volumeMounts:
        - name: npm-cache
          mountPath: /home/runner/.npm
      volumes:
        - name: npm-cache
          persistentVolumeClaim:
            claimName: npm-cache-pvc-large
      env:
        - name: RUNNER_FEATURE_FLAG_EPHEMERAL
          value: "true"
      # 节点选择器：指定运行在大规格节点上
      nodeSelector:
        node.kubernetes.io/instance-type: "m5.4xlarge"
      tolerations:
        - key: "runner-type"
          operator: "Equal"
          value: "large"
          effect: "NoSchedule"
---
apiVersion: actions-runner-controller.github.com/v1alpha1
kind: HorizontalRunnerAutoscaler
metadata:
  name: ci-large-runners-autoscaler
  namespace: arc-system
spec:
  scaleTargetRef:
    name: ci-large-runners
  minReplicas: 0
  maxReplicas: 5
  metrics:
    - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
      repositoryNames:
        - my-org/frontend-app
  scaleUpThreshold: 0.5
  scaleDownDelaySeconds: 300

# .github/workflows/self-hosted-ci.yml
name: Self-hosted CI
 
on:
  push:
    branches: [main]
  pull_request:
 
jobs:
  # 普通构建任务 - 使用标准 Runner
  test:
    runs-on: [self-hosted, linux, x64, ci]
    steps:
      - uses: actions/checkout@v4
 
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
 
      - run: npm ci
      - run: npm run lint
      - run: npm run test:ci
 
  # 大规格构建任务 - 使用 Large Runner
  build-frontend:
    runs-on: [self-hosted, linux, x64, ci-large]
    needs: test
    steps:
      - uses: actions/checkout@v4
 
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
 
      - run: npm ci
      - run: npm run build:production

# k8s/priority-classes.yaml
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: ci-critical
value: 1000
description: "Critical CI builds (main branch, releases)"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: ci-normal
value: 500
description: "Normal CI builds (PR builds)"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: ci-low
value: 100
preemptionPolicy: PreemptLowerPriority
description: "Low priority CI builds (feature branches, nightly)"
globalDefault: false
---
# 在 Tekton PipelineRun 中使用优先级
apiVersion: tekton.dev/v1
kind: PipelineRun
metadata:
  name: main-branch-build
  labels:
    priority: critical
spec:
  pipelineRef:
    name: elastic-ci-pipeline
  podTemplate:
    priorityClassName: ci-critical  # 关键构建使用高优先级

# k8s/resource-quotas.yaml
# CI 命名空间的资源配额
apiVersion: v1
kind: ResourceQuota
metadata:
  name: ci-resource-quota
  namespace: ci-system
spec:
  hard:
    requests.cpu: "100"       # 最多 100 核 CPU
    requests.memory: 200Gi    # 最多 200GB 内存
    limits.cpu: "200"         # CPU limit 上限
    limits.memory: 400Gi      # Memory limit 上限
    pods: "50"                # 最多 50 个 Pod
    persistentvolumeclaims: "20"  # 最多 20 个 PVC
---
# LimitRange - 单个 Pod 的资源限制
apiVersion: v1
kind: LimitRange
metadata:
  name: ci-limit-range
  namespace: ci-system
spec:
  limits:
    - type: Container
      default:              # 默认 limit
        cpu: "4"
        memory: "8Gi"
      defaultRequest:       # 默认 request
        cpu: "1"
        memory: "2Gi"
      max:                  # 最大 limit
        cpu: "16"
        memory: "64Gi"
      min:                  # 最小 request
        cpu: "0.5"
        memory: "1Gi"
    - type: Pod
      max:
        cpu: "32"
        memory: "128Gi"

# k8s/cluster-autoscaler-config.yaml
# Cluster Autoscaler 配置（通过启动参数）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-autoscaler
  namespace: kube-system
spec:
  template:
    spec:
      containers:
        - name: cluster-autoscaler
          image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.29.0
          command:
            - ./cluster-autoscaler
            - --cloud-provider=aws
            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/my-cluster
            - --scale-down-unneeded-time=5m      # 空闲 5 分钟后缩容
            - --scale-down-delay-after-add=5m    # 扩容后 5 分钟内不缩容
            - --scale-down-utilization-threshold=0.3  # 节点利用率低于 30% 可缩容
            - --balance-similar-node-groups=true
            - --expander=priority                # 按优先级选择节点组扩容
            - --max-graceful-termination-sec=600
            - --skip-nodes-with-system-pods=false  # CI Pod 可被驱逐
          resources:
            limits:
              cpu: 200m
              memory: 600Mi
            requests:
              cpu: 100m
              memory: 300Mi

# k8s/spot-node-config.yaml
# Spot 节点池配置（以 AWS EKS 为例）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: spot-termination-handler
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: spot-termination-handler
  template:
    metadata:
      labels:
        app: spot-termination-handler
    spec:
      serviceAccountName: spot-termination-handler-sa
      containers:
        - name: handler
          image: amazon/aws-node-termination-handler:latest
          args:
            - --node-name=$(HOSTNAME)
            - --drain-grace-period=120     # 给 Pod 120 秒优雅终止时间
            - --monitor-grace-period=60    # 监控信号后 60 秒执行排空
            - --skip-nodes-with-local-storage=false
          env:
            - name: HOSTNAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
      nodeSelector:
        eks.amazonaws.com/capacityType: SPOT
---
# CI Pod 配置 Spot 容忍度
apiVersion: v1
kind: Pod
metadata:
  name: ci-build-example
  labels:
    app: ci-build
spec:
  # 优先调度到 Spot 节点
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          preference:
            matchExpressions:
              - key: eks.amazonaws.com/capacityType
                operator: In
                values:
                  - SPOT
  tolerations:
    - key: "spot-instance"
      operator: "Equal"
      value: "true"
      effect: "NoSchedule"
  containers:
    - name: build
      image: node:20-alpine
      command: ["npm", "run", "build"]

# k8s/build-cache-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: npm-cache-pvc
  namespace: ci-system
spec:
  accessModes:
    - ReadWriteMany  # 多 Pod 共享缓存
  storageClassName: fast-ssd
  resources:
    requests:
      storage: 50Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: docker-cache-pvc
  namespace: ci-system
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: standard
  resources:
    requests:
      storage: 100Gi
---
# Tekton Workspace 使用 PVC 缓存
apiVersion: tekton.dev/v1
kind: PipelineRun
metadata:
  name: cached-build
spec:
  workspaces:
    - name: shared-workspace
      volumeClaimTemplate:
        spec:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 5Gi
    - name: npm-cache
      persistentVolumeClaim:
        claimName: npm-cache-pvc
    - name: docker-cache
      persistentVolumeClaim:
        claimName: docker-cache-pvc

# k8s/cost-monitoring.yaml
# 使用 Kubecost 监控构建成本
apiVersion: v1
kind: ConfigMap
metadata:
  name: cost-monitor-config
  namespace: ci-system
data:
  cost-model.json: |
    {
      "ci_namespace": "ci-system",
      "spot_discount": 0.7,
      "budget_monthly_usd": 5000,
      "alert_threshold": 0.8
    }
---
# 成本告警 Prometheus 规则
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: ci-cost-alerts
  namespace: ci-system
spec:
  groups:
    - name: ci-cost
      rules:
        - alert: CIBudgetApproaching
          expr: |
            sum(kube_pod_container_resource_requests{namespace="ci-system"} * on(node) group_left() node_cpu_hourly_cost) > 4000
          for: 1h
          labels:
            severity: warning
          annotations:
            summary: "CI resource cost approaching monthly budget"
            description: "Current monthly CI cost estimate exceeds 80% of budget ($5000)"
 
        - alert: CIQueueTooLong
          expr: |
            tekton_pipelines_running_count < tekton_pipelines_pending_count
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "CI build queue is longer than running count"
            description: "Pending builds exceed running builds for 15 minutes"

#!/bin/bash
# cost-analysis.sh - 弹性构建系统成本分析
 
echo "=== Elastic Build System Cost Analysis ==="
echo "Date: $(date -u +"%Y-%m-%d")"
echo ""
 
# 模拟参数
TEAM_SIZE=20
BUILDS_PER_DEV_PER_DAY=5
AVG_BUILD_DURATION_MIN=10
INSTANCE_TYPE="m5.xlarge"
ONDEMAND_HOURLY=0.192
SPOT_HOURLY=0.058
 
# 传统固定节点方案
FIXED_AGENTS=8  # 固定 8 个 Agent
FIXED_MONTHLY=$(echo "$FIXED_AGENTS * $ONDEMAND_HOURLY * 24 * 30" | bc)
echo "=== Traditional Fixed Agent Model ==="
echo "Fixed agents: $FIXED_AGENTS"
echo "Monthly cost: \$${FIXED_MONTHLY}"
echo "Peak capacity: $FIXED_AGENTS concurrent builds"
echo "Off-peak utilization: ~15%"
echo ""
 
# 弹性构建方案（70% Spot + 30% On-demand）
DAILY_BUILDS=$(echo "$TEAM_SIZE * $BUILDS_PER_DEV_PER_DAY" | bc)
TOTAL_BUILD_HOURS=$(echo "$DAILY_BUILDS * $AVG_BUILD_DURATION_MIN / 60" | bc)
SPOT_HOURS=$(echo "$TOTAL_BUILD_HOURS * 0.7" | bc)
ONDEMAND_HOURS=$(echo "$TOTAL_BUILD_HOURS * 0.3" | bc)
ELASTIC_DAILY=$(echo "$SPOT_HOURS * $SPOT_HOURLY + $ONDEMAND_HOURS * $ONDEMAND_HOURLY" | bc)
ELASTIC_MONTHLY=$(echo "$ELASTIC_DAILY * 30" | bc)
SAVINGS=$(echo "scale=1; ($FIXED_MONTHLY - $ELASTIC_MONTHLY) / $FIXED_MONTHLY * 100" | bc)
 
echo "=== Elastic Build Model ==="
echo "Daily builds: $DAILY_BUILDS"
echo "Total build hours/day: ${TOTAL_BUILD_HOURS}h"
echo "Spot hours/day: ${SPOT_HOURS}h"
echo "On-demand hours/day: ${ONDEMAND_HOURS}h"
echo "Monthly cost: \$${ELASTIC_MONTHLY}"
echo "Cost savings: ${SAVINGS}%"
echo "Peak capacity: 20+ concurrent builds (auto-scaled)"
echo ""
 
# 考虑缓存优化的方案
CACHE_HIT_RATE=0.7  # 70% 缓存命中
CACHED_BUILD_RATIO=$(echo "scale=2; 1 - $CACHE_HIT_RATE * 0.8" | bc)  # 缓存命中减少 80% 计算量
EFFECTIVE_BUILD_HOURS=$(echo "$TOTAL_BUILD_HOURS * $CACHED_BUILD_RATIO" | bc)
CACHED_SPOT_HOURS=$(echo "$EFFECTIVE_BUILD_HOURS * 0.7" | bc)
CACHED_ONDEMAND_HOURS=$(echo "$EFFECTIVE_BUILD_HOURS * 0.3" | bc)
CACHED_DAILY=$(echo "$CACHED_SPOT_HOURS * $SPOT_HOURLY + $CACHED_ONDEMAND_HOURS * $ONDEMAND_HOURLY" | bc)
CACHED_MONTHLY=$(echo "$CACHED_DAILY * 30" | bc)
TOTAL_SAVINGS=$(echo "scale=1; ($FIXED_MONTHLY - $CACHED_MONTHLY) / $FIXED_MONTHLY * 100" | bc)
 
echo "=== Elastic Build with Cache ==="
echo "Cache hit rate: 70%"
echo "Effective build hours/day: ${EFFECTIVE_BUILD_HOURS}h"
echo "Monthly cost: \$${CACHED_MONTHLY}"
echo "Total savings vs fixed: ${TOTAL_SAVINGS}%"

# grafana-dashboard-ci-metrics.yaml
# Grafana Dashboard ConfigMap（摘要）
apiVersion: v1
kind: ConfigMap
metadata:
  name: ci-metrics-dashboard
  namespace: monitoring
data:
  dashboard.json: |
    {
      "dashboard": {
        "title": "CI/CD Build System Metrics",
        "panels": [
          {
            "title": "Build Queue Wait Time",
            "targets": [
              {
                "expr": "histogram_quantile(0.95, sum(rate(tekton_pipelines_run_duration_seconds_bucket{namespace=\"ci-system\"}[5m])) by (le))"
              }
            ]
          },
          {
            "title": "Active Build Pods",
            "targets": [
              {
                "expr": "count(kube_pod_info{namespace=\"ci-system\"})"
              }
            ]
          },
          {
            "title": "Resource Utilization",
            "targets": [
              {
                "expr": "sum(kube_pod_container_resource_requests{namespace=\"ci-system\"}) / sum(kube_node_capacity)"
              }
            ]
          },
          {
            "title": "Spot Instance Usage",
            "targets": [
              {
                "expr": "count(kube_node_labels{label_eks_amazonaws_com_capacityType=\"SPOT\"}) / count(kube_node_info)"
              }
            ]
          },
          {
            "title": "Monthly Cost Estimate",
            "targets": [
              {
                "expr": "sum(kubecost_pod_cost_total{namespace=\"ci-system\"})"
              }
            ]
          }
        ]
      }
    }