如何搭建一套适合你的服务追踪系统

# OpenTelemetry Collector config — v0.153.0
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
 
processors:
  batch:
    send_batch_size: 1024
    timeout: 5s
  filter:
    error_mode: ignore
    traces:
      span:
        # 过滤健康检查等低价值 Span
        - 'attributes["http.route"] == "/healthz"'
        - 'attributes["http.route"] == "/readiness"'
  resource:
    attributes:
      - key: cluster.name
        value: prod-cn-east
        action: upsert
      - key: deployment.environment
        value: production
        action: upsert
 
exporters:
  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true
  otlp/jaeger:
    endpoint: jaeger-collector:4317
    tls:
      insecure: true
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, filter, resource]
      exporters: [otlp/tempo]

traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
             │  │─────────── TraceId ──────────────│ │──── SpanId ────│ │┐
             版本                                    │                 ││采样标志
                                                     ────────────────────┘
tracestate: vendor1=value1,vendor2=value2

# OpenTelemetry Java Agent v1.63.0
java -javaagent:opentelemetry-javaagent.jar \
     -Dotel.service.name=order-service \
     -Dotel.exporter.otlp.endpoint=http://otel-collector:4317 \
     -Dotel.exporter.otlp.protocol=grpc \
     -jar order-service.jar

# OpenTelemetry Operator v0.152.0 — Instrumentation CRD
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: otel-instrumentation
spec:
  exporter:
    endpoint: http://otel-collector:4317
  propagators:
    - tracecontext   # W3C Trace Context
    - b3             # 兼容 Zipkin B3
  sampler:
    type: parentbased_traceidratio
    argument: "0.1"  # 10% 采样率
  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:1.63.0
  python:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-python:0.50b0
  nodejs:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:0.55.0

metadata:
  annotations:
    instrumentation.opentelemetry.io/inject-java: "true"
    # 或 inject-python / inject-nodejs / inject-dotnet / inject-go

# OTel Collector — Tail-Based Sampling Processor
processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 100
    policies:
      - name: errors
        type: status_code
        status_code:
          status_codes:
            - ERROR
      - name: slow-traces
        type: latency
        latency:
          threshold_ms: 1000
      - name: sample-10pct
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

# otel-collector-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector
spec:
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.153.0
          ports:
            - containerPort: 4317  # OTLP gRPC
            - containerPort: 4318  # OTLP HTTP
          volumeMounts:
            - name: config
              mountPath: /etc/otelcol-contrib/config.yaml
              subPath: config.yaml
      volumes:
        - name: config
          configMap:
            name: otel-collector-config
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-config
data:
  config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
 
    processors:
      batch:
        send_batch_size: 2048
        timeout: 5s
      tail_sampling:
        decision_wait: 10s
        num_traces: 100000
        policies:
          - name: errors
            type: status_code
            status_code:
              status_codes: [ERROR]
          - name: slow
            type: latency
            latency:
              threshold_ms: 500
          - name: keep-10pct
            type: probabilistic
            probabilistic:
              sampling_percentage: 10
 
    exporters:
      otlphttp/tempo:
        endpoint: http://tempo:4318
      otlphttp/pyroscope:
        endpoint: http://pyroscope:4318
 
    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [batch, tail_sampling]
          exporters: [otlphttp/tempo]
        profiles:
          receivers: [otlp]
          processors: [batch]
          exporters: [otlphttp/pyroscope]

# tempo.yaml — Tempo v3.0
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tempo
spec:
  replicas: 3
  selector:
    matchLabels:
      app: tempo
  template:
    spec:
      containers:
        - name: tempo
          image: grafana/tempo:3.0.0
          args: ["-config.file=/etc/tempo/tempo.yaml"]
          volumeMounts:
            - name: config
              mountPath: /etc/tempo
      volumes:
        - name: config
          configMap:
            name: tempo-config
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: tempo-config
data:
  tempo.yaml: |
    server:
      http_listen_port: 3200
 
    distributor:
      receivers:
        otlp:
          protocols:
            grpc:
              endpoint: 0.0.0.0:4317
            http:
              endpoint: 0.0.0.0:4318
 
    ingester:
      max_block_duration: 30m
 
    storage:
      trace:
        backend: s3
        s3:
          bucket: tempo-traces
          endpoint: s3.amazonaws.com
          region: cn-east-1
        block:
          bloom_filter_false_positive: .05
          index_downsample_bytes: 1000
          encoding: zstd
 
    querier:
      max_search_duration: 0    # 禁用搜索超时
 
    metrics_generator:
      registry:
        external_labels:
          source: tempo
          cluster: prod-cn-east
      storage:
        path: /var/tempo/wal
        remote_write:
          - url: http://mimir:9009/api/v1/push

# Grafana datasources — 追踪 + 日志 + 剖析联动
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasources
data:
  datasources.yaml: |
    apiVersion: 1
    datasources:
      - name: Tempo
        type: tempo
        url: http://tempo:3200
        uid: tempo
        jsonData:
          tracesToMetrics:
            datasourceUid: prometheus
          tracesToLogsV2:
            datasourceUid: loki
            spanStartTimeShift: '-1h'
            spanEndTimeShift: '1h'
            filterByTraceID: true
            filterBySpanID: true
          serviceMap:
            datasourceUid: prometheus
      - name: Loki
        type: loki
        url: http://loki:3100
        uid: loki
        jsonData:
          derivedFields:
            - datasourceUid: tempo
              matcherRegex: '"trace_id":"(\w+)"'
              name: TraceID
              url: '$${__value.raw}'
      - name: Pyroscope
        type: grafana-pyroscope-datasource
        url: http://pyroscope:4040
        uid: pyroscope