三大支柱：日志、指标、链路追踪

metric_name{label1="value1", label2="value2"} value timestamp

http_request_duration_seconds{method="GET", path="/api/orders", status="200", le="0.5"} 120

traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
             ^  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^  ^^^^^^^^^^^^^^^^ ^^
             |  Trace ID                      Span ID          Flags
          Version

# otel-collector-config.yaml - OpenTelemetry Collector 配置
receivers:
  # 接收 OTLP 数据（来自应用程序）
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
 
  # 接收 Prometheus 格式的指标
  prometheus:
    config:
      scrape_configs:
        - job_name: 'otel-collector'
          scrape_interval: 15s
          static_configs:
            - targets: ['localhost:8888']
 
processors:
  # 批处理：将数据批量发送，减少网络开销
  batch:
    send_batch_size: 1024
    send_batch_max_size: 2048
    timeout: 5s
 
  # 内存限制：防止 OOM
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128
 
  # 资源属性：添加集群和环境信息
  resource:
    attributes:
      - key: cluster
        value: production
        action: upsert
      - key: environment
        value: prod
        action: upsert
 
  # 过滤：移除不需要的数据
  filter:
    error_mode: ignore
    traces:
      span:
        - 'attributes["http.route"] == "/health"'
        - 'attributes["http.route"] == "/metrics"'
    metrics:
      metric:
        - 'name == "process.runtime.go.gc.pause_ns"'
 
  # 属性处理：规范化标签名
  transform:
    error_mode: ignore
    metric_statements:
      - context: metric
        statements:
          - set(description, "") where name == "http.server.duration"
          - set(name, "http.server.request.duration") where name == "http.server.duration"
 
exporters:
  # 导出到 Prometheus（指标）
  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write
    resource_to_telemetry_conversion:
      enabled: true
 
  # 导出到 Loki（日志）
  loki:
    endpoint: http://loki:3100/loki/api/v1/push
    default_labels_enabled:
      exporter: false
      job: true
 
  # 导出到 Tempo（追踪）
  otlphttp:
    endpoint: http://tempo:4318
 
  # 调试输出（开发环境）
  debug:
    verbosity: detailed
 
service:
  pipelines:
    # 指标管线
    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, transform, filter, batch, resource]
      exporters: [prometheusremotewrite]
 
    # 日志管线
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [loki]
 
    # 追踪管线
    traces:
      receivers: [otlp]
      processors: [memory_limiter, filter, batch, resource]
      exporters: [otlphttp]
 
  # 扩展
  extensions: [health_check, pprof]
 
extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  pprof:
    endpoint: 0.0.0.0:1777

// cmd/server/main.go - 带可观测性的 HTTP 服务
package main
 
import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
 
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
	"go.opentelemetry.io/otel/trace"
	"go.uber.org/zap"
)
 
// Application 结构体持有所有依赖
type Application struct {
	httpClient    *http.Client
	logger        *zap.Logger
	tracer        trace.Tracer
	httpCounter   metric.Int64Counter
	httpHistogram metric.Float64Histogram
}
 
func main() {
	// 初始化结构化日志
	logger, _ := zap.NewProduction()
	defer logger.Sync()
 
	ctx := context.Background()
 
	// 创建资源（服务元数据）
	res, err := resource.New(ctx,
		resource.WithAttributes(
			semconv.ServiceNameKey.String("order-service"),
			semconv.ServiceVersionKey.String("1.0.0"),
			semconv.DeploymentEnvironmentKey.String("production"),
		),
	)
	if err != nil {
		log.Fatalf("Failed to create resource: %v", err)
	}
 
	// 初始化 Prometheus 指标导出器
	promExporter, err := prometheus.New()
	if err != nil {
		log.Fatalf("Failed to create Prometheus exporter: %v", err)
	}
 
	// 初始化 Meter Provider
	meterProvider := metric.NewMeterProvider(
		metric.WithResource(res),
		metric.WithReader(promExporter),
	)
	defer meterProvider.Shutdown(ctx)
 
	otel.SetMeterProvider(meterProvider)
 
	// 初始化 OTLP Trace 导出器
	traceExporter, err := otlptracegrpc.New(ctx,
		otlptracegrpc.WithEndpoint("otel-collector:4317"),
		otlptracegrpc.WithInsecure(),
	)
	if err != nil {
		log.Fatalf("Failed to create trace exporter: %v", err)
	}
 
	// 初始化 Tracer Provider
	traceProvider := sdktrace.NewTracerProvider(
		sdktrace.WithResource(res),
		sdktrace.WithBatcher(traceExporter),
		sdktrace.WithSampler(sdktrace.ParentBased(
			sdktrace.TraceIDRatioBased(0.1), // 10% 采样率
		)),
	)
	defer traceProvider.Shutdown(ctx)
 
	otel.SetTracerProvider(traceProvider)
 
	// 创建指标
	meter := meterProvider.Meter("order-service")
	httpCounter, _ := meter.Int64Counter(
		"http.server.request.total",
		metric.WithDescription("Total number of HTTP requests"),
	)
	httpHistogram, _ := meter.Float64Histogram(
		"http.server.request.duration",
		metric.WithDescription("HTTP request duration in seconds"),
		metric.WithUnit("s"),
	)
 
	app := &Application{
		httpClient:    &http.Client{Timeout: 10 * time.Second},
		logger:        logger,
		tracer:        traceProvider.Tracer("order-service"),
		httpCounter:   httpCounter,
		httpHistogram: httpHistogram,
	}
 
	// 创建 HTTP 路由
	mux := http.NewServeMux()
 
	// 业务端点
	mux.HandleFunc("/api/orders", app.handleOrders)
 
	// Prometheus 指标端点
	mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
		promExporter.Collect(r.Context(), w)
	})
 
	// 健康检查
	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
		fmt.Fprint(w, "ok")
	})
 
	server := &http.Server{
		Addr:    ":8080",
		Handler: mux,
	}
 
	// 优雅关闭
	go func() {
		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
		<-sigCh
		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
		defer cancel()
		server.Shutdown(ctx)
	}()
 
	logger.Info("Starting server on :8080")
	if err := server.ListenAndServe(); err != http.ErrServerClosed {
		log.Fatalf("Server error: %v", err)
	}
}
 
func (app *Application) handleOrders(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	ctx := r.Context()
 
	// 创建 Span
	ctx, span := app.tracer.Start(ctx, "handleOrders",
		trace.WithAttributes(
			attribute.String("http.method", r.Method),
			attribute.String("http.route", "/api/orders"),
		),
	)
	defer span.End()
 
	// 从请求头提取 Trace ID，用于日志关联
	spanCtx := span.SpanContext()
	traceID := spanCtx.TraceID().String()
 
	// 结构化日志，包含 Trace ID
	app.logger.Info("Processing order request",
		zap.String("trace_id", traceID),
		zap.String("method", r.Method),
		zap.String("path", r.URL.Path),
	)
 
	// 业务逻辑（模拟）
	if r.Method == http.MethodGet {
		app.processGetOrders(ctx, traceID)
	} else if r.Method == http.MethodPost {
		app.processCreateOrder(ctx, traceID)
	}
 
	// 记录指标
	duration := time.Since(startTime).Seconds()
	attrs := []attribute.KeyValue{
		attribute.String("method", r.Method),
		attribute.String("path", "/api/orders"),
		attribute.Int("status", http.StatusOK),
	}
	app.httpCounter.Add(ctx, 1, metric.WithAttributes(attrs...))
	app.httpHistogram.Record(ctx, duration, metric.WithAttributes(attrs...))
 
	// 添加 Exemplar（指标与追踪的关联）
	span.SetAttributes(
		attribute.String("http.status_code", "200"),
	)
 
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprint(w, `{"status":"ok"}`)
}
 
func (app *Application) processGetOrders(ctx context.Context, traceID string) {
	ctx, span := app.tracer.Start(ctx, "processGetOrders")
	defer span.End()
 
	app.logger.Info("Fetching orders from database",
		zap.String("trace_id", traceID),
	)
 
	// 模拟数据库查询
	time.Sleep(50 * time.Millisecond)
}
 
func (app *Application) processCreateOrder(ctx context.Context, traceID string) {
	ctx, span := app.tracer.Start(ctx, "processCreateOrder")
	defer span.End()
 
	app.logger.Info("Creating new order",
		zap.String("trace_id", traceID),
	)
 
	// 模拟业务处理
	time.Sleep(100 * time.Millisecond)
 
	// 模拟调用下游服务
	app.callPaymentService(ctx, traceID)
}
 
func (app *Application) callPaymentService(ctx context.Context, traceID string) {
	ctx, span := app.tracer.Start(ctx, "callPaymentService")
	defer span.End()
 
	app.logger.Info("Calling payment service",
		zap.String("trace_id", traceID),
	)
 
	// 模拟网络调用
	time.Sleep(30 * time.Millisecond)
 
	// 模拟错误
	if false {
		span.SetStatus(codes.Error, "payment service timeout")
		span.RecordError(fmt.Errorf("payment service timeout"))
		app.logger.Error("Payment service call failed",
			zap.String("trace_id", traceID),
			zap.String("error", "timeout"),
		)
	}
}

# promtail-config.yaml - Promtail 日志采集配置
server:
  http_listen_port: 9080
  grpc_listen_port: 0
 
positions:
  filename: /tmp/positions.yaml
 
clients:
  - url: http://loki:3100/loki/api/v1/push
 
scrape_configs:
  # 采集 Kubernetes Pod 日志
  - job_name: kubernetes-pods
    kubernetes_sd_configs:
      - role: pod
    pipeline_stages:
      # 解析 JSON 日志
      - json:
          expressions:
            level: level
            msg: msg
            trace_id: trace_id
            span_id: span_id
            service: service
            timestamp: ts
 
      # 设置日志时间戳
      - timestamp:
          source: timestamp
          format: RFC3339
 
      # 设置日志级别标签
      - labels:
          level:
          service:
 
      # 设置 Trace ID 标签（用于日志与追踪关联）
      - labels:
          trace_id:
 
      # 过滤健康检查日志
      - match:
          selector: '{http_route="/health"}'
          action: drop
 
      # 过滤指标端点日志
      - match:
          selector: '{http_route="/metrics"}'
          action: drop
 
    relabel_configs:
      # 只采集带有注解的 Pod
      - source_labels:
          - __meta_kubernetes_pod_annotation_prometheus_io_scrape
        action: keep
        regex: true
      # 使用 Pod 名称作为 job 标签
      - source_labels:
          - __meta_kubernetes_pod_label_app
        target_label: job

# HTTP 请求 P99 延迟（5 分钟窗口）
histogram_quantile(0.99,
  sum(rate(http_server_request_duration_seconds_bucket[5m])) by (le, method, path)
)
 
# HTTP 请求错误率（5xx 占比）
sum(rate(http_server_request_total{status=~"5.."}[5m]))
/
sum(rate(http_server_request_total[5m]))
 
# 服务 QPS（每秒请求数）
sum(rate(http_server_request_total[5m])) by (service)
 
# Pod 内存使用率
container_memory_working_set_bytes{container!="",pod!=""}
/
container_spec_memory_limit_bytes{container!="",pod!=""}
 
# CPU 使用率（基于 request）
sum(rate(container_cpu_usage_seconds_total{container!="",pod!=""}[5m])) by (pod)
/
sum(container_spec_cpu_quota{container!="",pod!=""}/container_spec_cpu_period{container!="",pod!=""}) by (pod)
 
# 请求延迟 Top 10 服务
topk(10,
  histogram_quantile(0.99,
    sum(rate(http_server_request_duration_seconds_bucket[5m])) by (le, service)
  )
)
 
# 通过 Trace ID 关联查询（Exemplar）
# 在 Grafana 中，指标面板支持展示 Exemplar，
# 点击数据点可以跳转到对应的 Trace

# 好的命名
http_server_request_duration_seconds
http_server_request_total
order_created_total
 
# 差的命名
httpLatency
http_requests
orders

# 好的标签设计
http_server_request_duration_seconds{method="GET", path="/api/orders", status="200"}
 
# 差的标签设计（user_id 基数可能达到数百万）
http_server_request_duration_seconds{method="GET", user_id="12345"}