APM 监控

┌─────────────────────────────────────────────────────────────────┐
│                        APM 系统架构                              │
├─────────────────────────────────────────────────────────────────┤
│                                                                 │
│  ┌──────────────┐    ┌──────────────┐    ┌──────────────┐      │
│  │   应用服务    │───▶│   Agent     │───▶│   Collector  │      │
│  │  (Node.js)   │    │  (数据采集)   │    │  (数据收集)   │      │
│  └──────────────┘    └──────────────┘    └──────────────┘      │
│                                                  │              │
│                                                  ▼              │
│                                          ┌──────────────┐      │
│                                          │   Storage    │      │
│                                          │  (时序数据库) │      │
│                                          └──────────────┘      │
│                                                  │              │
│                                                  ▼              │
│  ┌──────────────┐    ┌──────────────┐    ┌──────────────┐      │
│  │   告警系统    │◀───│   Analysis   │◀───│   Dashboard  │      │
│  │  (通知渠道)   │    │  (数据分析)   │    │  (可视化展示) │      │
│  └──────────────┘    └──────────────┘    └──────────────┘      │
│                                                                 │
└─────────────────────────────────────────────────────────────────┘

选择决策树：

需要错误追踪？ ──是──▶ Sentry
       │
       否
       │
       ▼
已有 ELK 栈？ ──是──▶ Elastic APM
       │
       否
       │
       ▼
微服务架构？ ──是──▶ Jaeger + Prometheus
       │
       否
       │
       ▼
企业预算充足？ ──是──▶ New Relic / Datadog
       │
       否
       │
       ▼
Prometheus + Grafana

┌──────────────────────────────────────────────────────────────────┐
│                    Prometheus + Grafana 架构                      │
├──────────────────────────────────────────────────────────────────┤
│                                                                  │
│  ┌────────────┐     ┌────────────┐     ┌────────────┐           │
│  │  App #1    │     │  App #2    │     │  App #3    │           │
│  │ /metrics   │     │ /metrics   │     │ /metrics   │           │
│  └─────┬──────┘     └─────┬──────┘     └─────┬──────┘           │
│        │                  │                  │                   │
│        └──────────────────┼──────────────────┘                   │
│                           │                                      │
│                           ▼                                      │
│                   ┌──────────────┐                               │
│                   │  Prometheus  │  ◀─── Pull 模式拉取指标       │
│                   │   (存储+查询) │                               │
│                   └──────┬───────┘                               │
│                          │                                       │
│                          ▼                                       │
│                   ┌──────────────┐                               │
│                   │   Grafana    │  ◀─── 可视化 + 告警           │
│                   │  (Dashboard) │                               │
│                   └──────────────┘                               │
│                                                                  │
└──────────────────────────────────────────────────────────────────┘

pnpm add prom-client

const client = require('prom-client')
const express = require('express')

const app = express()

// 启用默认指标（包含 Node.js 运行时指标）
const collectDefaultMetrics = client.collectDefaultMetrics
collectDefaultMetrics({ 
  register: client.register,
  // 自定义前缀
  prefix: 'myapp_',
  // 采集间隔（毫秒）
  gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5]
})

// 自定义计数器
const httpRequestsTotal = new client.Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'route', 'status']
})

// 自定义直方图
const httpRequestDuration = new client.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'route', 'status'],
  buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10]
})

// 中间件
app.use((req, res, next) => {
  const start = Date.now()
  
  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000
    const route = req.route ? req.route.path : req.path
    
    httpRequestsTotal.inc({
      method: req.method,
      route: route,
      status: res.statusCode
    })
    
    httpRequestDuration.observe(
      {
        method: req.method,
        route: route,
        status: res.statusCode
      },
      duration
    )
  })
  
  next()
})

// 暴露指标端点
app.get('/metrics', async (req, res) => {
  try {
    res.set('Content-Type', client.register.contentType)
    res.end(await client.register.metrics())
  } catch (error) {
    res.status(500).end(error.message)
  }
})

app.listen(3000, () => {
  console.log('Server running on port 3000')
})

// ==================== Counter 计数器 ====================
// 只增不减，用于累计计数
const ordersTotal = new client.Counter({
  name: 'orders_total',
  help: 'Total number of orders',
  labelNames: ['status', 'payment_method']
})

// 使用方式
ordersTotal.inc()                                    // +1
ordersTotal.inc({ status: 'completed', payment_method: 'credit' })  // 带标签 +1
ordersTotal.inc(10, { status: 'pending' })          // 指定增量

// ==================== Gauge 测量仪 ====================
// 可增可减，用于瞬时值
const activeConnections = new client.Gauge({
  name: 'active_connections',
  help: 'Number of active connections',
  labelNames: ['type']
})

// 使用方式
activeConnections.inc()                              // +1
activeConnections.dec()                              // -1
activeConnections.set(100, { type: 'websocket' })   // 设置为特定值
activeConnections.reset()                            // 重置为 0

// ==================== Histogram 直方图 ====================
// 分布统计，适合延迟、大小等
const responseSizes = new client.Histogram({
  name: 'response_size_bytes',
  help: 'Size of HTTP responses',
  buckets: [100, 500, 1000, 5000, 10000, 50000, 100000]  // 自定义桶
})

// ==================== Summary 摘要 ====================
// 分位数统计，适合 SLA 监控
const requestLatency = new client.Summary({
  name: 'request_latency_seconds',
  help: 'Request latency in seconds',
  percentiles: [0.5, 0.9, 0.95, 0.99],  // P50, P90, P95, P99
  maxAgeSeconds: 600,                    // 时间窗口
  ageBuckets: 5                          // 桶数量
})

// ==================== 实际业务示例 ====================
class MetricsService {
  constructor() {
    // 数据库查询监控
    this.dbQueryDuration = new client.Histogram({
      name: 'db_query_duration_seconds',
      help: 'Database query duration',
      labelNames: ['operation', 'table'],
      buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5]
    })

    // 缓存命中率
    this.cacheHits = new client.Counter({
      name: 'cache_hits_total',
      help: 'Cache hit/miss count',
      labelNames: ['result']  // hit | miss
    })

    // 队列大小
    this.queueSize = new client.Gauge({
      name: 'queue_size',
      help: 'Current queue size',
      labelNames: ['queue_name']
    })
  }

  // 记录数据库查询
  async trackDbQuery(operation, table, queryFn) {
    const timer = this.dbQueryDuration.startTimer({ operation, table })
    try {
      const result = await queryFn()
      return result
    } finally {
      timer()
    }
  }

  // 记录缓存命中
  recordCacheHit(isHit) {
    this.cacheHits.inc({ result: isHit ? 'hit' : 'miss' })
  }
}

module.exports = new MetricsService()

# prometheus.yml
global:
  scrape_interval: 15s      # 默认采集间隔
  evaluation_interval: 15s  # 规则评估间隔

# 告警管理器配置
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

# 告警规则文件
rule_files:
  - "alert.rules.yml"

# 采集目标配置
scrape_configs:
  # Prometheus 自身监控
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node.js 应用监控
  - job_name: 'nodejs-app'
    static_configs:
      - targets: ['app1:3000', 'app2:3000', 'app3:3000']
        labels:
          env: 'production'
          service: 'api'

  # 使用服务发现（Kubernetes）
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true

{
  "dashboard": {
    "title": "Node.js APM Dashboard",
    "panels": [
      {
        "title": "Request Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(http_requests_total[5m])",
            "legendFormat": "{{method}} {{route}}"
          }
        ]
      },
      {
        "title": "Response Time (P95)",
        "type": "graph",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
            "legendFormat": "P95"
          }
        ]
      },
      {
        "title": "Error Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(http_requests_total{status=~\"5..\"}[5m]) / rate(http_requests_total[5m])",
            "legendFormat": "Error Rate"
          }
        ]
      },
      {
        "title": "Memory Usage",
        "type": "graph",
        "targets": [
          {
            "expr": "process_resident_memory_bytes",
            "legendFormat": "RSS"
          },
          {
            "expr": "nodejs_heap_size_used_bytes",
            "legendFormat": "Heap Used"
          }
        ]
      }
    ]
  }
}

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alert.rules.yml:/etc/prometheus/alert.rules.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.enable-lifecycle'

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3001:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    depends_on:
      - prometheus

  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml

  nodejs-app:
    build: .
    ports:
      - "3000:3000"
    environment:
      - NODE_ENV=production

volumes:
  prometheus_data:
  grafana_data:

┌──────────────────────────────────────────────────────────────────┐
│                       Sentry 工作流程                             │
├──────────────────────────────────────────────────────────────────┤
│                                                                  │
│  ┌────────────┐      ┌────────────┐      ┌────────────┐         │
│  │  应用发生   │      │  SDK 上报  │      │  Sentry    │         │
│  │  错误/异常  │─────▶│  事件数据   │─────▶│  服务端    │         │
│  └────────────┘      └────────────┘      └─────┬──────┘         │
│                                                 │                │
│                                                 ▼                │
│                                          ┌────────────┐         │
│  ┌────────────┐      ┌────────────┐      │  事件处理  │         │
│  │  通知渠道   │◀─────│  告警规则   │◀─────│  聚合分析  │         │
│  │ (邮件/Slack)│      │  配置      │      │  去重存储  │         │
│  └────────────┘      └────────────┘      └────────────┘         │
│                                                                  │
└──────────────────────────────────────────────────────────────────┘

pnpm add @sentry/node @sentry/tracing

const Sentry = require('@sentry/node')
const { ProfilingIntegration } = require('@sentry/profiling-node')
const express = require('express')

// 在应用启动时初始化
Sentry.init({
  dsn: process.env.SENTRY_DSN,
  environment: process.env.NODE_ENV || 'development',
  
  // 采样率配置
  tracesSampleRate: process.env.NODE_ENV === 'production' ? 0.1 : 1.0,
  profilesSampleRate: 0.1,  // 性能分析采样率
  
  // 集成配置
  integrations: [
    // 启用 HTTP 集成
    new Sentry.Integrations.Http({ tracing: true }),
    // 启用 Express 集成
    new Sentry.Integrations.Express(),
    // 性能分析
    new ProfilingIntegration(),
  ],
  
  // 忽略特定错误
  ignoreErrors: [
    'NotFoundError',
    'UnauthorizedError',
    /NetworkError/,
  ],
  
  // 过滤事务
  ignoreTransactions: [
    'GET /health',
    'GET /metrics',
  ],
  
  // 发布版本追踪
  release: process.env.APP_VERSION || '1.0.0',
  
  // 服务器名称
  serverName: process.env.SERVER_NAME || 'node-server',
})

const app = express()

// 请求处理中间件（必须放在所有路由之前）
app.use(Sentry.Handlers.requestHandler({
  user: ['id', 'username', 'email'],  // 提取用户信息
  ip: true,                           // 记录 IP
  request: true,                      // 记录请求信息
  transaction: 'methodPath',          // 事务命名方式
}))

// 追踪中间件
app.use(Sentry.Handlers.tracingHandler())

// 业务路由
app.get('/', (req, res) => {
  res.send('Hello World')
})

app.get('/error', (req, res) => {
  throw new Error('Test error')
})

// 错误处理中间件（必须放在所有路由之后）
app.use(Sentry.Handlers.errorHandler({
  shouldHandleError(error) {
    // 只捕获 4xx 和 5xx 错误
    return true
  },
}))

app.listen(3000)

// ==================== 捕获异常 ====================
try {
  riskyOperation()
} catch (error) {
  Sentry.captureException(error)
}

// 异步错误捕获
async function fetchData() {
  try {
    const data = await fetch(url)
    return data
  } catch (error) {
    Sentry.captureException(error, {
      tags: { component: 'data-fetch' },
      extra: { url }
    })
    throw error
  }
}

// ==================== 捕获消息 ====================
// 不同级别的消息
Sentry.captureMessage('Something went wrong', 'warning')
Sentry.captureMessage('Critical error occurred', 'fatal')
Sentry.captureMessage('Info message', 'info')

// ==================== 添加上下文 ====================
// 设置用户信息（会附加到后续所有事件）
Sentry.setUser({ 
  id: '123', 
  username: 'john_doe',
  email: 'user@example.com',
  role: 'admin'
})

// 清除用户信息（如退出登录时）
Sentry.setUser(null)

// 添加标签（用于筛选和聚合）
Sentry.setTag('page', 'checkout')
Sentry.setTag('feature', 'payment')

// 添加额外数据
Sentry.setExtra('orderData', { 
  orderId: '456',
  items: [...],
  total: 99.99
})

// 设置面包屑（事件发生前的操作记录）
Sentry.addBreadcrumb({
  category: 'http',
  message: 'API request',
  level: 'info',
  data: {
    url: '/api/users',
    method: 'GET',
    status_code: 200
  }
})

// ==================== withScope 临时上下文 ====================
Sentry.withScope((scope) => {
  scope.setTag('custom-tag', 'value')
  scope.setExtra('custom-data', { foo: 'bar' })
  scope.setUser({ id: 'temp-user' })
  Sentry.captureException(new Error('Scoped error'))
})
// 离开 withScope 后，以上上下文设置不会影响全局

// ==================== 手动事务追踪 ====================
// 创建事务
const transaction = Sentry.startTransaction({
  op: 'task',
  name: 'Process Order',
})

try {
  // 子 Span：数据库查询
  const dbSpan = transaction.startChild({
    op: 'db.query',
    description: 'SELECT * FROM orders WHERE id = ?'
  })
  await queryDatabase()
  dbSpan.finish()
  
  // 子 Span：外部 API 调用
  const apiSpan = transaction.startChild({
    op: 'http.client',
    description: 'POST /payment/process'
  })
  await processPayment()
  apiSpan.finish()
  
  // 子 Span：发送通知
  const notifySpan = transaction.startChild({
    op: 'notify',
    description: 'Send order confirmation email'
  })
  await sendEmail()
  notifySpan.finish()
  
} finally {
  transaction.finish()  // 必须调用，否则事务不会发送
}

// ==================== 使用 Sentry 包装函数 ====================
const result = await Sentry.startSpan(
  {
    op: 'function',
    name: 'calculateTotal',
  },
  async (span) => {
    const total = await calculateOrderTotal()
    span.setAttribute('total_amount', total)
    return total
  }
)

// ==================== Express 中间件性能追踪 ====================
app.get('/api/users/:id', async (req, res) => {
  const transaction = Sentry.getActiveTransaction()
  
  if (transaction) {
    const span = transaction.startChild({
      op: 'db.query',
      description: 'Fetch user by ID'
    })
    
    try {
      const user = await User.findById(req.params.id)
      span.setAttribute('user.found', !!user)
      res.json(user)
    } finally {
      span.finish()
    }
  }
})

// sentry.client.config.js
module.exports = {
  org: 'your-org',
  project: 'your-project',
  authToken: process.env.SENTRY_AUTH_TOKEN,
  url: 'https://sentry.io/',
  
  // Source Maps 上传配置
  release: {
    name: process.env.APP_VERSION,
    create: true,
    finalize: true,
    setCommits: {
      auto: true,
    },
    dist: process.env.BUILD_ID,
  },
  
  // 上传配置
  include: ['./dist'],
  ignore: ['node_modules'],
  
  // 验证 Source Maps
  rewrite: true,
  stripPrefix: ['webpack:///'],
  urlPrefix: '~/static/',
}

// package.json
{
  "scripts": {
    "build": "webpack --mode production && sentry-cli sourcemaps inject ./dist && sentry-cli sourcemaps upload ./dist",
    "release": "sentry-cli releases new $npm_package_version && npm run build && sentry-cli releases finalize $npm_package_version"
  }
}

# Sentry 告警规则（在 Sentry 控制台配置）
rules:
  - name: "High Error Rate"
    conditions:
      - type: "event_frequency"
        comparison: "gt"
        value: 100
        timeframe: 1h
    actions:
      - type: "email"
        target: "team@example.com"
      - type: "slack"
        channel: "#alerts"
  
  - name: "New Error Type"
    conditions:
      - type: "new_issue"
    actions:
      - type: "slack"
        channel: "#errors"

┌──────────────────────────────────────────────────────────────────┐
│                      Elastic APM 架构                             │
├──────────────────────────────────────────────────────────────────┤
│                                                                  │
│  ┌────────────┐      ┌────────────┐      ┌────────────┐         │
│  │ Node.js    │      │ APM Agent  │      │ APM Server │         │
│  │ Application│─────▶│ (数据采集)  │─────▶│ (数据接收)  │         │
│  └────────────┘      └────────────┘      └─────┬──────┘         │
│                                                 │                │
│                                                 ▼                │
│                                          ┌────────────┐         │
│                                          │Elasticsearch│        │
│                                          │  (存储索引) │         │
│                                          └─────┬──────┘         │
│                                                 │                │
│                                                 ▼                │
│                                          ┌────────────┐         │
│                                          │   Kibana   │         │
│                                          │ (可视化分析) │         │
│                                          └────────────┘         │
│                                                                  │
└──────────────────────────────────────────────────────────────────┘

pnpm add elastic-apm-node

// 在应用最顶部引入（必须先于其他模块）
const apm = require('elastic-apm-node').start({
  // 服务配置
  serviceName: 'my-nodejs-service',
  serviceVersion: '1.0.0',
  serverUrl: process.env.ELASTIC_APM_SERVER_URL || 'http://localhost:8200',
  
  // 环境配置
  environment: process.env.NODE_ENV || 'development',
  
  // 采样配置
  transactionSampleRate: process.env.NODE_ENV === 'production' ? 0.1 : 1.0,
  
  // 栈追踪深度
  stackTraceLimit: 50,
  
  // 日志配置
  logLevel: 'info',
  logger: require('pino')({ level: 'info' }),
  
  // 忽略特定 URL
  ignoreUrls: ['/health', '/metrics', '/favicon.ico'],
  
  // 忽略特定 User Agent
  ignoreUserAgents: ['curl', 'pingdom'],
  
  // 错误配置
  captureErrorLogStackTraces: 'always',
  captureExceptions: true,
  
  // 性能配置
  metricsInterval: '30s',
  centralConfig: true,
  
  // 云提供商信息（可选）
  cloudProvider: 'aws',
  
  // Secret Token 或 API Key
  secretToken: process.env.ELASTIC_APM_SECRET_TOKEN,
  // apiKey: process.env.ELASTIC_APM_API_KEY,
})

const express = require('express')
const app = express()

app.get('/api/users', async (req, res) => {
  const users = await getUsers()
  res.json(users)
})

app.listen(3000)

// 导出 apm 实例供其他模块使用
module.exports.apm = apm

const apm = require('elastic-apm-node')

// ==================== 自定义事务 ====================
app.get('/api/orders/process', async (req, res) => {
  // 获取当前事务
  const transaction = apm.currentTransaction
  
  if (transaction) {
    // 设置自定义标签
    transaction.setLabel('order_type', 'premium')
    transaction.setLabel('customer_id', req.user.id)
    
    // 设置自定义上下文
    transaction.setCustomContext({
      order: {
        items: req.body.items.length,
        total: req.body.total
      }
    })
  }
  
  // 手动创建事务
  const manualTransaction = apm.startTransaction('Manual Transaction', 'custom')
  try {
    await processOrder()
    manualTransaction.result = 'success'
  } catch (error) {
    manualTransaction.result = 'error'
    apm.captureError(error)
    throw error
  } finally {
    manualTransaction.end()
  }
})

// ==================== 自定义 Span ====================
app.get('/api/orders', async (req, res) => {
  // 数据库查询 Span
  const dbSpan = apm.startSpan('query-orders', 'db')
  dbSpan?.addLabels({ operation: 'SELECT', table: 'orders' })
  
  try {
    const orders = await getOrders()
    dbSpan?.end()
    
    // 外部 API 调用 Span
    const apiSpan = apm.startSpan('validate-payment', 'external')
    await validatePayment(orders)
    apiSpan?.end()
    
    res.json(orders)
  } catch (error) {
    dbSpan?.end()
    apm.captureError(error)
    res.status(500).json({ error: 'Internal error' })
  }
})

// ==================== 错误捕获 ====================
try {
  await riskyOperation()
} catch (error) {
  apm.captureError(error, {
    tags: { component: 'payment' },
    custom: { orderId: '12345' }
  })
}

// 捕获自定义错误消息
apm.captureError(new Error('Something went wrong'), {
  user: {
    id: '123',
    email: 'user@example.com'
  },
  tags: {
    feature: 'checkout'
  },
  custom: {
    cartItems: 5
  }
})

// 查询慢请求
GET apm-*-transaction*/_search
{
  "query": {
    "range": {
      "transaction.duration.us": {
        "gte": 1000000  // 大于 1 秒
      }
    }
  },
  "size": 10,
  "sort": [
    { "@timestamp": "desc" }
  ]
}

// 查询错误趋势
GET apm-*-error*/_search
{
  "size": 0,
  "aggs": {
    "errors_over_time": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "1h"
      }
    }
  }
}

// OpenTelemetry 提供 vendor 中立的追踪方案
const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node')
const { SimpleSpanProcessor } = require('@opentelemetry/sdk-trace-base')
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger')
const { getResource } = require('@opentelemetry/resources')
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions')

// 配置 Provider
const provider = new NodeTracerProvider({
  resource: new getResource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'my-service',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
  }),
})

// 配置 Jaeger 导出器
const jaegerExporter = new JaegerExporter({
  endpoint: 'http://localhost:14268/api/traces',
})

provider.addSpanProcessor(new SimpleSpanProcessor(jaegerExporter))
provider.register()

// 在 Express 中使用
const { trace } = require('@opentelemetry/api')

app.get('/api/users/:id', async (req, res) => {
  const tracer = trace.getTracer('my-service')
  const span = tracer.startSpan('fetch-user')
  
  try {
    const user = await User.findById(req.params.id)
    span.setAttributes({ 'user.id': user.id })
    res.json(user)
  } catch (error) {
    span.recordException(error)
    res.status(500).json({ error: error.message })
  } finally {
    span.end()
  }
})

// ==================== 服务 A（发起请求） ====================
const axios = require('axios')
const { trace, propagation } = require('@opentelemetry/api')

async function callServiceB() {
  const tracer = trace.getTracer('service-a')
  
  return tracer.startActiveSpan('call-service-b', async (span) => {
    // 注入追踪上下文到请求头
    const headers = {}
    propagation.inject(trace.context.active(), headers)
    
    const response = await axios.get('http://service-b/api/data', { headers })
    
    span.end()
    return response.data
  })
}

// ==================== 服务 B（接收请求） ====================
const express = require('express')
const { trace, propagation } = require('@opentelemetry/api')

const app = express()

app.get('/api/data', async (req, res) => {
  // 从请求头提取追踪上下文
  const context = propagation.extract(trace.context.active(), req.headers)
  
  const tracer = trace.getTracer('service-b')
  
  return tracer.startActiveSpan('process-data', {}, context, async (span) => {
    const data = await processData()
    span.setAttribute('data.count', data.length)
    res.json(data)
    span.end()
  })
})

┌──────────────────────────────────────────────────────────────────┐
│                      健康检查端点                                 │
├──────────────────────────────────────────────────────────────────┤
│                                                                  │
│  /health        存活检查 (Liveness Probe)                        │
│  └─ 用途：Kubernetes 判断是否重启容器                             │
│  └─ 检查：应用进程是否存活                                         │
│  └─ 返回：{ status: 'ok' }                                       │
│                                                                  │
│  /health/ready  就绪检查 (Readiness Probe)                       │
│  └─ 用途：Kubernetes 判断是否转发流量                             │
│  └─ 检查：数据库、缓存、外部依赖                                    │
│  └─ 返回：{ status: 'ready' } 或 503                             │
│                                                                  │
│  /health/detail 详细健康检查                                      │
│  └─ 用途：运维排查问题                                             │
│  └─ 检查：所有组件详细状态                                          │
│  └─ 返回：完整的健康报告                                           │
│                                                                  │
└──────────────────────────────────────────────────────────────────┘

const express = require('express')
const app = express()

// 存活检查 - 仅检查进程是否存活
app.get('/health', (req, res) => {
  res.json({
    status: 'ok',
    timestamp: new Date().toISOString(),
    uptime: Math.floor(process.uptime())
  })
})

// 就绪检查 - 检查服务是否准备好接收流量
app.get('/health/ready', async (req, res) => {
  try {
    // 检查数据库连接
    await db.ping()
    
    // 检查 Redis 连接（可选）
    // await redis.ping()
    
    res.json({ status: 'ready' })
  } catch (error) {
    res.status(503).json({ 
      status: 'not ready',
      reason: error.message 
    })
  }
})

// 健康检查服务
class HealthCheckService {
  constructor() {
    this.checks = {
      database: this.checkDatabase.bind(this),
      redis: this.checkRedis.bind(this),
      external: this.checkExternalService.bind(this),
      disk: this.checkDiskSpace.bind(this)
    }
  }

  async checkDatabase() {
    const start = Date.now()
    try {
      await db.raw('SELECT 1')
      return {
        status: 'ok',
        latency: Date.now() - start,
        message: 'Database connection healthy'
      }
    } catch (error) {
      return {
        status: 'error',
        latency: Date.now() - start,
        message: error.message
      }
    }
  }

  async checkRedis() {
    const start = Date.now()
    try {
      await redis.ping()
      return {
        status: 'ok',
        latency: Date.now() - start,
        message: 'Redis connection healthy'
      }
    } catch (error) {
      return {
        status: 'error',
        latency: Date.now() - start,
        message: error.message
      }
    }
  }

  async checkExternalService() {
    const start = Date.now()
    try {
      const response = await axios.get('https://api.example.com/health', {
        timeout: 5000
      })
      return {
        status: response.status === 200 ? 'ok' : 'error',
        latency: Date.now() - start,
        message: 'External service reachable'
      }
    } catch (error) {
      return {
        status: 'error',
        latency: Date.now() - start,
        message: error.message
      }
    }
  }

  async checkDiskSpace() {
    const diskspace = require('diskspace')
    return new Promise((resolve) => {
      diskspace.check('/', (err, result) => {
        if (err) {
          resolve({ status: 'error', message: err.message })
        } else {
          const usedPercent = (result.used / result.total) * 100
          resolve({
            status: usedPercent < 90 ? 'ok' : 'warning',
            used: `${Math.round(usedPercent)}%`,
            free: `${Math.round((result.free / result.total) * 100)}%`
          })
        }
      })
    })
  }

  async runAllChecks() {
    const results = {}
    
    for (const [name, check] of Object.entries(this.checks)) {
      results[name] = await check()
    }
    
    return results
  }
}

const healthService = new HealthCheckService()

// 详细健康检查端点
app.get('/health/detail', async (req, res) => {
  const checks = await healthService.runAllChecks()
  
  const allHealthy = Object.values(checks).every(c => c.status === 'ok')
  const hasWarning = Object.values(checks).some(c => c.status === 'warning')
  
  let status = 'healthy'
  let statusCode = 200
  
  if (!allHealthy && !hasWarning) {
    status = 'unhealthy'
    statusCode = 503
  } else if (hasWarning) {
    status = 'degraded'
    statusCode = 200
  }
  
  res.status(statusCode).json({
    status,
    checks,
    system: {
      memory: process.memoryUsage(),
      uptime: Math.floor(process.uptime()),
      cpuUsage: process.cpuUsage(),
      nodeVersion: process.version,
      platform: process.platform
    },
    timestamp: new Date().toISOString()
  })
})

# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nodejs-app
spec:
  template:
    spec:
      containers:
        - name: app
          image: nodejs-app:latest
          
          # 存活探针
          livenessProbe:
            httpGet:
              path: /health
              port: 3000
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 3
          
          # 就绪探针
          readinessProbe:
            httpGet:
              path: /health/ready
              port: 3000
            initialDelaySeconds: 5
            periodSeconds: 5
            timeoutSeconds: 3
            failureThreshold: 3

┌──────────────────────────────────────────────────────────────────┐
│                     核心监控指标体系                               │
├──────────────────────────────────────────────────────────────────┤
│                                                                  │
│  【基础设施层】                                                   │
│  ├─ CPU 使用率        进程/系统 CPU 占用                          │
│  ├─ 内存使用率        RSS/Heap/External 内存                      │
│  ├─ 网络流量          入站/出站流量                                │
│  └─ 磁盘 I/O          读写速率                                    │
│                                                                  │
│  【应用层】                                                       │
│  ├─ 请求吞吐量        QPS/RPS                                    │
│  ├─ 响应时间          P50/P95/P99 延迟                            │
│  ├─ 错误率            4xx/5xx 比例                                │
│  └─ 并发连接数        活跃连接数                                   │
│                                                                  │
│  【业务层】                                                       │
│  ├─ 业务指标          订单量/用户数/交易额                         │
│  ├─ 业务成功率        支付成功率/下单成功率                        │
│  └─ 业务延迟          业务处理时长                                │
│                                                                  │
└──────────────────────────────────────────────────────────────────┘

const client = require('prom-client')

// 内存指标
const memoryGauge = new client.Gauge({
  name: 'nodejs_memory_usage_bytes',
  help: 'Memory usage in bytes',
  labelNames: ['type']
})

// 堆内存分布
const heapSizeGauge = new client.Gauge({
  name: 'nodejs_heap_space_size_bytes',
  help: 'Heap space size in bytes',
  labelNames: ['space', 'type']  // space: new/old/code/map, type: used/available
})

// 事件循环延迟
const eventLoopLag = new client.Gauge({
  name: 'nodejs_eventloop_lag_seconds',
  help: 'Event loop lag in seconds'
})

// GC 统计
const gcCount = new client.Counter({
  name: 'nodejs_gc_count',
  help: 'Garbage collection count',
  labelNames: ['kind']  // incremental, weak, full
})

const gcDuration = new client.Histogram({
  name: 'nodejs_gc_duration_seconds',
  help: 'Garbage collection duration',
  labelNames: ['kind'],
  buckets: [0.001, 0.01, 0.1, 0.5, 1, 2]
})

// 定期采集内存指标
function collectMemoryMetrics() {
  const mem = process.memoryUsage()
  memoryGauge.set({ type: 'rss' }, mem.rss)
  memoryGauge.set({ type: 'heapTotal' }, mem.heapTotal)
  memoryGauge.set({ type: 'heapUsed' }, mem.heapUsed)
  memoryGauge.set({ type: 'external' }, mem.external)
  memoryGauge.set({ type: 'arrayBuffers' }, mem.arrayBuffers || 0)
  
  // 堆空间详细信息（Node.js 12+）
  if (v8.getHeapSpaceStatistics) {
    const spaces = v8.getHeapSpaceStatistics()
    spaces.forEach(space => {
      const spaceName = space.space_name.toLowerCase().replace('_', '')
      heapSizeGauge.set({ space: spaceName, type: 'used' }, space.space_used_size)
      heapSizeGauge.set({ space: spaceName, type: 'available' }, space.space_available_size)
      heapSizeGauge.set({ space: spaceName, type: 'size' }, space.space_size)
    })
  }
}

// 监控事件循环延迟
function monitorEventLoopLag() {
  const start = process.hrtime.bigint()
  setImmediate(() => {
    const delta = Number(process.hrtime.bigint() - start)
    const lagSeconds = delta / 1e9 - 0.001  // 减去 setImmediate 的理论延迟
    eventLoopLag.set(lagSeconds)
  })
}

// 启动监控
setInterval(collectMemoryMetrics, 10000)      // 每 10 秒
setInterval(monitorEventLoopLag, 1000)       // 每 1 秒

# alertmanager.yml
global:
  # 默认通知配置
  resolve_timeout: 5m
  
  # SMTP 配置
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: 'alerts@example.com'
  smtp_auth_username: 'alerts@example.com'
  smtp_auth_password: 'password'

  # Slack 配置
  slack_api_url: 'https://hooks.slack.com/services/xxx'

# 路由配置
route:
  group_by: ['alertname', 'severity']
  group_wait: 30s        # 等待同组告警聚合
  group_interval: 5m     # 同组新告警间隔
  repeat_interval: 4h    # 重复告警间隔
  
  # 默认接收者
  receiver: 'team-email'
  
  # 子路由
  routes:
    # 严重告警 -> Slack + 邮件
    - match:
        severity: critical
      receiver: 'critical-alerts'
      continue: true
    
    # 警告级别 -> Slack
    - match:
        severity: warning
      receiver: 'team-slack'
    
    # 特定服务告警
    - match:
        service: payment
      receiver: 'payment-team'

# 接收者配置
receivers:
  - name: 'team-email'
    email_configs:
      - to: 'team@example.com'
        send_resolved: true
  
  - name: 'team-slack'
    slack_configs:
      - channel: '#alerts'
        send_resolved: true
        title: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}'
        text: >-
          {{ range .Alerts }}
          *Alert:* {{ .Labels.alertname }}
          *Severity:* {{ .Labels.severity }}
          *Description:* {{ .Annotations.description }}
          *Details:*
          {{ range .Labels.SortedPairs }} • *{{ .Name }}:* {{ .Value }}
          {{ end }}
          {{ end }}
  
  - name: 'critical-alerts'
    slack_configs:
      - channel: '#critical-alerts'
        send_resolved: true
    email_configs:
      - to: 'oncall@example.com'
        send_resolved: true
    # Webhook 配置
    webhook_configs:
      - url: 'https://pagerduty.com/webhook'
        send_resolved: true
  
  - name: 'payment-team'
    slack_configs:
      - channel: '#payment-alerts'
    email_configs:
      - to: 'payment-team@example.com'

# 静默配置（临时禁用告警）
inhibit_rules:
  # 当服务不可用时，抑制相关的其他告警
  - source_match:
      severity: 'critical'
      alertname: 'ServiceDown'
    target_match:
      severity: 'warning'
    equal: ['service']

# alert.rules.yml
groups:
  - name: nodejs-alerts
    rules:
      # ==================== 应用层告警 ====================
      
      # 高错误率
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) 
          / sum(rate(http_requests_total[5m])) > 0.01
        for: 5m
        labels:
          severity: critical
          service: api
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }} for more than 5 minutes"
      
      # 响应时间过长
      - alert: HighResponseTime
        expr: |
          histogram_quantile(0.95, 
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
          ) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High response time (P95)"
          description: "P95 response time is {{ $value | humanizeDuration }}"
      
      # 请求速率突降
      - alert: LowRequestRate
        expr: |
          sum(rate(http_requests_total[5m])) 
          < sum(rate(http_requests_total[5m] offset 1h)) * 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Request rate dropped significantly"
          description: "Request rate dropped by more than 50%"
      
      # ==================== 资源层告警 ====================
      
      # 高内存使用
      - alert: HighMemoryUsage
        expr: |
          (nodejs_memory_usage_bytes{type="heapUsed"} 
          / nodejs_memory_usage_bytes{type="heapTotal"}) > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage"
          description: "Heap usage is {{ $value | humanizePercentage }}"
      
      # 内存泄漏嫌疑
      - alert: PossibleMemoryLeak
        expr: |
          increase(nodejs_memory_usage_bytes{type="heapUsed"}[1h]) > 100000000
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Possible memory leak detected"
          description: "Heap usage increased by {{ $value | humanizeBytes }} in the last hour"
      
      # 事件循环阻塞
      - alert: EventLoopBlocked
        expr: nodejs_eventloop_lag_seconds > 0.5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Event loop is blocked"
          description: "Event loop lag is {{ $value | humanizeDuration }}"
      
      # ==================== 基础设施告警 ====================
      
      # 服务不可用
      - alert: ServiceDown
        expr: up{job="nodejs-app"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Service {{ $labels.instance }} is down"
          description: "Service has been down for more than 1 minute"
      
      # CPU 使用率过高
      - alert: HighCPUUsage
        expr: |
          100 - (avg by(instance) (irate(process_cpu_seconds_total[5m])) * 100) < 20
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage"
          description: "CPU usage is above 80% for {{ $labels.instance }}"

const winston = require('winston')
const PrometheusTransport = require('winston-prometheus-transport')

// 日志计数器
const logCounter = new client.Counter({
  name: 'app_logs_total',
  help: 'Total application logs',
  labelNames: ['level', 'component']
})

// Winston 配置
const logger = winston.createLogger({
  level: 'info',
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.errors({ stack: true }),
    winston.format.json()
  ),
  defaultMeta: { service: 'my-service' },
  transports: [
    // 控制台输出
    new winston.transports.Console({
      format: winston.format.combine(
        winston.format.colorize(),
        winston.format.simple()
      )
    }),
    
    // 文件输出
    new winston.transports.File({ 
      filename: 'logs/error.log', 
      level: 'error' 
    }),
    new winston.transports.File({ 
      filename: 'logs/combined.log' 
    }),
  ],
})

// 添加日志计数中间件
const originalLog = logger.log
logger.log = function(level, message, meta = {}) {
  logCounter.inc({ level, component: meta.component || 'default' })
  return originalLog.call(this, level, message, meta)
}

// ❌ 错误：高基数标签
const counter = new Counter({
  name: 'requests_total',
  labelNames: ['user_id']  // 可能有数百万用户
})

// ✅ 正确：使用低基数标签
const counter = new Counter({
  name: 'requests_total',
  labelNames: ['method', 'route', 'status']  // 有限的组合
})

# 配置告警聚合
route:
  group_by: ['alertname', 'severity']
  group_wait: 30s      # 等待同组告警
  group_interval: 5m   # 发送间隔
  
# 配置静默规则
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['service']

// 注入追踪头
const headers = {}
propagation.inject(trace.context.active(), headers)

// 提取追踪头
const context = propagation.extract(trace.context.active(), headers)