性能监控

┌─────────────────────────────────────────────────────────────────────┐
│                        监控系统架构                                   │
├─────────────────────────────────────────────────────────────────────┤
│                                                                     │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐            │
│  │  数据采集层  │ -> │  数据存储层  │ -> │  可视化层   │            │
│  └─────────────┘    └─────────────┘    └─────────────┘            │
│         │                  │                  │                     │
│         │                  │                  │                     │
│         v                  v                  v                     │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐            │
│  │ 系统指标    │    │ Prometheus  │    │  Grafana    │            │
│  │ - CPU      │    │ InfluxDB    │    │  Kibana     │            │
│  │ - 内存     │    │ Elasticsearch│   │  自定义面板  │            │
│  │ - 磁盘     │    └─────────────┘    └─────────────┘            │
│  │ - 网络     │                                                      │
│  └─────────────┘    ┌─────────────┐    ┌─────────────┐            │
│  ┌─────────────┐    │  告警层     │    │  通知渠道   │            │
│  │ 应用指标    │ -> │ - 规则引擎  │ -> │ - 邮件     │            │
│  │ - QPS      │    │ - 阈值判断  │    │ - 钉钉     │            │
│  │ - 响应时间  │    │ - 聚合分析  │    │ - 企业微信  │            │
│  │ - 错误率   │    └─────────────┘    │ - Slack    │            │
│  │ - 事件循环  │                       │ - 短信     │            │
│  └─────────────┘                       └─────────────┘            │
│  ┌─────────────┐                                                   │
│  │ 业务指标    │                                                   │
│  │ - 订单量   │                                                   │
│  │ - 用户数   │                                                   │
│  │ - 转化率   │                                                   │
│  └─────────────┘                                                   │
│                                                                     │
└─────────────────────────────────────────────────────────────────────┘

const os = require('os');

/**
 * CPU 使用率采集器
 * 通过对比两次采样计算 CPU 使用率
 */
class CPUMonitor {
  constructor() {
    this.previousCPUInfo = null;
  }
  
  /**
   * 获取当前 CPU 时间信息
   * @returns {Object} CPU 时间信息
   */
  getCPUInfo() {
    const cpus = os.cpus();
    
    let totalIdle = 0;
    let totalTick = 0;
    
    cpus.forEach(cpu => {
      for (const type in cpu.times) {
        totalTick += cpu.times[type];
      }
      totalIdle += cpu.times.idle;
    });
    
    return {
      totalTick,
      totalIdle,
      usage: totalTick - totalIdle,
      cores: cpus.length,
      model: cpus[0].model
    };
  }
  
  /**
   * 计算采样间隔内的 CPU 使用率
   * @returns {Object} CPU 使用率信息
   */
  getUsage() {
    const currentInfo = this.getCPUInfo();
    
    if (this.previousCPUInfo === null) {
      this.previousCPUInfo = currentInfo;
      return {
        usage: '0.00%',
        cores: currentInfo.cores,
        model: currentInfo.model
      };
    }
    
    const tickDiff = currentInfo.totalTick - this.previousCPUInfo.totalTick;
    const idleDiff = currentInfo.totalIdle - this.previousCPUInfo.totalIdle;
    const usageDiff = tickDiff - idleDiff;
    
    const usagePercent = tickDiff > 0 
      ? (usageDiff / tickDiff * 100).toFixed(2) 
      : '0.00';
    
    this.previousCPUInfo = currentInfo;
    
    return {
      usage: `${usagePercent}%`,
      cores: currentInfo.cores,
      model: currentInfo.model,
      user: ((currentInfo.usage / currentInfo.totalTick) * 100).toFixed(2) + '%',
      idle: ((currentInfo.totalIdle / currentInfo.totalTick) * 100).toFixed(2) + '%'
    };
  }
  
  /**
   * 获取各核心独立使用率
   * @returns {Array} 各核心使用率数组
   */
  getPerCoreUsage() {
    const cpus = os.cpus();
    return cpus.map((cpu, index) => {
      const total = Object.values(cpu.times).reduce((a, b) => a + b, 0);
      const idle = cpu.times.idle;
      const usage = ((total - idle) / total * 100).toFixed(2);
      
      return {
        core: index,
        usage: `${usage}%`,
        model: cpu.model,
        speed: cpu.speed
      };
    });
  }
}

// 使用示例
const cpuMonitor = new CPUMonitor();

// 定期采集
setInterval(() => {
  console.log('CPU 使用率:', cpuMonitor.getUsage());
}, 5000);

/**
 * 内存监控器
 * 监控系统内存和进程内存使用情况
 */
class MemoryMonitor {
  /**
   * 获取系统内存信息
   * @returns {Object} 系统内存信息
   */
  getSystemMemory() {
    const totalMem = os.totalmem();
    const freeMem = os.freemem();
    const usedMem = totalMem - freeMem;
    
    return {
      total: this.formatBytes(totalMem),
      used: this.formatBytes(usedMem),
      free: this.formatBytes(freeMem),
      usage: `${(usedMem / totalMem * 100).toFixed(2)}%`,
      bytes: {
        total: totalMem,
        used: usedMem,
        free: freeMem
      }
    };
  }
  
  /**
   * 获取进程内存信息
   * @returns {Object} 进程内存信息
   */
  getProcessMemory() {
    const usage = process.memoryUsage();
    
    return {
      rss: this.formatBytes(usage.rss),              // 常驻内存
      heapTotal: this.formatBytes(usage.heapTotal),  // 堆总量
      heapUsed: this.formatBytes(usage.heapUsed),    // 堆使用量
      external: this.formatBytes(usage.external),    // 外部内存
      arrayBuffers: this.formatBytes(usage.arrayBuffers || 0), // 数组缓冲区
      bytes: usage
    };
  }
  
  /**
   * 格式化字节数
   * @param {number} bytes 字节数
   * @returns {string} 格式化后的字符串
   */
  formatBytes(bytes) {
    if (bytes === 0) return '0 B';
    const k = 1024;
    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
    const i = Math.floor(Math.log(bytes) / Math.log(k));
    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
  }
}

const memoryMonitor = new MemoryMonitor();

/**
 * 系统负载监控
 */
function getLoadAverage() {
  const load = os.loadavg();
  const cpuCount = os.cpus().length;
  
  return {
    '1分钟': {
      value: load[0].toFixed(2),
      normalized: (load[0] / cpuCount).toFixed(2),
      status: load[0] / cpuCount > 1 ? 'high' : 'normal'
    },
    '5分钟': {
      value: load[1].toFixed(2),
      normalized: (load[1] / cpuCount).toFixed(2),
      status: load[1] / cpuCount > 1 ? 'high' : 'normal'
    },
    '15分钟': {
      value: load[2].toFixed(2),
      normalized: (load[2] / cpuCount).toFixed(2),
      status: load[2] / cpuCount > 1 ? 'high' : 'normal'
    },
    cpuCount
  };
}

/**
 * 事件循环延迟监控器
 * 检测 Node.js 事件循环阻塞情况
 */
class EventLoopMonitor {
  constructor(options = {}) {
    this.sampleInterval = options.sampleInterval || 100; // 采样间隔 ms
    this.alertThreshold = options.alertThreshold || 100; // 告警阈值 ms
    this.history = [];
    this.maxHistory = options.maxHistory || 100;
    this.timer = null;
    this.lastTime = null;
  }
  
  /**
   * 开始监控
   */
  start() {
    this.lastTime = process.hrtime.bigint();
    
    this.timer = setInterval(() => {
      const currentTime = process.hrtime.bigint();
      const delay = Number(currentTime - this.lastTime) / 1e6 - this.sampleInterval;
      
      this.recordDelay(delay);
      this.lastTime = currentTime;
    }, this.sampleInterval);
    
    this.timer.unref(); // 不阻止进程退出
  }
  
  /**
   * 停止监控
   */
  stop() {
    if (this.timer) {
      clearInterval(this.timer);
      this.timer = null;
    }
  }
  
  /**
   * 记录延迟
   */
  recordDelay(delay) {
    this.history.push({
      timestamp: Date.now(),
      delay: delay
    });
    
    if (this.history.length > this.maxHistory) {
      this.history.shift();
    }
    
    // 高延迟告警
    if (delay > this.alertThreshold) {
      console.warn(`[事件循环阻塞] 延迟 ${delay.toFixed(2)}ms 超过阈值 ${this.alertThreshold}ms`);
    }
  }
  
  /**
   * 获取统计信息
   */
  getStats() {
    if (this.history.length === 0) return null;
    
    const delays = this.history.map(h => h.delay);
    const avg = delays.reduce((a, b) => a + b, 0) / delays.length;
    const max = Math.max(...delays);
    const min = Math.min(...delays);
    const sorted = [...delays].sort((a, b) => a - b);
    const p99 = sorted[Math.floor(sorted.length * 0.99)] || 0;
    
    return {
      current: delays[delays.length - 1].toFixed(2) + 'ms',
      average: avg.toFixed(2) + 'ms',
      max: max.toFixed(2) + 'ms',
      min: min.toFixed(2) + 'ms',
      p99: p99.toFixed(2) + 'ms',
      samples: this.history.length
    };
  }
}

// 使用示例
const eventLoopMonitor = new EventLoopMonitor({
  sampleInterval: 100,
  alertThreshold: 100,
  maxHistory: 1000
});
eventLoopMonitor.start();

/**
 * 进程句柄监控
 * 监控打开的文件描述符、网络连接等
 */
function getHandleStats() {
  return {
    // 活跃句柄数量
    activeHandles: process._getActiveHandles().length,
    
    // 活跃请求（如网络请求）数量
    activeRequests: process._getActiveRequests().length,
    
    // 资源使用情况
    resourceUsage: process.resourceUsage ? process.resourceUsage() : null
  };
}

// 详细句柄信息（调试用）
function getHandleDetails() {
  const handles = process._getActiveHandles();
  const handleTypes = {};
  
  handles.forEach(handle => {
    const type = handle.constructor.name;
    handleTypes[type] = (handleTypes[type] || 0) + 1;
  });
  
  return {
    total: handles.length,
    types: handleTypes
  };
}

/**
 * HTTP 请求统计类
 * 记录请求次数、响应时间、错误率等
 */
class RequestStats {
  constructor(options = {}) {
    this.maxSamples = options.maxSamples || 1000;
    this.requests = {
      total: 0,
      success: 0,
      error: 0,
      clientError: 0,
      serverError: 0,
      timeouts: 0
    };
    this.responseTimes = [];
    this.statusCodes = {};
    this.routes = {};
    this.startTime = Date.now();
  }
  
  /**
   * 记录请求
   * @param {Object} info 请求信息
   */
  recordRequest(info) {
    const { method, url, statusCode, responseTime, route } = info;
    
    this.requests.total++;
    
    // 按状态码分类
    this.statusCodes[statusCode] = (this.statusCodes[statusCode] || 0) + 1;
    
    if (statusCode >= 200 && statusCode < 400) {
      this.requests.success++;
    } else if (statusCode >= 400 && statusCode < 500) {
      this.requests.clientError++;
      this.requests.error++;
    } else if (statusCode >= 500) {
      this.requests.serverError++;
      this.requests.error++;
    } else if (statusCode === 408 || statusCode === 504) {
      this.requests.timeouts++;
      this.requests.error++;
    }
    
    // 记录响应时间
    this.responseTimes.push(responseTime);
    if (this.responseTimes.length > this.maxSamples) {
      this.responseTimes.shift();
    }
    
    // 按路由统计
    if (route) {
      const routeKey = `${method} ${route}`;
      if (!this.routes[routeKey]) {
        this.routes[routeKey] = {
          count: 0,
          totalTime: 0,
          errors: 0,
          responseTimes: []
        };
      }
      this.routes[routeKey].count++;
      this.routes[routeKey].totalTime += responseTime;
      if (statusCode >= 400) {
        this.routes[routeKey].errors++;
      }
      this.routes[routeKey].responseTimes.push(responseTime);
      if (this.routes[routeKey].responseTimes.length > 100) {
        this.routes[routeKey].responseTimes.shift();
      }
    }
  }
  
  /**
   * 计算百分位数
   * @param {number} p 百分位 (0-1)
   * @returns {number} 百分位数值
   */
  calculatePercentile(p) {
    if (this.responseTimes.length === 0) return 0;
    const sorted = [...this.responseTimes].sort((a, b) => a - b);
    return sorted[Math.floor(sorted.length * p)] || 0;
  }
  
  /**
   * 获取统计信息
   */
  getStats() {
    const uptime = Date.now() - this.startTime;
    const avgTime = this.responseTimes.length > 0
      ? this.responseTimes.reduce((a, b) => a + b, 0) / this.responseTimes.length
      : 0;
    
    const errorRate = this.requests.total > 0
      ? (this.requests.error / this.requests.total * 100).toFixed(2)
      : '0.00';
    
    return {
      uptime: uptime,
      uptimeFormatted: this.formatUptime(uptime),
      requests: this.requests,
      qps: (this.requests.total / (uptime / 1000)).toFixed(2),
      errorRate: `${errorRate}%`,
      responseTime: {
        avg: `${avgTime.toFixed(2)}ms`,
        p50: `${this.calculatePercentile(0.5).toFixed(2)}ms`,
        p90: `${this.calculatePercentile(0.9).toFixed(2)}ms`,
        p95: `${this.calculatePercentile(0.95).toFixed(2)}ms`,
        p99: `${this.calculatePercentile(0.99).toFixed(2)}ms`
      },
      statusCodes: this.statusCodes,
      topRoutes: this.getTopRoutes(10)
    };
  }
  
  /**
   * 获取最慢路由
   */
  getTopRoutes(limit = 10) {
    const routes = Object.entries(this.routes)
      .map(([route, data]) => ({
        route,
        count: data.count,
        avgTime: (data.totalTime / data.count).toFixed(2) + 'ms',
        errorRate: (data.errors / data.count * 100).toFixed(2) + '%'
      }))
      .sort((a, b) => parseFloat(b.avgTime) - parseFloat(a.avgTime))
      .slice(0, limit);
    
    return routes;
  }
  
  /**
   * 格式化运行时间
   */
  formatUptime(ms) {
    const seconds = Math.floor(ms / 1000);
    const minutes = Math.floor(seconds / 60);
    const hours = Math.floor(minutes / 60);
    const days = Math.floor(hours / 24);
    
    if (days > 0) return `${days}d ${hours % 24}h ${minutes % 60}m`;
    if (hours > 0) return `${hours}h ${minutes % 60}m ${seconds % 60}s`;
    if (minutes > 0) return `${minutes}m ${seconds % 60}s`;
    return `${seconds}s`;
  }
  
  /**
   * 重置统计
   */
  reset() {
    this.requests = {
      total: 0,
      success: 0,
      error: 0,
      clientError: 0,
      serverError: 0,
      timeouts: 0
    };
    this.responseTimes = [];
    this.statusCodes = {};
    this.routes = {};
    this.startTime = Date.now();
  }
}

const requestStats = new RequestStats();

/**
 * 完整的 Express 监控中间件
 */
function createMonitoringMiddleware(options = {}) {
  const {
    slowRequestThreshold = 3000,  // 慢请求阈值 ms
    excludeRoutes = ['/health', '/metrics', '/favicon.ico'], // 排除的路由
    logger = console
  } = options;
  
  return function monitoringMiddleware(req, res, next) {
    const start = Date.now();
    
    // 排除特定路由
    if (excludeRoutes.some(route => req.path.startsWith(route))) {
      return next();
    }
    
    // 请求开始
    const requestInfo = {
      method: req.method,
      url: req.url,
      path: req.path,
      route: null,
      userAgent: req.get('User-Agent'),
      ip: req.ip || req.connection.remoteAddress
    };
    
    // 记录响应完成
    res.on('finish', () => {
      const duration = Date.now() - start;
      
      // 获取匹配的路由模式
      const route = req.route ? req.route.path : req.path;
      
      requestStats.recordRequest({
        method: req.method,
        url: req.url,
        route: route,
        statusCode: res.statusCode,
        responseTime: duration
      });
      
      // 慢请求告警
      if (duration > slowRequestThreshold) {
        logger.warn(
          `[慢请求] ${req.method} ${req.url} - ${duration}ms (状态码: ${res.statusCode})`
        );
      }
      
      // 设置响应头记录请求 ID
      const requestId = req.headers['x-request-id'] || generateRequestId();
      res.set('X-Request-Id', requestId);
      res.set('X-Response-Time', `${duration}ms`);
    });
    
    next();
  };
}

/**
 * 生成请求 ID
 */
function generateRequestId() {
  return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
}

// 使用示例
const express = require('express');
const app = express();

app.use(createMonitoringMiddleware({
  slowRequestThreshold: 2000,
  excludeRoutes: ['/health', '/metrics']
}));

/**
 * 健康检查端点
 * 用于负载均衡和服务发现
 */
function setupHealthCheck(app) {
  // 简单存活检查
  app.get('/health', (req, res) => {
    res.json({ status: 'ok', timestamp: new Date().toISOString() });
  });
  
  // 详细健康检查
  app.get('/health/detail', async (req, res) => {
    const checks = {
      server: { status: 'ok' },
      memory: checkMemory(),
      cpu: checkCPU(),
      disk: await checkDisk(),
      database: await checkDatabase(),
      redis: await checkRedis()
    };
    
    const allHealthy = Object.values(checks).every(c => c.status === 'ok');
    
    res.status(allHealthy ? 200 : 503).json({
      status: allHealthy ? 'healthy' : 'unhealthy',
      timestamp: new Date().toISOString(),
      uptime: process.uptime(),
      checks
    });
  });
}

function checkMemory() {
  const used = process.memoryUsage();
  const heapUsedMB = used.heapUsed / 1024 / 1024;
  const threshold = 500; // MB
  
  return {
    status: heapUsedMB < threshold ? 'ok' : 'warning',
    heapUsed: `${heapUsedMB.toFixed(2)}MB`,
    threshold: `${threshold}MB`
  };
}

function checkCPU() {
  const load = os.loadavg()[0] / os.cpus().length;
  
  return {
    status: load < 1 ? 'ok' : 'warning',
    loadAverage: load.toFixed(2)
  };
}

async function checkDisk() {
  // 实现磁盘检查逻辑
  return { status: 'ok' };
}

async function checkDatabase() {
  // 实现数据库连接检查
  try {
    // await db.ping();
    return { status: 'ok' };
  } catch (err) {
    return { status: 'error', message: err.message };
  }
}

async function checkRedis() {
  // 实现 Redis 连接检查
  try {
    // await redis.ping();
    return { status: 'ok' };
  } catch (err) {
    return { status: 'error', message: err.message };
  }
}

/**
 * 暴露监控数据的端点
 */
function setupMetricsEndpoint(app) {
  app.get('/metrics', (req, res) => {
    res.json({
      timestamp: new Date().toISOString(),
      process: {
        pid: process.pid,
        uptime: process.uptime(),
        uptimeFormatted: formatUptime(process.uptime()),
        memory: memoryMonitor.getProcessMemory(),
        handles: getHandleStats(),
        eventLoop: eventLoopMonitor.getStats()
      },
      system: {
        hostname: os.hostname(),
        platform: os.platform(),
        arch: os.arch(),
        cpus: cpuMonitor.getUsage(),
        memory: memoryMonitor.getSystemMemory(),
        load: getLoadAverage()
      },
      application: requestStats.getStats()
    });
  });
  
  // Prometheus 格式输出
  app.get('/metrics/prometheus', async (req, res) => {
    const register = require('prom-client').register;
    res.set('Content-Type', register.contentType);
    res.end(await register.metrics());
  });
}

function formatUptime(seconds) {
  const days = Math.floor(seconds / 86400);
  const hours = Math.floor((seconds % 86400) / 3600);
  const minutes = Math.floor((seconds % 3600) / 60);
  
  return `${days}d ${hours}h ${minutes}m`;
}

npm install newrelic

'use strict';

/**
 * New Relic APM 配置
 */
exports.config = {
  // 应用名称
  app_name: [process.env.NEW_RELIC_APP_NAME || 'My Node.js App'],
  
  // 许可证密钥
  license_key: process.env.NEW_RELIC_LICENSE_KEY,
  
  // 日志配置
  logging: {
    level: process.env.NODE_ENV === 'production' ? 'info' : 'debug',
    filepath: 'stdout'
  },
  
  // 错误收集
  error_collector: {
    enabled: true,
    ignore_status_codes: [404]
  },
  
  // 事务追踪
  transaction_tracer: {
    enabled: true,
    record_sql: 'obfuscated',
    explain_threshold: 500
  },
  
  // 应用性能分析
  application_logging: {
    enabled: true,
    forwarding: {
      enabled: true
    }
  },
  
  // 分布式追踪
  distributed_tracing: {
    enabled: true
  },
  
  // 自定义属性
  attributes: {
    exclude: ['request.headers.cookie', 'request.headers.authorization']
  }
};

// 在应用最顶部引入
require('newrelic');

const express = require('express');
const app = express();

// New Relic 会自动监控 Express 路由

// 自定义事务命名
app.get('/api/users/:id', (req, res) => {
  const transaction = require('newrelic').getTransaction();
  transaction.setName(`/api/users/:id`);
  
  // 业务逻辑
  res.json({ id: req.params.id });
});

// 自定义属性
app.get('/order', (req, res) => {
  require('newrelic').addCustomAttribute('orderId', req.query.orderId);
  require('newrelic').addCustomAttribute('userId', req.user.id);
  
  // 业务逻辑
});

npm install elastic-apm-node

/**
 * Elastic APM 配置
 */
module.exports = {
  // 服务名称
  serviceName: process.env.ELASTIC_APM_SERVICE_NAME || 'my-nodejs-service',
  
  // APM Server 地址
  serverUrl: process.env.ELASTIC_APM_SERVER_URL || 'http://localhost:8200',
  
  // 环境
  environment: process.env.NODE_ENV || 'development',
  
  // 采样率
  transactionSampleRate: 0.1, // 10% 采样
  
  // 日志级别
  logLevel: process.env.NODE_ENV === 'production' ? 'warn' : 'debug',
  
  // 错误收集
  captureErrorLogStackTraces: 'always',
  
  // 源码映射
  sourceLinesErrorAppFrames: 5,
  sourceLinesErrorLibraryFrames: 5
};

// 在应用最顶部启动
const apm = require('elastic-apm-node').start({
  serviceName: 'my-service',
  serverUrl: 'http://localhost:8200'
});

const express = require('express');
const app = express();

// 自定义 Span
app.get('/api/users/:id', async (req, res) => {
  // 自动创建事务
  
  // 数据库查询 Span
  const dbSpan = apm.startSpan('database-query', 'db');
  try {
    const user = await db.query('SELECT * FROM users WHERE id = ?', [req.params.id]);
    res.json(user);
  } catch (err) {
    apm.captureError(err);
    res.status(500).json({ error: err.message });
  } finally {
    if (dbSpan) dbSpan.end();
  }
});

// 错误追踪
app.use((err, req, res, next) => {
  apm.captureError(err);
  res.status(500).json({ error: 'Internal Server Error' });
});

// 自定义上下文
app.get('/checkout', (req, res) => {
  apm.setCustomContext({
    userId: req.user.id,
    cartItems: req.body.items.length
  });
  
  apm.setUserContext({
    id: req.user.id,
    email: req.user.email,
    username: req.user.name
  });
  
  // 业务逻辑
});

npm install prom-client

const client = require('prom-client');

// 创建 Registry（隔离指标命名空间）
const register = new client.Registry();

// 设置默认标签
register.setDefaultLabels({
  app: 'my-nodejs-app',
  environment: process.env.NODE_ENV || 'development'
});

// 收集默认指标
client.collectDefaultMetrics({
  register,
  prefix: 'nodejs_',
  gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5]
});

/**
 * 自定义指标定义
 */

// HTTP 请求计数器
const httpRequestsTotal = new client.Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'route', 'status_code'],
  registers: [register]
});

// HTTP 请求持续时间直方图
const httpRequestDuration = new client.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'route', 'status_code'],
  buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
  registers: [register]
});

// HTTP 请求进行中计数
const httpRequestsInProgress = new client.Gauge({
  name: 'http_requests_in_progress',
  help: 'Number of HTTP requests currently in progress',
  labelNames: ['method', 'route'],
  registers: [register]
});

// 数据库查询计数器
const dbQueriesTotal = new client.Counter({
  name: 'db_queries_total',
  help: 'Total number of database queries',
  labelNames: ['operation', 'table', 'status'],
  registers: [register]
});

// 数据库查询延迟
const dbQueryDuration = new client.Histogram({
  name: 'db_query_duration_seconds',
  help: 'Duration of database queries in seconds',
  labelNames: ['operation', 'table'],
  buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1],
  registers: [register]
});

// 缓存命中率
const cacheHits = new client.Counter({
  name: 'cache_hits_total',
  help: 'Total number of cache hits',
  labelNames: ['cache_name', 'result'],
  registers: [register]
});

// 业务指标 - 订单数量
const ordersTotal = new client.Counter({
  name: 'orders_total',
  help: 'Total number of orders',
  labelNames: ['status', 'payment_method'],
  registers: [register]
});

// 业务指标 - 活跃用户
const activeUsers = new client.Gauge({
  name: 'active_users_count',
  help: 'Number of currently active users',
  registers: [register]
});

/**
 * Express 中间件
 */
function prometheusMiddleware(req, res, next) {
  const start = Date.now();
  const route = req.route ? req.route.path : req.path;
  
  // 增加进行中请求计数
  httpRequestsInProgress.labels(req.method, route).inc();
  
  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    
    // 记录请求计数
    httpRequestsTotal.labels(req.method, route, res.statusCode).inc();
    
    // 记录请求持续时间
    httpRequestDuration.labels(req.method, route, res.statusCode).observe(duration);
    
    // 减少进行中请求计数
    httpRequestsInProgress.labels(req.method, route).dec();
  });
  
  next();
}

/**
 * 暴露 metrics 端点
 */
function setupPrometheusEndpoint(app) {
  app.use(prometheusMiddleware);
  
  app.get('/metrics', async (req, res) => {
    try {
      res.set('Content-Type', register.contentType);
      res.end(await register.metrics());
    } catch (err) {
      res.status(500).end(err.message);
    }
  });
}

// 导出指标和函数
module.exports = {
  register,
  setupPrometheusEndpoint,
  metrics: {
    httpRequestsTotal,
    httpRequestDuration,
    httpRequestsInProgress,
    dbQueriesTotal,
    dbQueryDuration,
    cacheHits,
    ordersTotal,
    activeUsers
  }
};

# grafana-dashboard.yaml
apiVersion: 1
providers:
  - name: 'Node.js Dashboard'
    folder: 'Monitoring'
    type: file
    options:
      path: /var/lib/grafana/dashboards

# 仪表板 JSON 配置示例
dashboard:
  title: "Node.js Application Monitoring"
  panels:
    - title: "Request Rate"
      type: "graph"
      targets:
        - expr: "rate(http_requests_total[5m])"
          legendFormat: "{{method}} {{route}}"
    
    - title: "Response Time (P95)"
      type: "graph"
      targets:
        - expr: "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"
          legendFormat: "P95"
    
    - title: "Error Rate"
      type: "graph"
      targets:
        - expr: "rate(http_requests_total{status_code=~\"5..\"}[5m])"
          legendFormat: "5xx errors"
    
    - title: "Memory Usage"
      type: "graph"
      targets:
        - expr: "nodejs_heap_size_used_bytes"
          legendFormat: "Heap Used"
        - expr: "nodejs_heap_size_total_bytes"
          legendFormat: "Heap Total"

npm install winston winston-elasticsearch

const winston = require('winston');
const { ElasticsearchTransport } = require('winston-elasticsearch');

/**
 * 日志管理器
 */
class LogManager {
  constructor(options = {}) {
    this.serviceName = options.serviceName || 'nodejs-app';
    this.environment = options.environment || process.env.NODE_ENV || 'development';
    this.elasticsearchConfig = options.elasticsearch;
    
    this.logger = this.createLogger();
  }
  
  createLogger() {
    const transports = [
      // 控制台输出
      new winston.transports.Console({
        format: winston.format.combine(
          winston.format.colorize(),
          winston.format.printf(({ level, message, timestamp, ...metadata }) => {
            let msg = `${timestamp} [${level}] ${message}`;
            if (Object.keys(metadata).length > 0) {
              msg += ` ${JSON.stringify(metadata)}`;
            }
            return msg;
          })
        )
      })
    ];
    
    // 生产环境添加 Elasticsearch
    if (this.elasticsearchConfig) {
      transports.push(
        new ElasticsearchTransport({
          level: 'info',
          clientOpts: this.elasticsearchConfig.clientOpts,
          index: this.elasticsearchConfig.index || 'nodejs-logs',
          transformer: (logData) => ({
            '@timestamp': new Date().toISOString(),
            message: logData.message,
            level: logData.level,
            service: this.serviceName,
            environment: this.environment,
            ...logData.meta
          })
        })
      );
    }
    
    return winston.createLogger({
      level: process.env.LOG_LEVEL || 'info',
      defaultMeta: {
        service: this.serviceName,
        environment: this.environment
      },
      format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.errors({ stack: true }),
        winston.format.json()
      ),
      transports
    });
  }
  
  /**
   * 创建请求上下文日志
   */
  createRequestLogger(requestId, userId = null) {
    return {
      info: (message, meta = {}) => this.logger.info(message, { requestId, userId, ...meta }),
      warn: (message, meta = {}) => this.logger.warn(message, { requestId, userId, ...meta }),
      error: (message, meta = {}) => this.logger.error(message, { requestId, userId, ...meta }),
      debug: (message, meta = {}) => this.logger.debug(message, { requestId, userId, ...meta })
    };
  }
}

/**
 * Express 请求日志中间件
 */
function requestLoggingMiddleware(logger) {
  return (req, res, next) => {
    const start = Date.now();
    const requestId = req.headers['x-request-id'] || generateRequestId();
    
    // 创建请求专属日志器
    req.log = logger.createRequestLogger(requestId, req.user?.id);
    
    // 记录请求开始
    req.log.info('Request started', {
      method: req.method,
      url: req.url,
      userAgent: req.get('User-Agent'),
      ip: req.ip
    });
    
    // 记录请求结束
    res.on('finish', () => {
      const duration = Date.now() - start;
      
      req.log.info('Request completed', {
        method: req.method,
        url: req.url,
        statusCode: res.statusCode,
        duration: `${duration}ms`,
        contentLength: res.get('Content-Length')
      });
    });
    
    // 设置响应头
    res.set('X-Request-Id', requestId);
    
    next();
  };
}

// 使用示例
const logManager = new LogManager({
  serviceName: 'my-api',
  environment: 'production',
  elasticsearch: {
    clientOpts: {
      node: 'http://localhost:9200'
    },
    index: 'nodejs-logs'
  }
});

app.use(requestLoggingMiddleware(logManager));

npm install morgan

const morgan = require('morgan');
const fs = require('fs');
const path = require('path');

/**
 * 自定义 Morgan 配置
 */
function setupMorgan(app) {
  // 自定义 token
  morgan.token('request-id', (req) => req.headers['x-request-id'] || '-');
  morgan.token('user-id', (req) => req.user?.id || '-');
  morgan.token('response-time-ms', (req, res) => {
    if (!res._header) return '-';
    const diff = process.hrtime(req._startTime);
    const ms = diff[0] * 1e3 + diff[1] * 1e-6;
    return ms.toFixed(3);
  });
  
  // 自定义格式
  const customFormat = ':date[iso] :request-id :user-id :method :url :status :response-time-ms ms :res[content-length]';
  
  // 开发环境
  if (process.env.NODE_ENV === 'development') {
    app.use(morgan('dev'));
  }
  
  // 生产环境
  if (process.env.NODE_ENV === 'production') {
    // 访问日志
    const accessLogStream = fs.createWriteStream(
      path.join(__dirname, 'logs', 'access.log'),
      { flags: 'a' }
    );
    app.use(morgan(customFormat, { stream: accessLogStream }));
    
    // 错误日志（单独文件）
    const errorLogStream = fs.createWriteStream(
      path.join(__dirname, 'logs', 'error.log'),
      { flags: 'a' }
    );
    app.use(morgan(customFormat, {
      stream: errorLogStream,
      skip: (req, res) => res.statusCode < 400
    }));
  }
}

/**
 * 告警管理器
 * 支持多渠道通知、告警聚合、静默期
 */
class AlertManager {
  constructor(options = {}) {
    this.thresholds = {
      memory: options.memoryThreshold || 500,       // MB
      cpu: options.cpuThreshold || 80,              // %
      responseTime: options.responseTimeThreshold || 3000, // ms
      errorRate: options.errorRateThreshold || 5,   // %
      eventLoopDelay: options.eventLoopDelayThreshold || 100 // ms
    };
    
    this.channels = [];
    this.alerts = [];
    this.silencePeriods = new Map(); // 静默期
    this.alertHistory = [];
    this.maxHistory = options.maxHistory || 1000;
  }
  
  /**
   * 添加告警渠道
   */
  addChannel(channel) {
    this.channels.push(channel);
  }
  
  /**
   * 检查并触发告警
   */
  check(metrics) {
    const alerts = [];
    
    // 内存检查
    if (metrics.memory && metrics.memory.heapUsedMB > this.thresholds.memory) {
      alerts.push(this.createAlert('memory', 
        `内存使用超过阈值: ${metrics.memory.heapUsedMB.toFixed(2)}MB > ${this.thresholds.memory}MB`,
        metrics.memory
      ));
    }
    
    // CPU 检查
    if (metrics.cpu && metrics.cpu.usagePercent > this.thresholds.cpu) {
      alerts.push(this.createAlert('cpu',
        `CPU 使用超过阈值: ${metrics.cpu.usagePercent}% > ${this.thresholds.cpu}%`,
        metrics.cpu
      ));
    }
    
    // 响应时间检查
    if (metrics.responseTime && metrics.responseTime.p95 > this.thresholds.responseTime) {
      alerts.push(this.createAlert('responseTime',
        `响应时间过长: P95 ${metrics.responseTime.p95}ms > ${this.thresholds.responseTime}ms`,
        metrics.responseTime
      ));
    }
    
    // 错误率检查
    if (metrics.errorRate > this.thresholds.errorRate) {
      alerts.push(this.createAlert('errorRate',
        `错误率过高: ${metrics.errorRate}% > ${this.thresholds.errorRate}%`,
        { errorRate: metrics.errorRate }
      ));
    }
    
    // 事件循环延迟检查
    if (metrics.eventLoop && metrics.eventLoop.avg > this.thresholds.eventLoopDelay) {
      alerts.push(this.createAlert('eventLoop',
        `事件循环延迟过高: ${metrics.eventLoop.avg}ms > ${this.thresholds.eventLoopDelay}ms`,
        metrics.eventLoop
      ));
    }
    
    // 发送告警
    alerts.forEach(alert => this.sendAlert(alert));
    
    return alerts;
  }
  
  /**
   * 创建告警对象
   */
  createAlert(type, message, data) {
    return {
      id: `${type}-${Date.now()}`,
      type,
      message,
      data,
      severity: this.getSeverity(type),
      timestamp: new Date().toISOString()
    };
  }
  
  /**
   * 获取告警级别
   */
  getSeverity(type) {
    const severityMap = {
      memory: 'warning',
      cpu: 'warning',
      responseTime: 'warning',
      errorRate: 'critical',
      eventLoop: 'warning'
    };
    return severityMap[type] || 'info';
  }
  
  /**
   * 发送告警
   */
  async sendAlert(alert) {
    // 检查静默期
    if (this.isSilenced(alert.type)) {
      console.log(`[告警静默] ${alert.type}: ${alert.message}`);
      return;
    }
    
    // 记录告警历史
    this.alertHistory.push(alert);
    if (this.alertHistory.length > this.maxHistory) {
      this.alertHistory.shift();
    }
    
    // 设置静默期（5分钟内同类型告警不重复发送）
    this.setSilence(alert.type, 5 * 60 * 1000);
    
    console.error(`[告警] [${alert.severity.toUpperCase()}] ${alert.message}`);
    
    // 发送到所有渠道
    for (const channel of this.channels) {
      try {
        await channel.send(alert);
      } catch (err) {
        console.error(`告警渠道 ${channel.name} 发送失败:`, err.message);
      }
    }
  }
  
  /**
   * 设置静默期
   */
  setSilence(type, duration) {
    this.silencePeriods.set(type, Date.now() + duration);
  }
  
  /**
   * 检查是否在静默期
   */
  isSilenced(type) {
    const silenceUntil = this.silencePeriods.get(type);
    if (!silenceUntil) return false;
    
    if (Date.now() < silenceUntil) {
      return true;
    }
    
    this.silencePeriods.delete(type);
    return false;
  }
  
  /**
   * 获取告警历史
   */
  getHistory(limit = 50) {
    return this.alertHistory.slice(-limit);
  }
}

/**
 * 告警渠道 - 钉钉
 */
class DingTalkChannel {
  constructor(webhook, options = {}) {
    this.name = 'DingTalk';
    this.webhook = webhook;
    this.atMobiles = options.atMobiles || [];
    this.isAtAll = options.isAtAll || false;
  }
  
  async send(alert) {
    const response = await fetch(this.webhook, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        msgtype: 'markdown',
        markdown: {
          title: `【${alert.severity.toUpperCase()}】${alert.type} 告警`,
          text: `### 性能告警\n\n` +
                `- **类型**: ${alert.type}\n` +
                `- **级别**: ${alert.severity}\n` +
                `- **消息**: ${alert.message}\n` +
                `- **时间**: ${alert.timestamp}\n` +
                `- **服务**: ${process.env.SERVICE_NAME || 'unknown'}\n`
        },
        at: {
          atMobiles: this.atMobiles,
          isAtAll: this.isAtAll
        }
      })
    });
    
    return response.json();
  }
}

/**
 * 告警渠道 - 企业微信
 */
class WeChatWorkChannel {
  constructor(webhook) {
    this.name = 'WeChatWork';
    this.webhook = webhook;
  }
  
  async send(alert) {
    const response = await fetch(this.webhook, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        msgtype: 'markdown',
        markdown: {
          content: `### 性能告警\n` +
                   `> 类型: ${alert.type}\n` +
                   `> 级别: ${alert.severity}\n` +
                   `> 消息: ${alert.message}\n` +
                   `> 时间: ${alert.timestamp}`
        }
      })
    });
    
    return response.json();
  }
}

/**
 * 告警渠道 - 邮件
 */
class EmailChannel {
  constructor(transporter, recipients) {
    this.name = 'Email';
    this.transporter = transporter;
    this.recipients = recipients;
  }
  
  async send(alert) {
    await this.transporter.sendMail({
      from: this.transporter.options.from,
      to: this.recipients.join(','),
      subject: `[${alert.severity.toUpperCase()}] ${alert.type} 性能告警`,
      text: alert.message,
      html: `
        <h2>性能告警</h2>
        <p><strong>类型:</strong> ${alert.type}</p>
        <p><strong>级别:</strong> ${alert.severity}</p>
        <p><strong>消息:</strong> ${alert.message}</p>
        <p><strong>时间:</strong> ${alert.timestamp}</p>
        <h3>详细信息:</h3>
        <pre>${JSON.stringify(alert.data, null, 2)}</pre>
      `
    });
  }
}

/**
 * 告警渠道 - Slack
 */
class SlackChannel {
  constructor(webhook, options = {}) {
    this.name = 'Slack';
    this.webhook = webhook;
    this.channel = options.channel;
  }
  
  async send(alert) {
    const color = alert.severity === 'critical' ? 'danger' : 'warning';
    
    await fetch(this.webhook, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        channel: this.channel,
        attachments: [{
          color: color,
          title: `【${alert.severity.toUpperCase()}】性能告警`,
          fields: [
            { title: '类型', value: alert.type, short: true },
            { title: '级别', value: alert.severity, short: true },
            { title: '消息', value: alert.message, short: false },
            { title: '时间', value: alert.timestamp, short: false }
          ]
        }]
      })
    });
  }
}

// 使用示例
const alertManager = new AlertManager({
  memoryThreshold: 500,
  cpuThreshold: 80,
  responseTimeThreshold: 2000,
  errorRateThreshold: 5
});

// 添加告警渠道
alertManager.addChannel(new DingTalkChannel(process.env.DINGTALK_WEBHOOK, {
  atMobiles: ['13800138000']
}));

alertManager.addChannel(new EmailChannel(transporter, ['admin@example.com']));

// 定期检查
setInterval(() => {
  const metrics = {
    memory: memoryMonitor.getProcessMemory(),
    cpu: cpuMonitor.getUsage(),
    responseTime: requestStats.getStats().responseTime,
    errorRate: parseFloat(requestStats.getStats().errorRate),
    eventLoop: eventLoopMonitor.getStats()
  };
  
  alertManager.check(metrics);
}, 60000);

/**
 * 日志规范配置
 */

// 日志级别定义
const LOG_LEVELS = {
  error: 0,   // 错误 - 需要立即处理
  warn: 1,    // 警告 - 潜在问题
  info: 2,    // 信息 - 重要业务事件
  http: 3,    // HTTP 请求日志
  debug: 4    // 调试 - 详细信息
};

// 日志字段规范
const LOG_FIELDS = {
  // 必需字段
  required: ['timestamp', 'level', 'message', 'service'],
  
  // 请求相关
  request: ['requestId', 'method', 'url', 'statusCode', 'responseTime'],
  
  // 用户相关
  user: ['userId', 'userRole'],
  
  // 错误相关
  error: ['errorName', 'errorMessage', 'stackTrace'],
  
  // 性能相关
  performance: ['duration', 'memoryUsage', 'cpuUsage']
};

// 敏感信息脱敏
function sanitizeLogData(data) {
  const sensitiveFields = ['password', 'token', 'secret', 'apiKey', 'creditCard'];
  const sanitized = { ...data };
  
  for (const field of sensitiveFields) {
    if (sanitized[field]) {
      sanitized[field] = '***REDACTED***';
    }
  }
  
  return sanitized;
}

// 结构化日志示例
function logStructured(logger, level, message, context = {}) {
  const logData = {
    timestamp: new Date().toISOString(),
    level,
    message,
    service: process.env.SERVICE_NAME,
    environment: process.env.NODE_ENV,
    hostname: os.hostname(),
    pid: process.pid,
    ...sanitizeLogData(context)
  };
  
  logger[level](message, logData);
}

┌─────────────────────────────────────────────────────────────────┐
│                    监控仪表板布局建议                             │
├─────────────────────────────────────────────────────────────────┤
│                                                                 │
│  ┌─────────────────┐  ┌─────────────────┐  ┌─────────────────┐ │
│  │  服务状态       │  │  QPS            │  │  错误率         │ │
│  │  ● 正常运行     │  │  1234 req/s     │  │  0.52%          │ │
│  │  Uptime: 5d 3h  │  │  ↑ 12%          │  │  ↓ 5%           │ │
│  └─────────────────┘  └─────────────────┘  └─────────────────┘ │
│                                                                 │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │                    响应时间趋势                               ││
│  │  P99 ────                                                    ││
│  │  P95 ----                                                    ││
│  │  P50 ------                                                  ││
│  │      时间轴 →                                                ││
│  └─────────────────────────────────────────────────────────────┘│
│                                                                 │
│  ┌──────────────────────┐  ┌──────────────────────┐            │
│  │    CPU 使用率         │  │    内存使用          │            │
│  │    ▓▓▓▓▓▓░░░░ 62%    │  │    ▓▓▓▓▓▓▓░░░ 420MB  │            │
│  └──────────────────────┘  └──────────────────────┘            │
│                                                                 │
│  ┌─────────────────────────────────────────────────────────────┐│
│  │                    热点路由 TOP 10                           ││
│  │  1. GET /api/users      234ms  1200 req/min                 ││
│  │  2. POST /api/orders    189ms   800 req/min                 ││
│  │  3. GET /api/products   145ms  2500 req/min                 ││
│  └─────────────────────────────────────────────────────────────┘│
│                                                                 │
└─────────────────────────────────────────────────────────────────┘

原始数据（秒级）: 保留 7 天
聚合数据（分钟级）: 保留 30 天
汇总数据（小时级）: 保留 1 年
统计数据（天级）: 永久保留