八、实战项目——02-部署与运维/02-监控告警与日志

08-实战项目/02-部署与运维/02-监控告警与日志

监控告警与日志系统

学习目标

• 掌握 Prometheus + Grafana 监控体系
• 理解自定义指标采集
• 实现告警规则配置
• 搭建 ELK 日志平台

前置知识

• 时序数据库概念
• 日志收集基础
• 告警策略设计

知识点列表

1. 监控体系架构

1.1 整体架构

# monitoring/architecture.yaml
监控架构:
指标采集:
– Node Exporter: 主机指标
– cAdvisor: 容器指标
– 应用指标: 业务指标

存储查询:
– Prometheus: 时序数据库
– Thanos: 长期存储

可视化:
– Grafana: 仪表盘
– 告警: AlertManager

日志系统:
– Filebeat: 日志采集
– Logstash: 日志处理
– Elasticsearch: 日志存储
– Kibana: 日志查询

2. Prometheus 监控

2.1 Prometheus 配置

# prometheus/prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
cluster: 'ecommerce'
environment: 'production'

alerting:
alertmanagers:
– static_configs:
– targets: ['alertmanager:9093']

rule_files:
– "alerts/*.yml"

scrape_configs:
– job_name: 'prometheus'
static_configs:
– targets: ['localhost:9090']

– job_name: 'node-exporter'
static_configs:
– targets: ['node-exporter:9100']

– job_name: 'cadvisor'
static_configs:
– targets: ['cadvisor:8080']

– job_name: 'user-service'
metrics_path: '/metrics'
static_configs:
– targets: ['user-service:3001']
relabel_configs:
– source_labels: [__address__]
target_label: instance
regex: '([^:]+):\\d+'
replacement: '${1}'

– job_name: 'product-service'
metrics_path: '/metrics'
static_configs:
– targets: ['product-service:3002']

– job_name: 'order-service'
metrics_path: '/metrics'
static_configs:
– targets: ['order-service:3003']

– job_name: 'payment-service'
metrics_path: '/metrics'
static_configs:
– targets: ['payment-service:3004']

– job_name: 'api-gateway'
metrics_path: '/metrics'
static_configs:
– targets: ['api-gateway:8080']

– job_name: 'postgres-exporter'
static_configs:
– targets: ['postgres-exporter:9187']

– job_name: 'redis-exporter'
static_configs:
– targets: ['redis-exporter:9121']

– job_name: 'rabbitmq-exporter'
static_configs:
– targets: ['rabbitmq-exporter:9419']

2.2 应用指标暴露

// monitoring/metrics.js
const client = require('prom-client');

// 创建 Registry
const register = new client.Registry();

// 添加默认指标
client.collectDefaultMetrics({ register });

// ============================================
// HTTP 指标
// ============================================
const httpRequestsTotal = new client.Counter({
name: 'http_requests_total',
help: 'Total number of HTTP requests',
labelNames: ['method', 'path', 'status'],
registers: [register]
});

const httpRequestDuration = new client.Histogram({
name: 'http_request_duration_seconds',
help: 'HTTP request duration in seconds',
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
labelNames: ['method', 'path'],
registers: [register]
});

const activeRequests = new client.Gauge({
name: 'http_active_requests',
help: 'Number of active HTTP requests',
registers: [register]
});

// ============================================
// 业务指标
// ============================================
const userRegistrations = new client.Counter({
name: 'user_registrations_total',
help: 'Total number of user registrations',
registers: [register]
});

const orderCreated = new client.Counter({
name: 'order_created_total',
help: 'Total number of orders created',
labelNames: ['status'],
registers: [register]
});

const orderAmount = new client.Histogram({
name: 'order_amount_histogram',
help: 'Order amount distribution',
buckets: [10, 50, 100, 500, 1000, 5000, 10000],
registers: [register]
});

const paymentSuccess = new client.Counter({
name: 'payment_success_total',
help: 'Total number of successful payments',
labelNames: ['channel'],
registers: [register]
});

const paymentFailed = new client.Counter({
name: 'payment_failed_total',
help: 'Total number of failed payments',
labelNames: ['channel', 'reason'],
registers: [register]
});

// ============================================
// 数据库指标
// ============================================
const dbQueryDuration = new client.Histogram({
name: 'db_query_duration_seconds',
help: 'Database query duration',
buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1],
labelNames: ['operation', 'table'],
registers: [register]
});

const dbConnectionPool = new client.Gauge({
name: 'db_connection_pool_size',
help: 'Database connection pool size',
registers: [register]
});

// ============================================
// 中间件
// ============================================
function metricsMiddleware(req, res, next) {
const start = Date.now();
activeRequests.inc();

res.on('finish', () => {
const duration = (Date.now() – start) / 1000;
const route = req.route?.path || req.path;

httpRequestsTotal.labels(req.method, route, res.statusCode).inc();
httpRequestDuration.labels(req.method, route).observe(duration);
activeRequests.dec();
});

next();
}

// ============================================
// 业务指标记录函数
// ============================================
function recordUserRegistration() {
userRegistrations.inc();
}

function recordOrder(order) {
orderCreated.labels(order.status).inc();
orderAmount.observe(order.amount);
}

function recordPayment(payment) {
if (payment.status === 'success') {
paymentSuccess.labels(payment.channel).inc();
} else {
paymentFailed.labels(payment.channel, payment.reason).inc();
}
}

async function recordDbQuery(queryFn, operation, table) {
const start = Date.now();
try {
return await queryFn();
} finally {
const duration = (Date.now() – start) / 1000;
dbQueryDuration.labels(operation, table).observe(duration);
}
}

// ============================================
// 指标端点
// ============================================
function metricsEndpoint(req, res) {
res.set('Content-Type', register.contentType);
register.metrics().then(metrics => res.send(metrics));
}

module.exports = {
metricsMiddleware,
metricsEndpoint,
recordUserRegistration,
recordOrder,
recordPayment,
recordDbQuery,
register
};

2.3 告警规则

# prometheus/alerts/service.yml
groups:
– name: service_alerts
interval: 30s
rules:
# 服务 Down 告警
– alert: ServiceDown
expr: up{job=~"user–service|product–service|order–service|payment–service"} == 0
for: 1m
labels:
severity: critical
team: backend
annotations:
summary: "{{ $labels.job }} 服务不可用"
description: "{{ $labels.job }} 服务已宕机超过 1 分钟"

# 高延迟告警
– alert: HighLatency
expr: |
histogram_quantile(0.99,
sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job)
) > 2
for: 5m
labels:
severity: warning
team: backend
annotations:
summary: "{{ $labels.job }} 延迟过高"
description: "p99 延迟已超过 2 秒"

# 高错误率告警
– alert: HighErrorRate
expr: |
sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
/ sum(rate(http_requests_total[5m])) by (job) > 0.05
for: 5m
labels:
severity: critical
team: backend
annotations:
summary: "{{ $labels.job }} 错误率过高"
description: "错误率已超过 5%"

– name: business_alerts
interval: 1m
rules:
# 订单创建速率下降
– alert: OrderRateDrop
expr: |
rate(order_created_total[30m]) < 10
for: 15m
labels:
severity: warning
team: business
annotations:
summary: "订单创建速率下降"
description: "过去 30 分钟订单创建速率低于 10/分钟"

# 支付成功率下降
– alert: PaymentSuccessRateDrop
expr: |
sum(rate(payment_success_total[5m]))
/ sum(rate(payment_success_total[5m]) + rate(payment_failed_total[5m])) < 0.95
for: 10m
labels:
severity: critical
team: business
annotations:
summary: "支付成功率下降"
description: "支付成功率已低于 95%"

– name: infrastructure_alerts
interval: 30s
rules:
# 高 CPU 使用率
– alert: HighCPUUsage
expr: 100 – (avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance) * 100) > 80
for: 10m
labels:
severity: warning
team: ops
annotations:
summary: "服务器 CPU 使用率过高"
description: "{{ $labels.instance }} CPU 使用率已超过 80%"

# 高内存使用率
– alert: HighMemoryUsage
expr: (1 – (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
for: 10m
labels:
severity: warning
team: ops
annotations:
summary: "服务器内存使用率过高"
description: "{{ $labels.instance }} 内存使用率已超过 90%"

# 磁盘空间不足
– alert: LowDiskSpace
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
for: 5m
labels:
severity: critical
team: ops
annotations:
summary: "磁盘空间不足"
description: "{{ $labels.instance }} 磁盘可用空间不足 10%"

– name: database_alerts
interval: 30s
rules:
# 数据库连接池耗尽
– alert: DatabaseConnectionPoolExhausted
expr: db_connection_pool_size > 80
for: 5m
labels:
severity: critical
team: backend
annotations:
summary: "数据库连接池即将耗尽"
description: "当前连接数 {{ $value }}"

# 慢查询告警
– alert: SlowQuery
expr: histogram_quantile(0.95, sum(rate(db_query_duration_seconds_bucket[5m])) by (le, operation)) > 1
for: 10m
labels:
severity: warning
team: backend
annotations:
summary: "慢查询告警"
description: "p95 查询耗时超过 1 秒"

2.4 AlertManager 配置

# alertmanager/config.yml
global:
resolve_timeout: 5m
slack_api_url: 'https://hooks.slack.com/services/xxx'

route:
group_by: ['alertname', 'severity']
group_wait: 10s
group_interval: 10s
repeat_interval: 4h
receiver: 'default'
routes:
– match:
severity: critical
receiver: critical
continue: true
– match:
severity: warning
receiver: warning
continue: true
– match:
team: business
receiver: business

receivers:
– name: 'default'
slack_configs:
– channel: '#alerts'
title: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'
text: |
{{ range .Alerts }}
*告警:* {{ .Annotations.summary }}
*描述:* {{ .Annotations.description }}
*严重级别:* {{ .Labels.severity }}
*实例:* {{ .Labels.instance }}
{{ end }}

– name: 'critical'
slack_configs:
– channel: '#critical-alerts'
title: '🚨 CRITICAL: {{ .CommonLabels.alertname }}'
text: |
{{ range .Alerts }}
*告警:* {{ .Annotations.summary }}
*描述:* {{ .Annotations.description }}
*时间:* {{ .StartsAt.Format "2006-01-02 15:04:05" }}
{{ end }}
webhook_configs:
– url: 'http://pagerduty:8080'
send_resolved: true

– name: 'warning'
slack_configs:
– channel: '#warning-alerts'
title: '⚠️ WARNING: {{ .CommonLabels.alertname }}'
text: |
{{ range .Alerts }}
*告警:* {{ .Annotations.summary }}
*描述:* {{ .Annotations.description }}
{{ end }}

– name: 'business'
slack_configs:
– channel: '#business-alerts'
title: '📊 BUSINESS: {{ .CommonLabels.alertname }}'
text: |
{{ range .Alerts }}
*告警:* {{ .Annotations.summary }}
*描述:* {{ .Annotations.description }}
{{ end }}

3. Grafana 仪表盘

3.1 仪表盘配置

{
"dashboard": {
"title": "E-commerce Platform Monitoring",
"tags": ["ecommerce", "production"],
"timezone": "browser",
"panels": [
{
"title": "服务健康状态",
"type": "stat",
"targets": [
{
"expr": "count(up{job=~\\"user-service|product-service|order-service|payment-service\\"} == 1)",
"legendFormat": "健康服务数"
}
],
"gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}
},
{
"title": "请求速率",
"type": "graph",
"targets": [
{
"expr": "sum(rate(http_requests_total[1m])) by (job)",
"legendFormat": "{{job}}"
}
],
"gridPos": {"x": 4, "y": 0, "w": 10, "h": 4}
},
{
"title": "请求延迟 (p99)",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job))",
"legendFormat": "{{job}}"
}
],
"gridPos": {"x": 14, "y": 0, "w": 10, "h": 4}
},
{
"title": "错误率",
"type": "graph",
"targets": [
{
"expr": "sum(rate(http_requests_total{status=~\\"5..\\"}[1m])) / sum(rate(http_requests_total[1m]))",
"legendFormat": "错误率"
}
],
"gridPos": {"x": 0, "y": 4, "w": 8, "h": 4}
},
{
"title": "活跃连接数",
"type": "graph",
"targets": [
{
"expr": "http_active_requests",
"legendFormat": "{{job}}"
}
],
"gridPos": {"x": 8, "y": 4, "w": 8, "h": 4}
},
{
"title": "CPU 使用率",
"type": "graph",
"targets": [
{
"expr": "100 – (avg(rate(node_cpu_seconds_total{mode=\\"idle\\"}[5m])) by (instance) * 100)",
"legendFormat": "{{instance}}"
}
],
"gridPos": {"x": 16, "y": 4, "w": 8, "h": 4}
},
{
"title": "订单统计",
"type": "stat",
"targets": [
{
"expr": "sum(rate(order_created_total[1h]))",
"legendFormat": "订单/小时"
}
],
"gridPos": {"x": 0, "y": 8, "w": 4, "h": 4}
},
{
"title": "支付成功率",
"type": "stat",
"targets": [
{
"expr": "sum(rate(payment_success_total[5m])) / (sum(rate(payment_success_total[5m])) + sum(rate(payment_failed_total[5m]))) * 100",
"legendFormat": "成功率"
}
],
"gridPos": {"x": 4, "y": 8, "w": 4, "h": 4}
},
{
"title": "内存使用",
"type": "graph",
"targets": [
{
"expr": "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100",
"legendFormat": "{{instance}}"
}
],
"gridPos": {"x": 8, "y": 8, "w": 8, "h": 4}
},
{
"title": "数据库连接池",
"type": "graph",
"targets": [
{
"expr": "db_connection_pool_size",
"legendFormat": "连接数"
}
],
"gridPos": {"x": 16, "y": 8, "w": 8, "h": 4}
}
],
"refresh": "30s",
"time": {
"from": "now-1h",
"to": "now"
}
}
}

4. ELK 日志系统

4.1 Filebeat 配置

# filebeat/filebeat.yml
filebeat.inputs:
– type: container
enabled: true
paths:
– /var/lib/docker/containers/*/*.log
json.keys_under_root: true
json.add_error_key: true
fields:
log_type: container
environment: production

– type: log
enabled: true
paths:
– /var/log/ecommerce/*.log
exclude_files: ['\\.gz$']
fields:
log_type: application
multiline:
type: pattern
pattern: '^\\d{4}-\\d{2}-\\d{2}'
negate: true
match: after

processors:
– add_host_metadata:
netinfo.enabled: true
– add_docker_metadata:
host: "unix:///var/run/docker.sock"
– add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
– logs_path:
logs_path: "/var/log/containers/"

output.logstash:
hosts: ["logstash:5044"]
worker: 2
loadbalance: true
compression_level: 3

logging.level: info
logging.to_files: true
logging.files:
path: /var/log/filebeat
name: filebeat
keepfiles: 7

4.2 Logstash 配置

# logstash/pipeline/main.conf
input {
beats {
port => 5044
client_inactivity_timeout => 60
}

tcp {
port => 5000
codec => json
}
}

filter {
# JSON 解析
json {
source => "message"
target => "parsed"
remove_field => ["message"]
}

# 日期解析
date {
match => ["[parsed][timestamp]", "ISO8601"]
target => "@timestamp"
}

# 服务字段处理
if [parsed][service] {
mutate {
add_field => { "service_name" => "%{[parsed][service]}" }
}
}

# 日志级别
if [parsed][level] {
mutate {
add_field => { "log_level" => "%{[parsed][level]}" }
}
}

# 错误提取
if [log_level] == "error" or [log_level] == "ERROR" {
grok {
match => { "message" => "\\[(?<error_code>[A-Z_]+)\\]" }
}
mutate {
add_tag => ["error"]
}
}

# 用户 ID 提取
if [parsed][userId] {
mutate {
add_field => { "user_id" => "%{[parsed][userId]}" }
}
}

# 请求 ID 提取
if [parsed][requestId] {
mutate {
add_field => { "request_id" => "%{[parsed][requestId]}" }
}
}

# 敏感信息脱敏
mutate {
gsub => [
"message", "(?<=password=)[^&]+", "***",
"message", "(?<=token=)[^&]+", "***"
]
}

# 删除冗余字段
mutate {
remove_field => ["agent", "host", "input", "ecs", "tags"]
}
}

output {
# 按服务分索引
if [service_name] {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "logs-%{[service_name]}-%{+YYYY.MM.dd}"
user => "${ELASTICSEARCH_USERNAME}"
password => "${ELASTICSEARCH_PASSWORD}"
}
} else {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "logs-default-%{+YYYY.MM.dd}"
}
}

# 错误日志单独索引
if "error" in [tags] {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "error-logs-%{+YYYY.MM.dd}"
}
}

stdout {
codec => rubydebug
enabled => false
}
}

4.3 Kibana 配置

// kibana/kibana.yml
server.name: kibana
server.host: "0.0.0.0"
server.port: 5601

elasticsearch.hosts: ["http://elasticsearch:9200"]
elasticsearch.username: ${ELASTICSEARCH_USERNAME}
elasticsearch.password: ${ELASTICSEARCH_PASSWORD}

kibana.index: ".kibana"

logging.dest: stdout
logging.json: true
logging.verbose: false

xpack.alerting.enabled: true
xpack.encryptedSavedObjects.encryptionKey: ${ENCRYPTION_KEY}
xpack.reporting.enabled: true

monitoring.ui.container.elasticsearch.enabled: true

server.publicBaseUrl: https://kibana.ecommerce.com

5. Docker Compose 监控配置

# docker-compose.monitoring.yml
version: '3.8'

services:
prometheus:
image: prom/prometheus:latest
container_name: prometheus
volumes:
– ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
– ./prometheus/alerts:/etc/prometheus/alerts
– prometheus_data:/prometheus
command:
– '–config.file=/etc/prometheus/prometheus.yml'
– '–storage.tsdb.path=/prometheus'
– '–web.enable-lifecycle'
ports:
– "9090:9090"
networks:
– monitoring
restart: unless–stopped

alertmanager:
image: prom/alertmanager:latest
container_name: alertmanager
volumes:
– ./alertmanager/config.yml:/etc/alertmanager/alertmanager.yml
– alertmanager_data:/alertmanager
command:
– '–config.file=/etc/alertmanager/alertmanager.yml'
– '–storage.path=/alertmanager'
ports:
– "9093:9093"
networks:
– monitoring
restart: unless–stopped

grafana:
image: grafana/grafana:latest
container_name: grafana
environment:
– GF_SECURITY_ADMIN_PASSWORD=admin
– GF_INSTALL_PLUGINS=grafana–piechart–panel
volumes:
– ./grafana/datasources:/etc/grafana/provisioning/datasources
– ./grafana/dashboards:/etc/grafana/provisioning/dashboards
– grafana_data:/var/lib/grafana
ports:
– "3000:3000"
networks:
– monitoring
restart: unless–stopped

node-exporter:
image: prom/node–exporter:latest
container_name: node–exporter
volumes:
– /proc:/host/proc:ro
– /sys:/host/sys:ro
– /:/rootfs:ro
command:
– '–path.procfs=/host/proc'
– '–path.sysfs=/host/sys'
– '–path.rootfs=/rootfs'
ports:
– "9100:9100"
networks:
– monitoring
restart: unless–stopped

cadvisor:
image: gcr.io/cadvisor/cadvisor:latest
container_name: cadvisor
volumes:
– /:/rootfs:ro
– /var/run:/var/run:ro
– /sys:/sys:ro
– /var/lib/docker/:/var/lib/docker:ro
– /dev/disk/:/dev/disk:ro
ports:
– "8080:8080"
privileged: true
networks:
– monitoring
restart: unless–stopped

postgres-exporter:
image: prometheuscommunity/postgres–exporter:latest
container_name: postgres–exporter
environment:
DATA_SOURCE_NAME: "postgresql://postgres:password@postgres:5432/postgres?sslmode=disable"
ports:
– "9187:9187"
networks:
– monitoring
restart: unless–stopped

redis-exporter:
image: oliver006/redis_exporter:latest
container_name: redis–exporter
environment:
REDIS_ADDR: "redis:6379"
REDIS_PASSWORD: ${REDIS_PASSWORD}
ports:
– "9121:9121"
networks:
– monitoring
restart: unless–stopped

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0
container_name: elasticsearch
environment:
– node.name=elasticsearch
– cluster.name=docker–cluster
– discovery.type=single–node
– bootstrap.memory_lock=true
– "ES_JAVA_OPTS=-Xms2g -Xmx2g"
– xpack.security.enabled=false
ulimits:
memlock:
soft: -1
hard: -1
volumes:
– elasticsearch_data:/usr/share/elasticsearch/data
ports:
– "9200:9200"
networks:
– monitoring
restart: unless–stopped

logstash:
image: docker.elastic.co/logstash/logstash:8.10.0
container_name: logstash
volumes:
– ./logstash/pipeline:/usr/share/logstash/pipeline
ports:
– "5000:5000"
– "5044:5044"
depends_on:
– elasticsearch
networks:
– monitoring
restart: unless–stopped

filebeat:
image: docker.elastic.co/beats/filebeat:8.10.0
container_name: filebeat
user: root
volumes:
– ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
– /var/lib/docker/containers:/var/lib/docker/containers:ro
– /var/run/docker.sock:/var/run/docker.sock:ro
– /var/log:/host/var/log:ro
command: ––strict.perms=false
depends_on:
– logstash
networks:
– monitoring
restart: unless–stopped

kibana:
image: docker.elastic.co/kibana/kibana:8.10.0
container_name: kibana
environment:
– ELASTICSEARCH_HOSTS=http://elasticsearch:9200
volumes:
– ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
ports:
– "5601:5601"
depends_on:
– elasticsearch
networks:
– monitoring
restart: unless–stopped

networks:
monitoring:
driver: bridge

volumes:
prometheus_data:
alertmanager_data:
grafana_data:
elasticsearch_data:

练习题

基础题

进阶题

挑战题

练习题参考答案

基础题1

scrape_configs:
– job_name: 'myapp'
static_configs:
– targets: ['myapp:3000']

基础题2

const httpRequestsTotal = new Counter({
name: 'http_requests_total',
help: 'Total HTTP requests'
});

基础题3

{
"title": "Request Rate",
"targets": [{"expr": "rate(http_requests_total[1m])"}]
}

下一节预告： 性能测试与优化