1
{
2
  "plugin": "kmsg",
3
  "logPath": "/dev/kmsg",
4
  "lookback": "5m",
5
  "bufferSize": 10,
6
  "source": "kernel-monitor",
7
  "conditions": [
8
    {
9
      "type": "KernelDeadlock",
10
      "reason": "KernelDeadlock",
11
      "message": "kernel: BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]"
12
    }
13
  ],
14
  "metrics": [
15
    {
16
      "name": "kernel_deadlock_total",
17
      "help": "Number of kernel deadlocks detected",
18
      "labels": ["reason"]
19
    }
20
  ]
21
}

1
{
2
  "plugin": "system-stats-monitor",
3
  "metrics": [
4
    {
5
      "name": "node_cpu_usage_percentage",
6
      "help": "CPU usage percentage",
7
      "type": "gauge"
8
    },
9
    {
10
      "name": "node_memory_usage_percentage",
11
      "help": "Memory usage percentage",
12
      "type": "gauge"
13
    }
14
  ]
15
}

1
{
2
  "plugin": "custom",
3
  "pluginConfigFile": "/etc/kubernetes/node-problem-detector/custom-plugin-config.json",
4
  "metrics": [
5
    {
6
      "name": "custom_plugin_check_total",
7
      "help": "Number of custom plugin checks",
8
      "labels": ["result"]
9
    }
10
  ]
11
}

1
#!/bin/bash
2
# NTP问题检测脚本
3

4
# 检查NTP服务状态
5
if ! systemctl is-active --quiet ntpd; then
6
    echo "NTP service is not running"
7
    exit 1
8
fi
9

10
# 检查NTP同步状态
11
ntpq -p | grep "*"
12
if [ $? -ne 0 ]; then
13
    echo "NTP is not synchronized"
14
    exit 1
15
fi
16

17
echo "NTP is healthy"
18
exit 0

1
# HELP node_problem_detector_problem_total Number of times a specific type of problem have occurred.
2
# TYPE node_problem_detector_problem_total counter
3
node_problem_detector_problem_total{reason="KernelDeadlock"} 1
4
node_problem_detector_problem_total{reason="ReadonlyFilesystem"} 0
5

6
# HELP node_problem_detector_problem_gauge Whether a specific type of problem is affecting node or not.
7
# TYPE node_problem_detector_problem_gauge gauge
8
node_problem_detector_problem_gauge{type="KernelDeadlock",reason="KernelDeadlock"} 1
9
node_problem_detector_problem_gauge{type="ReadonlyFilesystem",reason="ReadonlyFilesystem"} 0

1
{
2
  "project_id": "your-project-id",
3
  "monitored_resource_type": "gke_instance",
4
  "monitored_resource_labels": {
5
    "project_id": "your-project-id",
6
    "location": "us-central1-a",
7
    "cluster_name": "your-cluster",
8
    "instance_id": "node-1"
9
  }
10
}

1
// 注册问题守护进程工厂方法
2
func Register(problemDaemonType types.ProblemDaemonType, handler types.ProblemDaemonHandler) {
3
    handlers[problemDaemonType] = handler
4
}

1
// 根据配置创建所有问题守护进程
2
func NewProblemDaemons(monitorConfigPaths types.ProblemDaemonConfigPathMap) []types.Monitor {
3
    problemDaemonMap := make(map[string]types.Monitor)
4

5
    for problemDaemonType, configs := range monitorConfigPaths {
6
        for _, config := range *configs {
7
            if _, ok := problemDaemonMap[config]; ok {
8
                // 跳过重复配置
9
                klog.Warningf("Duplicated problem daemon configuration %q", config)
10
                continue
11
            }
12
            problemDaemonMap[config] = handlers[problemDaemonType].CreateProblemDaemonOrDie(config)
13
        }
14
    }
15

16
    problemDaemons := []types.Monitor{}
17
    for _, problemDaemon := range problemDaemonMap {
18
        problemDaemons = append(problemDaemons, problemDaemon)
19
    }
20
    return problemDaemons
21
}

1
// 启动问题检测器
2
func (p *problemDetector) Run(ctx context.Context) error {
3
    // 启动所有监控器
4
    var chans []<-chan *types.Status
5
    failureCount := 0
6

7
    for _, m := range p.monitors {
8
        ch, err := m.Start()
9
        if err != nil {
10
            klog.Errorf("Failed to start problem daemon %v: %v", m, err)
11
            failureCount++
12
            continue
13
        }
14
        if ch != nil {
15
            chans = append(chans, ch)
16
        }
17
    }
18

19
    allMonitors := p.monitors
20

21
    if len(allMonitors) == failureCount {
22
        return fmt.Errorf("no problem daemon is successfully setup")
23
    }
24

25
    defer func() {
26
        for _, m := range allMonitors {
27
            m.Stop()
28
        }
29
    }()
30

31
    ch := groupChannel(chans)
32
    klog.Info("Problem detector started")
33

34
    for {
35
        select {
36
        case <-ctx.Done():
37
            return nil
38
        case status := <-ch:
39
            for _, exporter := range p.exporters {
40
                exporter.ExportProblems(status)
41
            }
42
        }
43
    }
44
}

1
// 设置问题量规的值
2
func (pmm *ProblemMetricsManager) SetProblemGauge(problemType string, reason string, value bool) error {
3
    if pmm.problemGauge == nil {
4
        return errors.New("problem gauge is being set before initialized.")
5
    }
6

7
    pmm.problemTypeToReasonMutex.Lock()
8
    defer pmm.problemTypeToReasonMutex.Unlock()
9

10
    // 清除之前的原因，确保每个问题类型在任何时刻都最多只有一个原因被设置为1
11
    if lastReason, ok := pmm.problemTypeToReason[problemType]; ok {
12
        err := pmm.problemGauge.Record(map[string]string{"type": problemType, "reason": lastReason}, 0)
13
        if err != nil {
14
            return fmt.Errorf("failed to clear previous reason %q for type %q: %v",
15
                problemType, lastReason, err)
16
        }
17
    }
18

19
    pmm.problemTypeToReason[problemType] = reason
20

21
    var valueInt int64
22
    if value {
23
        valueInt = 1
24
    }
25
    return pmm.problemGauge.Record(map[string]string{"type": problemType, "reason": reason}, valueInt)
26
}

1
# 查看NPD Pod日志
2
kubectl logs -n kube-system -l app=node-problem-detector -f
3

4
# 查看特定节点的NPD日志
5
kubectl logs -n kube-system node-problem-detector-xxxxx -f

1
# 查看节点条件
2
kubectl describe node <node-name>
3

4
# 查看节点事件
5
kubectl get events --field-selector involvedObject.name=<node-name>

1
# 启动kubectl proxy
2
kubectl proxy --port=8080
3

4
# 访问Prometheus端点
5
curl http://<node-ip>:20257/metrics

1
# 在一个终端中监控事件
2
kubectl get events -w
3

4
# 在节点上注入测试消息
5
sudo sh -c "echo 'kernel: BUG: unable to handle kernel NULL pointer dereference at TESTING' >> /dev/kmsg"

1
# 在节点上注入测试消息
2
sudo sh -c "echo 'kernel: INFO: task docker:20744 blocked for more than 120 seconds.' >> /dev/kmsg"

1
NPD检测问题 → 标记节点为不可调度 → Draino排空节点 → Cluster Autoscaler终止节点 → 自动创建新节点

1
NPD检测问题 → 设置节点污点 → Descheduler驱逐Pod → Pod重新调度到健康节点

1
NPD检测问题 → NHC监控状态 → mediK8S执行修复 → Poison-Pill重启节点