Master-Node Stack & README
This commit is contained in:
162
k3s-monitoring/prometheus-config.yml
Normal file
162
k3s-monitoring/prometheus-config.yml
Normal file
@@ -0,0 +1,162 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: prometheus-config
|
||||
namespace: monitoring
|
||||
data:
|
||||
prometheus.yml: |
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['prometheus:9090']
|
||||
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets:
|
||||
- '100.104.27.47:9100'
|
||||
- '100.77.135.86:9100'
|
||||
- '100.90.177.14:9100'
|
||||
|
||||
- job_name: 'kube-state-metrics'
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'kube-state-metrics.kube-system.svc.cluster.local:8080'
|
||||
labels:
|
||||
service: 'kube-state-metrics'
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: ['alertmanager:9093']
|
||||
|
||||
rule_files:
|
||||
- "/etc/prometheus/alert.rules.yml"
|
||||
alert.rules.yml: |
|
||||
groups:
|
||||
# 1) 노드 상태 및 리소스 사용 관련 알람
|
||||
- name: node-health-rules
|
||||
rules:
|
||||
# ---- 노드 장애 / 다운 감지 ----
|
||||
- alert: NodeDown # 규칙 이름
|
||||
expr: up{job="node-exporter"} == 0
|
||||
# node-exporter가 응답하지 않으면 0으로 떨어짐
|
||||
for: 1m # 1분 연속으로 다운일 때 Alert
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} is down"
|
||||
description: "No response from node-exporter on {{ $labels.instance }} for more than 1 minute."
|
||||
|
||||
# ---- CPU 사용률 과도 ----
|
||||
- alert: HighCpuLoad
|
||||
expr: avg by (instance) (rate(node_cpu_seconds_total{mode!="idle",job="node-exporter"}[5m])) > 0.7
|
||||
# 최근 1분 동안 idle 이 아닌 CPU 비율의 평균이 70% 초과
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High CPU usage on {{ $labels.instance }}"
|
||||
description: "Average CPU usage over 5 minutes is above 70% on {{ $labels.instance }}."
|
||||
|
||||
# ---- 메모리 사용률 과도 ----
|
||||
- alert: HighMemoryUsage
|
||||
expr: 1 - (node_memory_MemAvailable_bytes{job="node-exporter"}
|
||||
/ node_memory_MemTotal_bytes{job="node-exporter"}) > 0.8
|
||||
# 사용률 = 1 - (사용 가능 / 전체) > 0.8 → 80% 이상 사용
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage is above 80% for more than 5 minutes on {{ $labels.instance }}."
|
||||
|
||||
# ---- 디스크 용량 부족 ----
|
||||
- alert: DiskAlmostFull
|
||||
expr: 1 - (
|
||||
node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|overlay"}
|
||||
/ node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|overlay"}
|
||||
) > 0.4
|
||||
# overlay, tmpfs 같은 일회성/메모리 파일시스템 제외
|
||||
# 사용률 > 85% 일 때 알람
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disk almost full on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
||||
description: "Filesystem {{ $labels.mountpoint }} is above 40% usage for more than 1 minutes on {{ $labels.instance }}."
|
||||
|
||||
# ---- 시스템 Load (코어 대비 과부하) ----
|
||||
- alert: HighSystemLoad
|
||||
expr: (
|
||||
avg by (instance) (node_load5{job="node-exporter"})
|
||||
/
|
||||
on (instance) count by (instance) (node_cpu_seconds_total{job="node-exporter",mode="idle"})
|
||||
) > 2
|
||||
# 1분 load average / CPU 코어 수 > 2 → 코어 수 대비 부하가 심한 경우
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High load average on {{ $labels.instance }}"
|
||||
description: "5-minute load average per CPU core is above 2 for more than 10 minutes on {{ $labels.instance }}."
|
||||
|
||||
# 2) Kubernetes 워크로드 상태 관련 알람 (kube-state-metrics 기준)
|
||||
- name: kubernetes-workload-rules
|
||||
rules:
|
||||
# ---- K8s 노드가 NotReady 상태 ----
|
||||
- alert: KubeNodeNotReady
|
||||
expr: kube_node_status_condition{condition="Ready",status="true",job="kube-state-metrics"} == 0
|
||||
# Ready=true 인 시리즈가 0이면, 사실상 NotReady
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Kubernetes node {{ $labels.node }} is NotReady"
|
||||
description: "Kubernetes node {{ $labels.node }} has been in NotReady state for more than 5 minutes."
|
||||
|
||||
# ---- Pod가 CrashLooping (재시작 반복) ----
|
||||
- alert: PodCrashLooping
|
||||
expr: increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) > 3
|
||||
# 5분 동안 같은 컨테이너가 3번 이상 재시작
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
|
||||
description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} restarted more than 3 times in the last 5 minutes."
|
||||
|
||||
# ---- Running인데 Ready가 아닌 Pod ----
|
||||
- alert: PodNotReady
|
||||
expr: (
|
||||
kube_pod_status_ready{condition="true",job="kube-state-metrics"} == 0
|
||||
)
|
||||
and on (namespace, pod)
|
||||
(
|
||||
kube_pod_status_phase{phase="Running",job="kube-state-metrics"} == 1
|
||||
)
|
||||
# Running 상태인데 Ready = false 인 경우
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is not Ready"
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is Running but not Ready for more than 10 minutes."
|
||||
|
||||
# ---- Deployment: 원하는 replicas와 실제 가용 replicas 불일치 ----
|
||||
- alert: DeploymentReplicasMismatch
|
||||
expr: kube_deployment_status_replicas_available{job="kube-state-metrics"}
|
||||
< kube_deployment_spec_replicas{job="kube-state-metrics"}
|
||||
# 가용 replicas < 설정 replicas → 정상적인 replica 수 유지 실패
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has unavailable replicas"
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has fewer available replicas than desired for more than 5 minutes."
|
||||
|
||||
|
||||
Reference in New Issue
Block a user