163 lines
6.9 KiB
YAML
163 lines
6.9 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
namespace: monitoring
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 30s
|
|
evaluation_interval: 30s
|
|
|
|
scrape_configs:
|
|
- job_name: 'prometheus'
|
|
static_configs:
|
|
- targets: ['prometheus:9090']
|
|
|
|
- job_name: 'node-exporter'
|
|
static_configs:
|
|
- targets:
|
|
- '100.104.27.47:9100'
|
|
- '100.77.135.86:9100'
|
|
- '100.90.177.14:9100'
|
|
|
|
- job_name: 'kube-state-metrics'
|
|
scrape_interval: 30s
|
|
static_configs:
|
|
- targets:
|
|
- 'kube-state-metrics.kube-system.svc.cluster.local:8080'
|
|
labels:
|
|
service: 'kube-state-metrics'
|
|
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs:
|
|
- targets: ['alertmanager:9093']
|
|
|
|
rule_files:
|
|
- "/etc/prometheus/alert.rules.yml"
|
|
alert.rules.yml: |
|
|
groups:
|
|
# 1) 노드 상태 및 리소스 사용 관련 알람
|
|
- name: node-health-rules
|
|
rules:
|
|
# ---- 노드 장애 / 다운 감지 ----
|
|
- alert: NodeDown # 규칙 이름
|
|
expr: up{job="node-exporter"} == 0
|
|
# node-exporter가 응답하지 않으면 0으로 떨어짐
|
|
for: 1m # 1분 연속으로 다운일 때 Alert
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Node {{ $labels.instance }} is down"
|
|
description: "No response from node-exporter on {{ $labels.instance }} for more than 1 minute."
|
|
|
|
# ---- CPU 사용률 과도 ----
|
|
- alert: HighCpuLoad
|
|
expr: avg by (instance) (rate(node_cpu_seconds_total{mode!="idle",job="node-exporter"}[5m])) > 0.7
|
|
# 최근 1분 동안 idle 이 아닌 CPU 비율의 평균이 70% 초과
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "High CPU usage on {{ $labels.instance }}"
|
|
description: "Average CPU usage over 5 minutes is above 70% on {{ $labels.instance }}."
|
|
|
|
# ---- 메모리 사용률 과도 ----
|
|
- alert: HighMemoryUsage
|
|
expr: 1 - (node_memory_MemAvailable_bytes{job="node-exporter"}
|
|
/ node_memory_MemTotal_bytes{job="node-exporter"}) > 0.8
|
|
# 사용률 = 1 - (사용 가능 / 전체) > 0.8 → 80% 이상 사용
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "High memory usage on {{ $labels.instance }}"
|
|
description: "Memory usage is above 80% for more than 5 minutes on {{ $labels.instance }}."
|
|
|
|
# ---- 디스크 용량 부족 ----
|
|
- alert: DiskAlmostFull
|
|
expr: 1 - (
|
|
node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|overlay"}
|
|
/ node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|overlay"}
|
|
) > 0.4
|
|
# overlay, tmpfs 같은 일회성/메모리 파일시스템 제외
|
|
# 사용률 > 85% 일 때 알람
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Disk almost full on {{ $labels.instance }} ({{ $labels.mountpoint }})"
|
|
description: "Filesystem {{ $labels.mountpoint }} is above 40% usage for more than 1 minutes on {{ $labels.instance }}."
|
|
|
|
# ---- 시스템 Load (코어 대비 과부하) ----
|
|
- alert: HighSystemLoad
|
|
expr: (
|
|
avg by (instance) (node_load5{job="node-exporter"})
|
|
/
|
|
on (instance) count by (instance) (node_cpu_seconds_total{job="node-exporter",mode="idle"})
|
|
) > 2
|
|
# 1분 load average / CPU 코어 수 > 2 → 코어 수 대비 부하가 심한 경우
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "High load average on {{ $labels.instance }}"
|
|
description: "5-minute load average per CPU core is above 2 for more than 10 minutes on {{ $labels.instance }}."
|
|
|
|
# 2) Kubernetes 워크로드 상태 관련 알람 (kube-state-metrics 기준)
|
|
- name: kubernetes-workload-rules
|
|
rules:
|
|
# ---- K8s 노드가 NotReady 상태 ----
|
|
- alert: KubeNodeNotReady
|
|
expr: kube_node_status_condition{condition="Ready",status="true",job="kube-state-metrics"} == 0
|
|
# Ready=true 인 시리즈가 0이면, 사실상 NotReady
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Kubernetes node {{ $labels.node }} is NotReady"
|
|
description: "Kubernetes node {{ $labels.node }} has been in NotReady state for more than 5 minutes."
|
|
|
|
# ---- Pod가 CrashLooping (재시작 반복) ----
|
|
- alert: PodCrashLooping
|
|
expr: increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) > 3
|
|
# 5분 동안 같은 컨테이너가 3번 이상 재시작
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
|
|
description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} restarted more than 3 times in the last 5 minutes."
|
|
|
|
# ---- Running인데 Ready가 아닌 Pod ----
|
|
- alert: PodNotReady
|
|
expr: (
|
|
kube_pod_status_ready{condition="true",job="kube-state-metrics"} == 0
|
|
)
|
|
and on (namespace, pod)
|
|
(
|
|
kube_pod_status_phase{phase="Running",job="kube-state-metrics"} == 1
|
|
)
|
|
# Running 상태인데 Ready = false 인 경우
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is not Ready"
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is Running but not Ready for more than 10 minutes."
|
|
|
|
# ---- Deployment: 원하는 replicas와 실제 가용 replicas 불일치 ----
|
|
- alert: DeploymentReplicasMismatch
|
|
expr: kube_deployment_status_replicas_available{job="kube-state-metrics"}
|
|
< kube_deployment_spec_replicas{job="kube-state-metrics"}
|
|
# 가용 replicas < 설정 replicas → 정상적인 replica 수 유지 실패
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has unavailable replicas"
|
|
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has fewer available replicas than desired for more than 5 minutes."
|
|
|
|
|