tobias/infrastructure
1
0
Fork 0
Code Issues 5 Pull Requests Projects Activity

Adjust prometheus rules to leave time for backups

Browse Source
This commit is contained in:
Tobias Manske 2024-01-13 00:09:45 +01:00
parent ac0335c9b5
commit 450b6c160d
Signed by: tobias
GPG Key ID: 9164B527694A0709
2 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ansible/plays/services/mimir/rules/cadvisor.yaml
View File

@ -35,14 +35,14 @@ groups:
annotations: annotations:
summary: Container Memory usage (instance {{ $labels.instance }}) summary: Container Memory usage (instance {{ $labels.instance }})
description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# - alert: ContainerVolumeUsage - alert: ContainerVolumeUsage
# expr: '(1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total) BY (instance))) * 100 > 80' expr: '(1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total) BY (instance))) * 100 > 80'
# for: 2m for: 2m
# labels: labels:
# severity: warning severity: warning
# annotations: annotations:
# summary: Container Volume usage (instance {{ $labels.instance }}) summary: Container Volume usage (instance {{ $labels.instance }})
# description: "Container Volume usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" description: "Container Volume usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ContainerHighThrottleRate - alert: ContainerHighThrottleRate
expr: 'rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1' expr: 'rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1'
for: 2m for: 2m

4
ansible/plays/services/mimir/rules/node.yaml
View File

@ -12,7 +12,7 @@ groups:
summary: Host out of memory (instance {{ $labels.instance }}) summary: Host out of memory (instance {{ $labels.instance }})
description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostMemoryUnderMemoryPressure - alert: HostMemoryUnderMemoryPressure
expr: 'rate(node_vmstat_pgmajfault[1m]) > 1000' expr: 'rate(node_vmstat_pgmajfault[1m]) > 1000 unless on() hour()>=0 <=3'
for: 2m for: 2m
labels: labels:
severity: warning severity: warning
@ -108,7 +108,7 @@ groups:
summary: Host unusual disk write latency (instance {{ $labels.instance }}) summary: Host unusual disk write latency (instance {{ $labels.instance }})
description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostHighCpuLoad - alert: HostHighCpuLoad
expr: '(100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100)) > 80' expr: '(100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100)) > 80 unless on() hour()>=0 <=3'
for: 2m for: 2m
labels: labels:
severity: warning severity: warning