92 lines
3.1 KiB
YAML
92 lines
3.1 KiB
YAML
|
apiVersion: operator.victoriametrics.com/v1beta1
|
||
|
|
kind: VMAlert
|
||
|
|
metadata:
|
||
|
|
name: health
|
||
|
|
namespace: victoriametrics
|
||
|
|
spec:
|
||
|
|
replicaCount: 2
|
||
|
|
image:
|
||
|
|
tag: v1.155.0
|
||
|
|
useStrictSecurity: true
|
||
|
|
logFormat: json
|
||
|
|
ruleSelector:
|
||
|
|
matchLabels:
|
||
|
|
app.kubernetes.io/part-of: cluster-health
|
||
|
|
datasource:
|
||
|
|
url: http://vmselect-main.victoriametrics.svc.cluster.local:8481/select/0/prometheus
|
||
|
|
remoteWrite:
|
||
|
|
url: http://vminsert-main.victoriametrics.svc.cluster.local:8480/insert/0/prometheus
|
||
|
|
extraArgs:
|
||
|
|
notifier.blackhole: "true"
|
||
|
|
podDisruptionBudget:
|
||
|
|
minAvailable: 1
|
||
|
|
affinity:
|
||
|
|
podAntiAffinity:
|
||
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||
|
|
- labelSelector:
|
||
|
|
matchLabels:
|
||
|
|
app.kubernetes.io/name: vmalert
|
||
|
|
app.kubernetes.io/instance: health
|
||
|
|
topologyKey: kubernetes.io/hostname
|
||
|
|
resources:
|
||
|
|
requests:
|
||
|
|
cpu: 10m
|
||
|
|
memory: 64Mi
|
||
|
|
limits:
|
||
|
|
memory: 128Mi
|
||
|
|
---
|
||
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
||
|
|
kind: VMRule
|
||
|
|
metadata:
|
||
|
|
name: health-days
|
||
|
|
namespace: victoriametrics
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/part-of: cluster-health
|
||
|
|
spec:
|
||
|
|
groups:
|
||
|
|
- name: health-days
|
||
|
|
interval: 1d
|
||
|
|
eval_offset: 10m
|
||
|
|
rules:
|
||
|
|
- record: node_health_day
|
||
|
|
expr: >-
|
||
|
|
label_replace(
|
||
|
|
avg by (node) (
|
||
|
|
avg_over_time(up{job="node-exporter-prometheus-node-exporter"}[1d] offset 10m)
|
||
|
|
),
|
||
|
|
"node", "$1", "node", "ms2k-(.+)"
|
||
|
|
)
|
||
|
|
- record: public_health_day
|
||
|
|
expr: >-
|
||
|
|
WITH (
|
||
|
|
names = max by (namespace, service, label_health_quorate_dev_name) (
|
||
|
|
kube_service_labels{label_health_quorate_dev_name!=""} @ now()
|
||
|
|
),
|
||
|
|
public = max by (namespace, service) (kube_service_labels)
|
||
|
|
and on (namespace, service) names,
|
||
|
|
slices = label_replace(
|
||
|
|
max by (namespace, endpointslice, label_kubernetes_io_service_name) (
|
||
|
|
kube_endpointslice_labels
|
||
|
|
),
|
||
|
|
"service", "$1", "label_kubernetes_io_service_name", "(.+)"
|
||
|
|
) and on (namespace, service) public,
|
||
|
|
ready = max by (namespace, service) (
|
||
|
|
slices * on (namespace, endpointslice) group_right (service)
|
||
|
|
max by (namespace, endpointslice) (
|
||
|
|
kube_endpointslice_endpoints{ready="true"}
|
||
|
|
)
|
||
|
|
),
|
||
|
|
health = (ready or on (namespace, service) (0 * public))
|
||
|
|
and on (namespace, service) public,
|
||
|
|
daily = avg_over_time(health[1d:30s] offset 10m)
|
||
|
|
or on (namespace, service) max by (namespace, service) (
|
||
|
|
last_over_time(service_health_day[1d] offset 10m)
|
||
|
|
)
|
||
|
|
)
|
||
|
|
label_keep(
|
||
|
|
label_replace(
|
||
|
|
daily * on (namespace, service) group_left (label_health_quorate_dev_name) names,
|
||
|
|
"name", "$1", "label_health_quorate_dev_name", "(.+)"
|
||
|
|
),
|
||
|
|
"name"
|
||
|
|
)
|