apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: health
  namespace: victoriametrics
spec:
  replicaCount: 2
  image:
    tag: v1.155.0
  useStrictSecurity: true
  logFormat: json
  ruleSelector:
    matchLabels:
      app.kubernetes.io/part-of: cluster-health
  datasource:
    url: http://vmselect-main.victoriametrics.svc.cluster.local:8481/select/0/prometheus
  remoteWrite:
    url: http://vminsert-main.victoriametrics.svc.cluster.local:8480/insert/0/prometheus
  extraArgs:
    notifier.blackhole: "true"
  podDisruptionBudget:
    minAvailable: 1
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              app.kubernetes.io/name: vmalert
              app.kubernetes.io/instance: health
          topologyKey: kubernetes.io/hostname
  resources:
    requests:
      cpu: 10m
      memory: 64Mi
    limits:
      memory: 128Mi
---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: health-days
  namespace: victoriametrics
  labels:
    app.kubernetes.io/part-of: cluster-health
spec:
  groups:
    - name: health-days
      interval: 1d
      eval_offset: 10m
      rules:
        - record: node_health_day
          expr: >-
            label_replace(
              avg by (node) (
                avg_over_time(up{job="node-exporter-prometheus-node-exporter"}[1d] offset 10m)
              ),
              "node", "$1", "node", "ms2k-(.+)"
            )
        - record: public_health_day
          expr: >-
            WITH (
              names = max by (namespace, service, label_health_quorate_dev_name) (
                kube_service_labels{label_health_quorate_dev_name!=""} @ now()
              ),
              public = max by (namespace, service) (kube_service_labels)
                and on (namespace, service) names,
              slices = label_replace(
                max by (namespace, endpointslice, label_kubernetes_io_service_name) (
                  kube_endpointslice_labels
                ),
                "service", "$1", "label_kubernetes_io_service_name", "(.+)"
              ) and on (namespace, service) public,
              ready = max by (namespace, service) (
                slices * on (namespace, endpointslice) group_right (service)
                  max by (namespace, endpointslice) (
                    kube_endpointslice_endpoints{ready="true"}
                  )
              ),
              health = (ready or on (namespace, service) (0 * public))
                and on (namespace, service) public,
              daily = avg_over_time(health[1d:30s] offset 10m)
                or on (namespace, service) max by (namespace, service) (
                  last_over_time(service_health_day[1d] offset 10m)
                )
            )
            label_keep(
              label_replace(
                daily * on (namespace, service) group_left (label_health_quorate_dev_name) names,
                "name", "$1", "label_health_quorate_dev_name", "(.+)"
              ),
              "name"
            )