250 lines
7.4 KiB
YAML
250 lines
7.4 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["pods", "services"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: [""]
|
|
resources: ["pods/exec"]
|
|
verbs: ["create"]
|
|
- apiGroups: ["discovery.k8s.io"]
|
|
resources: ["endpointslices"]
|
|
verbs: ["get", "list"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: cilium-service-map-monitor
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
data:
|
|
monitor.sh: |
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
interval="${CHECK_INTERVAL_SECONDS:-30}"
|
|
state_dir="/tmp/cilium-service-map-monitor"
|
|
mkdir -p "${state_dir}"
|
|
|
|
log() {
|
|
printf '%s %s\n' "$(date -Iseconds)" "$*"
|
|
}
|
|
|
|
endpoint_addresses() {
|
|
kubectl -n kube-system get endpointslice \
|
|
-l kubernetes.io/service-name=kube-dns \
|
|
-o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\n"}{end}' |
|
|
sed '/^$/d' |
|
|
sort -u
|
|
}
|
|
|
|
expected_backends() {
|
|
local port_proto="$1"
|
|
endpoint_addresses | while read -r ip; do
|
|
printf '%s:%s\n' "${ip}" "${port_proto}"
|
|
done
|
|
}
|
|
|
|
extract_backends() {
|
|
local frontend="$1"
|
|
awk -v frontend="${frontend}" '
|
|
/^[[:space:]]*[0-9]+[[:space:]]/ {
|
|
active = 0
|
|
}
|
|
index($0, frontend) {
|
|
active = 1
|
|
print
|
|
next
|
|
}
|
|
active {
|
|
print
|
|
}
|
|
' |
|
|
grep -oE '=>[[:space:]]+[0-9.]+:[0-9]+/[A-Z]+' |
|
|
sed -E 's/^=>[[:space:]]+//' |
|
|
sort -u
|
|
}
|
|
|
|
relevant_service_lines() {
|
|
awk '
|
|
/^[[:space:]]*[0-9]+[[:space:]]/ {
|
|
active = 0
|
|
}
|
|
/10\.240\.10\.0:(53|9153)\// {
|
|
active = 1
|
|
print
|
|
next
|
|
}
|
|
active && /=>/ {
|
|
print
|
|
}
|
|
'
|
|
}
|
|
|
|
check_port() {
|
|
local pod="$1"
|
|
local node="$2"
|
|
local service_list="$3"
|
|
local frontend="$4"
|
|
local port_proto="$5"
|
|
local state_file="${state_dir}/${pod}-${port_proto//\//-}"
|
|
|
|
local actual expected unexpected missing issue
|
|
actual="$(printf '%s\n' "${service_list}" | extract_backends "${frontend}")"
|
|
expected="$(expected_backends "${port_proto}")"
|
|
|
|
unexpected="$(comm -23 <(printf '%s\n' "${actual}" | sed '/^$/d' | sort -u) <(printf '%s\n' "${expected}" | sed '/^$/d' | sort -u) || true)"
|
|
missing="$(comm -13 <(printf '%s\n' "${actual}" | sed '/^$/d' | sort -u) <(printf '%s\n' "${expected}" | sed '/^$/d' | sort -u) || true)"
|
|
|
|
if [[ -n "${unexpected}" || -n "${missing}" ]]; then
|
|
issue="unexpected=${unexpected//$'\n'/,} missing=${missing//$'\n'/,}"
|
|
if [[ ! -f "${state_file}" || "$(cat "${state_file}")" != "${issue}" ]]; then
|
|
printf '%s' "${issue}" > "${state_file}"
|
|
log "CILIUM_SERVICE_MAP_DRIFT pod=${pod} node=${node} frontend=${frontend}"
|
|
log "expected_backends=$(printf '%s' "${expected}" | paste -sd ',' -)"
|
|
log "actual_backends=$(printf '%s' "${actual}" | paste -sd ',' -)"
|
|
log "unexpected_backends=$(printf '%s' "${unexpected}" | paste -sd ',' -)"
|
|
log "missing_backends=$(printf '%s' "${missing}" | paste -sd ',' -)"
|
|
log "kube_dns_service_lines_begin pod=${pod}"
|
|
printf '%s\n' "${service_list}" | relevant_service_lines
|
|
log "kube_dns_service_lines_end pod=${pod}"
|
|
log "kube_dns_endpointslice_begin"
|
|
kubectl -n kube-system get endpointslice -l kubernetes.io/service-name=kube-dns -o wide
|
|
log "kube_dns_endpointslice_end"
|
|
fi
|
|
elif [[ -f "${state_file}" ]]; then
|
|
rm -f "${state_file}"
|
|
log "CILIUM_SERVICE_MAP_RECOVERED pod=${pod} node=${node} frontend=${frontend}"
|
|
fi
|
|
}
|
|
|
|
log "starting cilium service-map monitor interval=${interval}s"
|
|
|
|
while true; do
|
|
kube_dns_ip="$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')"
|
|
cilium_pods="$(kubectl -n kube-system get pods -l k8s-app=cilium \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.nodeName}{"\n"}{end}')"
|
|
|
|
while read -r pod node; do
|
|
[[ -n "${pod:-}" ]] || continue
|
|
if service_list="$(kubectl -n kube-system exec "${pod}" -- cilium-dbg service list 2>&1)"; then
|
|
check_port "${pod}" "${node}" "${service_list}" "${kube_dns_ip}:53/UDP" "53/UDP"
|
|
check_port "${pod}" "${node}" "${service_list}" "${kube_dns_ip}:53/TCP" "53/TCP"
|
|
check_port "${pod}" "${node}" "${service_list}" "${kube_dns_ip}:9153/TCP" "9153/TCP"
|
|
else
|
|
log "CILIUM_SERVICE_MAP_CHECK_FAILED pod=${pod} node=${node} error=$(printf '%s' "${service_list}" | tr '\n' ' ')"
|
|
fi
|
|
done <<< "${cilium_pods}"
|
|
|
|
sleep "${interval}"
|
|
done
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/name: cilium-service-map-monitor
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: cilium-service-map-monitor
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
ms2k.io/config-version: "2026-05-15-01"
|
|
labels:
|
|
app.kubernetes.io/name: cilium-service-map-monitor
|
|
spec:
|
|
serviceAccountName: cilium-service-map-monitor
|
|
automountServiceAccountToken: true
|
|
containers:
|
|
- name: monitor
|
|
image: docker.io/alpine/k8s:1.37.1@sha256:712dc5385d1c1a3f83b747fa9e40f64d433a97f72ff42f9755ff7dd1f62846e4
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/bash", "/monitor/monitor.sh"]
|
|
env:
|
|
- name: CHECK_INTERVAL_SECONDS
|
|
value: "30"
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
- name: monitor-script
|
|
mountPath: /monitor
|
|
readOnly: true
|
|
- name: tmp
|
|
mountPath: /tmp
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 64Mi
|
|
limits:
|
|
memory: 256Mi
|
|
volumes:
|
|
- name: monitor-script
|
|
configMap:
|
|
name: cilium-service-map-monitor
|
|
defaultMode: 0755
|
|
- name: tmp
|
|
emptyDir: {}
|
|
---
|
|
apiVersion: cilium.io/v2
|
|
kind: CiliumNetworkPolicy
|
|
metadata:
|
|
name: cilium-service-map-monitor
|
|
namespace: kube-system
|
|
spec:
|
|
endpointSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: cilium-service-map-monitor
|
|
ingress: []
|
|
egress:
|
|
- toEntities:
|
|
- kube-apiserver
|
|
toPorts:
|
|
- ports:
|
|
- port: "6443"
|
|
protocol: TCP
|
|
- toEndpoints:
|
|
- matchLabels:
|
|
k8s:io.kubernetes.pod.namespace: kube-system
|
|
k8s-app: kube-dns
|
|
toPorts:
|
|
- ports:
|
|
- port: "53"
|
|
protocol: UDP
|
|
- port: "53"
|
|
protocol: TCP
|