ce7cb702218693d08ea6d83464d2457c9250027c

This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
5 changed files with 145336 additions and 0 deletions

View file

@ -0,0 +1,250 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cilium-service-map-monitor
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: cilium-service-map-monitor
namespace: kube-system
rules:
- apiGroups: [""]
resources: ["pods", "services"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: cilium-service-map-monitor
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: cilium-service-map-monitor
subjects:
- kind: ServiceAccount
name: cilium-service-map-monitor
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: cilium-service-map-monitor
namespace: kube-system
data:
monitor.sh: |
#!/usr/bin/env bash
set -euo pipefail
interval="${CHECK_INTERVAL_SECONDS:-30}"
state_dir="/tmp/cilium-service-map-monitor"
mkdir -p "${state_dir}"
log() {
printf '%s %s\n' "$(date -Iseconds)" "$*"
}
endpoint_addresses() {
kubectl -n kube-system get endpointslice \
-l kubernetes.io/service-name=kube-dns \
-o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\n"}{end}' |
sed '/^$/d' |
sort -u
}
expected_backends() {
local port_proto="$1"
endpoint_addresses | while read -r ip; do
printf '%s:%s\n' "${ip}" "${port_proto}"
done
}
extract_backends() {
local frontend="$1"
awk -v frontend="${frontend}" '
/^[[:space:]]*[0-9]+[[:space:]]/ {
active = 0
}
index($0, frontend) {
active = 1
print
next
}
active {
print
}
' |
grep -oE '=>[[:space:]]+[0-9.]+:[0-9]+/[A-Z]+' |
sed -E 's/^=>[[:space:]]+//' |
sort -u
}
relevant_service_lines() {
awk '
/^[[:space:]]*[0-9]+[[:space:]]/ {
active = 0
}
/10\.240\.10\.0:(53|9153)\// {
active = 1
print
next
}
active && /=>/ {
print
}
'
}
check_port() {
local pod="$1"
local node="$2"
local service_list="$3"
local frontend="$4"
local port_proto="$5"
local state_file="${state_dir}/${pod}-${port_proto//\//-}"
local actual expected unexpected missing issue
actual="$(printf '%s\n' "${service_list}" | extract_backends "${frontend}")"
expected="$(expected_backends "${port_proto}")"
unexpected="$(comm -23 <(printf '%s\n' "${actual}" | sed '/^$/d' | sort -u) <(printf '%s\n' "${expected}" | sed '/^$/d' | sort -u) || true)"
missing="$(comm -13 <(printf '%s\n' "${actual}" | sed '/^$/d' | sort -u) <(printf '%s\n' "${expected}" | sed '/^$/d' | sort -u) || true)"
if [[ -n "${unexpected}" || -n "${missing}" ]]; then
issue="unexpected=${unexpected//$'\n'/,} missing=${missing//$'\n'/,}"
if [[ ! -f "${state_file}" || "$(cat "${state_file}")" != "${issue}" ]]; then
printf '%s' "${issue}" > "${state_file}"
log "CILIUM_SERVICE_MAP_DRIFT pod=${pod} node=${node} frontend=${frontend}"
log "expected_backends=$(printf '%s' "${expected}" | paste -sd ',' -)"
log "actual_backends=$(printf '%s' "${actual}" | paste -sd ',' -)"
log "unexpected_backends=$(printf '%s' "${unexpected}" | paste -sd ',' -)"
log "missing_backends=$(printf '%s' "${missing}" | paste -sd ',' -)"
log "kube_dns_service_lines_begin pod=${pod}"
printf '%s\n' "${service_list}" | relevant_service_lines
log "kube_dns_service_lines_end pod=${pod}"
log "kube_dns_endpointslice_begin"
kubectl -n kube-system get endpointslice -l kubernetes.io/service-name=kube-dns -o wide
log "kube_dns_endpointslice_end"
fi
elif [[ -f "${state_file}" ]]; then
rm -f "${state_file}"
log "CILIUM_SERVICE_MAP_RECOVERED pod=${pod} node=${node} frontend=${frontend}"
fi
}
log "starting cilium service-map monitor interval=${interval}s"
while true; do
kube_dns_ip="$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')"
cilium_pods="$(kubectl -n kube-system get pods -l k8s-app=cilium \
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.nodeName}{"\n"}{end}')"
while read -r pod node; do
[[ -n "${pod:-}" ]] || continue
if service_list="$(kubectl -n kube-system exec "${pod}" -- cilium-dbg service list 2>&1)"; then
check_port "${pod}" "${node}" "${service_list}" "${kube_dns_ip}:53/UDP" "53/UDP"
check_port "${pod}" "${node}" "${service_list}" "${kube_dns_ip}:53/TCP" "53/TCP"
check_port "${pod}" "${node}" "${service_list}" "${kube_dns_ip}:9153/TCP" "9153/TCP"
else
log "CILIUM_SERVICE_MAP_CHECK_FAILED pod=${pod} node=${node} error=$(printf '%s' "${service_list}" | tr '\n' ' ')"
fi
done <<< "${cilium_pods}"
sleep "${interval}"
done
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cilium-service-map-monitor
namespace: kube-system
labels:
app.kubernetes.io/name: cilium-service-map-monitor
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: cilium-service-map-monitor
template:
metadata:
annotations:
ms2k.io/config-version: "2026-05-15-01"
labels:
app.kubernetes.io/name: cilium-service-map-monitor
spec:
serviceAccountName: cilium-service-map-monitor
automountServiceAccountToken: true
containers:
- name: monitor
image: docker.io/alpine/k8s:1.37.1@sha256:712dc5385d1c1a3f83b747fa9e40f64d433a97f72ff42f9755ff7dd1f62846e4
imagePullPolicy: IfNotPresent
command: ["/bin/bash", "/monitor/monitor.sh"]
env:
- name: CHECK_INTERVAL_SECONDS
value: "30"
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: monitor-script
mountPath: /monitor
readOnly: true
- name: tmp
mountPath: /tmp
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
memory: 256Mi
volumes:
- name: monitor-script
configMap:
name: cilium-service-map-monitor
defaultMode: 0755
- name: tmp
emptyDir: {}
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: cilium-service-map-monitor
namespace: kube-system
spec:
endpointSelector:
matchLabels:
app.kubernetes.io/name: cilium-service-map-monitor
ingress: []
egress:
- toEntities:
- kube-apiserver
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP

View file

@ -0,0 +1,26 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns-custom
namespace: kube-system
labels:
app.kubernetes.io/name: coredns
app.kubernetes.io/instance: kube-dns
data:
quorate-dev.server: |
quorate.dev:53 {
view zitadel-relay {
expr metadata('kubernetes/client-label/quorate.dev/zitadel-relay') == 'true'
}
metadata
rewrite stop name exact auth.quorate.dev. zitadel-oidc-internal.zitadel.svc.cluster.local.
kubernetes cluster.local {
pods verified
}
}
quorate.dev:53 {
errors
cache 30
forward . 10.28.147.227
}

View file

@ -0,0 +1,233 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: cluster-dns
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: cluster-dns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: cluster-dns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-dns
subjects:
- kind: ServiceAccount
name: cluster-dns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: dns-config
namespace: kube-system
data:
Corefile: |
.:53 {
errors
health
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
}
hosts /etc/coredns/NodeHosts {
ttl 60
reload 15s
fallthrough
}
prometheus :9153
cache 30
loop
reload
loadbalance
import /etc/coredns/custom/*.override
forward . /etc/resolv.conf
}
import /etc/coredns/custom/*.server
NodeHosts: |
10.210.99.194 ********-n100
10.133.210.151 ********-opi5p
10.149.70.251 ********-2697
10.100.10.13 ********-y520
10.151.204.207 ********-nas
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: CoreDNS
spec:
clusterIP: 10.97.104.110
ports:
- name: dns
port: 53
protocol: UDP
targetPort: 53
- name: dns-tcp
port: 53
protocol: TCP
targetPort: 53
- name: metrics
port: 9153
protocol: TCP
targetPort: 9153
selector:
k8s-app: kube-dns
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cluster-dns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/name: CoreDNS
spec:
replicas: 2
revisionHistoryLimit: 0
selector:
matchLabels:
k8s-app: kube-dns
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
labels:
k8s-app: kube-dns
spec:
containers:
- name: coredns
image: rancher/mirrored-coredns-coredns:1.15.0@sha256:0a9d8c8ddf34347eb59aad9a8784f0710de88d6d112726d11d271f9bc6daca26
imagePullPolicy: IfNotPresent
args:
- -conf
- /etc/coredns/Corefile
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /ready
port: 8181
scheme: HTTP
periodSeconds: 2
timeoutSeconds: 1
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /etc/coredns
name: config-volume
readOnly: true
- mountPath: /etc/coredns/custom
name: custom-config-volume
readOnly: true
dnsPolicy: Default
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: cluster-dns
terminationGracePeriodSeconds: 30
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
topologySpreadConstraints:
- labelSelector:
matchLabels:
k8s-app: kube-dns
maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
- labelSelector:
matchLabels:
k8s-app: kube-dns
maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
volumes:
- configMap:
items:
- key: Corefile
path: Corefile
- key: NodeHosts
path: NodeHosts
name: dns-config
name: config-volume
- configMap:
name: coredns-custom
optional: true
name: custom-config-volume
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: cluster-dns
namespace: kube-system
spec:
minAvailable: 1
selector:
matchLabels:
k8s-app: kube-dns

View file

@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- cilium-service-map-monitor.yaml
- network-policies.yaml
- coredns.yaml
- coredns-custom.yaml

View file

@ -0,0 +1,331 @@
---
# Cilium Gateway: reserved:ingress is the identity Cilium assigns to L7 LB listeners
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: gateway-internal
spec:
endpointSelector:
matchLabels:
"reserved:ingress": ""
ingress:
- fromCIDR:
- 10.115.187.218/24
- 10.63.81.107/24
- 10.162.8.60/24
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "443"
protocol: TCP
- fromEntities:
- host
- remote-node
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "443"
protocol: TCP
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# Relay gateway: allow public traffic to the shared reserved:ingress identity.
# Cilium uses the same ingress identity for Gateway API listeners, so this
# must be additive with the internal gateway policy above. Local testing and
# internal clients can also reach the relay listener from trusted subnets or host.
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: gateway-relay
spec:
endpointSelector:
matchLabels:
"reserved:ingress": ""
ingress:
- fromCIDR:
- 10.115.187.218/24
- 10.63.81.107/24
- 10.162.8.60/24
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "443"
protocol: TCP
- fromEntities:
- host
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "443"
protocol: TCP
- fromEntities:
- ingress
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "443"
protocol: TCP
- fromEntities:
- world
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "443"
protocol: TCP
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# CoreDNS: accept DNS queries from all pods cluster-wide,
# and reach out to the apiserver for zone sync.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: coredns
namespace: kube-system
spec:
endpointSelector:
matchLabels:
k8s-app: kube-dns
ingress:
- fromEntities:
- cluster
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
- fromEntities:
- host
toPorts:
- ports:
- port: "8080"
protocol: TCP
- port: "8181"
protocol: TCP
egress:
- toEntities:
- kube-apiserver
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: network-dns
app.kubernetes.io/name: coredns
app.kubernetes.io/instance: network-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
- toEntities:
- world
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# metrics-server: scraped by kubelet via host on 10250.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: metrics-server
namespace: kube-system
spec:
endpointSelector:
matchLabels:
k8s-app: metrics-server
ingress:
- fromEntities:
- host
- remote-node
- kube-apiserver
toPorts:
- ports:
- port: "10250"
protocol: TCP
egress:
- toEntities:
- kube-apiserver
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEntities:
- host
- remote-node
- kube-apiserver
toPorts:
- ports:
- port: "10250"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
---
# local-path-provisioner: only needs the apiserver.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: local-path-provisioner
namespace: kube-system
spec:
endpointSelector:
matchLabels:
app: local-path-provisioner
ingress: []
egress:
- toEntities:
- kube-apiserver
- host
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
---
# Hubble relay: reached by the Hubble CLI/observer on 4222 from the host.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: hubble-relay
namespace: kube-system
spec:
endpointSelector:
matchLabels:
app.kubernetes.io/name: hubble-relay
ingress:
- fromEntities:
- host
toPorts:
- ports:
- port: "4222"
protocol: TCP
- fromEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s:app.kubernetes.io/name: hubble-ui
toPorts:
- ports:
- port: "4245"
protocol: TCP
egress:
- toEntities:
- host
- remote-node
toPorts:
- ports:
- port: "4244"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
---
# Hubble UI backend: talks to hubble-relay.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: hubble-ui
namespace: kube-system
spec:
endpointSelector:
matchLabels:
app.kubernetes.io/name: hubble-ui
ingress:
- fromEntities:
- host
toPorts:
- ports:
- port: "8081"
protocol: TCP
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
app.kubernetes.io/name: hubble-relay
toPorts:
- ports:
- port: "4245"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
---
# Host egress to kube-system pods (DNS, metrics, hubble relay).
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: host-egress-kube-system
spec:
nodeSelector: {}
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
- port: "10250"
protocol: TCP
- port: "8181"
protocol: TCP
- port: "8081"
protocol: TCP
- port: "4222"
protocol: TCP
- port: "8080"
protocol: TCP