ce7cb702218693d08ea6d83464d2457c9250027c

This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
7 changed files with 145336 additions and 0 deletions

View file

@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- storageclass.yaml
- nodes.yaml

View file

@ -0,0 +1,106 @@
apiVersion: longhorn.io/v1beta2
kind: Node
metadata:
name: "********-nas"
namespace: longhorn
spec:
allowScheduling: true
evictionRequested: false
instanceManagerCPURequest: 0
name: "********-nas"
tags: []
disks:
default-disk-********:
allowScheduling: true
diskDriver: ""
diskType: filesystem
evictionRequested: false
path: /var/lib/longhorn/
storageReserved: 294387022233
tags:
- ssd
---
apiVersion: longhorn.io/v1beta2
kind: Node
metadata:
name: "********-opi5p"
namespace: longhorn
spec:
allowScheduling: true
evictionRequested: false
instanceManagerCPURequest: 0
name: "********-opi5p"
tags: []
disks:
default-disk-********:
allowScheduling: true
diskDriver: ""
diskType: filesystem
evictionRequested: false
path: /var/lib/longhorn/
storageReserved: 150871284940
tags:
- ssd
---
apiVersion: longhorn.io/v1beta2
kind: Node
metadata:
name: "********-2697"
namespace: longhorn
spec:
allowScheduling: true
evictionRequested: false
instanceManagerCPURequest: 0
name: "********-2697"
tags: []
disks:
default-disk-********:
allowScheduling: true
diskDriver: ""
diskType: filesystem
evictionRequested: false
path: /var/lib/longhorn/
storageReserved: 150265206374
tags:
- ssd
data-hdd:
allowScheduling: true
diskDriver: ""
diskType: filesystem
evictionRequested: false
path: /var/lib/data-hdd/longhorn/
storageReserved: 15994994073
tags:
- hdd
---
apiVersion: longhorn.io/v1beta2
kind: Node
metadata:
name: "********-y520"
namespace: longhorn
spec:
allowScheduling: true
evictionRequested: false
instanceManagerCPURequest: 0
name: "********-y520"
tags: []
disks:
# Root SSD is too small for general scheduling; never tag as ssd.
default-disk-********:
allowScheduling: false
diskDriver: ""
diskType: filesystem
evictionRequested: false
path: /var/lib/longhorn/
storageReserved: 37016972083
tags:
- y520-ssd
data-hdd:
allowScheduling: true
diskDriver: ""
diskType: filesystem
evictionRequested: false
path: /var/lib/data-hdd/longhorn/
storageReserved: 49984986931
tags:
- hdd

View file

@ -0,0 +1,74 @@
---
# For workloads with application-level replication.
# Longhorn replication is redundant for these -- replica=1 avoids double-writing.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-single
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
parameters:
numberOfReplicas: "1"
staleReplicaTimeout: "30"
fsType: "xfs"
dataLocality: "strict-local"
---
# For workloads that do not replicate data at the application layer.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-double
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
parameters:
numberOfReplicas: "2"
staleReplicaTimeout: "30"
fsType: "xfs"
dataLocality: "best-effort"
---
# For shared filesystem workloads that need true cross-node RWX.
# The default longhorn class is migratable and creates a blockdev volume, not
# a share-manager backed RWX filesystem.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-rwx
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
parameters:
numberOfReplicas: "3"
staleReplicaTimeout: "30"
fsType: "xfs"
dataLocality: "best-effort"
diskSelector: "ssd"
migratable: "false"
---
# Block-replicated volumes on hdd-tagged disks.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-hdd-double
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
parameters:
numberOfReplicas: "2"
staleReplicaTimeout: "30"
fsType: "xfs"
dataLocality: "best-effort"
diskSelector: "hdd"

View file

@ -0,0 +1,58 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: longhorn
namespace: longhorn
spec:
interval: 15m
releaseName: longhorn
targetNamespace: longhorn
chart:
spec:
chart: longhorn
version: 1.13.0
sourceRef:
kind: HelmRepository
name: longhorn
namespace: flux-system
install:
createNamespace: true
remediation:
retries: 3
upgrade:
remediation:
retries: 3
postRenderers:
- kustomize:
patches:
- target:
kind: Deployment
name: longhorn-driver-deployer
patch: |
- op: replace
path: /spec/replicas
value: 2
values:
image:
csi:
snapshotter:
tag: v8.7.0@sha256:540c1e04e5e0206015f0552fc86efb1f9e292b1f9892b8439a6f091255857148
defaultSettings:
guaranteedInstanceManagerCPU: 1
upgradeChecker: false
allowCollectingLonghornUsageMetrics: false
persistence:
reclaimPolicy: Retain
defaultFsType: xfs
defaultDataLocality: best-effort
volumeBindingMode: WaitForFirstConsumer
migratable: true
defaultDiskSelector:
enable: true
selector: ssd
longhornUI:
replicas: 2
tolerations:
- key: workload
value: arm
effect: PreferNoSchedule

View file

@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- helm-release.yaml
- network-policies.yaml

View file

@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
gateway.quorate.internal/allow-internal: "true"
name: longhorn

View file

@ -0,0 +1,405 @@
---
# longhorn-manager: intra-ns communication, apiserver, metrics from host.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-manager
namespace: longhorn
spec:
endpointSelector:
matchLabels:
app: longhorn-manager
ingress:
- fromEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
toPorts:
- ports:
- port: "9500"
protocol: TCP
- port: "9502"
protocol: TCP
- port: "9503"
protocol: TCP
- port: "8501"
protocol: TCP
- port: "8503"
protocol: TCP
- port: "8500"
protocol: TCP
- fromEntities:
- host
- remote-node
toPorts:
- ports:
- port: "9502"
protocol: TCP
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
toPorts:
- ports:
- port: "9500"
protocol: TCP
- port: "9502"
protocol: TCP
- port: "8501"
protocol: TCP
- port: "8503"
protocol: TCP
- port: "8500"
protocol: TCP
- port: "9600"
protocol: TCP
- toEntities:
- kube-apiserver
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toFQDNs:
- matchName: longhorn-upgrade-responder.rancher.io
toPorts:
- ports:
- port: "443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-manager-ingress-vmagent
namespace: longhorn
spec:
endpointSelector:
matchLabels:
app: longhorn-manager
ingress:
- fromEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: victoriametrics
app.kubernetes.io/name: vmagent
app.kubernetes.io/instance: main
toPorts:
- ports:
- port: "9500"
protocol: TCP
---
# longhorn-driver-deployer: installs CSI driver, needs manager API, apiserver + DNS.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-driver-deployer
namespace: longhorn
spec:
endpointSelector:
matchLabels:
app: longhorn-driver-deployer
ingress: []
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
app: longhorn-manager
toPorts:
- ports:
- port: "9500"
protocol: TCP
- toEntities:
- kube-apiserver
- remote-node
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# CSI sidecar pods (attacher, provisioner, resizer): only need apiserver + DNS.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-csi-sidecars
namespace: longhorn
spec:
endpointSelector:
matchExpressions:
- key: app
operator: In
values:
- csi-attacher
- csi-provisioner
- csi-resizer
- csi-snapshotter
ingress: []
egress:
- toEntities:
- kube-apiserver
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# longhorn-csi-plugin: metrics from host, talks to longhorn-manager.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-csi-plugin
namespace: longhorn
spec:
endpointSelector:
matchLabels:
app: longhorn-csi-plugin
ingress:
- fromEntities:
- host
toPorts:
- ports:
- port: "9808"
protocol: TCP
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
app: longhorn-manager
toPorts:
- ports:
- port: "9500"
protocol: TCP
- toEntities:
- kube-apiserver
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# Host egress to longhorn pods (iSCSI, CSI plugin metrics, manager metrics).
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: host-egress-longhorn
spec:
nodeSelector: {}
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
toPorts:
- ports:
- port: "3260"
protocol: TCP
- port: "9808"
protocol: TCP
- port: "9502"
protocol: TCP
- port: "2049"
protocol: TCP
- port: "9505"
protocol: TCP
---
# share-manager: longhorn-manager gRPC health/control and host NFS clients.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-share-manager
namespace: longhorn
spec:
endpointSelector:
matchLabels:
longhorn.io/component: share-manager
ingress:
- fromEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
app: longhorn-manager
toPorts:
- ports:
- port: "9600"
protocol: TCP
- fromEntities:
- host
- remote-node
toPorts:
- ports:
- port: "2049"
protocol: TCP
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
app: longhorn-manager
toPorts:
- ports:
- port: "9503"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# longhorn-ui: frontend that talks to longhorn-manager API.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-ui
namespace: longhorn
spec:
endpointSelector:
matchLabels:
app: longhorn-ui
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
app: longhorn-manager
toPorts:
- ports:
- port: "9500"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
# instance-managers: iSCSI from host, intra-ns engine/replica RPC (10000-20000),
# longhorn-manager control plane (8501, 8503), DNS.
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-instance-manager
namespace: longhorn
spec:
endpointSelector:
matchLabels:
longhorn.io/component: instance-manager
ingress:
- fromEntities:
- host
- remote-node
toPorts:
- ports:
- port: "3260"
protocol: TCP
- fromEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
toPorts:
- ports:
- port: "8500"
protocol: TCP
- port: "8501"
protocol: TCP
- port: "8503"
protocol: TCP
- port: "10000"
endPort: 20000
protocol: TCP
egress:
- toEntities:
- host
- remote-node
toPorts:
- ports:
- port: "3260"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: longhorn
toPorts:
- ports:
- port: "10000"
endPort: 20000
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-global
namespace: longhorn
spec:
endpointSelector:
matchLabels:
app: longhorn-global-manager
ingress:
- fromEntities: [host, remote-node]
toPorts:
- ports:
- port: "9505"
protocol: TCP
egress:
- toEntities: [kube-apiserver]
toPorts:
- ports:
- port: "6443"
protocol: TCP
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP