ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
5 changed files with 145336 additions and 0 deletions
7
scripts/k3s/config.agent-nas.yaml
Normal file
7
scripts/k3s/config.agent-nas.yaml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
server: https://k3s.ms2k.internal:6443
|
||||
|
||||
kubelet-arg:
|
||||
- max-pods=250
|
||||
|
||||
node-taint:
|
||||
- workload=nas:PreferNoSchedule
|
||||
23
scripts/k3s/config.server.yaml
Normal file
23
scripts/k3s/config.server.yaml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
server: https://********
|
||||
|
||||
flannel-backend: none
|
||||
disable-network-policy: true
|
||||
disable-kube-proxy: true
|
||||
|
||||
tls-san:
|
||||
- "********"
|
||||
- 10.72.123.235
|
||||
|
||||
cluster-cidr: 172.19.28.14/12
|
||||
service-cidr: 10.74.150.93/12
|
||||
cluster-dns: 10.97.104.110
|
||||
|
||||
secrets-encryption: true
|
||||
|
||||
kubelet-arg:
|
||||
- max-pods=250
|
||||
|
||||
disable:
|
||||
- coredns
|
||||
- traefik
|
||||
- servicelb
|
||||
74
scripts/k3s/install.sh
Executable file
74
scripts/k3s/install.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env bash
|
||||
# Usage: install.sh server|agent <ssh-target>
|
||||
set -euo pipefail
|
||||
set +o xtrace 2>/dev/null || true
|
||||
|
||||
VAULT_PATH="${K3S_VAULT_PATH:-secret/k3s}"
|
||||
VAULT_FIELD="${K3S_VAULT_FIELD:-server-token}"
|
||||
INSTALL_K3S_VERSION="${INSTALL_K3S_VERSION:-v1.36.2+k3s1}"
|
||||
REMOTE_CONFIG=/etc/rancher/k3s/config.yaml
|
||||
|
||||
die() { echo "error: $*" >&2; exit 1; }
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
Usage: install.sh server|agent <ssh-target>
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
scrub() {
|
||||
token=''
|
||||
unset token 2>/dev/null || true
|
||||
}
|
||||
|
||||
trap scrub EXIT
|
||||
|
||||
[[ $# -eq 2 ]] || usage
|
||||
role=$1
|
||||
target=$2
|
||||
|
||||
case "$role" in
|
||||
server|agent) ;;
|
||||
*) die "role must be server or agent (got: $role)" ;;
|
||||
esac
|
||||
[[ -n "$target" ]] || die "ssh-target is empty"
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
case "$role" in
|
||||
server) config=$script_dir/config.server.yaml ;;
|
||||
agent) config=$script_dir/config.agent-nas.yaml ;;
|
||||
esac
|
||||
[[ -f "$config" ]] || die "missing config: $config"
|
||||
|
||||
if grep -qE '^[[:space:]]*token:[[:space:]]' "$config"; then
|
||||
die "config must not contain token: ($config)"
|
||||
fi
|
||||
|
||||
command -v vault >/dev/null || die "vault CLI not found"
|
||||
command -v ssh >/dev/null || die "ssh not found"
|
||||
command -v scp >/dev/null || die "scp not found"
|
||||
|
||||
token=$(vault kv get -field="$VAULT_FIELD" "$VAULT_PATH") \
|
||||
|| die "failed to read $VAULT_FIELD from $VAULT_PATH"
|
||||
[[ -n "$token" ]] || die "empty token from $VAULT_PATH field $VAULT_FIELD"
|
||||
|
||||
echo "role=$role target=$target config=$config version=$INSTALL_K3S_VERSION"
|
||||
|
||||
ssh -o BatchMode=yes "$target" "install -d -m 0755 /etc/rancher/k3s"
|
||||
scp -o BatchMode=yes -q "$config" "${target}:/tmp/k3s-config.yaml"
|
||||
ssh -o BatchMode=yes "$target" "install -m 0600 /tmp/k3s-config.yaml $REMOTE_CONFIG && rm -f /tmp/k3s-config.yaml"
|
||||
|
||||
# K3S_TOKEN must not appear in ssh/remote argv (visible in ps).
|
||||
ssh -o BatchMode=yes "$target" 'bash -s' <<EOF
|
||||
set -euo pipefail
|
||||
set +o xtrace 2>/dev/null || true
|
||||
export INSTALL_K3S_VERSION=$(printf '%q' "$INSTALL_K3S_VERSION")
|
||||
export K3S_TOKEN=$(printf '%q' "$token")
|
||||
curl -sfL https://get.k3s.io | sh -s - $(printf '%q' "$role")
|
||||
unset K3S_TOKEN
|
||||
EOF
|
||||
|
||||
scrub
|
||||
echo "install finished on $target ($role)"
|
||||
234
scripts/k3s/prepare-node.sh
Executable file
234
scripts/k3s/prepare-node.sh
Executable file
|
|
@ -0,0 +1,234 @@
|
|||
#!/usr/bin/env bash
|
||||
# Usage: prepare-node.sh [--defer-restart] <ssh-target>
|
||||
# Host packages, Longhorn dir, runsc, internal CA + registries.yaml. Idempotent; before install.sh.
|
||||
set -euo pipefail
|
||||
|
||||
CA_URL="${K3S_CA_URL:-https://vault.quorate.internal/v1/pki/ca/pem}"
|
||||
CA_REMOTE=/etc/rancher/k3s/quorate-internal-ca.crt
|
||||
REG_REMOTE=/etc/rancher/k3s/registries.yaml
|
||||
|
||||
die() { echo "error: $*" >&2; exit 1; }
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
Usage: prepare-node.sh [--defer-restart] <ssh-target>
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
defer_restart=0
|
||||
if [[ ${1:-} == --defer-restart ]]; then
|
||||
defer_restart=1
|
||||
shift
|
||||
fi
|
||||
|
||||
[[ $# -eq 1 ]] || usage
|
||||
target=$1
|
||||
[[ -n "$target" ]] || die "ssh-target is empty"
|
||||
|
||||
command -v ssh >/dev/null || die "ssh not found"
|
||||
command -v scp >/dev/null || die "scp not found"
|
||||
command -v curl >/dev/null || die "curl not found"
|
||||
command -v openssl >/dev/null || die "openssl not found"
|
||||
|
||||
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
registries_src=$script_dir/registries.yaml
|
||||
[[ -f $registries_src ]] || die "missing $registries_src"
|
||||
|
||||
workdir=$(mktemp -d)
|
||||
trap 'rm -rf "$workdir"' EXIT
|
||||
ca_local=$workdir/quorate-internal-ca.crt
|
||||
|
||||
curl -fsSL --max-time 30 "$CA_URL" -o "$ca_local" \
|
||||
|| die "failed to fetch CA from $CA_URL"
|
||||
openssl x509 -in "$ca_local" -noout 2>/dev/null \
|
||||
|| die "fetched CA is not a valid PEM certificate"
|
||||
openssl x509 -in "$ca_local" -noout -text | grep -q 'CA:TRUE' \
|
||||
|| die "fetched certificate is not a CA"
|
||||
|
||||
echo "prepare-node target=$target"
|
||||
|
||||
ssh -o BatchMode=yes "$target" "install -d -m 0755 /etc/rancher/k3s"
|
||||
scp -o BatchMode=yes -q "$ca_local" "${target}:/tmp/quorate-internal-ca.crt"
|
||||
scp -o BatchMode=yes -q "$registries_src" "${target}:/tmp/k3s-registries.yaml"
|
||||
|
||||
ssh -o BatchMode=yes "$target" "bash -s -- $defer_restart" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
defer_restart=$1
|
||||
|
||||
if ! command -v apt-get >/dev/null; then
|
||||
echo "error: only Debian-family apt hosts are supported" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pkg_installed() {
|
||||
dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q 'install ok installed'
|
||||
}
|
||||
|
||||
need_k3s_restart=0
|
||||
|
||||
pkgs=(open-iscsi nfs-common xfsprogs smartmontools)
|
||||
missing=()
|
||||
for p in "${pkgs[@]}"; do
|
||||
pkg_installed "$p" || missing+=("$p")
|
||||
done
|
||||
if ((${#missing[@]})); then
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq "${missing[@]}"
|
||||
fi
|
||||
|
||||
if systemctl cat iscsid.service >/dev/null 2>&1; then
|
||||
systemctl is-enabled --quiet iscsid 2>/dev/null || systemctl enable iscsid >/dev/null
|
||||
systemctl is-active --quiet iscsid 2>/dev/null || systemctl start iscsid
|
||||
fi
|
||||
|
||||
install -d -m 0755 /var/lib/longhorn
|
||||
|
||||
inotify_file=/etc/sysctl.d/90-k3s-inotify.conf
|
||||
inotify_limit=1024
|
||||
inotify_setting="fs.inotify.max_user_instances = $inotify_limit"
|
||||
install -d -m 0755 /etc/sysctl.d
|
||||
if [[ ! -f $inotify_file ]] || ! cmp -s <(printf '%s\n' "$inotify_setting") "$inotify_file"; then
|
||||
printf '%s\n' "$inotify_setting" >"$inotify_file"
|
||||
fi
|
||||
chmod 0644 "$inotify_file"
|
||||
if [[ $(sysctl -n fs.inotify.max_user_instances) != "$inotify_limit" ]]; then
|
||||
sysctl -p "$inotify_file"
|
||||
fi
|
||||
|
||||
CA_REMOTE=/etc/rancher/k3s/quorate-internal-ca.crt
|
||||
REG_REMOTE=/etc/rancher/k3s/registries.yaml
|
||||
SYS_CA=/usr/local/share/ca-certificates/internal-root-ca.crt
|
||||
|
||||
install -d -m 0755 /etc/rancher/k3s
|
||||
if [[ -f $CA_REMOTE ]] && cmp -s /tmp/quorate-internal-ca.crt "$CA_REMOTE"; then
|
||||
:
|
||||
else
|
||||
install -m 0644 /tmp/quorate-internal-ca.crt "$CA_REMOTE"
|
||||
need_k3s_restart=1
|
||||
fi
|
||||
|
||||
if [[ -f $REG_REMOTE ]] && cmp -s /tmp/k3s-registries.yaml "$REG_REMOTE"; then
|
||||
:
|
||||
else
|
||||
install -m 0644 /tmp/k3s-registries.yaml "$REG_REMOTE"
|
||||
need_k3s_restart=1
|
||||
fi
|
||||
|
||||
if [[ -f $SYS_CA ]] && cmp -s /tmp/quorate-internal-ca.crt "$SYS_CA"; then
|
||||
:
|
||||
else
|
||||
install -d -m 0755 /usr/local/share/ca-certificates
|
||||
install -m 0644 /tmp/quorate-internal-ca.crt "$SYS_CA"
|
||||
update-ca-certificates >/dev/null
|
||||
fi
|
||||
rm -f /tmp/quorate-internal-ca.crt /tmp/k3s-registries.yaml
|
||||
|
||||
if ! command -v runsc >/dev/null 2>&1 || ! command -v containerd-shim-runsc-v1 >/dev/null 2>&1; then
|
||||
for p in ca-certificates curl gnupg; do
|
||||
pkg_installed "$p" || { apt-get update -qq; apt-get install -y -qq ca-certificates curl gnupg; break; }
|
||||
done
|
||||
|
||||
if [[ ! -f /usr/share/keyrings/gvisor-archive-keyring.gpg ]]; then
|
||||
curl -fsSL https://gvisor.dev/archive.key | gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg
|
||||
fi
|
||||
|
||||
arch=$(dpkg --print-architecture)
|
||||
gvisor_list=/etc/apt/sources.list.d/gvisor.list
|
||||
gvisor_line="deb [arch=${arch} signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main"
|
||||
if [[ ! -f $gvisor_list ]] || ! grep -qxF "$gvisor_line" "$gvisor_list"; then
|
||||
printf '%s\n' "$gvisor_line" >"$gvisor_list"
|
||||
fi
|
||||
|
||||
if ! { apt-get update -qq && apt-get install -y -qq runsc \
|
||||
&& command -v runsc >/dev/null && command -v containerd-shim-runsc-v1 >/dev/null; }; then
|
||||
ARCH=$(uname -m)
|
||||
URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}
|
||||
tmp=$(mktemp -d)
|
||||
curl -fsSL -o "$tmp/gvisor.tar.bz2" "${URL}/gvisor.tar.bz2"
|
||||
curl -fsSL -o "$tmp/gvisor.tar.bz2.sha512" "${URL}/gvisor.tar.bz2.sha512"
|
||||
(cd "$tmp" && sha512sum -c gvisor.tar.bz2.sha512)
|
||||
tar -xjf "$tmp/gvisor.tar.bz2" -C /usr/local/bin
|
||||
rm -rf "$tmp"
|
||||
fi
|
||||
need_k3s_restart=1
|
||||
fi
|
||||
|
||||
command -v runsc >/dev/null || { echo "error: runsc not on PATH" >&2; exit 1; }
|
||||
command -v containerd-shim-runsc-v1 >/dev/null || {
|
||||
echo "error: containerd-shim-runsc-v1 not on PATH" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ctd_dir=/var/lib/rancher/k3s/agent/etc/containerd
|
||||
tmpl=$ctd_dir/config-v3.toml.tmpl
|
||||
read -r -d '' desired_tmpl <<'TMPL' || true
|
||||
{{ template "base" . }}
|
||||
|
||||
[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runsc]
|
||||
runtime_type = "io.containerd.runsc.v1"
|
||||
pod_annotations = ["dev.gvisor.spec.mount.var-run-argo.share", "dev.gvisor.spec.mount.var-run-argo.type", "dev.gvisor.spec.mount.var-run-argo.options"]
|
||||
|
||||
[plugins.'io.containerd.nri.v1.nri']
|
||||
disable = false
|
||||
disable_connections = false
|
||||
plugin_config_path = "/etc/nri/conf.d"
|
||||
plugin_path = "/opt/nri/plugins"
|
||||
plugin_registration_timeout = "5s"
|
||||
plugin_request_timeout = "2s"
|
||||
socket_path = "/var/run/nri/nri.sock"
|
||||
TMPL
|
||||
|
||||
mkdir -p "$ctd_dir"
|
||||
if [[ -f $tmpl ]] && cmp -s <(printf '%s\n' "$desired_tmpl") "$tmpl"; then
|
||||
:
|
||||
else
|
||||
printf '%s\n' "$desired_tmpl" >"$tmpl"
|
||||
chmod 644 "$tmpl"
|
||||
need_k3s_restart=1
|
||||
fi
|
||||
|
||||
if [[ -f $ctd_dir/config.toml ]]; then
|
||||
grep -q 'runtimes.runsc' "$ctd_dir/config.toml" || need_k3s_restart=1
|
||||
grep -q 'io.containerd.nri.v1.nri' "$ctd_dir/config.toml" || need_k3s_restart=1
|
||||
fi
|
||||
|
||||
k3s_unit=
|
||||
if systemctl is-active --quiet k3s 2>/dev/null || systemctl is-enabled --quiet k3s 2>/dev/null; then
|
||||
k3s_unit=k3s
|
||||
elif systemctl is-active --quiet k3s-agent 2>/dev/null || systemctl is-enabled --quiet k3s-agent 2>/dev/null; then
|
||||
k3s_unit=k3s-agent
|
||||
fi
|
||||
|
||||
if [[ $need_k3s_restart -eq 1 ]]; then
|
||||
if [[ $defer_restart -eq 1 ]]; then
|
||||
echo "k3s restart: deferred"
|
||||
elif [[ -n $k3s_unit ]]; then
|
||||
systemctl restart "$k3s_unit"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $defer_restart -eq 0 && -n $k3s_unit ]]; then
|
||||
grep -q 'io.containerd.nri.v1.nri' "$ctd_dir/config.toml"
|
||||
grep -q 'socket_path = "/var/run/nri/nri.sock"' "$ctd_dir/config.toml"
|
||||
for _ in {1..30}; do
|
||||
[[ -S /var/run/nri/nri.sock ]] && break
|
||||
sleep 1
|
||||
done
|
||||
[[ -S /var/run/nri/nri.sock ]] || {
|
||||
echo "error: NRI socket not available after $k3s_unit restart" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
echo "packages: ${pkgs[*]}"
|
||||
echo "longhorn: /var/lib/longhorn"
|
||||
echo "runsc: $(runsc --version 2>&1 | head -1)"
|
||||
echo "registry CA: $CA_REMOTE"
|
||||
echo "registries: $REG_REMOTE"
|
||||
echo "system CA: $SYS_CA"
|
||||
echo "prepare-node finished on $(hostname)"
|
||||
REMOTE
|
||||
|
||||
echo "prepare-node finished on $target"
|
||||
14
scripts/k3s/registries.yaml
Normal file
14
scripts/k3s/registries.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
mirrors:
|
||||
zot.quorate.internal:
|
||||
endpoint:
|
||||
- "https://zot.quorate.internal"
|
||||
registry.quorate.internal:
|
||||
endpoint:
|
||||
- "https://registry.quorate.internal"
|
||||
configs:
|
||||
zot.quorate.internal:
|
||||
tls:
|
||||
ca_file: /etc/rancher/k3s/quorate-internal-ca.crt
|
||||
registry.quorate.internal:
|
||||
tls:
|
||||
ca_file: /etc/rancher/k3s/quorate-internal-ca.crt
|
||||
Loading…
Reference in a new issue