#!/usr/bin/env bash # Usage: prepare-node.sh [--defer-restart] # Host packages, Longhorn dir, runsc, internal CA + registries.yaml. Idempotent; before install.sh. set -euo pipefail CA_URL="${K3S_CA_URL:-https://vault.quorate.internal/v1/pki/ca/pem}" CA_REMOTE=/etc/rancher/k3s/quorate-internal-ca.crt REG_REMOTE=/etc/rancher/k3s/registries.yaml die() { echo "error: $*" >&2; exit 1; } usage() { cat >&2 <<'EOF' Usage: prepare-node.sh [--defer-restart] EOF exit 2 } defer_restart=0 if [[ ${1:-} == --defer-restart ]]; then defer_restart=1 shift fi [[ $# -eq 1 ]] || usage target=$1 [[ -n "$target" ]] || die "ssh-target is empty" command -v ssh >/dev/null || die "ssh not found" command -v scp >/dev/null || die "scp not found" command -v curl >/dev/null || die "curl not found" command -v openssl >/dev/null || die "openssl not found" script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) registries_src=$script_dir/registries.yaml [[ -f $registries_src ]] || die "missing $registries_src" workdir=$(mktemp -d) trap 'rm -rf "$workdir"' EXIT ca_local=$workdir/quorate-internal-ca.crt curl -fsSL --max-time 30 "$CA_URL" -o "$ca_local" \ || die "failed to fetch CA from $CA_URL" openssl x509 -in "$ca_local" -noout 2>/dev/null \ || die "fetched CA is not a valid PEM certificate" openssl x509 -in "$ca_local" -noout -text | grep -q 'CA:TRUE' \ || die "fetched certificate is not a CA" echo "prepare-node target=$target" ssh -o BatchMode=yes "$target" "install -d -m 0755 /etc/rancher/k3s" scp -o BatchMode=yes -q "$ca_local" "${target}:/tmp/quorate-internal-ca.crt" scp -o BatchMode=yes -q "$registries_src" "${target}:/tmp/k3s-registries.yaml" ssh -o BatchMode=yes "$target" "bash -s -- $defer_restart" <<'REMOTE' set -euo pipefail export DEBIAN_FRONTEND=noninteractive defer_restart=$1 if ! command -v apt-get >/dev/null; then echo "error: only Debian-family apt hosts are supported" >&2 exit 1 fi pkg_installed() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q 'install ok installed' } need_k3s_restart=0 pkgs=(open-iscsi nfs-common xfsprogs smartmontools) missing=() for p in "${pkgs[@]}"; do pkg_installed "$p" || missing+=("$p") done if ((${#missing[@]})); then apt-get update -qq apt-get install -y -qq "${missing[@]}" fi if systemctl cat iscsid.service >/dev/null 2>&1; then systemctl is-enabled --quiet iscsid 2>/dev/null || systemctl enable iscsid >/dev/null systemctl is-active --quiet iscsid 2>/dev/null || systemctl start iscsid fi install -d -m 0755 /var/lib/longhorn inotify_file=/etc/sysctl.d/90-k3s-inotify.conf inotify_limit=1024 inotify_setting="fs.inotify.max_user_instances = $inotify_limit" install -d -m 0755 /etc/sysctl.d if [[ ! -f $inotify_file ]] || ! cmp -s <(printf '%s\n' "$inotify_setting") "$inotify_file"; then printf '%s\n' "$inotify_setting" >"$inotify_file" fi chmod 0644 "$inotify_file" if [[ $(sysctl -n fs.inotify.max_user_instances) != "$inotify_limit" ]]; then sysctl -p "$inotify_file" fi CA_REMOTE=/etc/rancher/k3s/quorate-internal-ca.crt REG_REMOTE=/etc/rancher/k3s/registries.yaml SYS_CA=/usr/local/share/ca-certificates/internal-root-ca.crt install -d -m 0755 /etc/rancher/k3s if [[ -f $CA_REMOTE ]] && cmp -s /tmp/quorate-internal-ca.crt "$CA_REMOTE"; then : else install -m 0644 /tmp/quorate-internal-ca.crt "$CA_REMOTE" need_k3s_restart=1 fi if [[ -f $REG_REMOTE ]] && cmp -s /tmp/k3s-registries.yaml "$REG_REMOTE"; then : else install -m 0644 /tmp/k3s-registries.yaml "$REG_REMOTE" need_k3s_restart=1 fi if [[ -f $SYS_CA ]] && cmp -s /tmp/quorate-internal-ca.crt "$SYS_CA"; then : else install -d -m 0755 /usr/local/share/ca-certificates install -m 0644 /tmp/quorate-internal-ca.crt "$SYS_CA" update-ca-certificates >/dev/null fi rm -f /tmp/quorate-internal-ca.crt /tmp/k3s-registries.yaml if ! command -v runsc >/dev/null 2>&1 || ! command -v containerd-shim-runsc-v1 >/dev/null 2>&1; then for p in ca-certificates curl gnupg; do pkg_installed "$p" || { apt-get update -qq; apt-get install -y -qq ca-certificates curl gnupg; break; } done if [[ ! -f /usr/share/keyrings/gvisor-archive-keyring.gpg ]]; then curl -fsSL https://gvisor.dev/archive.key | gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg fi arch=$(dpkg --print-architecture) gvisor_list=/etc/apt/sources.list.d/gvisor.list gvisor_line="deb [arch=${arch} signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" if [[ ! -f $gvisor_list ]] || ! grep -qxF "$gvisor_line" "$gvisor_list"; then printf '%s\n' "$gvisor_line" >"$gvisor_list" fi if ! { apt-get update -qq && apt-get install -y -qq runsc \ && command -v runsc >/dev/null && command -v containerd-shim-runsc-v1 >/dev/null; }; then ARCH=$(uname -m) URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH} tmp=$(mktemp -d) curl -fsSL -o "$tmp/gvisor.tar.bz2" "${URL}/gvisor.tar.bz2" curl -fsSL -o "$tmp/gvisor.tar.bz2.sha512" "${URL}/gvisor.tar.bz2.sha512" (cd "$tmp" && sha512sum -c gvisor.tar.bz2.sha512) tar -xjf "$tmp/gvisor.tar.bz2" -C /usr/local/bin rm -rf "$tmp" fi need_k3s_restart=1 fi command -v runsc >/dev/null || { echo "error: runsc not on PATH" >&2; exit 1; } command -v containerd-shim-runsc-v1 >/dev/null || { echo "error: containerd-shim-runsc-v1 not on PATH" >&2 exit 1 } ctd_dir=/var/lib/rancher/k3s/agent/etc/containerd tmpl=$ctd_dir/config-v3.toml.tmpl read -r -d '' desired_tmpl <<'TMPL' || true {{ template "base" . }} [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runsc] runtime_type = "io.containerd.runsc.v1" pod_annotations = ["dev.gvisor.spec.mount.var-run-argo.share", "dev.gvisor.spec.mount.var-run-argo.type", "dev.gvisor.spec.mount.var-run-argo.options"] [plugins.'io.containerd.nri.v1.nri'] disable = false disable_connections = false plugin_config_path = "/etc/nri/conf.d" plugin_path = "/opt/nri/plugins" plugin_registration_timeout = "5s" plugin_request_timeout = "2s" socket_path = "/var/run/nri/nri.sock" TMPL mkdir -p "$ctd_dir" if [[ -f $tmpl ]] && cmp -s <(printf '%s\n' "$desired_tmpl") "$tmpl"; then : else printf '%s\n' "$desired_tmpl" >"$tmpl" chmod 644 "$tmpl" need_k3s_restart=1 fi if [[ -f $ctd_dir/config.toml ]]; then grep -q 'runtimes.runsc' "$ctd_dir/config.toml" || need_k3s_restart=1 grep -q 'io.containerd.nri.v1.nri' "$ctd_dir/config.toml" || need_k3s_restart=1 fi k3s_unit= if systemctl is-active --quiet k3s 2>/dev/null || systemctl is-enabled --quiet k3s 2>/dev/null; then k3s_unit=k3s elif systemctl is-active --quiet k3s-agent 2>/dev/null || systemctl is-enabled --quiet k3s-agent 2>/dev/null; then k3s_unit=k3s-agent fi if [[ $need_k3s_restart -eq 1 ]]; then if [[ $defer_restart -eq 1 ]]; then echo "k3s restart: deferred" elif [[ -n $k3s_unit ]]; then systemctl restart "$k3s_unit" fi fi if [[ $defer_restart -eq 0 && -n $k3s_unit ]]; then grep -q 'io.containerd.nri.v1.nri' "$ctd_dir/config.toml" grep -q 'socket_path = "/var/run/nri/nri.sock"' "$ctd_dir/config.toml" for _ in {1..30}; do [[ -S /var/run/nri/nri.sock ]] && break sleep 1 done [[ -S /var/run/nri/nri.sock ]] || { echo "error: NRI socket not available after $k3s_unit restart" >&2 exit 1 } fi echo "packages: ${pkgs[*]}" echo "longhorn: /var/lib/longhorn" echo "runsc: $(runsc --version 2>&1 | head -1)" echo "registry CA: $CA_REMOTE" echo "registries: $REG_REMOTE" echo "system CA: $SYS_CA" echo "prepare-node finished on $(hostname)" REMOTE echo "prepare-node finished on $target"