quorate/scripts/k3s/prepare-node.sh

#!/usr/bin/env bash
# Usage: prepare-node.sh [--defer-restart] <ssh-target>
# Host packages, Longhorn dir, runsc, internal CA + registries.yaml. Idempotent; before install.sh.
set -euo pipefail

CA_URL="${K3S_CA_URL:-https://vault.quorate.internal/v1/pki/ca/pem}"
CA_REMOTE=/etc/rancher/k3s/quorate-internal-ca.crt
REG_REMOTE=/etc/rancher/k3s/registries.yaml

die() { echo "error: $*" >&2; exit 1; }

usage() {
  cat >&2 <<'EOF'
Usage: prepare-node.sh [--defer-restart] <ssh-target>
EOF
  exit 2
}

defer_restart=0
if [[ ${1:-} == --defer-restart ]]; then
  defer_restart=1
  shift
fi

[[ $# -eq 1 ]] || usage
target=$1
[[ -n "$target" ]] || die "ssh-target is empty"

command -v ssh >/dev/null || die "ssh not found"
command -v scp >/dev/null || die "scp not found"
command -v curl >/dev/null || die "curl not found"
command -v openssl >/dev/null || die "openssl not found"

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
registries_src=$script_dir/registries.yaml
[[ -f $registries_src ]] || die "missing $registries_src"

workdir=$(mktemp -d)
trap 'rm -rf "$workdir"' EXIT
ca_local=$workdir/quorate-internal-ca.crt

curl -fsSL --max-time 30 "$CA_URL" -o "$ca_local" \
  || die "failed to fetch CA from $CA_URL"
openssl x509 -in "$ca_local" -noout 2>/dev/null \
  || die "fetched CA is not a valid PEM certificate"
openssl x509 -in "$ca_local" -noout -text | grep -q 'CA:TRUE' \
  || die "fetched certificate is not a CA"

echo "prepare-node target=$target"

ssh -o BatchMode=yes "$target" "install -d -m 0755 /etc/rancher/k3s"
scp -o BatchMode=yes -q "$ca_local" "${target}:/tmp/quorate-internal-ca.crt"
scp -o BatchMode=yes -q "$registries_src" "${target}:/tmp/k3s-registries.yaml"

ssh -o BatchMode=yes "$target" "bash -s -- $defer_restart" <<'REMOTE'
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
defer_restart=$1

if ! command -v apt-get >/dev/null; then
  echo "error: only Debian-family apt hosts are supported" >&2
  exit 1
fi

pkg_installed() {
  dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q 'install ok installed'
}

need_k3s_restart=0

pkgs=(open-iscsi nfs-common xfsprogs smartmontools)
missing=()
for p in "${pkgs[@]}"; do
  pkg_installed "$p" || missing+=("$p")
done
if ((${#missing[@]})); then
  apt-get update -qq
  apt-get install -y -qq "${missing[@]}"
fi

if systemctl cat iscsid.service >/dev/null 2>&1; then
  systemctl is-enabled --quiet iscsid 2>/dev/null || systemctl enable iscsid >/dev/null
  systemctl is-active --quiet iscsid 2>/dev/null || systemctl start iscsid
fi

install -d -m 0755 /var/lib/longhorn

inotify_file=/etc/sysctl.d/90-k3s-inotify.conf
inotify_limit=1024
inotify_setting="fs.inotify.max_user_instances = $inotify_limit"
install -d -m 0755 /etc/sysctl.d
if [[ ! -f $inotify_file ]] || ! cmp -s <(printf '%s\n' "$inotify_setting") "$inotify_file"; then
  printf '%s\n' "$inotify_setting" >"$inotify_file"
fi
chmod 0644 "$inotify_file"
if [[ $(sysctl -n fs.inotify.max_user_instances) != "$inotify_limit" ]]; then
  sysctl -p "$inotify_file"
fi

CA_REMOTE=/etc/rancher/k3s/quorate-internal-ca.crt
REG_REMOTE=/etc/rancher/k3s/registries.yaml
SYS_CA=/usr/local/share/ca-certificates/internal-root-ca.crt

install -d -m 0755 /etc/rancher/k3s
if [[ -f $CA_REMOTE ]] && cmp -s /tmp/quorate-internal-ca.crt "$CA_REMOTE"; then
  :
else
  install -m 0644 /tmp/quorate-internal-ca.crt "$CA_REMOTE"
  need_k3s_restart=1
fi

if [[ -f $REG_REMOTE ]] && cmp -s /tmp/k3s-registries.yaml "$REG_REMOTE"; then
  :
else
  install -m 0644 /tmp/k3s-registries.yaml "$REG_REMOTE"
  need_k3s_restart=1
fi

if [[ -f $SYS_CA ]] && cmp -s /tmp/quorate-internal-ca.crt "$SYS_CA"; then
  :
else
  install -d -m 0755 /usr/local/share/ca-certificates
  install -m 0644 /tmp/quorate-internal-ca.crt "$SYS_CA"
  update-ca-certificates >/dev/null
fi
rm -f /tmp/quorate-internal-ca.crt /tmp/k3s-registries.yaml

if ! command -v runsc >/dev/null 2>&1 || ! command -v containerd-shim-runsc-v1 >/dev/null 2>&1; then
  for p in ca-certificates curl gnupg; do
    pkg_installed "$p" || { apt-get update -qq; apt-get install -y -qq ca-certificates curl gnupg; break; }
  done

  if [[ ! -f /usr/share/keyrings/gvisor-archive-keyring.gpg ]]; then
    curl -fsSL https://gvisor.dev/archive.key | gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg
  fi

  arch=$(dpkg --print-architecture)
  gvisor_list=/etc/apt/sources.list.d/gvisor.list
  gvisor_line="deb [arch=${arch} signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main"
  if [[ ! -f $gvisor_list ]] || ! grep -qxF "$gvisor_line" "$gvisor_list"; then
    printf '%s\n' "$gvisor_line" >"$gvisor_list"
  fi

  if ! { apt-get update -qq && apt-get install -y -qq runsc \
      && command -v runsc >/dev/null && command -v containerd-shim-runsc-v1 >/dev/null; }; then
    ARCH=$(uname -m)
    URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}
    tmp=$(mktemp -d)
    curl -fsSL -o "$tmp/gvisor.tar.bz2" "${URL}/gvisor.tar.bz2"
    curl -fsSL -o "$tmp/gvisor.tar.bz2.sha512" "${URL}/gvisor.tar.bz2.sha512"
    (cd "$tmp" && sha512sum -c gvisor.tar.bz2.sha512)
    tar -xjf "$tmp/gvisor.tar.bz2" -C /usr/local/bin
    rm -rf "$tmp"
  fi
  need_k3s_restart=1
fi

command -v runsc >/dev/null || { echo "error: runsc not on PATH" >&2; exit 1; }
command -v containerd-shim-runsc-v1 >/dev/null || {
  echo "error: containerd-shim-runsc-v1 not on PATH" >&2
  exit 1
}

ctd_dir=/var/lib/rancher/k3s/agent/etc/containerd
tmpl=$ctd_dir/config-v3.toml.tmpl
read -r -d '' desired_tmpl <<'TMPL' || true
{{ template "base" . }}

[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runsc]
  runtime_type = "io.containerd.runsc.v1"
  pod_annotations = ["dev.gvisor.spec.mount.var-run-argo.share", "dev.gvisor.spec.mount.var-run-argo.type", "dev.gvisor.spec.mount.var-run-argo.options"]

[plugins.'io.containerd.nri.v1.nri']
  disable = false
  disable_connections = false
  plugin_config_path = "/etc/nri/conf.d"
  plugin_path = "/opt/nri/plugins"
  plugin_registration_timeout = "5s"
  plugin_request_timeout = "2s"
  socket_path = "/var/run/nri/nri.sock"
TMPL

mkdir -p "$ctd_dir"
if [[ -f $tmpl ]] && cmp -s <(printf '%s\n' "$desired_tmpl") "$tmpl"; then
  :
else
  printf '%s\n' "$desired_tmpl" >"$tmpl"
  chmod 644 "$tmpl"
  need_k3s_restart=1
fi

if [[ -f $ctd_dir/config.toml ]]; then
  grep -q 'runtimes.runsc' "$ctd_dir/config.toml" || need_k3s_restart=1
  grep -q 'io.containerd.nri.v1.nri' "$ctd_dir/config.toml" || need_k3s_restart=1
fi

k3s_unit=
if systemctl is-active --quiet k3s 2>/dev/null || systemctl is-enabled --quiet k3s 2>/dev/null; then
  k3s_unit=k3s
elif systemctl is-active --quiet k3s-agent 2>/dev/null || systemctl is-enabled --quiet k3s-agent 2>/dev/null; then
  k3s_unit=k3s-agent
fi

if [[ $need_k3s_restart -eq 1 ]]; then
  if [[ $defer_restart -eq 1 ]]; then
    echo "k3s restart: deferred"
  elif [[ -n $k3s_unit ]]; then
    systemctl restart "$k3s_unit"
  fi
fi

if [[ $defer_restart -eq 0 && -n $k3s_unit ]]; then
  grep -q 'io.containerd.nri.v1.nri' "$ctd_dir/config.toml"
  grep -q 'socket_path = "/var/run/nri/nri.sock"' "$ctd_dir/config.toml"
  for _ in {1..30}; do
    [[ -S /var/run/nri/nri.sock ]] && break
    sleep 1
  done
  [[ -S /var/run/nri/nri.sock ]] || {
    echo "error: NRI socket not available after $k3s_unit restart" >&2
    exit 1
  }
fi

echo "packages: ${pkgs[*]}"
echo "longhorn: /var/lib/longhorn"
echo "runsc: $(runsc --version 2>&1 | head -1)"
echo "registry CA: $CA_REMOTE"
echo "registries: $REG_REMOTE"
echo "system CA: $SYS_CA"
echo "prepare-node finished on $(hostname)"
REMOTE

echo "prepare-node finished on $target"