nuthatch/tests/kind/lib.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed

set -a
source "$TESTBED/versions.env"
set +a

user_kubeconfig_digest() {
  local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
  local path
  local paths
  IFS=: read -r -a paths <<< "$config_value"
  {
    for path in "${paths[@]}"; do
      if [[ -f "$path" ]]; then
        sha256sum "$path"
      else
        printf 'missing %s\n' "$path"
      fi
    done
  } | sha256sum | cut -d' ' -f1
}

download_tool() {
  local destination="$1"
  local checksum="$2"
  local url="$3"
  if [[ -x "$destination" ]] &&
     [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
    return
  fi
  curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
  test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
  chmod 0755 "$destination.tmp"
  mv "$destination.tmp" "$destination"
}

prepare_tools() {
  mkdir -p "$BIN"
  download_tool \
    "$BIN/kind" \
    "$KIND_SHA256" \
    "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
  download_tool \
    "$BIN/kubectl" \
    "$KUBECTL_SHA256" \
    "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
  test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
  test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
  test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
  test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
  test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}

acquire_fixture_lock() {
  mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
  exec 9>"$FIXTURE_LOCK_PATH"
  if ! flock --nonblock 9; then
    printf 'kind test fixture is already in use\n' >&2
    return 1
  fi
}

configure_provider() {
  if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
    NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)"
  fi
  export NUTHATCH_REAL_NERDCTL
  export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
  export KIND_EXPERIMENTAL_PROVIDER=nerdctl
  export PATH="$TESTBED/provider:$PATH"
}

runtime_ctr() {
  ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}

runtime_namespace_exists() {
  "$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}

cluster_exists() {
  runtime_namespace_exists &&
    "$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}

delete_cluster() {
  local container
  local containers
  if cluster_exists; then
    "$BIN/kind" delete cluster --name "$CLUSTER" || true
  fi
  containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
    --all \
    --quiet \
    --filter "label=io.x-k8s.kind.cluster=$CLUSTER")"
  while IFS= read -r container; do
    [[ -z "$container" ]] ||
      "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container"
  done <<< "$containers"
  if cluster_exists; then
    "$BIN/kind" delete cluster --name "$CLUSTER" || true
  fi
  if cluster_exists; then
    printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
    return 1
  fi
}

remove_runtime_namespace() {
  local status=0
  local digest
  local digests
  local lease
  local leases
  local network
  for network in kind nuthatch-kind; do
    if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
      "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
    fi
  done
  if ! runtime_namespace_exists; then
    return "$status"
  fi
  "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
  leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
  while IFS= read -r lease; do
    [[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
  done <<< "$leases"
  digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
  while IFS= read -r digest; do
    [[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
  done <<< "$digests"
  "$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
  runtime_namespace_exists && status=1
  return "$status"
}

prepare_provider() {
  configure_provider
  delete_cluster
  remove_runtime_namespace
  "$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}

load_external_image() {
  local name="$1"
  local source="$2"
  local digest="${source##*@}"
  local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
  local archive="$RUNTIME/image-archives/$digest.tar"
  "$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
  "$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
  "$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
  printf 'Loading %s from %s\n' "$image" "$source" >&2
  "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
  printf '%s\n' "$image"
}

render_manifests() {
  local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
  local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
  sed \
    -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
    -e "s|@@REDIS_IMAGE@@|$redis|g" \
    -e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
    -e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
    -e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
    -e "s|@@GARAGE_IMAGE@@|$garage|g" \
    -e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
    -e "s|@@COREDNS_IMAGE@@|$coredns|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
    "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
  sed \
    -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
    -e "s|@@REDIS_IMAGE@@|$redis|g" \
    -e "s|@@CURL_IMAGE@@|$curl|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
    "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
}

kube() {
  "$BIN/kubectl" \
    --kubeconfig "$KUBECONFIG_PATH" \
    --context "$CONTEXT" \
    "$@"
}

wait_for_job() {
  local job="$1"
  local deadline=$((SECONDS + 300))
  local complete
  local failed
  while ((SECONDS < deadline)); do
    complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
    if [[ "$complete" = True ]]; then
      kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
      return
    fi
    failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
    if [[ "$failed" = True ]]; then
      kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
      return 1
    fi
    sleep 1
  done
  printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
  return 1
}

setup_fixture() {
  mkdir -p "$RUNTIME"
  rm -rf "$RUNTIME/image-archives"
  mkdir -p "$RUNTIME/image-archives"
  prepare_tools
  prepare_provider
  "$BIN/kind" create cluster \
    --name "$CLUSTER" \
    --image "$NODE_IMAGE" \
    --config "$TESTBED/cluster.yaml" \
    --kubeconfig "$KUBECONFIG_PATH" \
    --wait 180s
  local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli
  postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
  redis="$(load_external_image redis "$REDIS_IMAGE")"
  dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
  keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
  zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
  garage="$(load_external_image garage "$GARAGE_IMAGE")"
  versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
  coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
  curl="$(load_external_image curl "$CURL_IMAGE")"
  aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
  render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
    "$garage" "$versitygw" "$coredns" "$curl" "$aws_cli"
  kube apply --filename "$RUNTIME/dependencies.yaml"
  local deployment
  for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do
    kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
  done
}

verify_dependencies() {
  kube apply --filename "$RUNTIME/probe.yaml"
  local job
  for job in state-probe oidc-probe garage-probe versitygw-probe; do
    wait_for_job "$job"
  done
}

destroy_fixture() {
  configure_provider
  delete_cluster
  remove_runtime_namespace
  rm -rf "$RUNTIME/image-archives"
  rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
}