Add isolated kind dependency testbed

This commit is contained in:
ms2k 2026-07-17 18:30:34 +09:00
commit 5759938f3f
10 changed files with 1155 additions and 0 deletions

7
tests/kind/cluster.yaml Normal file
View file

@ -0,0 +1,7 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
apiServerAddress: 127.0.0.1
apiServerPort: 36443
nodes:
- role: control-plane

View file

@ -0,0 +1,463 @@
apiVersion: v1
kind: Namespace
metadata:
name: nuthatch-testbed
---
apiVersion: v1
kind: ConfigMap
metadata:
name: postgres-init
namespace: nuthatch-testbed
data:
01-keycloak.sql: |
CREATE USER keycloak WITH PASSWORD 'synthetic-keycloak-db';
CREATE DATABASE keycloak OWNER keycloak;
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: postgres}
template:
metadata:
labels: {app: postgres}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
env:
- {name: POSTGRES_DB, value: nuthatch}
- {name: POSTGRES_USER, value: nuthatch}
- {name: POSTGRES_PASSWORD, value: synthetic-postgres}
ports:
- {name: postgres, containerPort: 5432}
readinessProbe:
exec:
command: [pg_isready, -U, nuthatch, -d, nuthatch]
periodSeconds: 2
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {cpu: "2", memory: 1Gi}
volumeMounts:
- {name: data, mountPath: /var/lib/postgresql}
- {name: init, mountPath: /docker-entrypoint-initdb.d}
volumes:
- name: data
emptyDir: {}
- name: init
configMap: {name: postgres-init}
---
apiVersion: v1
kind: Service
metadata:
name: postgres
namespace: nuthatch-testbed
spec:
selector: {app: postgres}
ports:
- {name: postgres, port: 5432, targetPort: postgres}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: redis}
template:
metadata:
labels: {app: redis}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: redis
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
args: [redis-server, --save, "", --appendonly, "no"]
ports:
- {name: redis, containerPort: 6379}
readinessProbe:
exec:
command: [redis-cli, ping]
periodSeconds: 2
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 256Mi}
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: nuthatch-testbed
spec:
selector: {app: redis}
ports:
- {name: redis, port: 6379, targetPort: redis}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dragonfly
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: dragonfly}
template:
metadata:
labels: {app: dragonfly}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: dragonfly
image: @@DRAGONFLY_IMAGE@@
imagePullPolicy: Never
args: [--proactor_threads=1, --maxmemory=256mb]
ports:
- {name: redis, containerPort: 6379}
readinessProbe:
tcpSocket: {port: redis}
periodSeconds: 2
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 384Mi}
---
apiVersion: v1
kind: Service
metadata:
name: dragonfly
namespace: nuthatch-testbed
spec:
selector: {app: dragonfly}
ports:
- {name: redis, port: 6379, targetPort: redis}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: keycloak
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: keycloak}
template:
metadata:
labels: {app: keycloak}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: keycloak
image: @@KEYCLOAK_IMAGE@@
imagePullPolicy: Never
args: [start-dev, --hostname=http://keycloak.nuthatch.test:8080]
env:
- {name: KC_BOOTSTRAP_ADMIN_USERNAME, value: nuthatch-admin}
- {name: KC_BOOTSTRAP_ADMIN_PASSWORD, value: synthetic-keycloak}
- {name: KC_DB, value: postgres}
- {name: KC_DB_URL, value: jdbc:postgresql://postgres:5432/keycloak}
- {name: KC_DB_USERNAME, value: keycloak}
- {name: KC_DB_PASSWORD, value: synthetic-keycloak-db}
- {name: JAVA_OPTS_KC_HEAP, value: "-XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=30 -XX:MaxRAMPercentage=65"}
ports:
- {name: http, containerPort: 8080}
readinessProbe:
httpGet: {path: /realms/master, port: http}
periodSeconds: 3
failureThreshold: 60
resources:
requests: {cpu: 200m, memory: 512Mi}
limits: {cpu: "2", memory: 1Gi}
---
apiVersion: v1
kind: Service
metadata:
name: keycloak
namespace: nuthatch-testbed
spec:
clusterIP: 10.96.0.55
selector: {app: keycloak}
ports:
- {name: http, port: 8080, targetPort: http}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: zitadel
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: zitadel}
template:
metadata:
labels: {app: zitadel}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: zitadel
image: @@ZITADEL_IMAGE@@
imagePullPolicy: Never
args: [start-from-init, --masterkeyFromEnv, --tlsMode, disabled]
env:
- {name: ZITADEL_MASTERKEY, value: synthetic-masterkey-32-character}
- {name: ZITADEL_DATABASE_POSTGRES_DSN, value: "postgresql://nuthatch:synthetic-postgres@postgres:5432/postgres?sslmode=disable"}
- {name: ZITADEL_EXTERNALSECURE, value: "false"}
- {name: ZITADEL_EXTERNALDOMAIN, value: zitadel.nuthatch.test}
- {name: ZITADEL_EXTERNALPORT, value: "8080"}
- {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME, value: nuthatch-admin}
- {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD, value: Synthetic-Zitadel-1!}
ports:
- {name: http, containerPort: 8080}
readinessProbe:
httpGet:
path: /.well-known/openid-configuration
port: http
httpHeaders:
- {name: Host, value: zitadel.nuthatch.test}
periodSeconds: 3
failureThreshold: 80
resources:
requests: {cpu: 200m, memory: 512Mi}
limits: {cpu: "2", memory: 1Gi}
---
apiVersion: v1
kind: Service
metadata:
name: zitadel
namespace: nuthatch-testbed
spec:
clusterIP: 10.96.0.56
selector: {app: zitadel}
ports:
- {name: http, port: 8080, targetPort: http}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: garage-config
namespace: nuthatch-testbed
data:
garage.toml: |
metadata_dir = "/data/meta"
data_dir = "/data/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "garage:3901"
rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.nuthatch.test"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "synthetic-garage-admin"
metrics_token = "synthetic-garage-metrics"
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: garage
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: garage}
template:
metadata:
labels: {app: garage}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: garage
image: @@GARAGE_IMAGE@@
imagePullPolicy: Never
command: [/garage]
args: [server, --single-node, --default-bucket]
env:
- {name: GARAGE_DEFAULT_ACCESS_KEY, value: GK00000000000000000000000000000000}
- {name: GARAGE_DEFAULT_SECRET_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: GARAGE_DEFAULT_BUCKET, value: nuthatch-fixture}
ports:
- {name: s3, containerPort: 3900}
resources:
requests: {cpu: 100m, memory: 128Mi}
limits: {cpu: "2", memory: 512Mi}
volumeMounts:
- {name: config, mountPath: /etc/garage.toml, subPath: garage.toml}
- {name: data, mountPath: /data}
- name: s3-readiness
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["while :; do sleep 3600; done"]
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
readinessProbe:
exec:
command: [aws, --endpoint-url, "http://127.0.0.1:3900", s3api, head-bucket, --bucket, nuthatch-fixture]
periodSeconds: 2
failureThreshold: 60
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
volumes:
- name: config
configMap: {name: garage-config}
- name: data
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: garage
namespace: nuthatch-testbed
spec:
selector: {app: garage}
ports:
- {name: s3, port: 3900, targetPort: s3}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: versitygw
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: versitygw}
template:
metadata:
labels: {app: versitygw}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: versitygw
image: @@VERSITYGW_IMAGE@@
imagePullPolicy: Never
args: [--port, ":10000", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
env:
- {name: ROOT_ACCESS_KEY, value: synthetic-s3-access}
- {name: ROOT_SECRET_KEY, value: synthetic-s3-secret}
ports:
- {name: s3, containerPort: 10000}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 512Mi}
volumeMounts:
- {name: data, mountPath: /data}
- {name: versions, mountPath: /versions}
- name: s3-readiness
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["while :; do sleep 3600; done"]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
readinessProbe:
exec:
command: [aws, --endpoint-url, "http://127.0.0.1:10000", s3api, list-buckets]
periodSeconds: 2
failureThreshold: 60
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
volumes:
- name: data
emptyDir: {}
- name: versions
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: versitygw
namespace: nuthatch-testbed
spec:
selector: {app: versitygw}
ports:
- {name: s3, port: 10000, targetPort: s3}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: controlled-dns
namespace: nuthatch-testbed
data:
Corefile: |
.:53 {
errors
hosts {
10.96.0.55 keycloak.nuthatch.test
10.96.0.56 zitadel.nuthatch.test
fallthrough
}
forward . 10.96.0.10
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: controlled-dns
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: controlled-dns}
template:
metadata:
labels: {app: controlled-dns}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: coredns
image: @@COREDNS_IMAGE@@
imagePullPolicy: Never
args: [-conf, /etc/coredns/Corefile]
ports:
- {name: dns-udp, containerPort: 53, protocol: UDP}
- {name: dns-tcp, containerPort: 53, protocol: TCP}
readinessProbe:
tcpSocket: {port: dns-tcp}
periodSeconds: 2
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumeMounts:
- {name: config, mountPath: /etc/coredns}
volumes:
- name: config
configMap: {name: controlled-dns}
---
apiVersion: v1
kind: Service
metadata:
name: controlled-dns
namespace: nuthatch-testbed
spec:
clusterIP: 10.96.0.53
selector: {app: controlled-dns}
ports:
- {name: dns-udp, port: 53, targetPort: dns-udp, protocol: UDP}
- {name: dns-tcp, port: 53, targetPort: dns-tcp, protocol: TCP}

9
tests/kind/destroy.sh Executable file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
acquire_fixture_lock
prepare_tools
destroy_fixture

22
tests/kind/lib-test.sh Executable file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
if kube get namespaces >/dev/null 2>&1; then
printf 'kube helper unexpectedly reached a cluster without fixture setup\n' >&2
exit 1
fi
FIXTURE_LOCK_DIR="$RUNTIME/lock-test"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
acquire_fixture_lock
if XDG_RUNTIME_DIR="$FIXTURE_LOCK_DIR" bash -c \
'source "$1"; acquire_fixture_lock' _ "$ROOT/tests/kind/lib.sh" 9>&-; then
printf 'singleton fixture lock admitted a second worktree process\n' >&2
exit 1
fi
exec 9>&-
printf 'fixture isolation tests passed\n'

269
tests/kind/lib.sh Executable file
View file

@ -0,0 +1,269 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed
set -a
source "$TESTBED/versions.env"
set +a
user_kubeconfig_digest() {
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
local path
local paths
IFS=: read -r -a paths <<< "$config_value"
{
for path in "${paths[@]}"; do
if [[ -f "$path" ]]; then
sha256sum "$path"
else
printf 'missing %s\n' "$path"
fi
done
} | sha256sum | cut -d' ' -f1
}
download_tool() {
local destination="$1"
local checksum="$2"
local url="$3"
if [[ -x "$destination" ]] &&
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
return
fi
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
chmod 0755 "$destination.tmp"
mv "$destination.tmp" "$destination"
}
prepare_tools() {
mkdir -p "$BIN"
download_tool \
"$BIN/kind" \
"$KIND_SHA256" \
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
download_tool \
"$BIN/kubectl" \
"$KUBECTL_SHA256" \
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}
acquire_fixture_lock() {
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
exec 9>"$FIXTURE_LOCK_PATH"
if ! flock --nonblock 9; then
printf 'kind test fixture is already in use\n' >&2
return 1
fi
}
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)"
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
export PATH="$TESTBED/provider:$PATH"
}
runtime_ctr() {
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}
runtime_namespace_exists() {
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}
cluster_exists() {
runtime_namespace_exists &&
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}
delete_cluster() {
local container
local containers
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")"
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container"
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
if cluster_exists; then
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
return 1
fi
}
remove_runtime_namespace() {
local status=0
local digest
local digests
local lease
local leases
local network
for network in kind nuthatch-kind; do
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
fi
done
if ! runtime_namespace_exists; then
return "$status"
fi
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
while IFS= read -r lease; do
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
done <<< "$leases"
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
while IFS= read -r digest; do
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
done <<< "$digests"
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
runtime_namespace_exists && status=1
return "$status"
}
prepare_provider() {
configure_provider
delete_cluster
remove_runtime_namespace
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}
load_external_image() {
local name="$1"
local source="$2"
local digest="${source##*@}"
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
local archive="$RUNTIME/image-archives/$digest.tar"
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
printf 'Loading %s from %s\n' "$image" "$source" >&2
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
printf '%s\n' "$image"
}
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
}
kube() {
"$BIN/kubectl" \
--kubeconfig "$KUBECONFIG_PATH" \
--context "$CONTEXT" \
"$@"
}
wait_for_job() {
local job="$1"
local deadline=$((SECONDS + 300))
local complete
local failed
while ((SECONDS < deadline)); do
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
return
fi
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
return 1
fi
sleep 1
done
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1
}
setup_fixture() {
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli"
kube apply --filename "$RUNTIME/dependencies.yaml"
local deployment
for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
done
}
verify_dependencies() {
kube apply --filename "$RUNTIME/probe.yaml"
local job
for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$job"
done
}
destroy_fixture() {
configure_provider
delete_cluster
remove_runtime_namespace
rm -rf "$RUNTIME/image-archives"
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
}

226
tests/kind/probe.yaml.in Normal file
View file

@ -0,0 +1,226 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: probe-payload
namespace: nuthatch-testbed
data:
payload: nuthatch-dependency-compatibility-probe
---
apiVersion: batch/v1
kind: Job
metadata:
name: state-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [psql]
args:
- -h
- postgres
- -U
- nuthatch
- -d
- nuthatch
- -v
- ON_ERROR_STOP=1
- -c
- "CREATE TABLE dependency_probe (value text NOT NULL)"
- -c
- "INSERT INTO dependency_probe VALUES ('passed')"
- -c
- "SELECT value = 'passed' FROM dependency_probe"
env:
- {name: PGPASSWORD, value: synthetic-postgres}
- name: redis
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["redis-cli -h redis SET nuthatch-probe passed && test \"$(redis-cli -h redis GET nuthatch-probe)\" = passed"]
containers:
- name: dragonfly
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["redis-cli -h dragonfly SET nuthatch-probe passed && test \"$(redis-cli -h dragonfly GET nuthatch-probe)\" = passed && printf 'PostgreSQL and Redis-compatible stores passed\\n'"]
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
containers:
- name: curl
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
probe() {
url="$1"
pattern="$2"
attempt=0
until curl --fail --silent --show-error "$url" | grep -q "$pattern"; do
attempt=$((attempt + 1))
test "$attempt" -lt 30
sleep 1
done
}
probe http://keycloak.nuthatch.test:8080/realms/master/.well-known/openid-configuration '"issuer"'
probe http://keycloak.nuthatch.test:8080/realms/master/protocol/openid-connect/certs '"keys"'
probe http://zitadel.nuthatch.test:8080/.well-known/openid-configuration '"issuer"'
probe http://zitadel.nuthatch.test:8080/oauth/v2/keys '"keys"'
printf 'Keycloak and ZITADEL discovery passed through controlled DNS\n'
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: garage-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: upload
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
attempt=0
until aws --endpoint-url http://garage:3900 s3api put-object --bucket nuthatch-fixture --key probe/payload --body /fixture/payload; do
attempt=$((attempt + 1))
test "$attempt" -lt 30
sleep 1
done
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
volumeMounts:
- {name: fixture, mountPath: /fixture}
- name: download
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://garage:3900", s3api, get-object, --bucket, nuthatch-fixture, --key, probe/payload, /work/result]
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
volumeMounts:
- {name: work, mountPath: /work}
containers:
- name: compare
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["cmp /fixture/payload /work/result && printf 'Garage put/get passed\\n'"]
volumeMounts:
- {name: fixture, mountPath: /fixture}
- {name: work, mountPath: /work}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture
configMap: {name: probe-payload}
- name: work
emptyDir: {}
---
apiVersion: batch/v1
kind: Job
metadata:
name: versitygw-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: bucket
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
aws --endpoint-url http://versitygw:10000 s3api create-bucket --bucket nuthatch-fixture ||
aws --endpoint-url http://versitygw:10000 s3api head-bucket --bucket nuthatch-fixture
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
- name: upload
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://versitygw:10000", s3api, put-object, --bucket, nuthatch-fixture, --key, probe/payload, --body, /fixture/payload]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
volumeMounts:
- {name: fixture, mountPath: /fixture}
- name: download
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://versitygw:10000", s3api, get-object, --bucket, nuthatch-fixture, --key, probe/payload, /work/result]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
volumeMounts:
- {name: work, mountPath: /work}
containers:
- name: compare
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["cmp /fixture/payload /work/result && printf 'VersityGW put/get passed\\n'"]
volumeMounts:
- {name: fixture, mountPath: /fixture}
- {name: work, mountPath: /work}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture
configMap: {name: probe-payload}
- name: work
emptyDir: {}

39
tests/kind/provider/nerdctl Executable file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${1:-}" = run ]]; then
shift
arguments=()
for argument in "$@"; do
if [[ "$argument" = kind ]]; then
arguments+=(nuthatch-kind)
else
arguments+=("$argument")
fi
done
exec "$NUTHATCH_REAL_NERDCTL" run --memory 20g --memory-swap 28g --cpus 8 "${arguments[@]}"
fi
# kind otherwise gives every rootless namespace the same bridge and IPv6 subnet.
if [[ "${1:-}" = network ]]; then
arguments=()
for argument in "$@"; do
case "$argument" in
kind)
arguments+=(nuthatch-kind)
;;
fc00:f853:ccd:e793::/64)
arguments+=(fc00:f853:ccd:e794::/64)
;;
--subnet=fc00:f853:ccd:e793::/64)
arguments+=(--subnet=fc00:f853:ccd:e794::/64)
;;
*)
arguments+=("$argument")
;;
esac
done
exec "$NUTHATCH_REAL_NERDCTL" "${arguments[@]}"
fi
exec "$NUTHATCH_REAL_NERDCTL" "$@"

38
tests/kind/run.sh Executable file
View file

@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ -n "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
if [[ "$status" = 0 ]] && [[ "$SUCCEEDED" = true ]]; then
printf 'Nuthatch dependency testbed passed\n'
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
setup_fixture
verify_dependencies
SUCCEEDED=true

31
tests/kind/setup.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "$SUCCEEDED" != true ]] && [[ "${NUTHATCH_TESTBED_RETAIN:-0}" != 1 ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
setup_fixture
SUCCEEDED=true
printf 'fixture kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'fixture context: %s\n' "$CONTEXT"

17
tests/kind/versions.env Normal file
View file

@ -0,0 +1,17 @@
KIND_VERSION=v0.32.0
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
KUBECTL_VERSION=v1.36.2
KUBECTL_SHA256=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
CONTAINERD_VERSION=2.3.3
NERDCTL_VERSION=2.3.4
NODE_IMAGE=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
POSTGRES_IMAGE=postgres@sha256:bb9940981eeb86356ec91bda430f8e0ba8729cc0b319615a2c2d7ac932fdf7bb
REDIS_IMAGE=redis@sha256:cd5f3ac681c77791c6a8eaa62de876ad2be043ee5a428afb7c0095aa08246277
DRAGONFLY_IMAGE=docker.dragonflydb.io/dragonflydb/dragonfly@sha256:665ee58607d035881384be45b20025490ecf87069418f61f715c19117ed5fb7d
KEYCLOAK_IMAGE=quay.io/keycloak/keycloak@sha256:1362a9d9f13ab325231ea133610cc905e12805804abc7acbef552dd613720aa6
ZITADEL_IMAGE=ghcr.io/zitadel/zitadel@sha256:d59554f17d20c06c9f6ed114bf9f2c3a9a63698003daea62d27bdae1e9667583
GARAGE_IMAGE=dxflrs/garage@sha256:dac0c92add4f1a0b41035e94b41036a270ffbe88a37c7ac9c3f19e6dc5bdccf2
VERSITYGW_IMAGE=versity/versitygw@sha256:4a2c613f6541410f87983c5134cd26a88dfc23f0ad555b7aa51d3d87fd2d347b
COREDNS_IMAGE=coredns/coredns@sha256:1ba6f47265602e2e50a9c4669e3a955e4298a0d30dc82f39293d4bf1a851e0ff
CURL_IMAGE=curlimages/curl@sha256:1ab04d023ece37e6ec991bf3306ad04e0ef0084e94a5c6b6563cfcb9563169db
AWS_CLI_IMAGE=amazon/aws-cli@sha256:10c3a6dff0fdcb3fbc86cd744709548ad041ebb6bbe14fbedc1595a5539d8d27