ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Add isolated kind dependency testbed

Browse source
This commit is contained in:
ms2k 2026-07-17 18:30:34 +09:00
commit 5759938f3f
13 changed files with 1155 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
/docs/protocol/states/ /docs/protocol/states/
/states/ /states/
/tests/kind/.runtime/
docs/journal docs/journal
scratchpad scratchpad

7
Makefile Normal file
View file

@ -0,0 +1,7 @@
.PHONY: check-dependencies check-fixture-lib
check-dependencies:
./tests/kind/run.sh
check-fixture-lib:
./tests/kind/lib-test.sh

26
docs/spec/testing.md Normal file
View file

@ -0,0 +1,26 @@
# Dependency testing
Backend integration tests execute inside an isolated kind cluster so they use
the same DNS and service routing boundaries as deployed Nuthatch components.
The fixture provides PostgreSQL, Redis and DragonflyDB, Keycloak and ZITADEL,
and Garage and VersityGW. Images and fixture tools are checksum or
digest-pinned.
`make check-dependencies` creates the complete fixture, verifies each released
dependency image through its product interface, and destroys the fixture.
`tests/kind/setup.sh` creates the same ready fixture for interactive test work;
`tests/kind/destroy.sh` removes it. `NUTHATCH_TESTBED_RETAIN=1` retains a failed
all-in-one run for inspection.
The fixture uses `tests/kind/.runtime/kubeconfig` and the
`kind-nuthatch-testbed` context. Every Kubernetes operation supplies both
explicitly. Fixture lifecycle operations are serialized by a host-runtime lock
shared across worktrees, and teardown is idempotent.
All fixture credentials and data are synthetic. Test workloads use ClusterIP
service names and do not require host port forwarding. The fixture does not
read or change the user's Kubernetes context.
The host provides rootless nerdctl and containerd at the versions recorded in
`tests/kind/versions.env`, together with `ctr`, `curl`, `flock`, `jq`, and
`sha256sum`. The fixture downloads its pinned kind and kubectl binaries.

7
tests/kind/cluster.yaml Normal file
View file

@ -0,0 +1,7 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
apiServerAddress: 127.0.0.1
apiServerPort: 36443
nodes:
- role: control-plane

463
tests/kind/dependencies.yaml.in Normal file
View file

@ -0,0 +1,463 @@
apiVersion: v1
kind: Namespace
metadata:
name: nuthatch-testbed
---
apiVersion: v1
kind: ConfigMap
metadata:
name: postgres-init
namespace: nuthatch-testbed
data:
01-keycloak.sql: |
CREATE USER keycloak WITH PASSWORD 'synthetic-keycloak-db';
CREATE DATABASE keycloak OWNER keycloak;
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: postgres}
template:
metadata:
labels: {app: postgres}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
env:
- {name: POSTGRES_DB, value: nuthatch}
- {name: POSTGRES_USER, value: nuthatch}
- {name: POSTGRES_PASSWORD, value: synthetic-postgres}
ports:
- {name: postgres, containerPort: 5432}
readinessProbe:
exec:
command: [pg_isready, -U, nuthatch, -d, nuthatch]
periodSeconds: 2
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {cpu: "2", memory: 1Gi}
volumeMounts:
- {name: data, mountPath: /var/lib/postgresql}
- {name: init, mountPath: /docker-entrypoint-initdb.d}
volumes:
- name: data
emptyDir: {}
- name: init
configMap: {name: postgres-init}
---
apiVersion: v1
kind: Service
metadata:
name: postgres
namespace: nuthatch-testbed
spec:
selector: {app: postgres}
ports:
- {name: postgres, port: 5432, targetPort: postgres}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: redis}
template:
metadata:
labels: {app: redis}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: redis
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
args: [redis-server, --save, "", --appendonly, "no"]
ports:
- {name: redis, containerPort: 6379}
readinessProbe:
exec:
command: [redis-cli, ping]
periodSeconds: 2
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 256Mi}
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: nuthatch-testbed
spec:
selector: {app: redis}
ports:
- {name: redis, port: 6379, targetPort: redis}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dragonfly
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: dragonfly}
template:
metadata:
labels: {app: dragonfly}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: dragonfly
image: @@DRAGONFLY_IMAGE@@
imagePullPolicy: Never
args: [--proactor_threads=1, --maxmemory=256mb]
ports:
- {name: redis, containerPort: 6379}
readinessProbe:
tcpSocket: {port: redis}
periodSeconds: 2
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 384Mi}
---
apiVersion: v1
kind: Service
metadata:
name: dragonfly
namespace: nuthatch-testbed
spec:
selector: {app: dragonfly}
ports:
- {name: redis, port: 6379, targetPort: redis}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: keycloak
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: keycloak}
template:
metadata:
labels: {app: keycloak}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: keycloak
image: @@KEYCLOAK_IMAGE@@
imagePullPolicy: Never
args: [start-dev, --hostname=http://keycloak.nuthatch.test:8080]
env:
- {name: KC_BOOTSTRAP_ADMIN_USERNAME, value: nuthatch-admin}
- {name: KC_BOOTSTRAP_ADMIN_PASSWORD, value: synthetic-keycloak}
- {name: KC_DB, value: postgres}
- {name: KC_DB_URL, value: jdbc:postgresql://postgres:5432/keycloak}
- {name: KC_DB_USERNAME, value: keycloak}
- {name: KC_DB_PASSWORD, value: synthetic-keycloak-db}
- {name: JAVA_OPTS_KC_HEAP, value: "-XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=30 -XX:MaxRAMPercentage=65"}
ports:
- {name: http, containerPort: 8080}
readinessProbe:
httpGet: {path: /realms/master, port: http}
periodSeconds: 3
failureThreshold: 60
resources:
requests: {cpu: 200m, memory: 512Mi}
limits: {cpu: "2", memory: 1Gi}
---
apiVersion: v1
kind: Service
metadata:
name: keycloak
namespace: nuthatch-testbed
spec:
clusterIP: 10.96.0.55
selector: {app: keycloak}
ports:
- {name: http, port: 8080, targetPort: http}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: zitadel
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: zitadel}
template:
metadata:
labels: {app: zitadel}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: zitadel
image: @@ZITADEL_IMAGE@@
imagePullPolicy: Never
args: [start-from-init, --masterkeyFromEnv, --tlsMode, disabled]
env:
- {name: ZITADEL_MASTERKEY, value: synthetic-masterkey-32-character}
- {name: ZITADEL_DATABASE_POSTGRES_DSN, value: "postgresql://nuthatch:synthetic-postgres@postgres:5432/postgres?sslmode=disable"}
- {name: ZITADEL_EXTERNALSECURE, value: "false"}
- {name: ZITADEL_EXTERNALDOMAIN, value: zitadel.nuthatch.test}
- {name: ZITADEL_EXTERNALPORT, value: "8080"}
- {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME, value: nuthatch-admin}
- {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD, value: Synthetic-Zitadel-1!}
ports:
- {name: http, containerPort: 8080}
readinessProbe:
httpGet:
path: /.well-known/openid-configuration
port: http
httpHeaders:
- {name: Host, value: zitadel.nuthatch.test}
periodSeconds: 3
failureThreshold: 80
resources:
requests: {cpu: 200m, memory: 512Mi}
limits: {cpu: "2", memory: 1Gi}
---
apiVersion: v1
kind: Service
metadata:
name: zitadel
namespace: nuthatch-testbed
spec:
clusterIP: 10.96.0.56
selector: {app: zitadel}
ports:
- {name: http, port: 8080, targetPort: http}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: garage-config
namespace: nuthatch-testbed
data:
garage.toml: |
metadata_dir = "/data/meta"
data_dir = "/data/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "garage:3901"
rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage.nuthatch.test"
[admin]
api_bind_addr = "[::]:3903"
admin_token = "synthetic-garage-admin"
metrics_token = "synthetic-garage-metrics"
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: garage
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: garage}
template:
metadata:
labels: {app: garage}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: garage
image: @@GARAGE_IMAGE@@
imagePullPolicy: Never
command: [/garage]
args: [server, --single-node, --default-bucket]
env:
- {name: GARAGE_DEFAULT_ACCESS_KEY, value: GK00000000000000000000000000000000}
- {name: GARAGE_DEFAULT_SECRET_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: GARAGE_DEFAULT_BUCKET, value: nuthatch-fixture}
ports:
- {name: s3, containerPort: 3900}
resources:
requests: {cpu: 100m, memory: 128Mi}
limits: {cpu: "2", memory: 512Mi}
volumeMounts:
- {name: config, mountPath: /etc/garage.toml, subPath: garage.toml}
- {name: data, mountPath: /data}
- name: s3-readiness
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["while :; do sleep 3600; done"]
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
readinessProbe:
exec:
command: [aws, --endpoint-url, "http://127.0.0.1:3900", s3api, head-bucket, --bucket, nuthatch-fixture]
periodSeconds: 2
failureThreshold: 60
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
volumes:
- name: config
configMap: {name: garage-config}
- name: data
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: garage
namespace: nuthatch-testbed
spec:
selector: {app: garage}
ports:
- {name: s3, port: 3900, targetPort: s3}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: versitygw
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: versitygw}
template:
metadata:
labels: {app: versitygw}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: versitygw
image: @@VERSITYGW_IMAGE@@
imagePullPolicy: Never
args: [--port, ":10000", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
env:
- {name: ROOT_ACCESS_KEY, value: synthetic-s3-access}
- {name: ROOT_SECRET_KEY, value: synthetic-s3-secret}
ports:
- {name: s3, containerPort: 10000}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 512Mi}
volumeMounts:
- {name: data, mountPath: /data}
- {name: versions, mountPath: /versions}
- name: s3-readiness
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["while :; do sleep 3600; done"]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
readinessProbe:
exec:
command: [aws, --endpoint-url, "http://127.0.0.1:10000", s3api, list-buckets]
periodSeconds: 2
failureThreshold: 60
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
volumes:
- name: data
emptyDir: {}
- name: versions
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: versitygw
namespace: nuthatch-testbed
spec:
selector: {app: versitygw}
ports:
- {name: s3, port: 10000, targetPort: s3}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: controlled-dns
namespace: nuthatch-testbed
data:
Corefile: |
.:53 {
errors
hosts {
10.96.0.55 keycloak.nuthatch.test
10.96.0.56 zitadel.nuthatch.test
fallthrough
}
forward . 10.96.0.10
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: controlled-dns
namespace: nuthatch-testbed
spec:
replicas: 1
selector:
matchLabels: {app: controlled-dns}
template:
metadata:
labels: {app: controlled-dns}
spec:
automountServiceAccountToken: false
enableServiceLinks: false
containers:
- name: coredns
image: @@COREDNS_IMAGE@@
imagePullPolicy: Never
args: [-conf, /etc/coredns/Corefile]
ports:
- {name: dns-udp, containerPort: 53, protocol: UDP}
- {name: dns-tcp, containerPort: 53, protocol: TCP}
readinessProbe:
tcpSocket: {port: dns-tcp}
periodSeconds: 2
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumeMounts:
- {name: config, mountPath: /etc/coredns}
volumes:
- name: config
configMap: {name: controlled-dns}
---
apiVersion: v1
kind: Service
metadata:
name: controlled-dns
namespace: nuthatch-testbed
spec:
clusterIP: 10.96.0.53
selector: {app: controlled-dns}
ports:
- {name: dns-udp, port: 53, targetPort: dns-udp, protocol: UDP}
- {name: dns-tcp, port: 53, targetPort: dns-tcp, protocol: TCP}

9
tests/kind/destroy.sh Executable file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
acquire_fixture_lock
prepare_tools
destroy_fixture

22
tests/kind/lib-test.sh Executable file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
if kube get namespaces >/dev/null 2>&1; then
printf 'kube helper unexpectedly reached a cluster without fixture setup\n' >&2
exit 1
fi
FIXTURE_LOCK_DIR="$RUNTIME/lock-test"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
acquire_fixture_lock
if XDG_RUNTIME_DIR="$FIXTURE_LOCK_DIR" bash -c \
'source "$1"; acquire_fixture_lock' _ "$ROOT/tests/kind/lib.sh" 9>&-; then
printf 'singleton fixture lock admitted a second worktree process\n' >&2
exit 1
fi
exec 9>&-
printf 'fixture isolation tests passed\n'

269
tests/kind/lib.sh Executable file
View file

@ -0,0 +1,269 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed
set -a
source "$TESTBED/versions.env"
set +a
user_kubeconfig_digest() {
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
local path
local paths
IFS=: read -r -a paths <<< "$config_value"
{
for path in "${paths[@]}"; do
if [[ -f "$path" ]]; then
sha256sum "$path"
else
printf 'missing %s\n' "$path"
fi
done
} | sha256sum | cut -d' ' -f1
}
download_tool() {
local destination="$1"
local checksum="$2"
local url="$3"
if [[ -x "$destination" ]] &&
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
return
fi
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
chmod 0755 "$destination.tmp"
mv "$destination.tmp" "$destination"
}
prepare_tools() {
mkdir -p "$BIN"
download_tool \
"$BIN/kind" \
"$KIND_SHA256" \
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
download_tool \
"$BIN/kubectl" \
"$KUBECTL_SHA256" \
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}
acquire_fixture_lock() {
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
exec 9>"$FIXTURE_LOCK_PATH"
if ! flock --nonblock 9; then
printf 'kind test fixture is already in use\n' >&2
return 1
fi
}
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)"
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
export PATH="$TESTBED/provider:$PATH"
}
runtime_ctr() {
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}
runtime_namespace_exists() {
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}
cluster_exists() {
runtime_namespace_exists &&
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}
delete_cluster() {
local container
local containers
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")"
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container"
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
if cluster_exists; then
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
return 1
fi
}
remove_runtime_namespace() {
local status=0
local digest
local digests
local lease
local leases
local network
for network in kind nuthatch-kind; do
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
fi
done
if ! runtime_namespace_exists; then
return "$status"
fi
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
while IFS= read -r lease; do
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
done <<< "$leases"
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
while IFS= read -r digest; do
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
done <<< "$digests"
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
runtime_namespace_exists && status=1
return "$status"
}
prepare_provider() {
configure_provider
delete_cluster
remove_runtime_namespace
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}
load_external_image() {
local name="$1"
local source="$2"
local digest="${source##*@}"
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
local archive="$RUNTIME/image-archives/$digest.tar"
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
printf 'Loading %s from %s\n' "$image" "$source" >&2
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
printf '%s\n' "$image"
}
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
}
kube() {
"$BIN/kubectl" \
--kubeconfig "$KUBECONFIG_PATH" \
--context "$CONTEXT" \
"$@"
}
wait_for_job() {
local job="$1"
local deadline=$((SECONDS + 300))
local complete
local failed
while ((SECONDS < deadline)); do
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
return
fi
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
return 1
fi
sleep 1
done
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1
}
setup_fixture() {
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli"
kube apply --filename "$RUNTIME/dependencies.yaml"
local deployment
for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
done
}
verify_dependencies() {
kube apply --filename "$RUNTIME/probe.yaml"
local job
for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$job"
done
}
destroy_fixture() {
configure_provider
delete_cluster
remove_runtime_namespace
rm -rf "$RUNTIME/image-archives"
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
}

226
tests/kind/probe.yaml.in Normal file
View file

@ -0,0 +1,226 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: probe-payload
namespace: nuthatch-testbed
data:
payload: nuthatch-dependency-compatibility-probe
---
apiVersion: batch/v1
kind: Job
metadata:
name: state-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [psql]
args:
- -h
- postgres
- -U
- nuthatch
- -d
- nuthatch
- -v
- ON_ERROR_STOP=1
- -c
- "CREATE TABLE dependency_probe (value text NOT NULL)"
- -c
- "INSERT INTO dependency_probe VALUES ('passed')"
- -c
- "SELECT value = 'passed' FROM dependency_probe"
env:
- {name: PGPASSWORD, value: synthetic-postgres}
- name: redis
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["redis-cli -h redis SET nuthatch-probe passed && test \"$(redis-cli -h redis GET nuthatch-probe)\" = passed"]
containers:
- name: dragonfly
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["redis-cli -h dragonfly SET nuthatch-probe passed && test \"$(redis-cli -h dragonfly GET nuthatch-probe)\" = passed && printf 'PostgreSQL and Redis-compatible stores passed\\n'"]
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
containers:
- name: curl
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
probe() {
url="$1"
pattern="$2"
attempt=0
until curl --fail --silent --show-error "$url" | grep -q "$pattern"; do
attempt=$((attempt + 1))
test "$attempt" -lt 30
sleep 1
done
}
probe http://keycloak.nuthatch.test:8080/realms/master/.well-known/openid-configuration '"issuer"'
probe http://keycloak.nuthatch.test:8080/realms/master/protocol/openid-connect/certs '"keys"'
probe http://zitadel.nuthatch.test:8080/.well-known/openid-configuration '"issuer"'
probe http://zitadel.nuthatch.test:8080/oauth/v2/keys '"keys"'
printf 'Keycloak and ZITADEL discovery passed through controlled DNS\n'
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: garage-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: upload
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
attempt=0
until aws --endpoint-url http://garage:3900 s3api put-object --bucket nuthatch-fixture --key probe/payload --body /fixture/payload; do
attempt=$((attempt + 1))
test "$attempt" -lt 30
sleep 1
done
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
volumeMounts:
- {name: fixture, mountPath: /fixture}
- name: download
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://garage:3900", s3api, get-object, --bucket, nuthatch-fixture, --key, probe/payload, /work/result]
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
volumeMounts:
- {name: work, mountPath: /work}
containers:
- name: compare
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["cmp /fixture/payload /work/result && printf 'Garage put/get passed\\n'"]
volumeMounts:
- {name: fixture, mountPath: /fixture}
- {name: work, mountPath: /work}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture
configMap: {name: probe-payload}
- name: work
emptyDir: {}
---
apiVersion: batch/v1
kind: Job
metadata:
name: versitygw-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: bucket
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
aws --endpoint-url http://versitygw:10000 s3api create-bucket --bucket nuthatch-fixture ||
aws --endpoint-url http://versitygw:10000 s3api head-bucket --bucket nuthatch-fixture
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
- name: upload
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://versitygw:10000", s3api, put-object, --bucket, nuthatch-fixture, --key, probe/payload, --body, /fixture/payload]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
volumeMounts:
- {name: fixture, mountPath: /fixture}
- name: download
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://versitygw:10000", s3api, get-object, --bucket, nuthatch-fixture, --key, probe/payload, /work/result]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
volumeMounts:
- {name: work, mountPath: /work}
containers:
- name: compare
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["cmp /fixture/payload /work/result && printf 'VersityGW put/get passed\\n'"]
volumeMounts:
- {name: fixture, mountPath: /fixture}
- {name: work, mountPath: /work}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture
configMap: {name: probe-payload}
- name: work
emptyDir: {}

39
tests/kind/provider/nerdctl Executable file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${1:-}" = run ]]; then
shift
arguments=()
for argument in "$@"; do
if [[ "$argument" = kind ]]; then
arguments+=(nuthatch-kind)
else
arguments+=("$argument")
fi
done
exec "$NUTHATCH_REAL_NERDCTL" run --memory 20g --memory-swap 28g --cpus 8 "${arguments[@]}"
fi
# kind otherwise gives every rootless namespace the same bridge and IPv6 subnet.
if [[ "${1:-}" = network ]]; then
arguments=()
for argument in "$@"; do
case "$argument" in
kind)
arguments+=(nuthatch-kind)
;;
fc00:f853:ccd:e793::/64)
arguments+=(fc00:f853:ccd:e794::/64)
;;
--subnet=fc00:f853:ccd:e793::/64)
arguments+=(--subnet=fc00:f853:ccd:e794::/64)
;;
*)
arguments+=("$argument")
;;
esac
done
exec "$NUTHATCH_REAL_NERDCTL" "${arguments[@]}"
fi
exec "$NUTHATCH_REAL_NERDCTL" "$@"

38
tests/kind/run.sh Executable file
View file

@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ -n "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
if [[ "$status" = 0 ]] && [[ "$SUCCEEDED" = true ]]; then
printf 'Nuthatch dependency testbed passed\n'
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
setup_fixture
verify_dependencies
SUCCEEDED=true

31
tests/kind/setup.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "$SUCCEEDED" != true ]] && [[ "${NUTHATCH_TESTBED_RETAIN:-0}" != 1 ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
setup_fixture
SUCCEEDED=true
printf 'fixture kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'fixture context: %s\n' "$CONTEXT"

17
tests/kind/versions.env Normal file
View file

@ -0,0 +1,17 @@
KIND_VERSION=v0.32.0
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
KUBECTL_VERSION=v1.36.2
KUBECTL_SHA256=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
CONTAINERD_VERSION=2.3.3
NERDCTL_VERSION=2.3.4
NODE_IMAGE=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
POSTGRES_IMAGE=postgres@sha256:bb9940981eeb86356ec91bda430f8e0ba8729cc0b319615a2c2d7ac932fdf7bb
REDIS_IMAGE=redis@sha256:cd5f3ac681c77791c6a8eaa62de876ad2be043ee5a428afb7c0095aa08246277
DRAGONFLY_IMAGE=docker.dragonflydb.io/dragonflydb/dragonfly@sha256:665ee58607d035881384be45b20025490ecf87069418f61f715c19117ed5fb7d
KEYCLOAK_IMAGE=quay.io/keycloak/keycloak@sha256:1362a9d9f13ab325231ea133610cc905e12805804abc7acbef552dd613720aa6
ZITADEL_IMAGE=ghcr.io/zitadel/zitadel@sha256:d59554f17d20c06c9f6ed114bf9f2c3a9a63698003daea62d27bdae1e9667583
GARAGE_IMAGE=dxflrs/garage@sha256:dac0c92add4f1a0b41035e94b41036a270ffbe88a37c7ac9c3f19e6dc5bdccf2
VERSITYGW_IMAGE=versity/versitygw@sha256:4a2c613f6541410f87983c5134cd26a88dfc23f0ad555b7aa51d3d87fd2d347b
COREDNS_IMAGE=coredns/coredns@sha256:1ba6f47265602e2e50a9c4669e3a955e4298a0d30dc82f39293d4bf1a851e0ff
CURL_IMAGE=curlimages/curl@sha256:1ab04d023ece37e6ec991bf3306ad04e0ef0084e94a5c6b6563cfcb9563169db
AWS_CLI_IMAGE=amazon/aws-cli@sha256:10c3a6dff0fdcb3fbc86cd744709548ad041ebb6bbe14fbedc1595a5539d8d27