ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Establish host properties and isolated kind matrix

Browse source
This commit is contained in:
ms2k 2026-07-18 00:54:47 +09:00
commit 06d1383456
1 changed files with 1076 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

453
tests/kind/lib.sh
View file

@ -75,7 +75,7 @@ acquire_fixture_lock() {
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)"
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
@ -105,10 +105,11 @@ delete_cluster() {
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")"
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
return
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
@ -169,9 +170,24 @@ load_external_image() {
printf '%s\n' "$image"
}
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
rm -f "$archive"
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
local keycloak_replicas="${14}" zitadel_replicas="${15}"
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
@ -182,6 +198,14 @@ render_manifests() {
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
@ -189,6 +213,18 @@ render_manifests() {
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
} > "$RUNTIME/fixture.env"
}
kube() {
@ -220,7 +256,83 @@ wait_for_job() {
return 1
}
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all) use_all=true ;;
base | durable | coordination | oidc | target) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target)
fi
for candidate in durable coordination oidc target; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then
coordination_providers=(redis dragonfly)
fi
if fixture_has_family oidc; then
oidc_providers=(keycloak zitadel)
fi
if fixture_has_family target; then
target_providers=(garage versitygw)
fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done
}
setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
@ -232,24 +344,74 @@ setup_fixture() {
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
local postgres=nuthatch-testbed/unused:postgres
local redis=nuthatch-testbed/unused:redis
local dragonfly=nuthatch-testbed/unused:dragonfly
local keycloak=nuthatch-testbed/unused:keycloak
local zitadel=nuthatch-testbed/unused:zitadel
local garage=nuthatch-testbed/unused:garage
local versitygw=nuthatch-testbed/unused:versitygw
local coredns=nuthatch-testbed/unused:coredns
local curl=nuthatch-testbed/unused:curl
local aws_cli=nuthatch-testbed/unused:aws-cli
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
local keycloak_replicas=0 zitadel_replicas=0
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
if fixture_has_family durable || fixture_has_family oidc; then
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
postgres_replicas=1
fi
if fixture_has_family coordination; then
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
redis_replicas=1
dragonfly_replicas=1
fi
if fixture_has_family oidc; then
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
keycloak_replicas=1
zitadel_replicas=1
coredns_replicas=1
fi
if fixture_has_family target; then
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
curl="$(load_external_image curl "$CURL_IMAGE")"
fi
garage_replicas=1
versitygw_replicas=1
fi
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli"
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
"$keycloak_replicas" "$zitadel_replicas" \
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
kube apply --filename "$RUNTIME/dependencies.yaml"
local deployment
for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
done
if fixture_has_family durable || fixture_has_family oidc; then
wait_for_deployment postgres
fi
if fixture_has_family coordination; then
wait_for_deployment redis
wait_for_deployment dragonfly
fi
if fixture_has_family oidc; then
wait_for_deployment keycloak
wait_for_deployment zitadel
wait_for_deployment controlled-dns
fi
if fixture_has_family target; then
wait_for_deployment garage
wait_for_deployment versitygw
fi
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}
verify_dependencies() {
@ -260,10 +422,253 @@ verify_dependencies() {
done
}
reset_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
reset_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
printf '%s\n' "$database"
}
target_endpoint() {
case "$1" in
garage) printf 'http://garage:3900\n' ;;
versitygw) printf 'http://versitygw:10000\n' ;;
*) return 1 ;;
esac
}
target_loopback_endpoint() {
case "$1" in
garage) printf 'http://127.0.0.1:3900\n' ;;
versitygw) printf 'http://127.0.0.1:10000\n' ;;
*) return 1 ;;
esac
}
reset_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >/dev/null || return
}
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile="nuthatch-fixture-lane-$lane"
local profile_file="$RUNTIME/$profile.env"
local database=
local coordination_database=
local oidc_issuer=
local target_url=
local target_bucket=
local target_region=
local target_access_key=
local target_secret_key=
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
fi
if fixture_has_family coordination; then
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
fi
if fixture_has_family oidc; then
case "$oidc" in
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
*) return 1 ;;
esac
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
reset_lane_target_store "$target" "$target_bucket" || return
target_url="$(target_endpoint "$target")" || return
case "$target" in
garage)
target_region=garage
target_access_key=GK00000000000000000000000000000000
target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
;;
versitygw)
target_region=us-east-1
target_access_key=synthetic-s3-access
target_secret_key=synthetic-s3-secret
;;
*) return 1 ;;
esac
fi
{
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
} > "$profile_file" || return
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
kube create configmap "$profile-file" \
--namespace "$FIXTURE_NAMESPACE" \
--from-file="profile.env=$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
printf '%s\n' "$profile"
}
render_lane_job() {
local job="$1"
local manifest="$2"
local image="$3"
local profile="$4"
source "$RUNTIME/fixture.env" || return
local coordination_image="$REDIS_LOCAL_IMAGE"
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
"$manifest" > "$RUNTIME/$job.yaml"
}
run_job_manifest() {
local job="$1"
local manifest="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
kube apply --filename "$manifest" >/dev/null || return
wait_for_job "$job"
}
cleanup_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >/dev/null
}
cleanup_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local target="$4"
local status=0
if fixture_has_family durable; then
cleanup_lane_durable_store "$lane" || status=1
fi
if fixture_has_family coordination; then
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
fi
if fixture_has_family target; then
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
fi
return "$status"
}
cleanup_lane_resources() {
local job="$1"
local profile="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
--ignore-not-found >/dev/null || return
rm -f "$RUNTIME/$profile.env" "$RUNTIME/$job.yaml"
}
run_job_matrix() {
local base_job="$1"
local manifest="$2"
local image="$3"
shift 3
local lane
local coordination
local oidc
local target
local profile
local job
select_fixture_families "$@" || return
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
job="$base_job-$lane"
render_lane_job "$job" "$manifest" "$image" "$profile" || return
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
cleanup_lane_resources "$job" "$profile" || return
done < <(expand_fixture_matrix "$@")
}
destroy_fixture() {
configure_provider
delete_cluster
remove_runtime_namespace
rm -rf "$RUNTIME/image-archives"
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
local status=0
configure_provider || return
delete_cluster || status=1
remove_runtime_namespace || status=1
rm -rf "$RUNTIME/image-archives" || status=1
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" || status=1
return "$status"
}