ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Establish host properties and isolated kind matrix

Browse source
This commit is contained in:
ms2k 2026-07-18 00:54:47 +09:00
commit 06d1383456
5 changed files with 1076 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

16
tests/kind/dependencies.yaml.in
View file

@ -19,7 +19,7 @@ metadata:
name: postgres
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@POSTGRES_REPLICAS@@
selector:
matchLabels: {app: postgres}
template:
@ -70,7 +70,7 @@ metadata:
name: redis
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@REDIS_REPLICAS@@
selector:
matchLabels: {app: redis}
template:
@ -110,7 +110,7 @@ metadata:
name: dragonfly
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@DRAGONFLY_REPLICAS@@
selector:
matchLabels: {app: dragonfly}
template:
@ -149,7 +149,7 @@ metadata:
name: keycloak
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@KEYCLOAK_REPLICAS@@
selector:
matchLabels: {app: keycloak}
template:
@ -198,7 +198,7 @@ metadata:
name: zitadel
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@ZITADEL_REPLICAS@@
selector:
matchLabels: {app: zitadel}
template:
@ -274,7 +274,7 @@ metadata:
name: garage
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@GARAGE_REPLICAS@@
selector:
matchLabels: {app: garage}
template:
@ -340,7 +340,7 @@ metadata:
name: versitygw
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@VERSITYGW_REPLICAS@@
selector:
matchLabels: {app: versitygw}
template:
@ -421,7 +421,7 @@ metadata:
name: controlled-dns
namespace: nuthatch-testbed
spec:
replicas: 1
replicas: @@COREDNS_REPLICAS@@
selector:
matchLabels: {app: controlled-dns}
template:

147
tests/kind/lib-test.sh
View file

@ -4,11 +4,156 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
assert_matrix() {
local expected="$1"
shift
local actual
actual="$(expand_fixture_matrix "$@")"
if [[ "$actual" != "$expected" ]]; then
printf 'unexpected matrix for %s\nexpected:\n%s\nactual:\n%s\n' \
"$*" "$expected" "$actual" >&2
return 1
fi
}
assert_matrix $'001\t-\t-\t-' durable
assert_matrix $'001\tredis\t-\t-\n002\tdragonfly\t-\t-' coordination
assert_matrix $'001\t-\tkeycloak\tgarage\n002\t-\tkeycloak\tversitygw\n003\t-\tzitadel\tgarage\n004\t-\tzitadel\tversitygw' oidc target
assert_matrix $'001\tredis\tkeycloak\tgarage\n002\tredis\tkeycloak\tversitygw\n003\tredis\tzitadel\tgarage\n004\tredis\tzitadel\tversitygw\n005\tdragonfly\tkeycloak\tgarage\n006\tdragonfly\tkeycloak\tversitygw\n007\tdragonfly\tzitadel\tgarage\n008\tdragonfly\tzitadel\tversitygw' all
if expand_fixture_matrix redis >/dev/null 2>&1; then
printf 'individual provider selector was accepted\n' >&2
exit 1
fi
select_fixture_families target durable target
if [[ "${FIXTURE_FAMILIES[*]}" != "durable target" ]]; then
printf 'families were not normalized: %s\n' "${FIXTURE_FAMILIES[*]}" >&2
exit 1
fi
if kube get namespaces >/dev/null 2>&1; then
printf 'kube helper unexpectedly reached a cluster without fixture setup\n' >&2
exit 1
fi
kube() {
return 1
}
select_fixture_families durable
if write_lane_profile 001 - - - >/dev/null 2>&1; then
printf 'lane provisioning failure was ignored\n' >&2
exit 1
fi
select_fixture_families base
if write_lane_profile 001 - - - >/dev/null 2>&1; then
printf 'ConfigMap provisioning failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
kube_calls=0
kube() {
kube_calls=$((kube_calls + 1))
case "$kube_calls" in
1) return 1 ;;
2) return 0 ;;
3) return 1 ;;
*) return 0 ;;
esac
}
if reset_lane_target_store garage nuthatch-lane-001 >/dev/null 2>&1; then
printf 'target-store emptying failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
write_lane_profile() {
printf 'nuthatch-fixture-lane-001\n'
}
render_lane_job() {
return 0
}
run_job_manifest() {
return 42
}
cleanup_called=false
cleanup_lane_resources() {
cleanup_called=true
}
cleanup_lane_state() {
cleanup_called=true
}
if run_job_matrix fixture-consumer /dev/null nuthatch-test:local durable; then
printf 'Job failure was ignored\n' >&2
exit 1
fi
if [[ "$cleanup_called" = true ]]; then
printf 'failed lane was not retained for inspection\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
kube() {
return 1
}
if run_job_manifest fixture-consumer /dev/null >/dev/null 2>&1; then
printf 'Job cleanup failure was ignored\n' >&2
exit 1
fi
if cleanup_lane_resources fixture-consumer nuthatch-fixture-lane-001 >/dev/null 2>&1; then
printf 'lane resource cleanup failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
select_fixture_families all
cleanup_calls=()
cleanup_lane_durable_store() {
cleanup_calls+=(durable)
return 1
}
cleanup_lane_coordination_store() {
cleanup_calls+=(coordination)
}
cleanup_lane_target_store() {
cleanup_calls+=(target)
}
if cleanup_lane_state 001 redis keycloak garage >/dev/null 2>&1; then
printf 'partial lane cleanup failure was ignored\n' >&2
exit 1
fi
if [[ "${cleanup_calls[*]}" != "durable coordination target" ]]; then
printf 'lane cleanup stopped early: %s\n' "${cleanup_calls[*]}" >&2
exit 1
fi
if ! declare -F load_local_image >/dev/null; then
printf 'local application image loading is unavailable\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
configure_provider() {
return 0
}
delete_cluster() {
return 1
}
runtime_cleanup_called=false
remove_runtime_namespace() {
runtime_cleanup_called=true
}
if destroy_fixture >/dev/null 2>&1; then
printf 'fixture teardown failure was ignored\n' >&2
exit 1
fi
if [[ "$runtime_cleanup_called" != true ]]; then
printf 'fixture teardown stopped before runtime cleanup\n' >&2
exit 1
fi
FIXTURE_LOCK_DIR="$RUNTIME/lock-test"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
acquire_fixture_lock
@ -19,4 +164,4 @@ if XDG_RUNTIME_DIR="$FIXTURE_LOCK_DIR" bash -c \
fi
exec 9>&-
printf 'fixture isolation tests passed\n'
printf 'fixture family matrix tests passed\n'

453
tests/kind/lib.sh
View file

@ -75,7 +75,7 @@ acquire_fixture_lock() {
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)"
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
@ -105,10 +105,11 @@ delete_cluster() {
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")"
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
return
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
@ -169,9 +170,24 @@ load_external_image() {
printf '%s\n' "$image"
}
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
rm -f "$archive"
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
local keycloak_replicas="${14}" zitadel_replicas="${15}"
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
@ -182,6 +198,14 @@ render_manifests() {
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
@ -189,6 +213,18 @@ render_manifests() {
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
} > "$RUNTIME/fixture.env"
}
kube() {
@ -220,7 +256,83 @@ wait_for_job() {
return 1
}
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all) use_all=true ;;
base | durable | coordination | oidc | target) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target)
fi
for candidate in durable coordination oidc target; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then
coordination_providers=(redis dragonfly)
fi
if fixture_has_family oidc; then
oidc_providers=(keycloak zitadel)
fi
if fixture_has_family target; then
target_providers=(garage versitygw)
fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done
}
setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
@ -232,24 +344,74 @@ setup_fixture() {
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
local postgres=nuthatch-testbed/unused:postgres
local redis=nuthatch-testbed/unused:redis
local dragonfly=nuthatch-testbed/unused:dragonfly
local keycloak=nuthatch-testbed/unused:keycloak
local zitadel=nuthatch-testbed/unused:zitadel
local garage=nuthatch-testbed/unused:garage
local versitygw=nuthatch-testbed/unused:versitygw
local coredns=nuthatch-testbed/unused:coredns
local curl=nuthatch-testbed/unused:curl
local aws_cli=nuthatch-testbed/unused:aws-cli
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
local keycloak_replicas=0 zitadel_replicas=0
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
if fixture_has_family durable || fixture_has_family oidc; then
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
postgres_replicas=1
fi
if fixture_has_family coordination; then
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
redis_replicas=1
dragonfly_replicas=1
fi
if fixture_has_family oidc; then
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
keycloak_replicas=1
zitadel_replicas=1
coredns_replicas=1
fi
if fixture_has_family target; then
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
curl="$(load_external_image curl "$CURL_IMAGE")"
fi
garage_replicas=1
versitygw_replicas=1
fi
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli"
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
"$keycloak_replicas" "$zitadel_replicas" \
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
kube apply --filename "$RUNTIME/dependencies.yaml"
local deployment
for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
done
if fixture_has_family durable || fixture_has_family oidc; then
wait_for_deployment postgres
fi
if fixture_has_family coordination; then
wait_for_deployment redis
wait_for_deployment dragonfly
fi
if fixture_has_family oidc; then
wait_for_deployment keycloak
wait_for_deployment zitadel
wait_for_deployment controlled-dns
fi
if fixture_has_family target; then
wait_for_deployment garage
wait_for_deployment versitygw
fi
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}
verify_dependencies() {
@ -260,10 +422,253 @@ verify_dependencies() {
done
}
reset_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
reset_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
printf '%s\n' "$database"
}
target_endpoint() {
case "$1" in
garage) printf 'http://garage:3900\n' ;;
versitygw) printf 'http://versitygw:10000\n' ;;
*) return 1 ;;
esac
}
target_loopback_endpoint() {
case "$1" in
garage) printf 'http://127.0.0.1:3900\n' ;;
versitygw) printf 'http://127.0.0.1:10000\n' ;;
*) return 1 ;;
esac
}
reset_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >/dev/null || return
}
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile="nuthatch-fixture-lane-$lane"
local profile_file="$RUNTIME/$profile.env"
local database=
local coordination_database=
local oidc_issuer=
local target_url=
local target_bucket=
local target_region=
local target_access_key=
local target_secret_key=
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
fi
if fixture_has_family coordination; then
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
fi
if fixture_has_family oidc; then
case "$oidc" in
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
*) return 1 ;;
esac
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
reset_lane_target_store "$target" "$target_bucket" || return
target_url="$(target_endpoint "$target")" || return
case "$target" in
garage)
target_region=garage
target_access_key=GK00000000000000000000000000000000
target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
;;
versitygw)
target_region=us-east-1
target_access_key=synthetic-s3-access
target_secret_key=synthetic-s3-secret
;;
*) return 1 ;;
esac
fi
{
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
} > "$profile_file" || return
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
kube create configmap "$profile-file" \
--namespace "$FIXTURE_NAMESPACE" \
--from-file="profile.env=$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
printf '%s\n' "$profile"
}
render_lane_job() {
local job="$1"
local manifest="$2"
local image="$3"
local profile="$4"
source "$RUNTIME/fixture.env" || return
local coordination_image="$REDIS_LOCAL_IMAGE"
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
"$manifest" > "$RUNTIME/$job.yaml"
}
run_job_manifest() {
local job="$1"
local manifest="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
kube apply --filename "$manifest" >/dev/null || return
wait_for_job "$job"
}
cleanup_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >/dev/null
}
cleanup_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local target="$4"
local status=0
if fixture_has_family durable; then
cleanup_lane_durable_store "$lane" || status=1
fi
if fixture_has_family coordination; then
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
fi
if fixture_has_family target; then
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
fi
return "$status"
}
cleanup_lane_resources() {
local job="$1"
local profile="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
--ignore-not-found >/dev/null || return
rm -f "$RUNTIME/$profile.env" "$RUNTIME/$job.yaml"
}
run_job_matrix() {
local base_job="$1"
local manifest="$2"
local image="$3"
shift 3
local lane
local coordination
local oidc
local target
local profile
local job
select_fixture_families "$@" || return
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
job="$base_job-$lane"
render_lane_job "$job" "$manifest" "$image" "$profile" || return
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
cleanup_lane_resources "$job" "$profile" || return
done < <(expand_fixture_matrix "$@")
}
destroy_fixture() {
configure_provider
delete_cluster
remove_runtime_namespace
rm -rf "$RUNTIME/image-archives"
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
local status=0
configure_provider || return
delete_cluster || status=1
remove_runtime_namespace || status=1
rm -rf "$RUNTIME/image-archives" || status=1
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" || status=1
return "$status"
}

103
tests/kind/matrix-job.yaml.in Normal file
View file

@ -0,0 +1,103 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [nuthatch-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
initContainers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [psql "$NUTHATCH_PG_CONN" --set ON_ERROR_STOP=1 --command "CREATE TABLE lane_canary (value text NOT NULL)" --command "INSERT INTO lane_canary VALUES ('$NUTHATCH_FIXTURE_PROFILE')"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: coordination
image: @@COORDINATION_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [redis-cli -u "$NUTHATCH_REDIS_URL" SET lane-canary "$NUTHATCH_FIXTURE_PROFILE" >/dev/null && test "$(redis-cli -u "$NUTHATCH_REDIS_URL" GET lane-canary)" = "$NUTHATCH_FIXTURE_PROFILE"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: oidc
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
discovery="$(curl --fail --silent --show-error "$NUTHATCH_OIDC_ISSUER/.well-known/openid-configuration")"
printf '%s' "$discovery" | grep -F '"issuer"' >/dev/null
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: target
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
export AWS_ACCESS_KEY_ID="$NUTHATCH_S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$NUTHATCH_S3_SECRET_KEY"
export AWS_DEFAULT_REGION="$NUTHATCH_S3_REGION"
printf '%s\n' "$NUTHATCH_FIXTURE_PROFILE" >/tmp/input
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api put-object --bucket "$NUTHATCH_S3_BUCKET" --key lane-canary --body /tmp/input >/dev/null
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api get-object --bucket "$NUTHATCH_S3_BUCKET" --key lane-canary /tmp/output >/dev/null
test "$(cat /tmp/input)" = "$(cat /tmp/output)"
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
containers:
- name: profile
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
test -n "$NUTHATCH_FIXTURE_PROFILE"
test "$NUTHATCH_S3_BUCKET" = "nuthatch-lane-$NUTHATCH_FIXTURE_LANE"
case "$NUTHATCH_PG_CONN" in
*"dbname=nuthatch_lane_$NUTHATCH_FIXTURE_LANE") ;;
*) exit 1 ;;
esac
grep -Fx "NUTHATCH_FIXTURE_PROFILE=$NUTHATCH_FIXTURE_PROFILE" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_COORDINATION_PROVIDER=$NUTHATCH_COORDINATION_PROVIDER" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_OIDC_PROVIDER=$NUTHATCH_OIDC_PROVIDER" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_TARGET_PROVIDER=$NUTHATCH_TARGET_PROVIDER" /run/nuthatch-fixture/profile.env
printf 'lane profile injection passed\n'
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
volumeMounts:
- {name: fixture-profile, mountPath: /run/nuthatch-fixture, readOnly: true}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture-profile
configMap:
name: @@PROFILE_NAME@@-file
items:
- {key: profile.env, path: profile.env}

43
tests/kind/matrix-run.sh Executable file
View file

@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture all
source "$RUNTIME/fixture.env"
run_job_matrix \
fixture-matrix \
"$TESTBED/matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
all
SUCCEEDED=true
printf 'Nuthatch fixture matrix passed\n'