ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Replace compose runtime with scoped kind deployment

Browse source
This commit is contained in:
ms2k 2026-07-18 16:16:24 +09:00
commit 0f888986d7
6 changed files with 457 additions and 146 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tests/kind/dependencies.yaml.in
View file

@ -257,7 +257,7 @@ data:
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "garage:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000"
[s3_api]
s3_region = "garage"
@ -353,10 +353,12 @@ spec:
- name: versitygw
image: @@VERSITYGW_IMAGE@@
imagePullPolicy: Never
args: [--port, ":10000", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
args: [--port, ":10000", --admin-port, "127.0.0.1:10001", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
env:
- {name: ROOT_ACCESS_KEY, value: synthetic-s3-access}
- {name: ROOT_SECRET_KEY, value: synthetic-s3-secret}
- {name: ADMIN_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: ADMIN_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
ports:
- {name: s3, containerPort: 10000}
resources:

27
tests/kind/lib-test.sh
View file

@ -53,18 +53,25 @@ if write_lane_profile 001 - - - >/dev/null 2>&1; then
fi
source "$ROOT/tests/kind/lib.sh"
kube_calls=0
kube() {
kube_calls=$((kube_calls + 1))
case "$kube_calls" in
1) return 1 ;;
2) return 0 ;;
3) return 1 ;;
*) return 0 ;;
esac
return 1
}
if reset_lane_target_store garage nuthatch-lane-001 >/dev/null 2>&1; then
printf 'target-store emptying failure was ignored\n' >&2
if ensure_target_scope_canary garage >/dev/null 2>&1; then
printf 'scope-canary provisioning failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
kube() {
return 1
}
if provision_lane_target_principal \
garage \
001 \
nuthatch-lane-001 \
access_key \
secret_key >/dev/null 2>&1; then
printf 'target-principal provisioning failure was ignored\n' >&2
exit 1
fi

87
tests/kind/lib.sh
View file

@ -461,18 +461,71 @@ target_loopback_endpoint() {
esac
}
reset_lane_target_store() {
ensure_target_scope_canary() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2; then
aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null || return
aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >/dev/null || return
sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
sh "$endpoint"
}
provision_lane_target_principal() {
local provider="$1"
local lane="$2"
local bucket="$3"
local access_key_var="$4"
local secret_key_var="$5"
local endpoint
local access_key
local secret_key
endpoint="$(target_loopback_endpoint "$provider")" || return
case "$provider" in
garage)
printf -v access_key 'GKWP05%028d' "$((10#$lane))"
printf -v secret_key '%064d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket create "$bucket" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow \
--read \
--write \
--owner \
"$bucket" \
--key GK00000000000000000000000000000000 >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
;;
versitygw)
access_key="nuthatch-wp05-$lane"
printf -v secret_key 'synthetic-wp05-%049d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-user \
--access "$access_key" \
--secret "$secret_key" \
--role user \
--user-id "$((10#$lane))" \
--group-id "$((10#$lane))" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-bucket \
--bucket "$bucket" \
--owner "$access_key" >/dev/null || return
;;
*) return 1 ;;
esac
printf -v "$access_key_var" '%s' "$access_key"
printf -v "$secret_key_var" '%s' "$secret_key"
}
write_lane_profile() {
@ -505,19 +558,17 @@ write_lane_profile() {
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
reset_lane_target_store "$target" "$target_bucket" || return
target_url="$(target_endpoint "$target")" || return
ensure_target_scope_canary "$target" || return
provision_lane_target_principal \
"$target" \
"$lane" \
"$target_bucket" \
target_access_key \
target_secret_key || return
case "$target" in
garage)
target_region=garage
target_access_key=GK00000000000000000000000000000000
target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
;;
versitygw)
target_region=us-east-1
target_access_key=synthetic-s3-access
target_secret_key=synthetic-s3-secret
;;
garage) target_region=garage ;;
versitygw) target_region=us-east-1 ;;
*) return 1 ;;
esac
fi
@ -669,6 +720,8 @@ destroy_fixture() {
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" || status=1
"$RUNTIME/fixture.env" \
"$RUNTIME"/nuthatch-fixture-lane-*.env \
"$RUNTIME"/wp05-app-*.yaml || status=1
return "$status"
}

1
tests/kind/wp04-job.yaml.in
View file

@ -21,6 +21,7 @@ spec:
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_JWT_SECRET, value: synthetic-wp04-jwt-secret}
- {name: NUTHATCH_REDIS_URL, value: "redis://127.0.0.1:6379/0"}
resources:
requests: {cpu: 100m, memory: 32Mi}

135
tests/kind/wp05-job.yaml.in Normal file
View file

@ -0,0 +1,135 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: credential-negative
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
export AWS_ACCESS_KEY_ID="$NUTHATCH_S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$NUTHATCH_S3_SECRET_KEY"
export AWS_DEFAULT_REGION="$NUTHATCH_S3_REGION"
printf denied-write >/tmp/denied-write
expect_denied() {
name="$1"
shift
if "$@" >/tmp/"$name".out 2>/tmp/"$name".err; then
exit 1
fi
grep -E 'AccessDenied|Forbidden|403' /tmp/"$name".err
}
expect_denied head \
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api head-bucket \
--bucket nuthatch-fixture
expect_denied list \
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api list-objects-v2 \
--bucket nuthatch-fixture
expect_denied get \
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api get-object \
--bucket nuthatch-fixture --key scope-canary /tmp/denied-read
expect_denied put \
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api put-object \
--bucket nuthatch-fixture --key denied-write --body /tmp/denied-write
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
containers:
- name: smoke
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
base="http://nuthatch-@@LANE@@:8080"
curl --fail --silent --show-error "$base/healthz" | grep -F '"ok":true'
curl --fail --silent --show-error "$base/readyz" | grep -F '"ready":true'
curl --fail --silent --show-error \
--cookie-jar /tmp/cookies \
--dump-header /tmp/login-headers \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--data '{"username":"wp05-smoke"}' \
"$base/login" >/tmp/login.json
csrf="$(awk 'BEGIN{IGNORECASE=1} $1=="X-CSRF-Token:"{gsub("\r","",$2); print $2}' /tmp/login-headers)"
test -n "$csrf"
exercise_root() {
root="$1"
suffix="$2"
first="first-$root"
second="second-$root"
printf '%s' "$first" >/tmp/body
size="$(wc -c </tmp/body | tr -d ' ')"
digest="$(sha256sum /tmp/body | cut -d' ' -f1)"
curl --fail --silent --show-error \
--cookie /tmp/cookies \
--header 'Content-Type: application/octet-stream' \
--header "X-Declared-Size: $size" \
--header "X-Content-SHA256: $digest" \
--header "X-CSRF-Token: $csrf" \
--data-binary @/tmp/body \
"$base/upload?root=$root&path=smoke-$suffix.txt&operation_id=00000000-0000-4000-8000-0000000000${suffix}" |
grep -F '"result":"succeeded"'
curl --fail --silent --show-error \
--cookie /tmp/cookies \
--header 'Accept: application/json' \
"$base/?root=$root" |
grep -F "smoke-$suffix.txt"
curl --fail --silent --show-error \
--cookie /tmp/cookies \
"$base/download?root=$root&path=smoke-$suffix.txt" >/tmp/download
test "$(cat /tmp/download)" = "$first"
printf '%s' "$second" >/tmp/body
size="$(wc -c </tmp/body | tr -d ' ')"
digest="$(sha256sum /tmp/body | cut -d' ' -f1)"
curl --fail --silent --show-error \
--cookie /tmp/cookies \
--header 'Content-Type: application/octet-stream' \
--header "X-Declared-Size: $size" \
--header "X-Content-SHA256: $digest" \
--header "X-CSRF-Token: $csrf" \
--data-binary @/tmp/body \
"$base/upload?root=$root&path=smoke-$suffix.txt&operation_id=10000000-0000-4000-8000-0000000000${suffix}" |
grep -F '"result":"succeeded"'
curl --fail --silent --show-error \
--cookie /tmp/cookies \
"$base/download?root=$root&path=smoke-$suffix.txt" >/tmp/download
test "$(cat /tmp/download)" = "$second"
curl --fail --silent --show-error \
--cookie /tmp/cookies \
--header 'Content-Type: application/json' \
--data "{\"root\":\"$root\",\"path\":\"smoke-$suffix.txt\",\"operation_id\":\"20000000-0000-4000-8000-0000000000${suffix}\",\"csrf\":\"$csrf\"}" \
"$base/delete" |
grep -F '"result":"succeeded"'
if curl --fail --silent \
--cookie /tmp/cookies \
"$base/download?root=$root&path=smoke-$suffix.txt" >/dev/null; then
exit 1
fi
}
exercise_root personal 01
exercise_root shared 02
printf 'WP-05 app smoke passed\n'
resources:
requests: {cpu: 50m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}

115
tests/kind/wp05-run.sh Executable file
View file

@ -0,0 +1,115 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-wp05:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/wp05-app-$lane.yaml"
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/wp05-app-$lane.yaml"
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/wp05-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
if [[ "${NUTHATCH_APP_INTERACTIVE:-0}" = 1 ]]; then
SUCCEEDED=true
OWNS_FIXTURE=false
printf 'interactive kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'interactive context: %s\n' "$CONTEXT"
printf 'interactive namespace: %s\n' "$FIXTURE_NAMESPACE"
printf 'interactive service: nuthatch-%s:8080\n' "$lane"
exit 0
fi
job="wp05-app-$lane"
render_lane_job \
"$job" \
"$TESTBED/wp05-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
printf 'Running lane %s: coordination=%s target=%s\n' "$lane" "$coordination" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
done < <(expand_fixture_matrix coordination target)
SUCCEEDED=true
printf 'Nuthatch WP-05 app gate passed\n'