ms2k
957d170150
Introduce remora-durable with pool/migrations, a RemoraApiMutation-shaped ledger, claim-fenced outbox projection, CAS helpers, and keyset cursors, plus a kind Job acceptance suite against the hermetic fixture.
321 lines
10 KiB
Shell
Executable file
321 lines
10 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
TESTBED="$ROOT/tests/kind"
|
|
RUNTIME="$TESTBED/.runtime"
|
|
BIN="$RUNTIME/bin"
|
|
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
|
|
CONTEXT=kind-remora-testbed
|
|
CLUSTER=remora-testbed
|
|
NERDCTL_NAMESPACE=remora-wp02
|
|
FIXTURE_NAMESPACE=remora-testbed
|
|
|
|
set -a
|
|
source "$TESTBED/versions.env"
|
|
set +a
|
|
|
|
user_kubeconfig_digest() {
|
|
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
|
|
local path
|
|
local paths
|
|
IFS=: read -r -a paths <<< "$config_value"
|
|
{
|
|
for path in "${paths[@]}"; do
|
|
if [[ -f "$path" ]]; then
|
|
sha256sum "$path"
|
|
else
|
|
printf 'missing %s\n' "$path"
|
|
fi
|
|
done
|
|
} | sha256sum | cut -d' ' -f1
|
|
}
|
|
|
|
download_tool() {
|
|
local destination="$1"
|
|
local checksum="$2"
|
|
local url="$3"
|
|
if [[ -x "$destination" ]] && [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
|
|
return
|
|
fi
|
|
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
|
|
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
|
|
chmod 0755 "$destination.tmp"
|
|
mv "$destination.tmp" "$destination"
|
|
}
|
|
|
|
prepare_tools() {
|
|
mkdir -p "$BIN"
|
|
download_tool \
|
|
"$BIN/kind" \
|
|
"$KIND_SHA256" \
|
|
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
|
|
download_tool \
|
|
"$BIN/kubectl" \
|
|
"$KUBECTL_SHA256" \
|
|
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
|
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
|
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
|
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
|
|
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
|
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
|
}
|
|
|
|
prepare_provider() {
|
|
configure_provider
|
|
delete_cluster
|
|
remove_runtime_namespace
|
|
"$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
|
}
|
|
|
|
acquire_fixture_lock() {
|
|
mkdir -p "$RUNTIME"
|
|
exec 9>"$RUNTIME/fixture.lock"
|
|
if ! flock --nonblock 9; then
|
|
printf 'kind test fixture is already in use\n' >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
configure_provider() {
|
|
if [[ -z "${REMORA_REAL_NERDCTL:-}" ]]; then
|
|
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
|
fi
|
|
export REMORA_REAL_NERDCTL
|
|
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
|
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
|
export PATH="$TESTBED/provider:$PATH"
|
|
}
|
|
|
|
remove_runtime_namespace() {
|
|
local status=0
|
|
local digests
|
|
local digest
|
|
local leases
|
|
local lease
|
|
if ! runtime_namespace_exists; then
|
|
return
|
|
fi
|
|
"$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
|
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
|
|
if [[ -n "$leases" ]]; then
|
|
while read -r lease; do
|
|
runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
|
|
done <<< "$leases"
|
|
fi
|
|
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
|
|
if [[ -n "$digests" ]]; then
|
|
while read -r digest; do
|
|
runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
|
|
done <<< "$digests"
|
|
fi
|
|
"$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
|
if runtime_namespace_exists; then
|
|
printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
|
|
status=1
|
|
fi
|
|
return "$status"
|
|
}
|
|
|
|
runtime_ctr() {
|
|
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
|
|
}
|
|
|
|
runtime_namespace_exists() {
|
|
"$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
|
}
|
|
|
|
cluster_exists() {
|
|
if ! runtime_namespace_exists; then
|
|
return 1
|
|
fi
|
|
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
|
|
}
|
|
|
|
delete_cluster() {
|
|
if cluster_exists; then
|
|
"$BIN/kind" delete cluster --name "$CLUSTER"
|
|
fi
|
|
if cluster_exists; then
|
|
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
load_external_image() {
|
|
local name="$1"
|
|
local source="$2"
|
|
local digest="${source##*@}"
|
|
local image="remora-testbed/$name:sha256-${digest#sha256:}"
|
|
local archive="$RUNTIME/image-archives/$digest.tar"
|
|
"$REMORA_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2 || return 1
|
|
"$REMORA_REAL_NERDCTL" image tag "$source" "$image" || return 1
|
|
"$REMORA_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image" || return 1
|
|
printf 'Loading %s from %s\n' "$image" "$source" >&2
|
|
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2 || return 1
|
|
printf '%s\n' "$image"
|
|
}
|
|
|
|
load_local_image() {
|
|
local image="$1"
|
|
local namespace="${2:-default}"
|
|
local archive="$RUNTIME/image-archives/package-test.tar"
|
|
rm -f "$archive"
|
|
"$REMORA_REAL_NERDCTL" --namespace "$namespace" image save \
|
|
--platform linux/amd64 \
|
|
--output "$archive" \
|
|
"$image"
|
|
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
|
|
}
|
|
|
|
render_manifests() {
|
|
local postgres_image="$1"
|
|
local redis_image="$2"
|
|
local dragonfly_image="$3"
|
|
local vault_image="$4"
|
|
local keycloak_image="$5"
|
|
local zitadel_image="$6"
|
|
local garage_image="$7"
|
|
local versitygw_image="$8"
|
|
local coredns_image="$9"
|
|
local curl_image="${10}"
|
|
local aws_cli_image="${11}"
|
|
sed \
|
|
-e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
|
|
-e "s|@@REDIS_IMAGE@@|$redis_image|g" \
|
|
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly_image|g" \
|
|
-e "s|@@VAULT_IMAGE@@|$vault_image|g" \
|
|
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak_image|g" \
|
|
-e "s|@@ZITADEL_IMAGE@@|$zitadel_image|g" \
|
|
-e "s|@@GARAGE_IMAGE@@|$garage_image|g" \
|
|
-e "s|@@VERSITYGW_IMAGE@@|$versitygw_image|g" \
|
|
-e "s|@@COREDNS_IMAGE@@|$coredns_image|g" \
|
|
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
|
|
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
|
|
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
|
|
sed \
|
|
-e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
|
|
-e "s|@@REDIS_IMAGE@@|$redis_image|g" \
|
|
-e "s|@@VAULT_IMAGE@@|$vault_image|g" \
|
|
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
|
|
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
|
|
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
|
{
|
|
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres_image"
|
|
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis_image"
|
|
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly_image"
|
|
printf 'VAULT_LOCAL_IMAGE=%q\n' "$vault_image"
|
|
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak_image"
|
|
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel_image"
|
|
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage_image"
|
|
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw_image"
|
|
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns_image"
|
|
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl_image"
|
|
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli_image"
|
|
} > "$RUNTIME/fixture.env"
|
|
}
|
|
|
|
setup_fixture() {
|
|
mkdir -p "$RUNTIME"
|
|
rm -rf "$RUNTIME/image-archives"
|
|
mkdir -p "$RUNTIME/image-archives"
|
|
prepare_tools
|
|
prepare_provider
|
|
|
|
"$BIN/kind" create cluster \
|
|
--name "$CLUSTER" \
|
|
--image "$NODE_IMAGE" \
|
|
--config "$TESTBED/cluster.yaml" \
|
|
--kubeconfig "$KUBECONFIG_PATH" \
|
|
--wait 180s
|
|
|
|
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
|
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
|
|
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
|
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
|
|
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
|
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
|
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
|
|
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
|
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
|
|
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
|
|
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
|
render_manifests \
|
|
"$POSTGRES_LOCAL_IMAGE" \
|
|
"$REDIS_LOCAL_IMAGE" \
|
|
"$DRAGONFLY_LOCAL_IMAGE" \
|
|
"$VAULT_LOCAL_IMAGE" \
|
|
"$KEYCLOAK_LOCAL_IMAGE" \
|
|
"$ZITADEL_LOCAL_IMAGE" \
|
|
"$GARAGE_LOCAL_IMAGE" \
|
|
"$VERSITYGW_LOCAL_IMAGE" \
|
|
"$COREDNS_LOCAL_IMAGE" \
|
|
"$CURL_LOCAL_IMAGE" \
|
|
"$AWS_CLI_LOCAL_IMAGE"
|
|
|
|
kube apply --filename "$RUNTIME/dependencies.yaml"
|
|
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
|
|
done
|
|
}
|
|
|
|
verify_dependencies() {
|
|
kube apply --filename "$RUNTIME/probe.yaml"
|
|
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
|
wait_for_job "$FIXTURE_NAMESPACE" "$job"
|
|
done
|
|
}
|
|
|
|
destroy_fixture() {
|
|
configure_provider
|
|
delete_cluster
|
|
remove_runtime_namespace
|
|
rm -rf "$RUNTIME/image-archives"
|
|
rm -f \
|
|
"$KUBECONFIG_PATH" \
|
|
"$RUNTIME/dependencies.yaml" \
|
|
"$RUNTIME/fixture.env" \
|
|
"$RUNTIME/probe.yaml" \
|
|
"$RUNTIME/sample-job.yaml" \
|
|
"$RUNTIME/durable-job.yaml"
|
|
}
|
|
|
|
kube() {
|
|
"$BIN/kubectl" \
|
|
--kubeconfig "$KUBECONFIG_PATH" \
|
|
--context "$CONTEXT" \
|
|
"$@"
|
|
}
|
|
|
|
wait_for_job() {
|
|
local namespace="$1"
|
|
local job="$2"
|
|
local deadline=$((SECONDS + 300))
|
|
local complete
|
|
local failed
|
|
while ((SECONDS < deadline)); do
|
|
complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
|
if [[ "$complete" = True ]]; then
|
|
kube --namespace "$namespace" logs "job/$job" --all-containers
|
|
return
|
|
fi
|
|
failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
|
if [[ "$failed" = True ]]; then
|
|
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
|
|
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
|
return 1
|
|
}
|
|
|
|
run_job_manifest() {
|
|
local namespace="$1"
|
|
local job="$2"
|
|
local manifest="$3"
|
|
kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true
|
|
kube --namespace "$namespace" apply --filename "$manifest"
|
|
wait_for_job "$namespace" "$job"
|
|
}
|