ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Establish host properties and isolated kind matrix

Browse source
This commit is contained in:
ms2k 2026-07-18 00:54:47 +09:00
commit 06d1383456
11 changed files with 1076 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Makefile
View file

@ -1,13 +1,19 @@
.PHONY: check-dependencies check-fixture-lib check-frontend build-frontend mvp-up mvp-down mvp-build mvp-logs mvp-ps .PHONY: test check-dependencies check-fixture-lib check-kind-matrix check-frontend build-frontend mvp-up mvp-down mvp-build mvp-logs mvp-ps
COMPOSE := nerdctl compose -f deploy/docker-compose.yml COMPOSE := nerdctl compose -f deploy/docker-compose.yml
test:
cabal test --offline nuthatch-unit
check-dependencies: check-dependencies:
./tests/kind/run.sh ./tests/kind/run.sh
check-fixture-lib: check-fixture-lib:
./tests/kind/lib-test.sh ./tests/kind/lib-test.sh
check-kind-matrix:
./tests/kind/matrix-run.sh
check-frontend: check-frontend:
npm --prefix web run check npm --prefix web run check

40
docs/spec/testing.md
View file

@ -1,4 +1,19 @@
# Dependency testing # Testing
## Host unit tests
The recorded host toolchain is GHC 9.10.3, cabal-install 3.16.1.0, and libpq
18beta1 or a later compatible libpq release. `make test` runs the
`nuthatch-unit` Tasty suite entirely on the host. It requires no network access
and no PostgreSQL, Redis-compatible, OIDC, S3-compatible, container, or
Kubernetes service.
The suite uses Tasty as its single explicit test tree, Tasty HUnit for named
examples, and Hedgehog through Tasty Hedgehog for generated properties.
Hedgehog failures include the seed needed to replay the shrunk counterexample.
Generators represent valid and invalid domains with distinct types.
## Kind dependency fixture
Backend integration tests execute inside an isolated kind cluster so they use Backend integration tests execute inside an isolated kind cluster so they use
the same DNS and service routing boundaries as deployed Nuthatch components. the same DNS and service routing boundaries as deployed Nuthatch components.
@ -8,9 +23,12 @@ digest-pinned.
`make check-dependencies` creates the complete fixture, verifies each released `make check-dependencies` creates the complete fixture, verifies each released
dependency image through its product interface, and destroys the fixture. dependency image through its product interface, and destroys the fixture.
`tests/kind/setup.sh` creates the same ready fixture for interactive test work; `make check-kind-matrix` runs a fixture-consumer Job sequentially across the
`tests/kind/destroy.sh` removes it. `NUTHATCH_TESTBED_RETAIN=1` retains a failed Redis/DragonflyDB × Keycloak/ZITADEL × Garage/VersityGW matrix without
all-in-one run for inspection. deploying Nuthatch. `tests/kind/setup.sh` creates the same ready fixture for
interactive test work; `tests/kind/destroy.sh` removes it.
`NUTHATCH_TESTBED_RETAIN=1` retains a failed all-in-one or matrix run for
inspection.
The fixture uses `tests/kind/.runtime/kubeconfig` and the The fixture uses `tests/kind/.runtime/kubeconfig` and the
`kind-nuthatch-testbed` context. Every Kubernetes operation supplies both `kind-nuthatch-testbed` context. Every Kubernetes operation supplies both
@ -21,6 +39,20 @@ All fixture credentials and data are synthetic. Test workloads use ClusterIP
service names and do not require host port forwarding. The fixture does not service names and do not require host port forwarding. The fixture does not
read or change the user's Kubernetes context. read or change the user's Kubernetes context.
Package runners select the `base`, `durable`, `coordination`, `oidc`, and
`target` fixture families; `all` selects every dependency family. The selected
families expand into provider lanes, which run sequentially and stop at the
first failure. Each lane receives an isolated PostgreSQL database,
Redis-compatible logical database, and target bucket as applicable.
Lane configuration is available both as environment variables from a
lane-specific ConfigMap and as `/run/nuthatch-fixture/profile.env`. Successful
lanes delete their Job, profile, database, coordination state, and bucket
before the next lane. Failed lanes remain intact when the enclosing runner is
retained. Package-owned application or test images are loaded with
`load_local_image`; released dependencies always use the digest-pinned images
from `tests/kind/versions.env`.
The host provides rootless nerdctl and containerd at the versions recorded in The host provides rootless nerdctl and containerd at the versions recorded in
`tests/kind/versions.env`, together with `ctr`, `curl`, `flock`, `jq`, and `tests/kind/versions.env`, together with `ctr`, `curl`, `flock`, `jq`, and
`sha256sum`. The fixture downloads its pinned kind and kubectl binaries. `sha256sum`. The fixture downloads its pinned kind and kubectl binaries.

5
nuthatch.cabal
View file

@ -64,6 +64,7 @@ common shared
, text >= 2.0 && < 2.2 , text >= 2.0 && < 2.2
, time >= 1.12 && < 1.15 , time >= 1.12 && < 1.15
, transformers >= 0.6 && < 0.7 , transformers >= 0.6 && < 0.7
, unicode-transforms >= 0.4 && < 0.5
, unordered-containers >= 0.2 && < 0.3 , unordered-containers >= 0.2 && < 0.3
, uuid >= 1.3 && < 1.4 , uuid >= 1.3 && < 1.4
, wai >= 3.2 && < 3.3 , wai >= 3.2 && < 3.3
@ -109,6 +110,10 @@ test-suite nuthatch-unit
, nuthatch , nuthatch
, base , base
, bytestring , bytestring
, hedgehog >= 1.7 && < 1.8
, tasty >= 1.5 && < 1.6
, tasty-hedgehog >= 1.4 && < 1.5
, tasty-hunit >= 0.10 && < 0.11
, text , text
ghc-options: ghc-options:
-threaded -threaded

12
src/Nuthatch/Paths.hs
View file

@ -14,17 +14,21 @@ import Data.Char (isControl)
import Data.Text (Text) import Data.Text (Text)
import qualified Data.Text as T import qualified Data.Text as T
import qualified Data.Text.Encoding as TE import qualified Data.Text.Encoding as TE
import qualified Data.Text.Normalize as TN
import Nuthatch.Types import Nuthatch.Types
validateRelativePath :: Text -> Either Text RelativePath validateRelativePath :: Text -> Either Text RelativePath
validateRelativePath raw = do validateRelativePath raw = do
let path = T.dropWhile (== '/') raw if T.null raw
if T.null path
then Left "path must not be empty" then Left "path must not be empty"
else if T.head raw == '/' || T.last raw == '/'
then Left "path must not begin or end with a separator"
else if TN.normalize TN.NFC raw /= raw
then Left "path must be NFC-normalized"
else do else do
let segments = T.splitOn "/" path let segments = T.splitOn "/" raw
mapM_ validateSegment segments mapM_ validateSegment segments
pure (RelativePath path) pure (RelativePath raw)
where where
validateSegment seg validateSegment seg
| T.null seg = Left "empty path segment" | T.null seg = Left "empty path segment"

27
src/Nuthatch/Protocol.hs
View file

@ -13,6 +13,11 @@ module Nuthatch.Protocol
, claimActiveBatch , claimActiveBatch
, lookupMutation , lookupMutation
, lookupObjectMeta , lookupObjectMeta
, sameFingerprint
, growthBytes
, quotaAllows
, terminalExpiryEligible
, recoveryEligible
, desiredMatches , desiredMatches
, priorMatches , priorMatches
) where ) where
@ -105,6 +110,21 @@ sameFingerprint rec kind key size msha =
&& mrDeclaredSize rec == size && mrDeclaredSize rec == size
&& mrContentSha256 rec == msha && mrContentSha256 rec == msha
growthBytes :: Integer -> Integer -> Integer
growthBytes priorSize desiredSize = max 0 (desiredSize - priorSize)
quotaAllows :: Integer -> Integer -> Integer -> Integer -> Bool
quotaAllows limitBytes usedBytes reservedBytes growth =
usedBytes + reservedBytes + growth <= limitBytes
terminalExpiryEligible :: MutationState -> ExternalState -> Bool
terminalExpiryEligible state external =
state `elem` [Succeeded, Failed, Rejected] && external == ExtIdle
recoveryEligible :: MutationState -> Integer -> Integer -> Bool
recoveryEligible state elapsedSeconds graceSeconds =
state `elem` [Prepared, Applying] && elapsedSeconds > graceSeconds
admitMutation admitMutation
:: Connection :: Connection
-> UserId -> UserId
@ -131,7 +151,7 @@ admitMutation conn uid kind op pool key rel size msha maxObject = do
desiredSize = case kind of desiredSize = case kind of
Put -> size Put -> size
Delete -> 0 Delete -> 0
growth = max 0 (desiredSize - priorSize) growth = growthBytes priorSize desiredSize
baseRec state = baseRec state =
MutationRecord MutationRecord
{ mrUserId = uid { mrUserId = uid
@ -416,13 +436,12 @@ desiredMatches rec = \case
Put -> False Put -> False
ObjectPresent sz msha -> ObjectPresent sz msha ->
case mrKind rec of case mrKind rec of
Delete -> sz == 0 Delete -> False
Put -> Put ->
sz == mrDeclaredSize rec sz == mrDeclaredSize rec
&& case (mrContentSha256 rec, msha) of && case (mrContentSha256 rec, msha) of
(Just want, Just got) -> want == got (Just want, Just got) -> want == got
-- Some stores omit custom metadata on HEAD. (Just _, Nothing) -> False
(Just _, Nothing) -> True
(Nothing, _) -> True (Nothing, _) -> True
priorMatches :: MutationRecord -> ObjectObservation -> Bool priorMatches :: MutationRecord -> ObjectObservation -> Bool

332
test/Spec.hs
View file

@ -1,9 +1,18 @@
{-# LANGUAGE LambdaCase #-}
module Main (main) where module Main (main) where
import Data.Either (isLeft)
import Data.Text (Text) import Data.Text (Text)
import qualified Data.Text as T import qualified Data.Text as T
import Hedgehog
( Gen
, Property
, assert
, forAll
, property
, (===)
)
import qualified Hedgehog.Gen as Gen
import qualified Hedgehog.Range as Range
import Nuthatch.Paths import Nuthatch.Paths
( buildObjectKey ( buildObjectKey
, encodeUserId , encodeUserId
@ -14,88 +23,273 @@ import Nuthatch.Protocol
( MutationRecord (..) ( MutationRecord (..)
, ObjectObservation (..) , ObjectObservation (..)
, desiredMatches , desiredMatches
, growthBytes
, priorMatches , priorMatches
, quotaAllows
, recoveryEligible
, sameFingerprint
, terminalExpiryEligible
) )
import Nuthatch.Types import Nuthatch.Types
import Test.Tasty (TestTree, defaultMain, testGroup)
import Test.Tasty.HUnit ((@?=), testCase)
import Test.Tasty.Hedgehog (testProperty)
main :: IO () main :: IO ()
main = do main = defaultMain tests
assertRight "simple path" (validateRelativePath "a/b.txt")
assertLeft "dotdot" (validateRelativePath "a/../b")
assertLeft "empty segment" (validateRelativePath "a//b")
assertLeft "slash only" (validateRelativePath "/")
let uid = encodeUserId "iss" "sub"
assertTrue "user id stable" (uid == encodeUserId "iss" "sub")
assertTrue "user id differs"
(encodeUserId "iss" "sub" /= encodeUserId "iss" "other")
let key = buildObjectKey Personal uid (RelativePath "docs/a.txt")
assertTrue "personal key prefix" $
T.isPrefixOf ("users/" <> unUserId uid <> "/") (unObjectKey key)
let shared = buildObjectKey Shared uid (RelativePath "docs/a.txt")
assertTrue "shared key" (unObjectKey shared == "shared/docs/a.txt")
assertTrue "list ancestors nested" $ tests :: TestTree
tests = testGroup "nuthatch-unit"
[ pathTests
, identityTests
, protocolTests
]
pathTests :: TestTree
pathTests = testGroup "canonical paths"
[ testProperty "valid paths parse and render without rewriting" propPathRoundTrip
, testProperty "invalid paths are rejected" propInvalidPaths
, testProperty "constructed object keys remain under their root" propRootConfinement
, testCase "nested cache ancestors include every containing listing" $
listCacheAncestorPrefixes (RelativePath "docs/a/b.txt") listCacheAncestorPrefixes (RelativePath "docs/a/b.txt")
== ["", "docs", "docs/a"] @?= ["", "docs", "docs/a"]
assertTrue "list ancestors root file" $ , testCase "root files invalidate the root listing" $
listCacheAncestorPrefixes (RelativePath "file.txt") == [""] listCacheAncestorPrefixes (RelativePath "file.txt") @?= [""]
]
let putRec = samplePut identityTests :: TestTree
assertTrue "put desired present" $ identityTests = testGroup "identities"
desiredMatches putRec (ObjectPresent 10 (Just (ContentSha256 "abc"))) [ testProperty "root labels parse back to the same root" propRootLabels
assertTrue "put not desired absent" $ , testProperty "mutation kind labels parse back to the same kind" propKindLabels
not (desiredMatches putRec ObjectAbsent) , testCase "issuer and subject identity encoding remains stable" $
assertTrue "put unavailable not desired" $ encodeUserId "https://issuer.example" "subject-α"
not (desiredMatches putRec (ObjectUnavailable "timeout")) @?= UserId "e0240ab33521a1882fb647a79f7fd3c74c9b6348b29bdb14e572204517882859"
assertTrue "put prior empty absent" $ , testProperty "issuer and subject inputs are separated" propUserIdentitySeparation
priorMatches putRec ObjectAbsent ]
assertTrue "put unavailable not prior" $
not (priorMatches putRec (ObjectUnavailable "timeout"))
let delRec = sampleDelete
assertTrue "delete desired absent" $
desiredMatches delRec ObjectAbsent
assertTrue "delete not desired present" $
not (desiredMatches delRec (ObjectPresent 10 Nothing))
putStrLn "nuthatch-unit: ok" protocolTests :: TestTree
protocolTests = testGroup "mutation protocol decisions"
[ testProperty "request fingerprints distinguish every bound field" propFingerprintDistinction
, testProperty "quota growth and admission preserve the limit" propQuotaInvariant
, testProperty "object observations match the independent classifier" propObservations
, testProperty "only settled terminal records are expiry eligible" propExpiryEligibility
, testProperty "only stale active records are recovery eligible" propRecoveryTiming
]
samplePut :: MutationRecord newtype ValidPath = ValidPath { getValidPath :: Text }
samplePut = MutationRecord deriving stock (Show)
newtype InvalidPath = InvalidPath { getInvalidPath :: Text }
deriving stock (Show)
genValidPath :: Gen ValidPath
genValidPath = do
segments <- Gen.list (Range.linear 1 6) genSegment
pure (ValidPath (T.intercalate "/" segments))
where
genSegment =
Gen.filter (`notElem` [".", ".."]) $
Gen.text (Range.linear 1 16) $
Gen.choice [Gen.alphaNum, Gen.element ("_-. " :: String)]
genInvalidPath :: Gen InvalidPath
genInvalidPath = do
ValidPath valid <- genValidPath
InvalidPath <$> Gen.element
[ ""
, "/" <> valid
, valid <> "/"
, valid <> "//tail"
, valid <> "/../tail"
, valid <> "/./tail"
, valid <> "/back\\slash"
, valid <> "/control\NUL"
, "e\x0301"
]
propPathRoundTrip :: Property
propPathRoundTrip = property $ do
ValidPath raw <- forAll genValidPath
validateRelativePath raw === Right (RelativePath raw)
propInvalidPaths :: Property
propInvalidPaths = property $ do
InvalidPath raw <- forAll genInvalidPath
assert (isLeft (validateRelativePath raw))
propRootConfinement :: Property
propRootConfinement = property $ do
ValidPath raw <- forAll genValidPath
let path = RelativePath raw
uid = encodeUserId "https://issuer.example" "subject"
personalPrefix = "users/" <> unUserId uid <> "/"
personalKey = unObjectKey (buildObjectKey Personal uid path)
sharedKey = unObjectKey (buildObjectKey Shared uid path)
assert (personalPrefix `T.isPrefixOf` personalKey)
assert ("shared/" `T.isPrefixOf` sharedKey)
assert (not ("shared/" `T.isPrefixOf` personalKey))
assert (not ("users/" `T.isPrefixOf` sharedKey))
propRootLabels :: Property
propRootLabels = property $ do
root <- forAll (Gen.element [Personal, Shared])
parseRoot (rootLabel root) === Just root
propKindLabels :: Property
propKindLabels = property $ do
kind <- forAll (Gen.element [Put, Delete])
parseKind (kindLabel kind) === Just kind
propUserIdentitySeparation :: Property
propUserIdentitySeparation = property $ do
issuer <- forAll genIdentityPart
subject <- forAll genIdentityPart
otherIssuer <- forAll (Gen.filter (/= issuer) genIdentityPart)
other <- forAll (Gen.filter (/= subject) genIdentityPart)
assert (encodeUserId issuer subject /= encodeUserId otherIssuer subject)
assert (encodeUserId issuer subject /= encodeUserId issuer other)
where
genIdentityPart = Gen.text (Range.linear 1 32) Gen.alphaNum
propFingerprintDistinction :: Property
propFingerprintDistinction = property $ do
ValidPath raw <- forAll genValidPath
size <- forAll (Gen.integral (Range.linear 1 1000000))
let uid = encodeUserId "issuer" "subject"
key = buildObjectKey Personal uid (RelativePath raw)
otherRootKey = buildObjectKey Shared uid (RelativePath raw)
otherPathKey = buildObjectKey Personal uid (RelativePath ("other/" <> raw))
sha = Just (ContentSha256 "content-a")
rec = sampleMutation Put key size sha
assert (sameFingerprint rec Put key size sha)
assert (not (sameFingerprint rec Delete key size sha))
assert (not (sameFingerprint rec Put otherRootKey size sha))
assert (not (sameFingerprint rec Put otherPathKey size sha))
assert (not (sameFingerprint rec Put key (size + 1) sha))
assert (not (sameFingerprint rec Put key size (Just (ContentSha256 "content-b"))))
propQuotaInvariant :: Property
propQuotaInvariant = property $ do
prior <- forAll nonNegativeBytes
desired <- forAll nonNegativeBytes
used <- forAll nonNegativeBytes
reserved <- forAll nonNegativeBytes
limit <- forAll nonNegativeBytes
let growth = growthBytes prior desired
modelGrowth = max 0 (desired - prior)
allowed = quotaAllows limit used reserved growth
growth === modelGrowth
assert (growth >= 0)
allowed === (used + reserved + modelGrowth <= limit)
if allowed
then assert (used + reserved + growth <= limit)
else assert (used + reserved + growth > limit)
where
nonNegativeBytes = Gen.integral (Range.linear 0 1000000000)
propObservations :: Property
propObservations = property $ do
kind <- forAll (Gen.element [Put, Delete])
desiredSize <- forAll (Gen.integral (Range.linear 1 1000000))
priorSize <- forAll (Gen.integral (Range.linear 0 1000000))
observation <- forAll genObservation
let desiredSha = Just (ContentSha256 "desired")
priorSha = if priorSize == 0 then Nothing else Just (ContentSha256 "prior")
rec = (sampleMutation kind (ObjectKey "users/u/file") desiredSize desiredSha)
{ mrPriorSize = priorSize
, mrPriorSha256 = priorSha
}
desiredMatches rec observation === modelDesired kind desiredSize desiredSha observation
priorMatches rec observation === modelPrior priorSize priorSha observation
propExpiryEligibility :: Property
propExpiryEligibility = property $ do
state <- forAll genMutationState
external <- forAll genExternalState
terminalExpiryEligible state external
=== (state `elem` [Succeeded, Failed, Rejected] && external == ExtIdle)
propRecoveryTiming :: Property
propRecoveryTiming = property $ do
state <- forAll genMutationState
elapsed <- forAll (Gen.integral (Range.linear 0 10000))
grace <- forAll (Gen.integral (Range.linear 0 10000))
recoveryEligible state elapsed grace
=== (state `elem` [Prepared, Applying] && elapsed > grace)
genObservation :: Gen ObjectObservation
genObservation = Gen.choice
[ pure ObjectAbsent
, pure (ObjectUnavailable "dependency unavailable")
, ObjectPresent
<$> Gen.integral (Range.linear 0 1000000)
<*> Gen.element
[ Nothing
, Just (ContentSha256 "desired")
, Just (ContentSha256 "prior")
, Just (ContentSha256 "other")
]
]
modelDesired
:: MutationKind
-> Integer
-> Maybe ContentSha256
-> ObjectObservation
-> Bool
modelDesired kind size sha observation =
case (kind, observation) of
(Delete, ObjectAbsent) -> True
(Put, ObjectPresent observedSize observedSha) ->
observedSize == size
&& case (sha, observedSha) of
(Just expected, Just actual) -> expected == actual
(Just _, Nothing) -> False
(Nothing, _) -> True
_ -> False
modelPrior
:: Integer
-> Maybe ContentSha256
-> ObjectObservation
-> Bool
modelPrior size sha observation =
case observation of
ObjectUnavailable _ -> False
ObjectAbsent -> size == 0 && sha == Nothing
ObjectPresent observedSize observedSha ->
observedSize == size
&& case (sha, observedSha) of
(Just expected, Just actual) -> expected == actual
(Nothing, Nothing) -> True
(Just _, Nothing) -> False
(Nothing, Just _) -> True
genMutationState :: Gen MutationState
genMutationState =
Gen.element [Prepared, Applying, Succeeded, Failed, Rejected]
genExternalState :: Gen ExternalState
genExternalState = Gen.element [ExtIdle, ExtPending, ExtSettled]
sampleMutation
:: MutationKind
-> ObjectKey
-> Integer
-> Maybe ContentSha256
-> MutationRecord
sampleMutation kind key size sha = MutationRecord
{ mrUserId = UserId "u" { mrUserId = UserId "u"
, mrKind = Put , mrKind = kind
, mrOperationId = OperationId "op" , mrOperationId = OperationId "op"
, mrPoolId = PoolId "personal:u" , mrPoolId = PoolId "personal:u"
, mrObjectKey = ObjectKey "users/u/file.txt" , mrObjectKey = key
, mrRelativePath = RelativePath "file.txt" , mrRelativePath = RelativePath "file"
, mrDeclaredSize = 10 , mrDeclaredSize = size
, mrContentSha256 = Just (ContentSha256 "abc") , mrContentSha256 = sha
, mrPriorSize = 0 , mrPriorSize = 0
, mrPriorSha256 = Nothing , mrPriorSha256 = Nothing
, mrGrowth = 10 , mrGrowth = size
, mrState = Applying , mrState = Applying
, mrExternal = ExtSettled , mrExternal = ExtSettled
} }
sampleDelete :: MutationRecord
sampleDelete = samplePut
{ mrKind = Delete
, mrDeclaredSize = 0
, mrContentSha256 = Nothing
, mrPriorSize = 10
, mrPriorSha256 = Just (ContentSha256 "abc")
, mrGrowth = 0
}
assertRight :: String -> Either Text a -> IO ()
assertRight label = \case
Right _ -> pure ()
Left err -> fail (label <> ": " <> T.unpack err)
assertLeft :: String -> Either Text a -> IO ()
assertLeft label = \case
Left _ -> pure ()
Right _ -> fail (label <> ": expected failure")
assertTrue :: String -> Bool -> IO ()
assertTrue _ True = pure ()
assertTrue label False = fail (label <> ": expected True")

16
tests/kind/dependencies.yaml.in
View file

@ -19,7 +19,7 @@ metadata:
name: postgres name: postgres
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@POSTGRES_REPLICAS@@
selector: selector:
matchLabels: {app: postgres} matchLabels: {app: postgres}
template: template:
@ -70,7 +70,7 @@ metadata:
name: redis name: redis
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@REDIS_REPLICAS@@
selector: selector:
matchLabels: {app: redis} matchLabels: {app: redis}
template: template:
@ -110,7 +110,7 @@ metadata:
name: dragonfly name: dragonfly
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@DRAGONFLY_REPLICAS@@
selector: selector:
matchLabels: {app: dragonfly} matchLabels: {app: dragonfly}
template: template:
@ -149,7 +149,7 @@ metadata:
name: keycloak name: keycloak
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@KEYCLOAK_REPLICAS@@
selector: selector:
matchLabels: {app: keycloak} matchLabels: {app: keycloak}
template: template:
@ -198,7 +198,7 @@ metadata:
name: zitadel name: zitadel
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@ZITADEL_REPLICAS@@
selector: selector:
matchLabels: {app: zitadel} matchLabels: {app: zitadel}
template: template:
@ -274,7 +274,7 @@ metadata:
name: garage name: garage
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@GARAGE_REPLICAS@@
selector: selector:
matchLabels: {app: garage} matchLabels: {app: garage}
template: template:
@ -340,7 +340,7 @@ metadata:
name: versitygw name: versitygw
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@VERSITYGW_REPLICAS@@
selector: selector:
matchLabels: {app: versitygw} matchLabels: {app: versitygw}
template: template:
@ -421,7 +421,7 @@ metadata:
name: controlled-dns name: controlled-dns
namespace: nuthatch-testbed namespace: nuthatch-testbed
spec: spec:
replicas: 1 replicas: @@COREDNS_REPLICAS@@
selector: selector:
matchLabels: {app: controlled-dns} matchLabels: {app: controlled-dns}
template: template:

147
tests/kind/lib-test.sh
View file

@ -4,11 +4,156 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)" ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh" source "$ROOT/tests/kind/lib.sh"
assert_matrix() {
local expected="$1"
shift
local actual
actual="$(expand_fixture_matrix "$@")"
if [[ "$actual" != "$expected" ]]; then
printf 'unexpected matrix for %s\nexpected:\n%s\nactual:\n%s\n' \
"$*" "$expected" "$actual" >&2
return 1
fi
}
assert_matrix $'001\t-\t-\t-' durable
assert_matrix $'001\tredis\t-\t-\n002\tdragonfly\t-\t-' coordination
assert_matrix $'001\t-\tkeycloak\tgarage\n002\t-\tkeycloak\tversitygw\n003\t-\tzitadel\tgarage\n004\t-\tzitadel\tversitygw' oidc target
assert_matrix $'001\tredis\tkeycloak\tgarage\n002\tredis\tkeycloak\tversitygw\n003\tredis\tzitadel\tgarage\n004\tredis\tzitadel\tversitygw\n005\tdragonfly\tkeycloak\tgarage\n006\tdragonfly\tkeycloak\tversitygw\n007\tdragonfly\tzitadel\tgarage\n008\tdragonfly\tzitadel\tversitygw' all
if expand_fixture_matrix redis >/dev/null 2>&1; then
printf 'individual provider selector was accepted\n' >&2
exit 1
fi
select_fixture_families target durable target
if [[ "${FIXTURE_FAMILIES[*]}" != "durable target" ]]; then
printf 'families were not normalized: %s\n' "${FIXTURE_FAMILIES[*]}" >&2
exit 1
fi
if kube get namespaces >/dev/null 2>&1; then if kube get namespaces >/dev/null 2>&1; then
printf 'kube helper unexpectedly reached a cluster without fixture setup\n' >&2 printf 'kube helper unexpectedly reached a cluster without fixture setup\n' >&2
exit 1 exit 1
fi fi
kube() {
return 1
}
select_fixture_families durable
if write_lane_profile 001 - - - >/dev/null 2>&1; then
printf 'lane provisioning failure was ignored\n' >&2
exit 1
fi
select_fixture_families base
if write_lane_profile 001 - - - >/dev/null 2>&1; then
printf 'ConfigMap provisioning failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
kube_calls=0
kube() {
kube_calls=$((kube_calls + 1))
case "$kube_calls" in
1) return 1 ;;
2) return 0 ;;
3) return 1 ;;
*) return 0 ;;
esac
}
if reset_lane_target_store garage nuthatch-lane-001 >/dev/null 2>&1; then
printf 'target-store emptying failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
write_lane_profile() {
printf 'nuthatch-fixture-lane-001\n'
}
render_lane_job() {
return 0
}
run_job_manifest() {
return 42
}
cleanup_called=false
cleanup_lane_resources() {
cleanup_called=true
}
cleanup_lane_state() {
cleanup_called=true
}
if run_job_matrix fixture-consumer /dev/null nuthatch-test:local durable; then
printf 'Job failure was ignored\n' >&2
exit 1
fi
if [[ "$cleanup_called" = true ]]; then
printf 'failed lane was not retained for inspection\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
kube() {
return 1
}
if run_job_manifest fixture-consumer /dev/null >/dev/null 2>&1; then
printf 'Job cleanup failure was ignored\n' >&2
exit 1
fi
if cleanup_lane_resources fixture-consumer nuthatch-fixture-lane-001 >/dev/null 2>&1; then
printf 'lane resource cleanup failure was ignored\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
select_fixture_families all
cleanup_calls=()
cleanup_lane_durable_store() {
cleanup_calls+=(durable)
return 1
}
cleanup_lane_coordination_store() {
cleanup_calls+=(coordination)
}
cleanup_lane_target_store() {
cleanup_calls+=(target)
}
if cleanup_lane_state 001 redis keycloak garage >/dev/null 2>&1; then
printf 'partial lane cleanup failure was ignored\n' >&2
exit 1
fi
if [[ "${cleanup_calls[*]}" != "durable coordination target" ]]; then
printf 'lane cleanup stopped early: %s\n' "${cleanup_calls[*]}" >&2
exit 1
fi
if ! declare -F load_local_image >/dev/null; then
printf 'local application image loading is unavailable\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
configure_provider() {
return 0
}
delete_cluster() {
return 1
}
runtime_cleanup_called=false
remove_runtime_namespace() {
runtime_cleanup_called=true
}
if destroy_fixture >/dev/null 2>&1; then
printf 'fixture teardown failure was ignored\n' >&2
exit 1
fi
if [[ "$runtime_cleanup_called" != true ]]; then
printf 'fixture teardown stopped before runtime cleanup\n' >&2
exit 1
fi
FIXTURE_LOCK_DIR="$RUNTIME/lock-test" FIXTURE_LOCK_DIR="$RUNTIME/lock-test"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock" FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
acquire_fixture_lock acquire_fixture_lock
@ -19,4 +164,4 @@ if XDG_RUNTIME_DIR="$FIXTURE_LOCK_DIR" bash -c \
fi fi
exec 9>&- exec 9>&-
printf 'fixture isolation tests passed\n' printf 'fixture family matrix tests passed\n'

437
tests/kind/lib.sh
View file

@ -75,7 +75,7 @@ acquire_fixture_lock() {
configure_provider() { configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
fi fi
export NUTHATCH_REAL_NERDCTL export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE" export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
@ -105,10 +105,11 @@ delete_cluster() {
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \ containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \ --all \
--quiet \ --quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" --filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
while IFS= read -r container; do while IFS= read -r container; do
[[ -z "$container" ]] || [[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
return
done <<< "$containers" done <<< "$containers"
if cluster_exists; then if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true "$BIN/kind" delete cluster --name "$CLUSTER" || true
@ -169,9 +170,24 @@ load_external_image() {
printf '%s\n' "$image" printf '%s\n' "$image"
} }
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
rm -f "$archive"
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}
render_manifests() { render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5" local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}" local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
local keycloak_replicas="${14}" zitadel_replicas="${15}"
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
sed \ sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \ -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \ -e "s|@@REDIS_IMAGE@@|$redis|g" \
@ -182,6 +198,14 @@ render_manifests() {
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \ -e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \ -e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml" "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \ sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \ -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
@ -189,6 +213,18 @@ render_manifests() {
-e "s|@@CURL_IMAGE@@|$curl|g" \ -e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml" "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
} > "$RUNTIME/fixture.env"
} }
kube() { kube() {
@ -220,7 +256,83 @@ wait_for_job() {
return 1 return 1
} }
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all) use_all=true ;;
base | durable | coordination | oidc | target) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target)
fi
for candidate in durable coordination oidc target; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then
coordination_providers=(redis dragonfly)
fi
if fixture_has_family oidc; then
oidc_providers=(keycloak zitadel)
fi
if fixture_has_family target; then
target_providers=(garage versitygw)
fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done
}
setup_fixture() { setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME" mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives" rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives" mkdir -p "$RUNTIME/image-archives"
@ -232,24 +344,74 @@ setup_fixture() {
--config "$TESTBED/cluster.yaml" \ --config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \ --kubeconfig "$KUBECONFIG_PATH" \
--wait 180s --wait 180s
local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli local postgres=nuthatch-testbed/unused:postgres
local redis=nuthatch-testbed/unused:redis
local dragonfly=nuthatch-testbed/unused:dragonfly
local keycloak=nuthatch-testbed/unused:keycloak
local zitadel=nuthatch-testbed/unused:zitadel
local garage=nuthatch-testbed/unused:garage
local versitygw=nuthatch-testbed/unused:versitygw
local coredns=nuthatch-testbed/unused:coredns
local curl=nuthatch-testbed/unused:curl
local aws_cli=nuthatch-testbed/unused:aws-cli
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
local keycloak_replicas=0 zitadel_replicas=0
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
if fixture_has_family durable || fixture_has_family oidc; then
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")" postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
postgres_replicas=1
fi
if fixture_has_family coordination; then
redis="$(load_external_image redis "$REDIS_IMAGE")" redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")" dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
redis_replicas=1
dragonfly_replicas=1
fi
if fixture_has_family oidc; then
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")" keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")" zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")" coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")" curl="$(load_external_image curl "$CURL_IMAGE")"
keycloak_replicas=1
zitadel_replicas=1
coredns_replicas=1
fi
if fixture_has_family target; then
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")" aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
curl="$(load_external_image curl "$CURL_IMAGE")"
fi
garage_replicas=1
versitygw_replicas=1
fi
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \ render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" "$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
"$keycloak_replicas" "$zitadel_replicas" \
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
kube apply --filename "$RUNTIME/dependencies.yaml" kube apply --filename "$RUNTIME/dependencies.yaml"
local deployment if fixture_has_family durable || fixture_has_family oidc; then
for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do wait_for_deployment postgres
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s fi
done if fixture_has_family coordination; then
wait_for_deployment redis
wait_for_deployment dragonfly
fi
if fixture_has_family oidc; then
wait_for_deployment keycloak
wait_for_deployment zitadel
wait_for_deployment controlled-dns
fi
if fixture_has_family target; then
wait_for_deployment garage
wait_for_deployment versitygw
fi
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
} }
verify_dependencies() { verify_dependencies() {
@ -260,10 +422,253 @@ verify_dependencies() {
done done
} }
reset_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
reset_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
printf '%s\n' "$database"
}
target_endpoint() {
case "$1" in
garage) printf 'http://garage:3900\n' ;;
versitygw) printf 'http://versitygw:10000\n' ;;
*) return 1 ;;
esac
}
target_loopback_endpoint() {
case "$1" in
garage) printf 'http://127.0.0.1:3900\n' ;;
versitygw) printf 'http://127.0.0.1:10000\n' ;;
*) return 1 ;;
esac
}
reset_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >/dev/null || return
}
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile="nuthatch-fixture-lane-$lane"
local profile_file="$RUNTIME/$profile.env"
local database=
local coordination_database=
local oidc_issuer=
local target_url=
local target_bucket=
local target_region=
local target_access_key=
local target_secret_key=
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
fi
if fixture_has_family coordination; then
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
fi
if fixture_has_family oidc; then
case "$oidc" in
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
*) return 1 ;;
esac
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
reset_lane_target_store "$target" "$target_bucket" || return
target_url="$(target_endpoint "$target")" || return
case "$target" in
garage)
target_region=garage
target_access_key=GK00000000000000000000000000000000
target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
;;
versitygw)
target_region=us-east-1
target_access_key=synthetic-s3-access
target_secret_key=synthetic-s3-secret
;;
*) return 1 ;;
esac
fi
{
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
} > "$profile_file" || return
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
kube create configmap "$profile-file" \
--namespace "$FIXTURE_NAMESPACE" \
--from-file="profile.env=$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
printf '%s\n' "$profile"
}
render_lane_job() {
local job="$1"
local manifest="$2"
local image="$3"
local profile="$4"
source "$RUNTIME/fixture.env" || return
local coordination_image="$REDIS_LOCAL_IMAGE"
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
"$manifest" > "$RUNTIME/$job.yaml"
}
run_job_manifest() {
local job="$1"
local manifest="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
kube apply --filename "$manifest" >/dev/null || return
wait_for_job "$job"
}
cleanup_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >/dev/null
}
cleanup_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local target="$4"
local status=0
if fixture_has_family durable; then
cleanup_lane_durable_store "$lane" || status=1
fi
if fixture_has_family coordination; then
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
fi
if fixture_has_family target; then
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
fi
return "$status"
}
cleanup_lane_resources() {
local job="$1"
local profile="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
--ignore-not-found >/dev/null || return
rm -f "$RUNTIME/$profile.env" "$RUNTIME/$job.yaml"
}
run_job_matrix() {
local base_job="$1"
local manifest="$2"
local image="$3"
shift 3
local lane
local coordination
local oidc
local target
local profile
local job
select_fixture_families "$@" || return
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
job="$base_job-$lane"
render_lane_job "$job" "$manifest" "$image" "$profile" || return
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
cleanup_lane_resources "$job" "$profile" || return
done < <(expand_fixture_matrix "$@")
}
destroy_fixture() { destroy_fixture() {
configure_provider local status=0
delete_cluster configure_provider || return
remove_runtime_namespace delete_cluster || status=1
rm -rf "$RUNTIME/image-archives" remove_runtime_namespace || status=1
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml" rm -rf "$RUNTIME/image-archives" || status=1
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" || status=1
return "$status"
} }

103
tests/kind/matrix-job.yaml.in Normal file
View file

@ -0,0 +1,103 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [nuthatch-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
initContainers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [psql "$NUTHATCH_PG_CONN" --set ON_ERROR_STOP=1 --command "CREATE TABLE lane_canary (value text NOT NULL)" --command "INSERT INTO lane_canary VALUES ('$NUTHATCH_FIXTURE_PROFILE')"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: coordination
image: @@COORDINATION_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [redis-cli -u "$NUTHATCH_REDIS_URL" SET lane-canary "$NUTHATCH_FIXTURE_PROFILE" >/dev/null && test "$(redis-cli -u "$NUTHATCH_REDIS_URL" GET lane-canary)" = "$NUTHATCH_FIXTURE_PROFILE"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: oidc
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
discovery="$(curl --fail --silent --show-error "$NUTHATCH_OIDC_ISSUER/.well-known/openid-configuration")"
printf '%s' "$discovery" | grep -F '"issuer"' >/dev/null
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: target
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
export AWS_ACCESS_KEY_ID="$NUTHATCH_S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$NUTHATCH_S3_SECRET_KEY"
export AWS_DEFAULT_REGION="$NUTHATCH_S3_REGION"
printf '%s\n' "$NUTHATCH_FIXTURE_PROFILE" >/tmp/input
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api put-object --bucket "$NUTHATCH_S3_BUCKET" --key lane-canary --body /tmp/input >/dev/null
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api get-object --bucket "$NUTHATCH_S3_BUCKET" --key lane-canary /tmp/output >/dev/null
test "$(cat /tmp/input)" = "$(cat /tmp/output)"
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
containers:
- name: profile
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
test -n "$NUTHATCH_FIXTURE_PROFILE"
test "$NUTHATCH_S3_BUCKET" = "nuthatch-lane-$NUTHATCH_FIXTURE_LANE"
case "$NUTHATCH_PG_CONN" in
*"dbname=nuthatch_lane_$NUTHATCH_FIXTURE_LANE") ;;
*) exit 1 ;;
esac
grep -Fx "NUTHATCH_FIXTURE_PROFILE=$NUTHATCH_FIXTURE_PROFILE" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_COORDINATION_PROVIDER=$NUTHATCH_COORDINATION_PROVIDER" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_OIDC_PROVIDER=$NUTHATCH_OIDC_PROVIDER" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_TARGET_PROVIDER=$NUTHATCH_TARGET_PROVIDER" /run/nuthatch-fixture/profile.env
printf 'lane profile injection passed\n'
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
volumeMounts:
- {name: fixture-profile, mountPath: /run/nuthatch-fixture, readOnly: true}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture-profile
configMap:
name: @@PROFILE_NAME@@-file
items:
- {key: profile.env, path: profile.env}

43
tests/kind/matrix-run.sh Executable file
View file

@ -0,0 +1,43 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture all
source "$RUNTIME/fixture.env"
run_job_matrix \
fixture-matrix \
"$TESTBED/matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
all
SUCCEEDED=true
printf 'Nuthatch fixture matrix passed\n'