Type ordinary integration probe HTTP traffic
This commit is contained in:
parent
390434bed1
commit
cc5c47f2ce
1 changed files with 1022 additions and 661 deletions
|
|
@ -31,13 +31,14 @@ import System.Process (readProcessWithExitCode)
|
|||
import Text.Read (readMaybe)
|
||||
import Nuthatch.Probe.Assert
|
||||
import Nuthatch.Probe.Curl
|
||||
import Nuthatch.Probe.Http
|
||||
import Nuthatch.Probe.Oidc
|
||||
import Nuthatch.Probe.ProductClient
|
||||
|
||||
main :: IO ()
|
||||
main =
|
||||
withSystemTempDirectory "nuthatch-streaming-probe" $ \directory -> do
|
||||
context <- newProbeContext directory
|
||||
context <- newProbeContext
|
||||
user <- requireEnv "NUTHATCH_OIDC_USER_A"
|
||||
session <- login context False user "streaming-a"
|
||||
getArgs >>= \case
|
||||
|
|
@ -58,14 +59,16 @@ verifyLarge context session directory = do
|
|||
uploaded <- uploadFileDeclared
|
||||
context session "streaming-large.bin"
|
||||
"70000000-0000-4000-8000-000000000001"
|
||||
largeBytes largeDigest largePath []
|
||||
largeBytes largeDigest largePath defaultRequestOptions
|
||||
requireSucceeded "large streaming upload" uploaded
|
||||
downloaded <- curlToFile downloadPath
|
||||
[ "--cookie", sessionCookies session
|
||||
, probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin"
|
||||
]
|
||||
assertEqual "large streaming download status" 200 (curlStatus downloaded)
|
||||
downloaded <- downloadTo
|
||||
(probeClient context)
|
||||
defaultRequestOptions
|
||||
{ optionCookies = Just (sessionCookies session) }
|
||||
downloadPath
|
||||
(probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin")
|
||||
assertEqual "large streaming download status" 200 (probeStatus downloaded)
|
||||
hashFile downloadPath >>= assertEqual "large download digest" largeDigest
|
||||
let tiny = "x"
|
||||
mapM_
|
||||
|
|
@ -73,25 +76,25 @@ verifyLarge context session directory = do
|
|||
replay <- uploadBytesDeclared
|
||||
context session "streaming-large.bin"
|
||||
"70000000-0000-4000-8000-000000000007"
|
||||
largeBytes largeDigest tiny []
|
||||
largeBytes largeDigest tiny defaultRequestOptions
|
||||
requireSucceeded ("prior/desired observation " <> label) replay)
|
||||
["first", "replay"]
|
||||
sizeMismatch <- uploadBytesDeclared
|
||||
context session "size-mismatch.bin"
|
||||
"70000000-0000-4000-8000-000000000002"
|
||||
2 (sha256Hex tiny) tiny []
|
||||
2 (sha256Hex tiny) tiny defaultRequestOptions
|
||||
requireError "size mismatch" 400 "size_mismatch" sizeMismatch
|
||||
identityMismatch <- uploadBytesDeclared
|
||||
context session "identity-mismatch.bin"
|
||||
"70000000-0000-4000-8000-000000000003"
|
||||
1 (BS.replicate 64 48) tiny []
|
||||
1 (BS.replicate 64 48) tiny defaultRequestOptions
|
||||
requireError "content identity mismatch"
|
||||
400 "content_identity_mismatch" identityMismatch
|
||||
tooLarge <- uploadFileDeclared
|
||||
context session "too-large.bin"
|
||||
"70000000-0000-4000-8000-000000000004"
|
||||
overLimit (sha256Hex tiny) largePath
|
||||
["--max-time", "5", "--limit-rate", "64k"]
|
||||
defaultRequestOptions { optionTimeoutMicros = Just 5000000 }
|
||||
requireError "object size limit" 413 "object_too_large" tooLarge
|
||||
aborted <- interruptedUpload
|
||||
context session "aborted-upload.bin"
|
||||
|
|
@ -99,14 +102,14 @@ verifyLarge context session directory = do
|
|||
largeBytes largeDigest largePath
|
||||
assert "aborted upload unexpectedly completed" (not aborted)
|
||||
missing <- download context session "personal" "aborted-upload.bin"
|
||||
assertEqual "aborted upload visibility" 404 (curlStatus missing)
|
||||
assertEqual "aborted upload visibility" 404 (probeStatus missing)
|
||||
replayAfterAbort <- uploadBytesDeclared
|
||||
context session "aborted-upload.bin"
|
||||
"70000000-0000-4000-8000-000000000005"
|
||||
largeBytes largeDigest tiny []
|
||||
assertEqual "aborted upload replay status" 200 (curlStatus replayAfterAbort)
|
||||
largeBytes largeDigest tiny defaultRequestOptions
|
||||
assertEqual "aborted upload replay status" 200 (probeStatus replayAfterAbort)
|
||||
assert "aborted upload replay lost active or failed identity" $
|
||||
any (`BS.isInfixOf` curlBody replayAfterAbort)
|
||||
any (`BS.isInfixOf` probeBody replayAfterAbort)
|
||||
["\"result\":\"pending\"", "\"result\":\"failed\""]
|
||||
abortedOverwrite <- interruptedUpload
|
||||
context session "streaming-large.bin"
|
||||
|
|
@ -114,12 +117,14 @@ verifyLarge context session directory = do
|
|||
largeBytes (BS.replicate 64 48) largePath
|
||||
assert "aborted overwrite unexpectedly completed" (not abortedOverwrite)
|
||||
preservedPath <- pure (directory <> "/preserved.bin")
|
||||
preserved <- curlToFile preservedPath
|
||||
[ "--cookie", sessionCookies session
|
||||
, probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin"
|
||||
]
|
||||
assertEqual "preserved object status" 200 (curlStatus preserved)
|
||||
preserved <- downloadTo
|
||||
(probeClient context)
|
||||
defaultRequestOptions
|
||||
{ optionCookies = Just (sessionCookies session) }
|
||||
preservedPath
|
||||
(probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin")
|
||||
assertEqual "preserved object status" 200 (probeStatus preserved)
|
||||
hashFile preservedPath >>= assertEqual "preserved object digest" largeDigest
|
||||
let partialPath = directory <> "/partial-download.bin"
|
||||
partial <- interruptedDownload context session partialPath
|
||||
|
|
@ -128,9 +133,9 @@ verifyLarge context session directory = do
|
|||
assert "aborted download emitted no bytes" (partialBytes > 0)
|
||||
assert "aborted download emitted the complete object" $
|
||||
partialBytes < largeBytes
|
||||
ready <- curl [probeBase context <> "/readyz"]
|
||||
ready <- get (probeClient context) (probeBase context <> "/readyz")
|
||||
requireStatus "readiness after aborts" 200 ready
|
||||
assertContains "readiness after aborts" "\"ready\":true" (curlBody ready)
|
||||
assertContains "readiness after aborts" "\"ready\":true" (probeBody ready)
|
||||
putStrLn "Large streaming and abort behavior passed"
|
||||
|
||||
verifyThrottle :: ProbeContext -> Session -> FilePath -> IO ()
|
||||
|
|
@ -141,35 +146,42 @@ verifyThrottle context session directory = do
|
|||
throttlePath = directory <> "/throttle-upload.bin"
|
||||
writeZeros jointPath 65536
|
||||
jointDigest <- hashFile jointPath
|
||||
joint <- uploadFileDeclared
|
||||
joint <- curlUploadFileDeclared
|
||||
context session "joint-throttle.bin"
|
||||
"70000000-0000-4000-8000-000000000013"
|
||||
65536 jointDigest jointPath []
|
||||
65536 jointDigest jointPath
|
||||
requireSucceeded "joint throttle setup" joint
|
||||
writeZeros throttlePath 1048576
|
||||
uploadDigest <- hashFile throttlePath
|
||||
throttled <- uploadFileDeclared
|
||||
throttled <- curlUploadFileDeclared
|
||||
context sessionB "throttled-upload.bin"
|
||||
"71000000-0000-4000-8000-000000000001"
|
||||
1048576 uploadDigest throttlePath []
|
||||
1048576 uploadDigest throttlePath
|
||||
requireError "incremental upload throttle"
|
||||
429 "bandwidth_exceeded" throttled
|
||||
prestart <- uploadBytesDeclared
|
||||
prestart <- curlUploadBytesDeclared
|
||||
context sessionB "prestart.bin"
|
||||
"71000000-0000-4000-8000-000000000002"
|
||||
1 uploadDigest "x" []
|
||||
assertEqual "pre-start upload throttle" 429 (curlStatus prestart)
|
||||
1 uploadDigest "x"
|
||||
assertEqual
|
||||
("pre-start upload throttle: " <> B8.unpack (probeBody prestart))
|
||||
429
|
||||
(probeStatus prestart)
|
||||
let firstPath = directory <> "/parallel-a"
|
||||
secondPath = directory <> "/parallel-b"
|
||||
downloadTo path =
|
||||
try (curlToFile path
|
||||
[ "--cookie", sessionCookies session
|
||||
, probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin"
|
||||
]) :: IO (Either SomeException CurlResponse)
|
||||
downloadAttempt path = do
|
||||
cookies <- cookieHeader (sessionCookies session)
|
||||
try
|
||||
(curlToFile path
|
||||
[ "--header", "Cookie: " <> B8.unpack cookies
|
||||
, probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin"
|
||||
])
|
||||
:: IO (Either SomeException ProbeResponse)
|
||||
BS.writeFile firstPath BS.empty
|
||||
BS.writeFile secondPath BS.empty
|
||||
(first, second) <- concurrently (downloadTo firstPath) (downloadTo secondPath)
|
||||
(first, second) <-
|
||||
concurrently (downloadAttempt firstPath) (downloadAttempt secondPath)
|
||||
assert "first throttled download completed" (isLeft first)
|
||||
assert "second throttled download completed" (isLeft second)
|
||||
firstBytes <- getFileSize firstPath
|
||||
|
|
@ -189,21 +201,21 @@ verifyRefill context session directory = do
|
|||
let bodyPath = directory <> "/refill-upload.bin"
|
||||
writeZeros bodyPath 65536
|
||||
digest <- hashFile bodyPath
|
||||
initial <- uploadFileDeclared
|
||||
initial <- curlUploadFileDeclared
|
||||
context session "refill-initial.bin"
|
||||
"72000000-0000-4000-8000-000000000001"
|
||||
65536 digest bodyPath []
|
||||
65536 digest bodyPath
|
||||
requireSucceeded "initial token use" initial
|
||||
exhausted <- uploadFileDeclared
|
||||
exhausted <- curlUploadFileDeclared
|
||||
context session "refill-exhausted.bin"
|
||||
"72000000-0000-4000-8000-000000000003"
|
||||
65536 digest bodyPath []
|
||||
65536 digest bodyPath
|
||||
requireStatus "exhausted token bucket" 429 exhausted
|
||||
threadDelay 2000000
|
||||
refilled <- uploadFileDeclared
|
||||
refilled <- curlUploadFileDeclared
|
||||
context session "refill-refilled.bin"
|
||||
"72000000-0000-4000-8000-000000000002"
|
||||
65536 digest bodyPath []
|
||||
65536 digest bodyPath
|
||||
requireSucceeded "refilled token bucket" refilled
|
||||
putStrLn "Token-bucket refill behavior passed"
|
||||
|
||||
|
|
@ -211,15 +223,15 @@ verifyUnavailable :: ProbeContext -> Session -> IO ()
|
|||
verifyUnavailable context session = do
|
||||
listing <- browse context session "?root=personal"
|
||||
requireStatus "browse without Redis" 200 listing
|
||||
assertContains "browse without Redis" "\"path\":" (curlBody listing)
|
||||
assertContains "browse without Redis" "\"path\":" (probeBody listing)
|
||||
let path =
|
||||
if "cross-user.txt" `BS.isInfixOf` curlBody listing
|
||||
if "cross-user.txt" `BS.isInfixOf` probeBody listing
|
||||
then "cross-user.txt"
|
||||
else "streaming-large.bin"
|
||||
query =
|
||||
if path == "cross-user.txt" then "CROSS-USER" else "STREAMING-LARGE"
|
||||
searched <- browse context session ("?root=personal&q=" <> query)
|
||||
assertContains "search without Redis" (B8.pack path) (curlBody searched)
|
||||
assertContains "search without Redis" (B8.pack path) (probeBody searched)
|
||||
unavailableUpload <- upload context session "personal" "unavailable.bin"
|
||||
"72000000-0000-4000-8000-000000000001" "x"
|
||||
requireError "upload without Redis"
|
||||
|
|
@ -249,14 +261,22 @@ uploadFileDeclared
|
|||
-> Integer
|
||||
-> BS.ByteString
|
||||
-> FilePath
|
||||
-> [String]
|
||||
-> IO CurlResponse
|
||||
uploadFileDeclared context session path operationId size digest bodyPath extra =
|
||||
curl
|
||||
( extra
|
||||
<> uploadArguments context session path operationId size digest
|
||||
<> ["--data-binary", "@" <> bodyPath]
|
||||
)
|
||||
-> RequestOptions
|
||||
-> IO ProbeResponse
|
||||
uploadFileDeclared context session path operationId size digest bodyPath options =
|
||||
postWith
|
||||
(probeClient context)
|
||||
options
|
||||
{ optionCookies = Just (sessionCookies session)
|
||||
, optionHeaders =
|
||||
[ ("Content-Type", "application/octet-stream")
|
||||
, ("X-Declared-Size", B8.pack (show size))
|
||||
, ("X-Content-SHA256", digest)
|
||||
, ("X-CSRF-Token", sessionCsrf session)
|
||||
] <> optionHeaders options
|
||||
, optionBody = FileBody bodyPath
|
||||
}
|
||||
(uploadUrl context path operationId)
|
||||
|
||||
uploadBytesDeclared
|
||||
:: ProbeContext
|
||||
|
|
@ -266,14 +286,22 @@ uploadBytesDeclared
|
|||
-> Integer
|
||||
-> BS.ByteString
|
||||
-> BS.ByteString
|
||||
-> [String]
|
||||
-> IO CurlResponse
|
||||
uploadBytesDeclared context session path operationId size digest body extra =
|
||||
withSystemTempFile "nuthatch-streaming-body" $ \bodyPath handle -> do
|
||||
BS.hPut handle body
|
||||
hClose handle
|
||||
uploadFileDeclared
|
||||
context session path operationId size digest bodyPath extra
|
||||
-> RequestOptions
|
||||
-> IO ProbeResponse
|
||||
uploadBytesDeclared context session path operationId size digest body options =
|
||||
postWith
|
||||
(probeClient context)
|
||||
options
|
||||
{ optionCookies = Just (sessionCookies session)
|
||||
, optionHeaders =
|
||||
[ ("Content-Type", "application/octet-stream")
|
||||
, ("X-Declared-Size", B8.pack (show size))
|
||||
, ("X-Content-SHA256", digest)
|
||||
, ("X-CSRF-Token", sessionCsrf session)
|
||||
] <> optionHeaders options
|
||||
, optionBody = ByteBody body
|
||||
}
|
||||
(uploadUrl context path operationId)
|
||||
|
||||
uploadArguments
|
||||
:: ProbeContext
|
||||
|
|
@ -282,19 +310,50 @@ uploadArguments
|
|||
-> String
|
||||
-> Integer
|
||||
-> BS.ByteString
|
||||
-> [String]
|
||||
uploadArguments context session path operationId size digest =
|
||||
[ "--cookie", sessionCookies session
|
||||
, "--header", "Content-Type: application/octet-stream"
|
||||
, "--header", "X-Declared-Size: " <> show size
|
||||
, "--header", "X-Content-SHA256: " <> B8.unpack digest
|
||||
, "--header", "X-CSRF-Token: " <> B8.unpack (sessionCsrf session)
|
||||
, probeBase context
|
||||
<> "/api/upload?root=personal&path="
|
||||
<> path
|
||||
<> "&operation_id="
|
||||
<> operationId
|
||||
]
|
||||
-> IO [String]
|
||||
uploadArguments context session path operationId size digest = do
|
||||
cookies <- cookieHeader (sessionCookies session)
|
||||
pure
|
||||
[ "--header", "Cookie: " <> B8.unpack cookies
|
||||
, "--header", "Content-Type: application/octet-stream"
|
||||
, "--header", "X-Declared-Size: " <> show size
|
||||
, "--header", "X-Content-SHA256: " <> B8.unpack digest
|
||||
, "--header", "X-CSRF-Token: " <> B8.unpack (sessionCsrf session)
|
||||
, probeBase context
|
||||
<> "/api/upload?root=personal&path="
|
||||
<> path
|
||||
<> "&operation_id="
|
||||
<> operationId
|
||||
]
|
||||
|
||||
curlUploadFileDeclared
|
||||
:: ProbeContext
|
||||
-> Session
|
||||
-> String
|
||||
-> String
|
||||
-> Integer
|
||||
-> BS.ByteString
|
||||
-> FilePath
|
||||
-> IO ProbeResponse
|
||||
curlUploadFileDeclared context session path operationId size digest bodyPath = do
|
||||
arguments <- uploadArguments context session path operationId size digest
|
||||
curl (arguments <> ["--data-binary", "@" <> bodyPath])
|
||||
|
||||
curlUploadBytesDeclared
|
||||
:: ProbeContext
|
||||
-> Session
|
||||
-> String
|
||||
-> String
|
||||
-> Integer
|
||||
-> BS.ByteString
|
||||
-> BS.ByteString
|
||||
-> IO ProbeResponse
|
||||
curlUploadBytesDeclared context session path operationId size digest body =
|
||||
withSystemTempFile "nuthatch-streaming-body" $ \bodyPath handle -> do
|
||||
BS.hPut handle body
|
||||
hClose handle
|
||||
curlUploadFileDeclared
|
||||
context session path operationId size digest bodyPath
|
||||
|
||||
interruptedUpload
|
||||
:: ProbeContext
|
||||
|
|
@ -306,6 +365,7 @@ interruptedUpload
|
|||
-> FilePath
|
||||
-> IO Bool
|
||||
interruptedUpload context session path operationId size digest bodyPath = do
|
||||
common <- uploadArguments context session path operationId size digest
|
||||
let arguments =
|
||||
[ "--insecure"
|
||||
, "--silent"
|
||||
|
|
@ -315,7 +375,7 @@ interruptedUpload context session path operationId size digest bodyPath = do
|
|||
, "--request", "POST"
|
||||
, "--upload-file", bodyPath
|
||||
]
|
||||
<> uploadArguments context session path operationId size digest
|
||||
<> common
|
||||
(exitCode, _, _) <- readProcessWithExitCode "curl" arguments ""
|
||||
pure (exitCode == ExitSuccess)
|
||||
|
||||
|
|
@ -325,13 +385,14 @@ interruptedDownload
|
|||
-> FilePath
|
||||
-> IO Bool
|
||||
interruptedDownload context session outputPath = do
|
||||
cookies <- cookieHeader (sessionCookies session)
|
||||
let arguments =
|
||||
[ "--insecure"
|
||||
, "--silent"
|
||||
, "--show-error"
|
||||
, "--max-time", "1"
|
||||
, "--limit-rate", "64k"
|
||||
, "--cookie", sessionCookies session
|
||||
, "--header", "Cookie: " <> B8.unpack cookies
|
||||
, "--output", outputPath
|
||||
, probeBase context
|
||||
<> "/api/download?root=personal&path=streaming-large.bin"
|
||||
|
|
@ -339,6 +400,14 @@ interruptedDownload context session outputPath = do
|
|||
(exitCode, _, _) <- readProcessWithExitCode "curl" arguments ""
|
||||
pure (exitCode == ExitSuccess)
|
||||
|
||||
uploadUrl :: ProbeContext -> String -> String -> String
|
||||
uploadUrl context path operationId =
|
||||
probeBase context
|
||||
<> "/api/upload?root=personal&path="
|
||||
<> path
|
||||
<> "&operation_id="
|
||||
<> operationId
|
||||
|
||||
writeZeros :: FilePath -> Integer -> IO ()
|
||||
writeZeros path byteCount =
|
||||
withBinaryFile path WriteMode $ \handle -> do
|
||||
|
|
@ -369,19 +438,19 @@ readEnvInteger name = do
|
|||
value <- requireEnv name
|
||||
maybe (fail ("invalid " <> name)) pure (readMaybe value)
|
||||
|
||||
requireSucceeded :: String -> CurlResponse -> IO ()
|
||||
requireSucceeded :: String -> ProbeResponse -> IO ()
|
||||
requireSucceeded label response = do
|
||||
requireStatus label 200 response
|
||||
assertContains label "\"result\":\"succeeded\"" (curlBody response)
|
||||
assertContains label "\"result\":\"succeeded\"" (probeBody response)
|
||||
|
||||
requireError :: String -> Int -> BS.ByteString -> CurlResponse -> IO ()
|
||||
requireError :: String -> Int -> BS.ByteString -> ProbeResponse -> IO ()
|
||||
requireError label status code response = do
|
||||
requireStatus label status response
|
||||
assertContains label ("\"error\":\"" <> code <> "\"") (curlBody response)
|
||||
assertContains label ("\"error\":\"" <> code <> "\"") (probeBody response)
|
||||
|
||||
requireStatus :: String -> Int -> CurlResponse -> IO ()
|
||||
requireStatus :: String -> Int -> ProbeResponse -> IO ()
|
||||
requireStatus label expected response =
|
||||
assertEqual (label <> " status") expected (curlStatus response)
|
||||
assertEqual (label <> " status") expected (probeStatus response)
|
||||
|
||||
isLeft :: Either left right -> Bool
|
||||
isLeft (Left _) = True
|
||||
|
|
|
|||
Loading…
Reference in a new issue