ms2k/remora
SHA256
Watch
0
Code Releases Activity

Add exact resumable HTTP source streaming

Browse source
This commit is contained in:
ms2k 2026-07-17 05:31:50 +09:00
commit 5b43b6c660
8 changed files with 1361 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

22
crates/remora-http-source/Cargo.toml Normal file
View file

@ -0,0 +1,22 @@
[package]
name = "remora-http-source"
edition.workspace = true
publish = false
rust-version.workspace = true
version.workspace = true
[dependencies]
bytes = "1.12.1"
http = "1.4.0"
httpdate = "1.0.3"
remora-domain.workspace = true
remora-outbound = { path = "../remora-outbound" }
remora-stream = { path = "../remora-stream" }
reqwest = { version = "0.12.28", default-features = false }
thiserror.workspace = true
tokio.workspace = true
[dev-dependencies]
ipnet = "2.12.0"
sha2.workspace = true
tokio = { workspace = true, features = ["io-util", "net"] }

73
crates/remora-http-source/src/body.rs Normal file
View file

@ -0,0 +1,73 @@
use bytes::Bytes;
use remora_stream::{SourceBody, StreamError};
pub struct HttpSourceBody {
response: reqwest::Response,
pending: Bytes,
pending_offset: usize,
remaining: Option<u64>,
finished: bool,
}
impl HttpSourceBody {
pub(crate) fn new(response: reqwest::Response, expected_length: Option<u64>) -> Self {
Self {
response,
pending: Bytes::new(),
pending_offset: 0,
remaining: expected_length,
finished: false,
}
}
fn copy_pending(&mut self, output: &mut [u8]) -> usize {
let available = &self.pending[self.pending_offset..];
let count = available.len().min(output.len());
output[..count].copy_from_slice(&available[..count]);
self.pending_offset += count;
if self.pending_offset == self.pending.len() {
self.pending = Bytes::new();
self.pending_offset = 0;
}
if let Some(remaining) = &mut self.remaining {
*remaining -= count as u64;
}
count
}
}
impl SourceBody for HttpSourceBody {
async fn read(&mut self, output: &mut [u8]) -> Result<usize, StreamError> {
if output.is_empty() {
return Ok(0);
}
if self.pending_offset < self.pending.len() {
return Ok(self.copy_pending(output));
}
if self.finished {
return Ok(0);
}
match self.response.chunk().await {
Ok(Some(chunk)) => {
if self
.remaining
.is_some_and(|remaining| chunk.len() as u64 > remaining)
{
return Err(StreamError::SourceChanged);
}
self.pending = chunk;
Ok(self.copy_pending(output))
}
Ok(None) => {
if self.remaining.is_some_and(|remaining| remaining != 0) {
return Err(StreamError::transient_source(
"HTTP response ended before its declared length",
));
}
self.finished = true;
Ok(0)
}
Err(_) => Err(StreamError::transient_source("HTTP response body failed")),
}
}
}

44
crates/remora-http-source/src/error.rs Normal file
View file

@ -0,0 +1,44 @@
use std::time::Duration;
use remora_stream::{StreamError, StreamSide};
#[derive(Clone, Debug, Eq, thiserror::Error, PartialEq)]
pub enum HttpSourceError {
#[error("HTTP source request is invalid")]
InvalidRequest,
#[error("HTTP source access is denied")]
Authorization,
#[error("HTTP source was not found")]
NotFound,
#[error("HTTP source representation changed")]
SourceChanged,
#[error("HTTP source response is invalid")]
InvalidResponse,
#[error("HTTP source is temporarily unavailable")]
Transient { retry_after: Option<Duration> },
}
impl HttpSourceError {
#[must_use]
pub fn retry_after(&self) -> Option<Duration> {
match self {
Self::Transient { retry_after } => *retry_after,
_ => None,
}
}
#[must_use]
pub fn into_stream_error(self) -> StreamError {
match self {
Self::Authorization => StreamError::Authorization {
side: StreamSide::Source,
},
Self::NotFound => StreamError::SourceNotFound,
Self::SourceChanged => StreamError::SourceChanged,
Self::Transient { .. } => StreamError::transient_source("HTTP source unavailable"),
Self::InvalidRequest | Self::InvalidResponse => {
StreamError::internal("HTTP source contract violation")
}
}
}
}

154
crates/remora-http-source/src/identity.rs Normal file
View file

@ -0,0 +1,154 @@
use std::fmt;
use remora_domain::MemberPath;
use remora_outbound::CanonicalUrl;
use remora_stream::IntegrityDigest;
use crate::HttpSourceError;
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct StrongEtag(String);
impl StrongEtag {
pub fn parse(value: &str) -> Result<Self, HttpSourceError> {
let bytes = value.as_bytes();
if value.starts_with("W/")
|| bytes.len() < 2
|| bytes.first() != Some(&b'"')
|| bytes.last() != Some(&b'"')
|| bytes[1..bytes.len() - 1]
.iter()
.any(|byte| *byte == b'"' || *byte < 0x21 || *byte == 0x7f)
{
return Err(HttpSourceError::InvalidResponse);
}
Ok(Self(value.to_owned()))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Debug for StrongEtag {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("StrongEtag([redacted])")
}
}
#[derive(Clone, Eq, PartialEq)]
pub struct HttpSourceIdentity {
requested_url: CanonicalUrl,
final_url: CanonicalUrl,
etag: StrongEtag,
representation_length: u64,
content_encoding: HttpContentEncoding,
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum HttpContentEncoding {
Identity,
}
impl HttpSourceIdentity {
#[must_use]
pub fn from_parts(
requested_url: CanonicalUrl,
final_url: CanonicalUrl,
etag: StrongEtag,
representation_length: u64,
) -> Self {
Self {
requested_url,
final_url,
etag,
representation_length,
content_encoding: HttpContentEncoding::Identity,
}
}
#[must_use]
pub fn requested_url(&self) -> &CanonicalUrl {
&self.requested_url
}
#[must_use]
pub fn final_url(&self) -> &CanonicalUrl {
&self.final_url
}
#[must_use]
pub fn etag(&self) -> &StrongEtag {
&self.etag
}
#[must_use]
pub fn representation_length(&self) -> u64 {
self.representation_length
}
#[must_use]
pub fn content_encoding(&self) -> HttpContentEncoding {
self.content_encoding
}
}
impl fmt::Debug for HttpSourceIdentity {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("HttpSourceIdentity")
.field("representation_length", &self.representation_length)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct HttpResumeCheckpoint {
identity: HttpSourceIdentity,
result_path: MemberPath,
offset: u64,
prefix_integrity: IntegrityDigest,
}
impl HttpResumeCheckpoint {
pub fn new(
identity: HttpSourceIdentity,
result_path: MemberPath,
offset: u64,
prefix_integrity: IntegrityDigest,
) -> Result<Self, HttpSourceError> {
if offset == 0
|| offset >= identity.representation_length
|| prefix_integrity.as_str().is_empty()
{
return Err(HttpSourceError::InvalidRequest);
}
Ok(Self {
identity,
result_path,
offset,
prefix_integrity,
})
}
#[must_use]
pub fn identity(&self) -> &HttpSourceIdentity {
&self.identity
}
#[must_use]
pub fn result_path(&self) -> &MemberPath {
&self.result_path
}
#[must_use]
pub fn offset(&self) -> u64 {
self.offset
}
#[must_use]
pub fn prefix_integrity(&self) -> &IntegrityDigest {
&self.prefix_integrity
}
}

11
crates/remora-http-source/src/lib.rs Normal file
View file

@ -0,0 +1,11 @@
//! HTTP source adapter with strong representation identity and validated range resumption.
mod body;
mod error;
mod identity;
mod source;
pub use body::HttpSourceBody;
pub use error::HttpSourceError;
pub use identity::{HttpContentEncoding, HttpResumeCheckpoint, HttpSourceIdentity, StrongEtag};
pub use source::{HttpSource, HttpSourceRequest, OpenedHttpSource};

335
crates/remora-http-source/src/source.rs Normal file
View file

@ -0,0 +1,335 @@
use std::time::{Duration, SystemTime};
use http::{HeaderMap, HeaderValue, StatusCode, header};
use remora_domain::MemberPath;
use remora_outbound::{
CanonicalUrl, OutboundClient, OutboundError, OutboundPolicy, OutboundRequest, Resolver,
};
use remora_stream::{IntegrityDigest, RetryTiming, capped_server_hint};
use tokio::time::timeout;
use crate::{
HttpResumeCheckpoint, HttpSourceBody, HttpSourceError, HttpSourceIdentity, StrongEtag,
};
pub struct HttpSourceRequest {
url: CanonicalUrl,
result_path: MemberPath,
credential_headers: HeaderMap,
}
impl HttpSourceRequest {
pub fn new(url: &str, result_path: &str) -> Result<Self, HttpSourceError> {
Ok(Self {
url: CanonicalUrl::parse(url).map_err(|_| HttpSourceError::InvalidRequest)?,
result_path: MemberPath::parse(result_path)
.map_err(|_| HttpSourceError::InvalidRequest)?,
credential_headers: HeaderMap::new(),
})
}
pub fn with_credential_headers(mut self, headers: HeaderMap) -> Result<Self, HttpSourceError> {
OutboundRequest::get(self.url.as_url().as_str())
.map_err(|_| HttpSourceError::InvalidRequest)?
.with_credential_headers(headers.clone())
.map_err(|_| HttpSourceError::InvalidRequest)?;
self.credential_headers = headers;
Ok(self)
}
}
impl std::fmt::Debug for HttpSourceRequest {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("HttpSourceRequest")
.field("result_path", &self.result_path.canonical())
.field("credential_header_count", &self.credential_headers.len())
.finish_non_exhaustive()
}
}
pub struct OpenedHttpSource {
result_path: MemberPath,
identity: Option<HttpSourceIdentity>,
body: HttpSourceBody,
}
impl std::fmt::Debug for OpenedHttpSource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("OpenedHttpSource")
.field("result_path", &self.result_path.canonical())
.field("has_resume_identity", &self.identity.is_some())
.finish_non_exhaustive()
}
}
impl OpenedHttpSource {
#[must_use]
pub fn result_path(&self) -> &MemberPath {
&self.result_path
}
#[must_use]
pub fn identity(&self) -> Option<&HttpSourceIdentity> {
self.identity.as_ref()
}
#[must_use]
pub fn into_body(self) -> HttpSourceBody {
self.body
}
}
pub struct HttpSource<R> {
client: OutboundClient<R>,
timing: RetryTiming,
}
impl<R: Resolver> HttpSource<R> {
#[must_use]
pub fn new(resolver: R, policy: OutboundPolicy, timing: RetryTiming) -> Self {
let timing = timing.sanitized();
Self {
client: OutboundClient::builder(resolver, policy)
.connect_timeout(timing.connect_timeout)
.build(),
timing,
}
}
pub async fn open(
&self,
request: HttpSourceRequest,
) -> Result<OpenedHttpSource, HttpSourceError> {
let requested_url = request.url.clone();
let outbound = build_request(&request.url, request.credential_headers, None)?;
let response = self.execute(outbound).await?;
let status = response.response().status();
if status != StatusCode::OK {
return Err(status_error(
status,
response.response().headers(),
&self.timing,
));
}
ensure_identity_encoding(response.response().headers())?;
let expected_length = content_length(response.response().headers())?;
let identity = resumable_identity(
requested_url,
response.final_url().clone(),
response.response().headers(),
expected_length,
);
Ok(OpenedHttpSource {
result_path: request.result_path,
identity,
body: HttpSourceBody::new(response.into_response(), expected_length),
})
}
pub async fn resume(
&self,
checkpoint: &HttpResumeCheckpoint,
observed_prefix: &IntegrityDigest,
credential_headers: HeaderMap,
) -> Result<OpenedHttpSource, HttpSourceError> {
if checkpoint.prefix_integrity() != observed_prefix {
return Err(HttpSourceError::SourceChanged);
}
let outbound = build_request(
checkpoint.identity().requested_url(),
credential_headers,
Some(checkpoint),
)?;
let response = self.execute(outbound).await?;
if response.response().status() != StatusCode::PARTIAL_CONTENT {
return Err(match response.response().status() {
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => HttpSourceError::Authorization,
StatusCode::NOT_FOUND | StatusCode::GONE => HttpSourceError::NotFound,
status if status.is_server_error() || status == StatusCode::TOO_MANY_REQUESTS => {
status_error(status, response.response().headers(), &self.timing)
}
_ => HttpSourceError::SourceChanged,
});
}
validate_resume_response(checkpoint, &response)?;
let remaining = checkpoint.identity().representation_length() - checkpoint.offset();
Ok(OpenedHttpSource {
result_path: checkpoint.result_path().clone(),
identity: Some(checkpoint.identity().clone()),
body: HttpSourceBody::new(response.into_response(), Some(remaining)),
})
}
async fn execute(
&self,
request: OutboundRequest,
) -> Result<remora_outbound::OutboundResponse, HttpSourceError> {
match timeout(
self.timing.response_start_timeout,
self.client.execute(request),
)
.await
{
Ok(Ok(response)) => Ok(response),
Ok(Err(error)) => Err(map_outbound_error(error)),
Err(_) => Err(HttpSourceError::Transient { retry_after: None }),
}
}
}
fn build_request(
url: &CanonicalUrl,
credentials: HeaderMap,
checkpoint: Option<&HttpResumeCheckpoint>,
) -> Result<OutboundRequest, HttpSourceError> {
let mut public = HeaderMap::new();
public.insert(
header::ACCEPT_ENCODING,
HeaderValue::from_static("identity"),
);
if let Some(checkpoint) = checkpoint {
public.insert(
header::RANGE,
HeaderValue::from_str(&format!("bytes={}-", checkpoint.offset()))
.map_err(|_| HttpSourceError::InvalidRequest)?,
);
public.insert(
header::IF_MATCH,
HeaderValue::from_str(checkpoint.identity().etag().as_str())
.map_err(|_| HttpSourceError::InvalidRequest)?,
);
}
OutboundRequest::get(url.as_url().as_str())
.and_then(|request| request.with_public_headers(public))
.and_then(|request| request.with_credential_headers(credentials))
.map_err(|_| HttpSourceError::InvalidRequest)
}
fn ensure_identity_encoding(headers: &HeaderMap) -> Result<(), HttpSourceError> {
match headers.get(header::CONTENT_ENCODING) {
None => Ok(()),
Some(value) if value.as_bytes().eq_ignore_ascii_case(b"identity") => Ok(()),
Some(_) => Err(HttpSourceError::InvalidResponse),
}
}
fn content_length(headers: &HeaderMap) -> Result<Option<u64>, HttpSourceError> {
headers
.get(header::CONTENT_LENGTH)
.map(|value| {
value
.to_str()
.map_err(|_| HttpSourceError::InvalidResponse)?
.parse::<u64>()
.map_err(|_| HttpSourceError::InvalidResponse)
})
.transpose()
}
fn resumable_identity(
requested_url: CanonicalUrl,
final_url: CanonicalUrl,
headers: &HeaderMap,
length: Option<u64>,
) -> Option<HttpSourceIdentity> {
let etag = headers
.get(header::ETAG)?
.to_str()
.ok()
.and_then(|value| StrongEtag::parse(value).ok())?;
Some(HttpSourceIdentity::from_parts(
requested_url,
final_url,
etag,
length?,
))
}
fn validate_resume_response(
checkpoint: &HttpResumeCheckpoint,
response: &remora_outbound::OutboundResponse,
) -> Result<(), HttpSourceError> {
ensure_identity_encoding(response.response().headers())
.map_err(|_| HttpSourceError::SourceChanged)?;
if response.final_url() != checkpoint.identity().final_url() {
return Err(HttpSourceError::SourceChanged);
}
let etag = response
.response()
.headers()
.get(header::ETAG)
.and_then(|value| value.to_str().ok())
.and_then(|value| StrongEtag::parse(value).ok())
.ok_or(HttpSourceError::SourceChanged)?;
if &etag != checkpoint.identity().etag() {
return Err(HttpSourceError::SourceChanged);
}
let total = checkpoint.identity().representation_length();
let remaining = total - checkpoint.offset();
if content_length(response.response().headers())? != Some(remaining) {
return Err(HttpSourceError::SourceChanged);
}
let range = response
.response()
.headers()
.get(header::CONTENT_RANGE)
.and_then(|value| value.to_str().ok())
.ok_or(HttpSourceError::SourceChanged)?;
let expected = format!("bytes {}-{}/{total}", checkpoint.offset(), total - 1);
if range != expected {
return Err(HttpSourceError::SourceChanged);
}
Ok(())
}
fn status_error(status: StatusCode, headers: &HeaderMap, timing: &RetryTiming) -> HttpSourceError {
match status {
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => HttpSourceError::Authorization,
StatusCode::NOT_FOUND | StatusCode::GONE => HttpSourceError::NotFound,
status if status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() => {
HttpSourceError::Transient {
retry_after: parse_retry_after(headers, timing, SystemTime::now()),
}
}
_ => HttpSourceError::InvalidResponse,
}
}
fn parse_retry_after(
headers: &HeaderMap,
timing: &RetryTiming,
now: SystemTime,
) -> Option<Duration> {
let value = headers.get(header::RETRY_AFTER)?.to_str().ok()?;
let hint = value
.parse::<u64>()
.ok()
.map(Duration::from_secs)
.or_else(|| {
httpdate::parse_http_date(value)
.ok()?
.duration_since(now)
.ok()
});
capped_server_hint(hint, timing)
}
fn map_outbound_error(error: OutboundError) -> HttpSourceError {
match error {
OutboundError::ResolutionFailed | OutboundError::Transport => {
HttpSourceError::Transient { retry_after: None }
}
OutboundError::InvalidUrl | OutboundError::InvalidHeaders => {
HttpSourceError::InvalidRequest
}
OutboundError::EmptyResolution
| OutboundError::AddressDenied
| OutboundError::SchemePortDenied
| OutboundError::HttpsDowngrade => HttpSourceError::Authorization,
OutboundError::InvalidRedirect | OutboundError::RedirectLimit => {
HttpSourceError::InvalidResponse
}
}
}

579
crates/remora-http-source/tests/http_source.rs Normal file
View file

@ -0,0 +1,579 @@
use std::{
collections::HashMap,
io,
net::IpAddr,
sync::{Arc, Mutex},
time::Duration,
};
use http::{HeaderMap, HeaderValue};
use ipnet::IpNet;
use remora_domain::MemberPath;
use remora_http_source::{
HttpContentEncoding, HttpResumeCheckpoint, HttpSource, HttpSourceError, HttpSourceIdentity,
HttpSourceRequest, StrongEtag,
};
use remora_outbound::{CanonicalUrl, OutboundPolicyBuilder, Resolver};
use remora_stream::{RetryTiming, Sha256Accountant, SourceBody, StreamError};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
task::JoinHandle,
};
#[derive(Clone)]
struct LocalResolver;
impl Resolver for LocalResolver {
async fn resolve(&self, _host: &str, _port: u16) -> io::Result<Vec<IpAddr>> {
Ok(vec!["127.0.0.1".parse().expect("address")])
}
}
#[derive(Debug)]
struct ObservedRequest {
path: String,
headers: HashMap<String, String>,
}
struct ResponseSpec {
status: &'static str,
headers: Vec<(&'static str, String)>,
body: Vec<u8>,
}
impl ResponseSpec {
fn new(status: &'static str, body: impl Into<Vec<u8>>) -> Self {
Self {
status,
headers: Vec::new(),
body: body.into(),
}
}
fn header(mut self, name: &'static str, value: impl Into<String>) -> Self {
self.headers.push((name, value.into()));
self
}
}
async fn serve(
connections: usize,
handler: impl Fn(usize, &ObservedRequest) -> ResponseSpec + Send + Sync + 'static,
) -> (u16, Arc<Mutex<Vec<ObservedRequest>>>, JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
let port = listener.local_addr().expect("address").port();
let (observed, task) = start_server(listener, connections, handler);
(port, observed, task)
}
async fn serve_on(
port: u16,
connections: usize,
handler: impl Fn(usize, &ObservedRequest) -> ResponseSpec + Send + Sync + 'static,
) -> (Arc<Mutex<Vec<ObservedRequest>>>, JoinHandle<()>) {
let listener = TcpListener::bind(("127.0.0.1", port)).await.expect("bind");
start_server(listener, connections, handler)
}
fn start_server(
listener: TcpListener,
connections: usize,
handler: impl Fn(usize, &ObservedRequest) -> ResponseSpec + Send + Sync + 'static,
) -> (Arc<Mutex<Vec<ObservedRequest>>>, JoinHandle<()>) {
let observed = Arc::new(Mutex::new(Vec::new()));
let task_observed = Arc::clone(&observed);
let handler = Arc::new(handler);
let task = tokio::spawn(async move {
for index in 0..connections {
let (mut stream, _) = listener.accept().await.expect("accept");
let request = read_request(&mut stream).await;
let response = handler(index, &request);
task_observed.lock().expect("observed").push(request);
let mut head = format!("HTTP/1.1 {}\r\nConnection: close\r\n", response.status);
for (name, value) in response.headers {
head.push_str(&format!("{name}: {value}\r\n"));
}
head.push_str("\r\n");
if let Err(error) = stream.write_all(head.as_bytes()).await {
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
continue;
}
if let Err(error) = stream.write_all(&response.body).await {
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
}
}
});
(observed, task)
}
async fn read_request(stream: &mut tokio::net::TcpStream) -> ObservedRequest {
let mut bytes = Vec::new();
let mut chunk = [0_u8; 1024];
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let count = stream.read(&mut chunk).await.expect("read");
assert_ne!(count, 0);
bytes.extend_from_slice(&chunk[..count]);
}
let request = String::from_utf8(bytes).expect("request text");
let mut lines = request.split("\r\n");
let path = lines
.next()
.and_then(|line| line.split_whitespace().nth(1))
.expect("path")
.to_owned();
let headers = lines
.filter_map(|line| line.split_once(':'))
.map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_owned()))
.collect();
ObservedRequest { path, headers }
}
fn source(port: u16) -> HttpSource<LocalResolver> {
let policy = OutboundPolicyBuilder::new()
.allow_scheme_port("http", port)
.allow_non_global("127.0.0.0/8".parse::<IpNet>().expect("network"))
.build();
HttpSource::new(LocalResolver, policy, RetryTiming::default_bounds())
}
fn request(port: u16) -> HttpSourceRequest {
HttpSourceRequest::new(
&format!("http://source.test:{port}/entry?secret=hidden"),
"dir/cafe\u{301}.bin",
)
.expect("request")
}
fn strong(body: &[u8], etag: &str) -> ResponseSpec {
ResponseSpec::new("200 OK", body)
.header("Content-Length", body.len().to_string())
.header("ETag", etag)
}
async fn read_all(mut body: remora_http_source::HttpSourceBody) -> Result<Vec<u8>, StreamError> {
let mut output = Vec::new();
let mut buffer = [0_u8; 3];
loop {
let count = body.read(&mut buffer).await?;
if count == 0 {
return Ok(output);
}
output.extend_from_slice(&buffer[..count]);
}
}
fn digest(bytes: &[u8]) -> remora_stream::IntegrityDigest {
let mut accountant = Sha256Accountant::new();
accountant.update(bytes);
accountant.finalize().digest().expect("digest").clone()
}
#[tokio::test]
async fn full_stream_exposes_strong_identity_and_explicit_canonical_result_path() {
let body = b"representation".to_vec();
let expected = body.clone();
let (port, observed, server) = serve(1, move |_, _| strong(&body, "\"v1\"")).await;
let opened = source(port).open(request(port)).await.expect("open");
assert_eq!(opened.result_path().canonical(), "dir/café.bin");
let identity = opened.identity().expect("strong identity");
assert_eq!(identity.etag().as_str(), "\"v1\"");
assert_eq!(identity.representation_length(), expected.len() as u64);
assert_eq!(identity.content_encoding(), HttpContentEncoding::Identity);
assert_eq!(read_all(opened.into_body()).await.expect("body"), expected);
server.await.expect("server");
let observed = observed.lock().expect("observed");
assert_eq!(observed[0].path, "/entry?secret=hidden");
assert_eq!(
observed[0]
.headers
.get("accept-encoding")
.map(String::as_str),
Some("identity")
);
}
#[tokio::test]
async fn matching_conditional_range_resumes_exact_suffix() {
let bytes = b"abcdefghij".to_vec();
let served = bytes.clone();
let (port, observed, server) = serve(2, move |index, request| {
if index == 0 {
strong(&served, "\"v1\"")
} else {
assert_eq!(
request.headers.get("range").map(String::as_str),
Some("bytes=4-")
);
assert_eq!(
request.headers.get("if-match").map(String::as_str),
Some("\"v1\"")
);
ResponseSpec::new("206 Partial Content", served[4..].to_vec())
.header("Content-Length", (served.len() - 4).to_string())
.header(
"Content-Range",
format!("bytes 4-{}/{}", served.len() - 1, served.len()),
)
.header("ETag", "\"v1\"")
}
})
.await;
let initial = source(port).open(request(port)).await.expect("initial");
let checkpoint = HttpResumeCheckpoint::new(
initial.identity().expect("identity").clone(),
initial.result_path().clone(),
4,
digest(&bytes[..4]),
)
.expect("checkpoint");
drop(initial);
let resumed = source(port)
.resume(&checkpoint, &digest(&bytes[..4]), HeaderMap::new())
.await
.expect("resume");
assert_eq!(
read_all(resumed.into_body()).await.expect("body"),
bytes[4..]
);
server.await.expect("server");
assert_eq!(observed.lock().expect("observed").len(), 2);
}
#[tokio::test]
async fn durable_identity_and_path_resume_after_worker_and_server_restart() {
let (port, _, first_server) = serve(2, |index, _| match index {
0 => ResponseSpec::new("302 Found", Vec::new())
.header("Content-Length", "0")
.header("Location", "/stable"),
1 => strong(b"abcdefghij", "\"v1\""),
_ => unreachable!(),
})
.await;
let initial = source(port).open(request(port)).await.expect("initial");
let identity = initial.identity().expect("identity");
let persisted = (
identity.requested_url().as_url().as_str().to_owned(),
identity.final_url().as_url().as_str().to_owned(),
identity.etag().as_str().to_owned(),
identity.representation_length(),
initial.result_path().canonical(),
4,
digest(b"abcd"),
);
drop(initial);
first_server.await.expect("first server");
let (observed, second_server) = serve_on(port, 2, |index, _| match index {
0 => ResponseSpec::new("302 Found", Vec::new())
.header("Content-Length", "0")
.header("Location", "/stable"),
1 => ResponseSpec::new("206 Partial Content", b"efghij".to_vec())
.header("Content-Length", "6")
.header("Content-Range", "bytes 4-9/10")
.header("ETag", "\"v1\""),
_ => unreachable!(),
})
.await;
let rehydrated = HttpSourceIdentity::from_parts(
CanonicalUrl::parse(&persisted.0).expect("requested URL"),
CanonicalUrl::parse(&persisted.1).expect("final URL"),
StrongEtag::parse(&persisted.2).expect("etag"),
persisted.3,
);
let checkpoint = HttpResumeCheckpoint::new(
rehydrated,
MemberPath::parse(&persisted.4).expect("result path"),
persisted.5,
persisted.6,
)
.expect("checkpoint");
let resumed = source(port)
.resume(&checkpoint, &digest(b"abcd"), HeaderMap::new())
.await
.expect("resume");
assert_eq!(resumed.result_path().canonical(), "dir/café.bin");
assert_eq!(
read_all(resumed.into_body()).await.expect("body"),
b"efghij"
);
second_server.await.expect("second server");
let observed = observed.lock().expect("observed");
assert_eq!(observed[0].path, "/entry?secret=hidden");
assert_eq!(observed[1].path, "/stable");
}
#[tokio::test]
async fn weak_absent_and_modification_time_only_responses_are_restart_only() {
for (etag, last_modified) in [
(Some("W/\"weak\""), None),
(None, None),
(None, Some("Wed, 21 Oct 2015 07:28:00 GMT")),
] {
let (port, _, server) = serve(1, move |_, _| {
let mut response =
ResponseSpec::new("200 OK", b"body".to_vec()).header("Content-Length", "4");
if let Some(etag) = etag {
response = response.header("ETag", etag);
}
if let Some(last_modified) = last_modified {
response = response.header("Last-Modified", last_modified);
}
response
})
.await;
let opened = source(port).open(request(port)).await.expect("open");
assert!(opened.identity().is_none());
assert_eq!(read_all(opened.into_body()).await.expect("body"), b"body");
server.await.expect("server");
}
}
#[tokio::test]
async fn changed_or_unproven_resume_responses_fail_as_source_changed() {
for response in [
ResponseSpec::new("412 Precondition Failed", Vec::new()).header("Content-Length", "0"),
ResponseSpec::new("200 OK", b"changed".to_vec()).header("Content-Length", "7"),
ResponseSpec::new("206 Partial Content", b"efghij".to_vec())
.header("Content-Length", "6")
.header("Content-Range", "bytes 5-9/10")
.header("ETag", "\"v1\""),
ResponseSpec::new("206 Partial Content", b"efghij".to_vec())
.header("Content-Length", "6")
.header("Content-Range", "bytes 4-9/10")
.header("ETag", "\"v2\""),
ResponseSpec::new("206 Partial Content", b"efghij".to_vec())
.header("Content-Length", "6")
.header("Content-Range", "bytes 4-9/10")
.header("ETag", "\"v1\"")
.header("Content-Encoding", "gzip"),
ResponseSpec::new("206 Partial Content", b"efghijk".to_vec())
.header("Content-Length", "7")
.header("Content-Range", "bytes 4-10/11")
.header("ETag", "\"v1\""),
] {
let response = Arc::new(Mutex::new(Some(response)));
let task_response = Arc::clone(&response);
let (port, _, server) = serve(2, move |index, _| {
if index == 0 {
strong(b"abcdefghij", "\"v1\"")
} else {
task_response
.lock()
.expect("response")
.take()
.expect("once")
}
})
.await;
let source = source(port);
let opened = source.open(request(port)).await.expect("initial");
let identity = opened.identity().expect("identity").clone();
let checkpoint =
HttpResumeCheckpoint::new(identity, opened.result_path().clone(), 4, digest(b"abcd"))
.expect("checkpoint");
drop(opened);
assert_eq!(
source
.resume(&checkpoint, &digest(b"abcd"), HeaderMap::new())
.await
.expect_err("changed"),
HttpSourceError::SourceChanged
);
server.await.expect("server");
}
}
#[tokio::test]
async fn changed_redirect_target_invalidates_resume_identity() {
let (port, _, server) = serve(4, |index, _| match index {
0 => ResponseSpec::new("302 Found", Vec::new())
.header("Content-Length", "0")
.header("Location", "/a"),
1 => strong(b"abcdefghij", "\"v1\""),
2 => ResponseSpec::new("302 Found", Vec::new())
.header("Content-Length", "0")
.header("Location", "/b"),
3 => ResponseSpec::new("206 Partial Content", b"efghij".to_vec())
.header("Content-Length", "6")
.header("Content-Range", "bytes 4-9/10")
.header("ETag", "\"v1\""),
_ => unreachable!(),
})
.await;
let source = source(port);
let opened = source.open(request(port)).await.expect("initial");
let checkpoint = HttpResumeCheckpoint::new(
opened.identity().expect("identity").clone(),
opened.result_path().clone(),
4,
digest(b"abcd"),
)
.expect("checkpoint");
drop(opened);
assert_eq!(
source
.resume(&checkpoint, &digest(b"abcd"), HeaderMap::new())
.await
.expect_err("redirect changed"),
HttpSourceError::SourceChanged
);
server.await.expect("server");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn response_start_timeout_is_retryable() {
let (port, _, server) = serve(1, |_, _| {
std::thread::sleep(Duration::from_millis(50));
strong(b"body", "\"v1\"")
})
.await;
let policy = OutboundPolicyBuilder::new()
.allow_scheme_port("http", port)
.allow_non_global("127.0.0.0/8".parse::<IpNet>().expect("network"))
.build();
let timing = RetryTiming {
response_start_timeout: Duration::from_millis(5),
..RetryTiming::default_bounds()
};
assert_eq!(
HttpSource::new(LocalResolver, policy, timing)
.open(request(port))
.await
.expect_err("timeout"),
HttpSourceError::Transient { retry_after: None }
);
server.await.expect("server");
}
#[tokio::test]
async fn durable_prefix_mismatch_fails_before_network_io() {
let (port, _, server) = serve(1, |_, _| strong(b"abcdefghij", "\"v1\"")).await;
let source = source(port);
let opened = source.open(request(port)).await.expect("initial");
let identity = opened.identity().expect("identity").clone();
let checkpoint =
HttpResumeCheckpoint::new(identity, opened.result_path().clone(), 4, digest(b"abcd"))
.expect("checkpoint");
drop(opened);
assert_eq!(
source
.resume(&checkpoint, &digest(b"abce"), HeaderMap::new())
.await
.expect_err("prefix"),
HttpSourceError::SourceChanged
);
server.await.expect("server");
}
#[tokio::test]
async fn truncated_stream_is_retryable_and_non_identity_encoding_fails_closed() {
let (port, _, server) = serve(1, |_, _| {
ResponseSpec::new("200 OK", b"short".to_vec())
.header("Content-Length", "10")
.header("ETag", "\"v1\"")
.header("Accept-Ranges", "bytes")
})
.await;
let opened = source(port).open(request(port)).await.expect("open");
assert!(matches!(
read_all(opened.into_body()).await.expect_err("truncated"),
StreamError::Transient { .. }
));
server.await.expect("server");
let (port, _, server) = serve(1, |_, _| {
strong(b"compressed", "\"gzip\"").header("Content-Encoding", "gzip")
})
.await;
assert_eq!(
source(port)
.open(request(port))
.await
.expect_err("encoding"),
HttpSourceError::InvalidResponse
);
server.await.expect("server");
}
#[tokio::test]
async fn truncated_resumed_stream_is_retryable() {
let (port, _, server) = serve(2, |index, _| {
if index == 0 {
strong(b"abcdefghij", "\"v1\"")
} else {
ResponseSpec::new("206 Partial Content", b"efg".to_vec())
.header("Content-Length", "6")
.header("Content-Range", "bytes 4-9/10")
.header("ETag", "\"v1\"")
}
})
.await;
let initial = source(port).open(request(port)).await.expect("initial");
let checkpoint = HttpResumeCheckpoint::new(
initial.identity().expect("identity").clone(),
initial.result_path().clone(),
4,
digest(b"abcd"),
)
.expect("checkpoint");
drop(initial);
let resumed = source(port)
.resume(&checkpoint, &digest(b"abcd"), HeaderMap::new())
.await
.expect("resume");
assert!(matches!(
read_all(resumed.into_body()).await.expect_err("truncated"),
StreamError::Transient { .. }
));
server.await.expect("server");
}
#[tokio::test]
async fn status_mapping_and_retry_hints_are_bounded() {
for (status, retry_after, expected) in [
("401 Unauthorized", None, HttpSourceError::Authorization),
("404 Not Found", None, HttpSourceError::NotFound),
(
"503 Service Unavailable",
Some("3"),
HttpSourceError::Transient {
retry_after: Some(Duration::from_secs(3)),
},
),
(
"429 Too Many Requests",
Some("600"),
HttpSourceError::Transient { retry_after: None },
),
] {
let (port, _, server) = serve(1, move |_, _| {
let mut response = ResponseSpec::new(status, Vec::new()).header("Content-Length", "0");
if let Some(value) = retry_after {
response = response.header("Retry-After", value);
}
response
})
.await;
assert_eq!(
source(port).open(request(port)).await.expect_err("status"),
expected
);
server.await.expect("server");
}
}
#[test]
fn request_debug_and_validation_do_not_expose_sensitive_url_or_headers() {
assert!(HttpSourceRequest::new("http://source.test/object", "../bad").is_err());
let mut headers = HeaderMap::new();
headers.insert("x-api-key", HeaderValue::from_static("top-secret"));
let request = HttpSourceRequest::new("https://source.test/object?token=secret", "artifact.bin")
.expect("request")
.with_credential_headers(headers)
.expect("credentials");
let debug = format!("{request:?}");
assert!(!debug.contains("token"));
assert!(!debug.contains("secret"));
assert!(!debug.contains("top-secret"));
}

13
crates/remora-outbound/src/transport.rs
View file

@ -1,4 +1,4 @@
use std::{fmt, net::SocketAddr};
use std::{fmt, net::SocketAddr, time::Duration};
use http::{HeaderMap, HeaderName, Method, StatusCode};
use reqwest::{Certificate, redirect::Policy};
@ -18,6 +18,7 @@ pub struct OutboundClient<R> {
resolver: R,
policy: OutboundPolicy,
roots: Vec<Certificate>,
connect_timeout: Option<Duration>,
}
#[derive(Clone, Debug)]
@ -54,6 +55,7 @@ impl<R: Resolver> OutboundClient<R> {
resolver,
policy,
roots: Vec::new(),
connect_timeout: None,
},
}
}
@ -109,6 +111,9 @@ impl<R: Resolver> OutboundClient<R> {
let mut builder = reqwest::Client::builder()
.redirect(Policy::none())
.no_proxy();
if let Some(connect_timeout) = self.connect_timeout {
builder = builder.connect_timeout(connect_timeout);
}
if url.literal_address().is_none() {
builder = builder.resolve(
url.host_str(),
@ -123,6 +128,12 @@ impl<R: Resolver> OutboundClient<R> {
}
impl<R> OutboundClientBuilder<R> {
#[must_use]
pub fn connect_timeout(mut self, timeout: Duration) -> Self {
self.client.connect_timeout = Some(timeout);
self
}
#[must_use]
pub fn add_root_certificate(mut self, certificate: Certificate) -> Self {
self.client.roots.push(certificate);