ms2k/remora
SHA256
Watch
0
Code Releases Activity

Ship production deployment and runtime observability

Browse source
This commit is contained in:
ms2k 2026-07-18 04:22:14 +09:00
commit 775dd61963
4 changed files with 3712 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

675
crates/remora-api/src/application.rs Normal file
View file

@ -0,0 +1,675 @@
use std::{
fs,
io::{self, BufReader},
net::{SocketAddr, TcpListener},
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use actix_web::{App, HttpServer, middleware, web};
use remora_admission::{AdmissionConfig, AdmissionService};
use remora_attempt::{
AttemptConfig, AttemptService, CleanupIdentity, RecoveryConfig, StagingMultipart, StagingObject,
};
use remora_coordination::{CoordinationStore, StoreConfig};
use remora_credentials::{CredentialRegistry, VaultClient, VaultConfig};
use remora_deletion::{DeletionConfig, DeletionService, DrainOutcome, TargetRegistry};
use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
use remora_durable::{PoolConfig, connect};
use remora_identity::{DefaultTarget, OidcConfig, OidcIdentityProvider};
use remora_runtime::{
ManagementConfig, RuntimeError, RuntimeStatus, env_duration, env_parse, env_string,
required_env, spawn_periodic, start_management_server, trace_request,
};
use remora_target::{MultipartIdentity, ObjectIdentity, TargetStore, TargetStoreConfig};
use rustls::{
RootCertStore, ServerConfig,
pki_types::{CertificateDer, PrivateKeyDer},
server::WebPkiClientVerifier,
};
use sqlx::PgPool;
use tokio::task::JoinHandle;
use crate::{
PublicApiConfig, PublicApiServices, PublicApiState, WorkerApiState, authenticate_rustls_worker,
configure_public_api, configure_worker_api,
};
#[derive(Clone)]
pub struct ApiApplicationConfig {
pub public_bind: SocketAddr,
pub worker_bind: SocketAddr,
pub management: ManagementConfig,
pub database_url: String,
pub database_connections: u32,
pub coordination_url: String,
pub coordination_namespace: String,
pub coordination: StoreConfig,
pub attempt: AttemptConfig,
pub recovery: RecoveryConfig,
pub recovery_interval: Duration,
pub credential_reconcile_interval: Duration,
pub deletion_interval: Duration,
pub default_target_prefix: String,
pub reserved_target_segments: Vec<String>,
pub admission_policy_version: String,
pub public_api: PublicApiConfig,
pub oidc: OidcConfig,
pub target: TargetStoreConfig,
pub vault: VaultConfig,
pub worker_tls: WorkerTlsConfig,
}
#[derive(Clone, Debug)]
pub struct WorkerTlsConfig {
pub certificate: PathBuf,
pub private_key: PathBuf,
pub client_ca: PathBuf,
}
pub struct ApiApplication {
config: ApiApplicationConfig,
}
impl ApiApplicationConfig {
pub fn from_env() -> Result<Self, RuntimeError> {
let reserved_target_segments = comma_values(
"REMORA_RESERVED_TARGET_SEGMENTS",
&env_string("REMORA_RESERVED_TARGET_SEGMENTS", "_remora"),
)?;
let oidc_audience = required_env("REMORA_OIDC_AUDIENCE")?;
let oidc_resource_client = env_string("REMORA_OIDC_RESOURCE_CLIENT", &oidc_audience);
let mut oidc = OidcConfig::new(required_env("REMORA_OIDC_ISSUER")?, oidc_audience)
.map_err(runtime_error)?
.with_resource_access_client(oidc_resource_client)
.with_reserved_segments(reserved_target_segments.clone());
for claim in comma_values(
"REMORA_OIDC_PROJECT_ROLE_CLAIMS",
&env_string(
"REMORA_OIDC_PROJECT_ROLE_CLAIMS",
remora_identity::ZITADEL_PROJECT_ROLES_CLAIM,
),
)? {
oidc = oidc.with_project_role_claim(claim);
}
let event_origins = comma_values(
"REMORA_EVENT_ALLOWED_ORIGINS",
&required_env("REMORA_EVENT_ALLOWED_ORIGINS")?,
)?;
let cursor_secret_path = PathBuf::from(required_env("REMORA_CURSOR_SECRET_FILE")?);
let cursor_secret = fs::read(&cursor_secret_path)
.map_err(|error| file_error("REMORA_CURSOR_SECRET_FILE", error))?;
let global_limit = env_parse("REMORA_GLOBAL_ATTEMPT_LIMIT", "64")?;
let lease_ttl = env_duration("REMORA_LEASE_TTL_SECONDS", 30)?;
let target = TargetStoreConfig::from_env().map_err(runtime_error)?;
let vault = VaultConfig::from_env().map_err(runtime_error)?;
Ok(Self {
public_bind: env_parse("REMORA_PUBLIC_BIND", "0.0.0.0:8080")?,
worker_bind: env_parse("REMORA_WORKER_BIND", "0.0.0.0:8443")?,
management: ManagementConfig::from_env(crate::COMPONENT_NAME)?,
database_url: required_env("DATABASE_URL")?,
database_connections: env_parse("REMORA_DATABASE_MAX_CONNECTIONS", "32")?,
coordination_url: required_env("REMORA_COORDINATION_URL")?,
coordination_namespace: required_env("REMORA_COORDINATION_NAMESPACE")?,
coordination: StoreConfig {
global_limit,
lease_ttl,
reserve_ttl: env_duration("REMORA_RESERVE_TTL_SECONDS", 30)?,
progress_ttl: env_duration("REMORA_PROGRESS_TTL_SECONDS", 60)?,
cancel_ttl: None,
},
attempt: AttemptConfig {
max_attempts: env_parse("REMORA_MAX_ATTEMPTS", "3")?,
},
recovery: RecoveryConfig {
retry_delay: env_duration("REMORA_RETRY_DELAY_SECONDS", 1)?,
cleanup_claim_ttl: env_duration("REMORA_CLEANUP_CLAIM_TTL_SECONDS", 30)?,
batch_size: env_parse("REMORA_RECOVERY_BATCH_SIZE", "128")?,
},
recovery_interval: env_duration("REMORA_RECOVERY_INTERVAL_SECONDS", 2)?,
credential_reconcile_interval: env_duration(
"REMORA_CREDENTIAL_RECONCILE_INTERVAL_SECONDS",
10,
)?,
deletion_interval: env_duration("REMORA_DELETION_INTERVAL_SECONDS", 2)?,
default_target_prefix: env_string("REMORA_DEFAULT_TARGET_PREFIX", ""),
reserved_target_segments,
admission_policy_version: env_string(
"REMORA_ADMISSION_POLICY_VERSION",
"deployment-v1",
),
public_api: PublicApiConfig {
cursor_secret,
member_access_ttl: env_duration("REMORA_MEMBER_ACCESS_TTL_SECONDS", 60)?,
event_retention: env_duration("REMORA_EVENT_RETENTION_SECONDS", 3600)?,
event_poll_interval: duration_millis("REMORA_EVENT_POLL_MILLIS", 250)?,
event_heartbeat_interval: env_duration("REMORA_EVENT_HEARTBEAT_SECONDS", 15)?,
event_ticket_ttl: env_duration("REMORA_EVENT_TICKET_TTL_SECONDS", 30)?,
event_allowed_origins: event_origins,
},
oidc,
target,
vault,
worker_tls: WorkerTlsConfig {
certificate: PathBuf::from(required_env("REMORA_WORKER_TLS_CERT_FILE")?),
private_key: PathBuf::from(required_env("REMORA_WORKER_TLS_KEY_FILE")?),
client_ca: PathBuf::from(required_env("REMORA_WORKER_TLS_CLIENT_CA_FILE")?),
},
})
}
}
impl ApiApplication {
#[must_use]
pub fn new(config: ApiApplicationConfig) -> Self {
Self { config }
}
pub async fn run(self) -> Result<(), RuntimeError> {
let _ = rustls::crypto::ring::default_provider().install_default();
let pool = connect(
&self.config.database_url,
PoolConfig {
max_connections: self.config.database_connections,
acquire_timeout: Duration::from_secs(10),
idle_timeout: Some(Duration::from_secs(60)),
},
)
.await
.map_err(runtime_error)?;
migrate_all(&pool).await?;
let coordination = CoordinationStore::connect(
&self.config.coordination_url,
&self.config.coordination_namespace,
self.config.coordination.clone(),
)
.await
.map_err(runtime_error)?;
let target = TargetStore::connect(self.config.target.clone()).map_err(runtime_error)?;
let readiness_target = target.clone();
let target_registry = TargetRegistry::new([target.clone()]);
let vault = VaultClient::new(self.config.vault.clone()).map_err(runtime_error)?;
let readiness_vault = vault.clone();
let credentials = CredentialRegistry::new(pool.clone(), vault);
let attempts = AttemptService::new(
pool.clone(),
coordination.clone(),
self.config.attempt.clone(),
)
.map_err(runtime_error)?;
let deletion = DeletionService::new(
pool.clone(),
target_registry.clone(),
DeletionConfig::default(),
)
.map_err(runtime_error)?;
let state = PublicApiState::new(
pool.clone(),
PublicApiServices {
identity: OidcIdentityProvider::new(self.config.oidc.clone())
.map_err(runtime_error)?,
admission: admission_service(&self.config, &pool)?,
attempts: attempts.clone(),
credentials: credentials.clone(),
deletion: deletion.clone(),
targets: target_registry.clone(),
},
self.config.public_api.clone(),
)
.map_err(runtime_error)?;
let status = RuntimeStatus::new(crate::COMPONENT_NAME, pool.clone(), coordination.clone())
.with_readiness_check(move || {
let target = readiness_target.clone();
async move { target.check().await.is_ok() }
})
.with_readiness_check(move || {
let vault = readiness_vault.clone();
async move { vault.check().await.is_ok() }
});
let maintenance = start_maintenance(
&self.config,
&pool,
&status,
attempts.clone(),
credentials,
deletion,
target_registry,
);
let public = public_server(self.config.public_bind, state)?;
let worker = worker_server(
self.config.worker_bind,
WorkerApiState::new(attempts),
&self.config.worker_tls,
)?;
let management = start_management_server(self.config.management, status.clone())
.map_err(runtime_error)?;
status.set_accepting_traffic(true);
tracing::info!(
public_bind = %self.config.public_bind,
worker_bind = %self.config.worker_bind,
"api application ready"
);
wait_for_shutdown().await?;
status.set_accepting_traffic(false);
public.stop(true).await;
worker.stop(true).await;
management.stop(true).await;
for task in maintenance {
task.abort();
}
pool.close().await;
Ok(())
}
}
fn admission_service(
config: &ApiApplicationConfig,
pool: &PgPool,
) -> Result<AdmissionService, RuntimeError> {
let store =
TargetStoreId::parse(config.target.target_store_id.clone()).map_err(runtime_error)?;
let bucket = BucketName::parse(config.target.bucket.clone()).map_err(runtime_error)?;
let prefix = TargetPrefix::parse(
&config.default_target_prefix,
config.reserved_target_segments.iter().map(String::as_str),
)
.map_err(runtime_error)?;
AdmissionService::new(
pool.clone(),
AdmissionConfig {
default_target: DefaultTarget {
store,
bucket,
prefix,
},
reserved_target_segments: config.reserved_target_segments.clone(),
policy_version: config.admission_policy_version.clone(),
},
)
.map_err(runtime_error)
}
async fn migrate_all(pool: &PgPool) -> Result<(), RuntimeError> {
remora_durable::migrate(pool).await.map_err(runtime_error)?;
remora_identity::migrate(pool)
.await
.map_err(runtime_error)?;
remora_credentials::migrate(pool)
.await
.map_err(runtime_error)?;
remora_admission::migrate(pool)
.await
.map_err(runtime_error)?;
remora_attempt::migrate(pool).await.map_err(runtime_error)?;
remora_result::migrate(pool).await.map_err(runtime_error)?;
remora_deletion::migrate(pool)
.await
.map_err(runtime_error)?;
crate::migrate(pool).await.map_err(runtime_error)
}
fn start_maintenance(
config: &ApiApplicationConfig,
pool: &PgPool,
status: &RuntimeStatus,
attempts: AttemptService,
credentials: CredentialRegistry,
deletion: DeletionService,
targets: TargetRegistry,
) -> Vec<JoinHandle<()>> {
let recovery = config.recovery.clone();
let cleanup_attempts = attempts.clone();
let cleanup_recovery = recovery.clone();
let cleanup_targets = targets.clone();
let attempt_task = spawn_periodic(
"attempt_recovery",
config.recovery_interval,
status.clone(),
move || {
let attempts = attempts.clone();
let recovery = recovery.clone();
async move {
attempts
.reconcile_once(&recovery)
.await
.map(|_| ())
.map_err(|error| error.to_string())
}
},
);
let credential_task = spawn_periodic(
"credential_recovery",
config.credential_reconcile_interval,
status.clone(),
move || {
let credentials = credentials.clone();
async move {
credentials
.reconcile_secret_write_intents()
.await
.map(|_| ())
.map_err(|error| error.to_string())
}
},
);
let deletion_pool = pool.clone();
let deletion_task = spawn_periodic(
"result_deletion",
config.deletion_interval,
status.clone(),
move || {
let deletion = deletion.clone();
let pool = deletion_pool.clone();
async move { process_deletions(&pool, &deletion).await }
},
);
let cleanup_task = spawn_periodic(
"staging_cleanup",
config.recovery_interval,
status.clone(),
move || {
let attempts = cleanup_attempts.clone();
let recovery = cleanup_recovery.clone();
let targets = cleanup_targets.clone();
async move { process_staging_cleanup(&attempts, &recovery, &targets).await }
},
);
let event_pool = pool.clone();
let retention = config.public_api.event_retention;
let event_task = spawn_periodic(
"event_history_prune",
Duration::from_secs(60),
status.clone(),
move || {
let pool = event_pool.clone();
async move {
let retention =
time::Duration::try_from(retention).map_err(|error| error.to_string())?;
crate::prune_live_event_history(&pool, time::OffsetDateTime::now_utc() - retention)
.await
.map(|_| ())
.map_err(|error| error.to_string())
}
},
);
vec![
attempt_task,
credential_task,
deletion_task,
cleanup_task,
event_task,
]
}
async fn process_deletions(pool: &PgPool, deletion: &DeletionService) -> Result<(), String> {
let operation_ids = sqlx::query_scalar::<_, String>(
"SELECT operation_id FROM deletion_operations
WHERE state IN ('queued', 'running', 'retry_wait', 'blocked')
ORDER BY updated_at, operation_id LIMIT 32",
)
.fetch_all(pool)
.await
.map_err(|error| error.to_string())?;
for value in operation_ids {
let operation_id = OpaqueId::parse(value).map_err(|error| error.to_string())?;
match deletion.process_next(&operation_id).await {
Ok(
DrainOutcome::Idle
| DrainOutcome::ItemCompleted
| DrainOutcome::Blocked
| DrainOutcome::Completed,
) => {}
Err(error) => return Err(error.to_string()),
}
}
Ok(())
}
async fn process_staging_cleanup(
attempts: &AttemptService,
recovery: &RecoveryConfig,
targets: &TargetRegistry,
) -> Result<(), String> {
let Some(claim) = attempts
.claim_cleanup(recovery)
.await
.map_err(|error| error.to_string())?
else {
return Ok(());
};
let (target_store, bucket) = cleanup_target(&claim.identity);
let Some(target) = targets.resolve(target_store, bucket) else {
attempts
.retry_cleanup(&claim, "target_configuration")
.await
.map_err(|error| error.to_string())?;
return Err("cleanup target configuration is unavailable".into());
};
let effect = match &claim.identity {
CleanupIdentity::Object(object) => target
.delete_exact(&ObjectIdentity {
target_store_id: object.target_store.clone(),
bucket: object.bucket.clone(),
key: object.key.clone(),
version_id: object.version_id.clone(),
})
.await
.map(|_| None),
CleanupIdentity::Multipart(multipart) => target
.abort_multipart_exact(&MultipartIdentity {
target_store_id: multipart.target_store.clone(),
bucket: multipart.bucket.clone(),
key: multipart.key.clone(),
upload_id: multipart.upload_id.clone(),
})
.await
.map(|_| None),
CleanupIdentity::WriteIntent(intent) => {
target.discover_staging_write(&intent.key).await.map(Some)
}
};
match effect {
Ok(Some(discovered)) => {
let objects = discovered
.objects
.into_iter()
.map(|object| StagingObject {
target_store: object.target_store_id,
bucket: object.bucket,
key: object.key,
version_id: object.version_id,
})
.collect();
let multiparts = discovered
.multiparts
.into_iter()
.map(|multipart| StagingMultipart {
target_store: multipart.target_store_id,
bucket: multipart.bucket,
key: multipart.key,
upload_id: multipart.upload_id,
})
.collect();
attempts
.reconcile_write_intent(&claim, objects, multiparts)
.await
.map(|_| ())
.map_err(|error| error.to_string())
}
Ok(None) => attempts
.complete_cleanup(&claim)
.await
.map(|_| ())
.map_err(|error| error.to_string()),
Err(_) => {
attempts
.retry_cleanup(&claim, "target_unavailable")
.await
.map_err(|error| error.to_string())?;
Err("cleanup target operation failed".into())
}
}
}
fn cleanup_target(identity: &CleanupIdentity) -> (&str, &str) {
match identity {
CleanupIdentity::WriteIntent(intent) => (&intent.target_store, &intent.bucket),
CleanupIdentity::Object(object) => (&object.target_store, &object.bucket),
CleanupIdentity::Multipart(multipart) => (&multipart.target_store, &multipart.bucket),
}
}
fn public_server(
bind: SocketAddr,
state: PublicApiState,
) -> Result<actix_web::dev::ServerHandle, RuntimeError> {
let listener = TcpListener::bind(bind).map_err(runtime_error)?;
let server = HttpServer::new(move || {
App::new()
.wrap(middleware::Logger::new("%a \"%m %U\" %s %b %D"))
.wrap(middleware::from_fn(trace_request))
.app_data(web::Data::new(state.clone()))
.configure(configure_public_api)
})
.disable_signals()
.listen(listener)
.map_err(runtime_error)?
.run();
let handle = server.handle();
tokio::spawn(server);
Ok(handle)
}
fn worker_server(
bind: SocketAddr,
state: WorkerApiState,
tls: &WorkerTlsConfig,
) -> Result<actix_web::dev::ServerHandle, RuntimeError> {
let listener = TcpListener::bind(bind).map_err(runtime_error)?;
let server = HttpServer::new(move || {
App::new()
.wrap(middleware::Logger::new("%a \"%m %U\" %s %b %D"))
.wrap(middleware::from_fn(trace_request))
.app_data(web::Data::new(state.clone()))
.configure(configure_worker_api)
})
.disable_signals()
.on_connect(authenticate_rustls_worker)
.listen_rustls_0_23(listener, server_tls(tls)?)
.map_err(runtime_error)?
.run();
let handle = server.handle();
tokio::spawn(server);
Ok(handle)
}
fn server_tls(config: &WorkerTlsConfig) -> Result<ServerConfig, RuntimeError> {
let server_certificates = certificates(&config.certificate)?;
let key = private_key(&config.private_key)?;
let mut roots = RootCertStore::empty();
for certificate in certificates(&config.client_ca)? {
roots.add(certificate).map_err(runtime_error)?;
}
let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
.build()
.map_err(runtime_error)?;
ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(server_certificates, key)
.map_err(runtime_error)
}
fn certificates(path: &Path) -> Result<Vec<CertificateDer<'static>>, RuntimeError> {
let file = fs::File::open(path).map_err(|error| path_error(path, error))?;
let certificates = rustls_pemfile::certs(&mut BufReader::new(file))
.collect::<Result<Vec<_>, _>>()
.map_err(runtime_error)?;
if certificates.is_empty() {
return Err(RuntimeError::message(format!(
"{} contains no certificates",
path.display()
)));
}
Ok(certificates)
}
fn private_key(path: &Path) -> Result<PrivateKeyDer<'static>, RuntimeError> {
let file = fs::File::open(path).map_err(|error| path_error(path, error))?;
rustls_pemfile::private_key(&mut BufReader::new(file))
.map_err(runtime_error)?
.ok_or_else(|| RuntimeError::message(format!("{} contains no private key", path.display())))
}
async fn wait_for_shutdown() -> Result<(), RuntimeError> {
#[cfg(unix)]
{
let mut terminate =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.map_err(runtime_error)?;
tokio::select! {
result = tokio::signal::ctrl_c() => result.map_err(runtime_error),
_ = terminate.recv() => Ok(()),
}
}
#[cfg(not(unix))]
tokio::signal::ctrl_c().await.map_err(runtime_error)
}
fn comma_values(name: &str, input: &str) -> Result<Vec<String>, RuntimeError> {
let values = input
.split(',')
.map(str::trim)
.map(str::to_owned)
.collect::<Vec<_>>();
if values.is_empty() || values.iter().any(String::is_empty) {
return Err(RuntimeError::message(format!(
"{name} must contain non-empty comma-separated values"
)));
}
Ok(values)
}
fn duration_millis(name: &str, default: u64) -> Result<Duration, RuntimeError> {
let milliseconds = env_parse(name, &default.to_string())?;
if milliseconds == 0 {
return Err(RuntimeError::message(format!(
"{name} must be greater than zero"
)));
}
Ok(Duration::from_millis(milliseconds))
}
fn runtime_error(error: impl std::fmt::Display) -> RuntimeError {
RuntimeError::message(error.to_string())
}
fn file_error(name: &str, error: io::Error) -> RuntimeError {
RuntimeError::message(format!("cannot read {name}: {}", error.kind()))
}
fn path_error(path: &Path, error: io::Error) -> RuntimeError {
RuntimeError::message(format!("cannot read {}: {}", path.display(), error.kind()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rejects_empty_comma_values() {
assert!(comma_values("TEST", "one,,two").is_err());
}
#[test]
fn file_errors_do_not_include_file_contents() {
let error = file_error(
"REMORA_CURSOR_SECRET_FILE",
io::Error::from(io::ErrorKind::NotFound),
);
assert_eq!(
error.to_string(),
"cannot read REMORA_CURSOR_SECRET_FILE: entity not found"
);
}
}

2
crates/remora-api/src/lib.rs
View file

@ -1,7 +1,9 @@
mod application;
mod migrate;
mod public;
mod worker;
pub use application::{ApiApplication, ApiApplicationConfig};
pub use migrate::{migrate, migrator};
pub use public::{
PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,

10
crates/remora-api/src/main.rs
View file

@ -1,3 +1,9 @@
fn main() {
println!("{}", remora_api::COMPONENT_NAME);
use remora_api::{ApiApplication, ApiApplicationConfig};
#[actix_web::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
remora_runtime::init_logging()?;
let config = ApiApplicationConfig::from_env()?;
ApiApplication::new(config).run().await?;
Ok(())
}

5
crates/remora-api/src/worker.rs
View file

@ -96,6 +96,7 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
.limit(1024 * 1024)
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
)
.route("/health/ready", web::get().to(worker_ready))
.route("/jobs/{job_id}/claim", web::post().to(claim))
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
@ -131,6 +132,10 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
);
}
async fn worker_ready(_: AuthenticatedWorker) -> HttpResponse {
HttpResponse::Ok().finish()
}
async fn register_staging_object(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,