Exclude image preparation from client startup deadline

This commit is contained in:
ms2k 2026-07-28 22:39:10 +09:00
commit 55eed08db7
9 changed files with 106 additions and 22 deletions

View file

@ -4,8 +4,8 @@ use std::path::PathBuf;
use clap::{Args, Parser, Subcommand}; use clap::{Args, Parser, Subcommand};
use lazaret_core::records::StateResetScopeV1; use lazaret_core::records::StateResetScopeV1;
use crate::ClientCommand;
use crate::state::{StateCommand, StateOperation}; use crate::state::{StateCommand, StateOperation};
use crate::{ClientCommand, DEFAULT_STARTUP_TIMEOUT};
#[derive(Debug, Parser)] #[derive(Debug, Parser)]
#[command( #[command(
@ -36,6 +36,14 @@ enum PublicCommand {
struct RunArguments { struct RunArguments {
#[command(flatten)] #[command(flatten)]
domain: DomainArguments, domain: DomainArguments,
/// Maximum time to wait for Domain publication, in seconds.
#[arg(
long,
value_name = "SECONDS",
default_value_t = DEFAULT_STARTUP_TIMEOUT.as_secs(),
value_parser = clap::value_parser!(u64).range(1..)
)]
startup_timeout_seconds: u64,
/// Command and arguments to run instead of the default `/bin/bash`. /// Command and arguments to run instead of the default `/bin/bash`.
#[arg(last = true, num_args = 1.., value_name = "COMMAND")] #[arg(last = true, num_args = 1.., value_name = "COMMAND")]
command: Vec<OsString>, command: Vec<OsString>,
@ -121,6 +129,7 @@ pub fn parse(
profile: arguments.domain.profile, profile: arguments.domain.profile,
workspace: arguments.domain.workspace.unwrap_or(current_workspace), workspace: arguments.domain.workspace.unwrap_or(current_workspace),
configuration: arguments.domain.config, configuration: arguments.domain.config,
startup_timeout: std::time::Duration::from_secs(arguments.startup_timeout_seconds),
argv: command_argv(arguments.command), argv: command_argv(arguments.command),
}), }),
PublicCommand::State { operation } => { PublicCommand::State { operation } => {
@ -195,6 +204,7 @@ fn state_command(operation: StateArguments, current_workspace: PathBuf) -> State
profile: domain.profile, profile: domain.profile,
workspace: domain.workspace.unwrap_or(current_workspace), workspace: domain.workspace.unwrap_or(current_workspace),
configuration: domain.config, configuration: domain.config,
startup_timeout: DEFAULT_STARTUP_TIMEOUT,
argv: Vec::new(), argv: Vec::new(),
}, },
machine, machine,

View file

@ -157,8 +157,22 @@ impl CoordinatorClientBackend {
let mut child = child_broker() let mut child = child_broker()
.spawn_bootstrap(&target, File::from(reader)) .spawn_bootstrap(&target, File::from(reader))
.map_err(failure)?; .map_err(failure)?;
match child.acquisition_status().map_err(failure)? { match child.bootstrap_status().map_err(failure)? {
b'L' => { b'L' => {
match child.bootstrap_status().map_err(failure)? {
b'R' => {}
b'E' => {
let result = child.reap().map_err(failure)?;
return Err(failure(format!(
"Host Coordinator refused image preparation with {result:?}"
)));
}
_ => {
return Err(failure(
"Host Coordinator sent an invalid image-preparation status",
));
}
}
self.coordinator = Some(child); self.coordinator = Some(child);
self.bootstrap_owner = Some(writer); self.bootstrap_owner = Some(writer);
Ok(BootstrapAcquisition::Leader) Ok(BootstrapAcquisition::Leader)
@ -205,7 +219,16 @@ impl CoordinatorClientBackend {
} else { } else {
None None
}; };
let mut observer = DomainObservation::new(initial_path, SystemObservationClock::start()); let startup_timeout = self
.command
.as_ref()
.ok_or_else(|| ClientError::Backend(failure("client command was not validated")))?
.startup_timeout;
let mut observer = DomainObservation::new(
initial_path,
SystemObservationClock::start(),
startup_timeout,
);
let mut waiter_transfer = None; let mut waiter_transfer = None;
loop { loop {
let (state, endpoint) = match self.discover().map_err(ClientError::Backend)? { let (state, endpoint) = match self.discover().map_err(ClientError::Backend)? {

View file

@ -40,20 +40,16 @@ pub(crate) enum ObservationFailure {
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
struct ObservationPolicy { struct ObservationPolicy {
retry: Duration, retry: Duration,
deadline: Duration,
} }
const BOOTSTRAP_POLICY: ObservationPolicy = ObservationPolicy { const BOOTSTRAP_POLICY: ObservationPolicy = ObservationPolicy {
retry: Duration::from_millis(10), retry: Duration::from_millis(10),
deadline: Duration::from_secs(30),
}; };
const WAITING_POLICY: ObservationPolicy = ObservationPolicy { const WAITING_POLICY: ObservationPolicy = ObservationPolicy {
retry: Duration::from_millis(10), retry: Duration::from_millis(10),
deadline: Duration::from_secs(30),
}; };
const TRANSFER_POLICY: ObservationPolicy = ObservationPolicy { const TRANSFER_POLICY: ObservationPolicy = ObservationPolicy {
retry: Duration::from_millis(1), retry: Duration::from_millis(1),
deadline: Duration::from_secs(30),
}; };
pub(crate) trait ObservationClock { pub(crate) trait ObservationClock {
@ -91,11 +87,16 @@ impl ObservationClock for SystemObservationClock {
pub(crate) struct DomainObservation<C> { pub(crate) struct DomainObservation<C> {
path: ObservationPath, path: ObservationPath,
clock: C, clock: C,
deadline: Duration,
} }
impl<C: ObservationClock> DomainObservation<C> { impl<C: ObservationClock> DomainObservation<C> {
pub(crate) fn new(path: ObservationPath, clock: C) -> Self { pub(crate) fn new(path: ObservationPath, clock: C, deadline: Duration) -> Self {
Self { path, clock } Self {
path,
clock,
deadline,
}
} }
pub(crate) fn path(&self) -> ObservationPath { pub(crate) fn path(&self) -> ObservationPath {
@ -149,7 +150,7 @@ impl<C: ObservationClock> DomainObservation<C> {
} }
pub(crate) fn remaining(&self) -> Result<Duration, ObservationFailure> { pub(crate) fn remaining(&self) -> Result<Duration, ObservationFailure> {
let remaining = self.policy().deadline.saturating_sub(self.clock.elapsed()); let remaining = self.deadline.saturating_sub(self.clock.elapsed());
if remaining.is_zero() { if remaining.is_zero() {
Err(ObservationFailure::DeadlineExceeded) Err(ObservationFailure::DeadlineExceeded)
} else { } else {
@ -177,6 +178,8 @@ mod tests {
use super::*; use super::*;
use lazaret_core::protocol::BootstrapWaitOutcome; use lazaret_core::protocol::BootstrapWaitOutcome;
const TEST_DEADLINE: Duration = Duration::from_secs(47);
#[derive(Default)] #[derive(Default)]
struct FakeClock { struct FakeClock {
elapsed: Duration, elapsed: Duration,
@ -227,7 +230,7 @@ mod tests {
), ),
]; ];
for (path, expected) in cases { for (path, expected) in cases {
let observer = DomainObservation::new(path, FakeClock::default()); let observer = DomainObservation::new(path, FakeClock::default(), TEST_DEADLINE);
assert_eq!(observer.discovery(DiscoveryState::Ready), expected[0]); assert_eq!(observer.discovery(DiscoveryState::Ready), expected[0]);
assert_eq!(observer.discovery(DiscoveryState::Stale), expected[1]); assert_eq!(observer.discovery(DiscoveryState::Stale), expected[1]);
assert_eq!(observer.discovery(DiscoveryState::Absent), expected[2]); assert_eq!(observer.discovery(DiscoveryState::Absent), expected[2]);
@ -236,7 +239,11 @@ mod tests {
#[test] #[test]
fn waiter_lock_and_coordinator_outcomes_are_closed() { fn waiter_lock_and_coordinator_outcomes_are_closed() {
let observer = DomainObservation::new(ObservationPath::WaitingClient, FakeClock::default()); let observer = DomainObservation::new(
ObservationPath::WaitingClient,
FakeClock::default(),
TEST_DEADLINE,
);
assert_eq!( assert_eq!(
observer.waiter(BootstrapWaitOutcome::Waiting, false), observer.waiter(BootstrapWaitOutcome::Waiting, false),
WaiterAction::AwaitTerminal WaiterAction::AwaitTerminal
@ -263,7 +270,11 @@ mod tests {
); );
assert_eq!(observer.lock_available(false), None); assert_eq!(observer.lock_available(false), None);
let leader = DomainObservation::new(ObservationPath::BootstrapLeader, FakeClock::default()); let leader = DomainObservation::new(
ObservationPath::BootstrapLeader,
FakeClock::default(),
TEST_DEADLINE,
);
assert!(leader.coordinator_exited(true)); assert!(leader.coordinator_exited(true));
assert!(!leader.coordinator_exited(false)); assert!(!leader.coordinator_exited(false));
assert!(!observer.coordinator_exited(true)); assert!(!observer.coordinator_exited(true));
@ -271,11 +282,14 @@ mod tests {
#[test] #[test]
fn retry_deadlines_and_transfer_restart_are_deterministic() { fn retry_deadlines_and_transfer_restart_are_deterministic() {
let mut observer = let mut observer = DomainObservation::new(
DomainObservation::new(ObservationPath::WaitingClient, FakeClock::default()); ObservationPath::WaitingClient,
FakeClock::default(),
TEST_DEADLINE,
);
observer.retry().unwrap(); observer.retry().unwrap();
assert_eq!(observer.clock.waits, [Duration::from_millis(10)]); assert_eq!(observer.clock.waits, [Duration::from_millis(10)]);
observer.clock.elapsed = WAITING_POLICY.deadline; observer.clock.elapsed = TEST_DEADLINE;
assert_eq!(observer.retry(), Err(ObservationFailure::DeadlineExceeded)); assert_eq!(observer.retry(), Err(ObservationFailure::DeadlineExceeded));
observer.transfer_publication(); observer.transfer_publication();
assert_eq!(observer.path(), ObservationPath::PublicationTransfer); assert_eq!(observer.path(), ObservationPath::PublicationTransfer);

View file

@ -3,6 +3,7 @@ use std::fs::File;
use std::io::{self, Write}; use std::io::{self, Write};
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Mutex; use std::sync::Mutex;
use std::time::Duration;
use ed25519_dalek::SigningKey; use ed25519_dalek::SigningKey;
use lazaret_control_client::{ControlClient, ControlClientError, PendingClientEnrollment}; use lazaret_control_client::{ControlClient, ControlClientError, PendingClientEnrollment};
@ -25,11 +26,14 @@ pub mod private_roles;
pub mod state; pub mod state;
pub mod terminal; pub mod terminal;
pub const DEFAULT_STARTUP_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct ClientCommand { pub struct ClientCommand {
pub profile: String, pub profile: String,
pub workspace: PathBuf, pub workspace: PathBuf,
pub configuration: Option<PathBuf>, pub configuration: Option<PathBuf>,
pub startup_timeout: Duration,
pub argv: Vec<Vec<u8>>, pub argv: Vec<Vec<u8>>,
} }

View file

@ -6,7 +6,8 @@ use std::os::unix::ffi::OsStringExt;
use std::path::PathBuf; use std::path::PathBuf;
use lazaret_client::{ use lazaret_client::{
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, DomainStatus, run, BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, DEFAULT_STARTUP_TIMEOUT,
DomainStatus, run,
}; };
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@ -75,6 +76,7 @@ fn command() -> ClientCommand {
profile: "agent".into(), profile: "agent".into(),
workspace: PathBuf::from("/workspace"), workspace: PathBuf::from("/workspace"),
configuration: None, configuration: None,
startup_timeout: DEFAULT_STARTUP_TIMEOUT,
argv: vec![b"sh".to_vec(), b"-c".to_vec(), b"exit 0".to_vec()], argv: vec![b"sh".to_vec(), b"-c".to_vec(), b"exit 0".to_vec()],
} }
} }
@ -243,6 +245,7 @@ fn command_line_accepts_profile_entry_and_preserves_explicit_command_bytes() {
assert_eq!(parsed.profile, "agent"); assert_eq!(parsed.profile, "agent");
assert_eq!(parsed.workspace, PathBuf::from("/current")); assert_eq!(parsed.workspace, PathBuf::from("/current"));
assert_eq!(parsed.configuration, None); assert_eq!(parsed.configuration, None);
assert_eq!(parsed.startup_timeout, DEFAULT_STARTUP_TIMEOUT);
assert_eq!(parsed.argv, [b"/bin/bash".to_vec()]); assert_eq!(parsed.argv, [b"/bin/bash".to_vec()]);
let parsed = lazaret_client::cli::parse( let parsed = lazaret_client::cli::parse(
@ -263,6 +266,8 @@ fn command_line_accepts_profile_entry_and_preserves_explicit_command_bytes() {
"/project", "/project",
"--config", "--config",
"/host/config.toml", "/host/config.toml",
"--startup-timeout-seconds",
"180",
"--", "--",
"tool", "tool",
"argument", "argument",
@ -280,6 +285,7 @@ fn command_line_accepts_profile_entry_and_preserves_explicit_command_bytes() {
parsed.configuration, parsed.configuration,
Some(PathBuf::from("/host/config.toml")) Some(PathBuf::from("/host/config.toml"))
); );
assert_eq!(parsed.startup_timeout.as_secs(), 180);
assert_eq!(parsed.argv, [b"tool".to_vec(), b"argument".to_vec()]); assert_eq!(parsed.argv, [b"tool".to_vec(), b"argument".to_vec()]);
assert!( assert!(
@ -336,6 +342,7 @@ fn public_help_explains_commands_and_options() {
&[ &[
"Agent profile to use", "Agent profile to use",
"Project directory to share", "Project directory to share",
"Maximum time to wait for Domain publication",
"default `/bin/bash`", "default `/bin/bash`",
], ],
), ),
@ -374,6 +381,13 @@ fn clap_rejects_missing_profile_and_ambiguous_reset_scope() {
for arguments in [ for arguments in [
vec!["run", "--", "tool"], vec!["run", "--", "tool"],
vec!["state", "reset", "--profile", "agent"], vec!["state", "reset", "--profile", "agent"],
vec![
"run",
"--profile",
"agent",
"--startup-timeout-seconds",
"0",
],
vec![ vec![
"state", "state",
"reset", "reset",

View file

@ -127,6 +127,10 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
.inspect_err(|_| signal(b'E'))?, .inspect_err(|_| signal(b'E'))?,
); );
signal(b'L'); signal(b'L');
eprintln!(
"Lazaret is preparing sandbox image {}",
configuration.fingerprint_record().image
);
let executable = std::env::current_exe().map_err(failure)?; let executable = std::env::current_exe().map_err(failure)?;
let installed = executable let installed = executable
@ -146,7 +150,10 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
runsc: &installed.join(RUNSC_EXECUTABLE_NAME), runsc: &installed.join(RUNSC_EXECUTABLE_NAME),
}, },
)) ))
.map_err(failure)?; .map_err(failure)
.inspect_err(|_| signal(b'E'))?;
eprintln!("Lazaret sandbox image is ready");
signal(b'R');
let inputs = selection let inputs = selection
.cold_start_inputs(&configuration) .cold_start_inputs(&configuration)
.map_err(failure)?; .map_err(failure)?;

View file

@ -306,11 +306,11 @@ pub struct BootstrapChild {
} }
impl BootstrapChild { impl BootstrapChild {
/// Reads the private bootstrap acquisition status byte. /// Reads the next private bootstrap status byte.
/// ///
/// # Errors /// # Errors
/// Returns an error when the child closes or corrupts its status channel. /// Returns an error when the child closes or corrupts its status channel.
pub fn acquisition_status(&mut self) -> io::Result<u8> { pub fn bootstrap_status(&mut self) -> io::Result<u8> {
let mut status = [0_u8; 1]; let mut status = [0_u8; 1];
self.status.read_exact(&mut status)?; self.status.read_exact(&mut status)?;
Ok(status[0]) Ok(status[0])

View file

@ -56,12 +56,13 @@ fn bootstrap_child_retains_exact_pipe_owner_and_pidfd_reap() {
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap(); let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap();
let target = TargetCommand::new("/bin/sh") let target = TargetCommand::new("/bin/sh")
.argument("-c") .argument("-c")
.argument("printf L; read -r _ || exit 0") .argument("printf LR; read -r _ || exit 0")
.with_lifetime_pipe_parent_edge(); .with_lifetime_pipe_parent_edge();
let mut child = child_broker() let mut child = child_broker()
.spawn_bootstrap(&target, File::from(reader)) .spawn_bootstrap(&target, File::from(reader))
.unwrap(); .unwrap();
assert_eq!(child.acquisition_status().unwrap(), b'L'); assert_eq!(child.bootstrap_status().unwrap(), b'L');
assert_eq!(child.bootstrap_status().unwrap(), b'R');
assert!(!child.has_exited().unwrap()); assert!(!child.has_exited().unwrap());
drop(writer); drop(writer);
let status = child.reap().unwrap(); let status = child.reap().unwrap();

View file

@ -220,6 +220,17 @@ select another workspace or configuration source. The explicit
equivalent compatibility interface. `state` is reserved as the maintenance equivalent compatibility interface. `state` is reserved as the maintenance
command name; a profile with that name is launched through the explicit form. command name; a profile with that name is launched through the explicit form.
The Client publication timeout defaults to 30 seconds. The run-only
`--startup-timeout-seconds <seconds>` option accepts a nonzero per-invocation
override and does not enter Domain identity or the Configuration Fingerprint.
For the bootstrap leader, Host Coordinator image acquisition and preparation
precede this timeout and emit start and completion messages on the developer's
standard error. The timeout begins only after the Host Coordinator confirms
that image preparation completed. A Client joining another in-progress
bootstrap applies its configured timeout to its waiter and publication
observation because it does not own the leader's private bootstrap status
channel.
The explicit configuration path follows this refinement's strict host-path grammar. The explicit configuration path follows this refinement's strict host-path grammar.
The loader resolves any declared symlink once to a canonical target and builds The loader resolves any declared symlink once to a canonical target and builds
a no-symlink confined plan to that target. The source is limited to 1,048,576 a no-symlink confined plan to that target. The source is limited to 1,048,576