Decompose oversized Rust files by responsibility
This commit is contained in:
parent
03bfe92ca0
commit
ee9201b057
2 changed files with 18648 additions and 18513 deletions
565
crates/lazaret-host-coordinator/src/runtime/acquisition.rs
Normal file
565
crates/lazaret-host-coordinator/src/runtime/acquisition.rs
Normal file
|
|
@ -0,0 +1,565 @@
|
|||
pub struct AwaitingAttachedRunsc {
|
||||
supervisor: Option<RetainedProcess>,
|
||||
control: Option<CoordinatorChannel>,
|
||||
lifetime: Option<std::os::fd::OwnedFd>,
|
||||
staged: Option<StagedProcessOwnership>,
|
||||
helper_staged: Option<StagedProcessOwnership>,
|
||||
git_routing_required: bool,
|
||||
git_routing_mount: Option<MountIdentity>,
|
||||
network_evidence: Option<NetworkReadinessEvidence>,
|
||||
runsc_command: lazaret_runtime_supervisor::sandbox::RunscCommand,
|
||||
}
|
||||
|
||||
impl AwaitingAttachedRunsc {
|
||||
#[must_use]
|
||||
pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
|
||||
self.supervisor.as_ref().map(RetainedProcess::identity)
|
||||
}
|
||||
|
||||
/// Requests RS-owned State Anchor acquisition and commits exact staged and mount evidence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing intent, event mismatch, changed ownership, or channel failure.
|
||||
pub fn mount_state_anchor(
|
||||
&mut self,
|
||||
coordinator: &StateAnchorCoordinator<'_>,
|
||||
prepared_artifact_id: PreparedArtifactId,
|
||||
) -> Result<MountIdentity, SandboxCoordinatorError> {
|
||||
let control = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?;
|
||||
control.mount_state_anchor()?;
|
||||
let staged = match control.receive()? {
|
||||
SupervisorEvent::StateAnchorStaged(staged) => staged,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not stage the State Anchor daemon",
|
||||
));
|
||||
}
|
||||
};
|
||||
coordinator
|
||||
.stage_daemon(staged)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
let acknowledgment = coordinator
|
||||
.acknowledge_staged_daemon(staged)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
control.acknowledge_state_anchor(acknowledgment)?;
|
||||
let (ready_staged, user_namespace, mount_namespace, mount) = match control.receive()? {
|
||||
SupervisorEvent::StateAnchorReady {
|
||||
staged,
|
||||
user_namespace,
|
||||
mount_namespace,
|
||||
mount,
|
||||
} => (staged, user_namespace, mount_namespace, mount),
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not report State Anchor readiness",
|
||||
));
|
||||
}
|
||||
};
|
||||
if ready_staged != staged {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"State Anchor readiness changed daemon ownership",
|
||||
));
|
||||
}
|
||||
let readiness =
|
||||
lazaret_state_anchor::VerifiedStateAnchorReadiness::from_parts(ready_staged, mount);
|
||||
coordinator
|
||||
.commit_daemon_ready(readiness, user_namespace, mount_namespace)
|
||||
.and_then(|()| {
|
||||
coordinator.commit_prepared(staged.generation_id(), prepared_artifact_id, mount)
|
||||
})
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
Ok(mount)
|
||||
}
|
||||
|
||||
/// Requests generation-local routing and commits exact staged and mount evidence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing intent, event mismatch, changed ownership, or channel failure.
|
||||
pub fn start_git_routing(
|
||||
&mut self,
|
||||
coordinator: &GitRoutingCoordinator<'_>,
|
||||
) -> Result<MountIdentity, SandboxCoordinatorError> {
|
||||
if !self.git_routing_required || self.git_routing_mount.is_some() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Git routing is not awaiting start",
|
||||
));
|
||||
}
|
||||
let control = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?;
|
||||
control.start_git_routing()?;
|
||||
let staged = match control.receive()? {
|
||||
SupervisorEvent::GitRoutingStaged(staged) => staged,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not stage the Git router",
|
||||
));
|
||||
}
|
||||
};
|
||||
coordinator
|
||||
.stage_daemon(staged)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
let acknowledgment = coordinator
|
||||
.acknowledge_daemon(staged)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
control.acknowledge_git_routing(acknowledgment)?;
|
||||
let (ready_staged, mount) = match control.receive()? {
|
||||
SupervisorEvent::GitRoutingReady { staged, mount } => (staged, mount),
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not report Git-routing readiness",
|
||||
));
|
||||
}
|
||||
};
|
||||
if ready_staged != staged {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Git-routing readiness changed daemon ownership",
|
||||
));
|
||||
}
|
||||
coordinator
|
||||
.commit_ready(staged, mount)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
self.git_routing_mount = Some(mount);
|
||||
Ok(mount)
|
||||
}
|
||||
|
||||
/// Releases the RS-owned sidecar install path after exact State Anchor readiness.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an unexpected RS response or channel failure.
|
||||
pub fn start_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
let control = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?;
|
||||
control.start_sidecar()?;
|
||||
if control.receive()? == SupervisorEvent::SidecarStarted {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not start the sidecar path",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Launches the RS through the HC broker and commits its staged ownership before execution.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses any pipe, channel, broker, staging, or acknowledgment failure.
|
||||
pub fn launch(
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
launch: &RuntimeSupervisorLaunch,
|
||||
) -> Result<Self, SandboxCoordinatorError> {
|
||||
if launch.network_sidecar_id().is_some() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"networked Runtime Supervisor requires helper intent",
|
||||
));
|
||||
}
|
||||
Self::launch_inner(coordinator, None, launch)
|
||||
}
|
||||
|
||||
/// Launches the RS only after independent durable helper intent is recorded.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a non-network launch or any sandbox/helper staging failure.
|
||||
pub fn launch_networked(
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
network: &NetworkCoordinator<'_>,
|
||||
launch: &RuntimeSupervisorLaunch,
|
||||
) -> Result<Self, SandboxCoordinatorError> {
|
||||
if launch.network_sidecar_id().is_none() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor launch has no network policy",
|
||||
));
|
||||
}
|
||||
Self::launch_inner(coordinator, Some(network), launch)
|
||||
}
|
||||
|
||||
fn launch_inner(
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
network: Option<&NetworkCoordinator<'_>>,
|
||||
launch: &RuntimeSupervisorLaunch,
|
||||
) -> Result<Self, SandboxCoordinatorError> {
|
||||
let parent = current_process_identity()?;
|
||||
let (coordinator_control, supervisor_control) = private_seqpacket_pair()?;
|
||||
let (supervisor_lifetime, coordinator_lifetime) =
|
||||
pipe_with(PipeFlags::CLOEXEC).map_err(io::Error::from)?;
|
||||
let process = child_broker().acquire_with_stdio(
|
||||
parent,
|
||||
launch.ownership(),
|
||||
launch.target(),
|
||||
launch.child_standard_io(supervisor_control, supervisor_lifetime)?,
|
||||
|staged| {
|
||||
coordinator
|
||||
.stage_runtime_supervisor(staged)
|
||||
.map_err(io::Error::other)?;
|
||||
if let Some(network) = network {
|
||||
network
|
||||
.begin_helper_intent(
|
||||
staged.generation_id(),
|
||||
launch.network_sidecar_id().ok_or_else(|| {
|
||||
io::Error::other("network sidecar identity is absent")
|
||||
})?,
|
||||
staged.identity(),
|
||||
)
|
||||
.map_err(io::Error::other)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|staged| {
|
||||
coordinator
|
||||
.acknowledge_runtime_supervisor(staged)
|
||||
.map_err(io::Error::other)
|
||||
},
|
||||
)?;
|
||||
Ok(Self {
|
||||
supervisor: Some(process),
|
||||
control: Some(CoordinatorChannel::new(coordinator_control)),
|
||||
lifetime: Some(coordinator_lifetime),
|
||||
staged: None,
|
||||
helper_staged: None,
|
||||
git_routing_required: launch.has_git_routing(),
|
||||
git_routing_mount: None,
|
||||
network_evidence: None,
|
||||
runsc_command: launch.runsc_command().clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Receives, persists, and acknowledges the exact helper child acquisition.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses unexpected events, changed ownership, or a missing network launch.
|
||||
pub fn receive_helper_staged(
|
||||
&mut self,
|
||||
network: &NetworkCoordinator<'_>,
|
||||
) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
|
||||
if self.helper_staged.is_some() || self.network_evidence.is_some() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"helper stage was already received",
|
||||
));
|
||||
}
|
||||
let control = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?;
|
||||
let staged = match control.receive()? {
|
||||
SupervisorEvent::HelperStaged(staged) => staged,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not stage the helper first",
|
||||
));
|
||||
}
|
||||
};
|
||||
let runtime_supervisor = self
|
||||
.supervisor
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor handle is absent",
|
||||
))?
|
||||
.identity();
|
||||
network.stage_helper(staged, runtime_supervisor)?;
|
||||
let acknowledgment = network.acknowledge_helper(staged)?;
|
||||
control.acknowledge_helper(acknowledgment)?;
|
||||
self.helper_staged = Some(staged);
|
||||
Ok(staged)
|
||||
}
|
||||
|
||||
/// Commits helper readiness, independently observes the namespace, and validates attestation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses event ordering, namespace mutation, or any attestation mismatch.
|
||||
pub fn receive_network_readiness<BuildProbeRequest>(
|
||||
&mut self,
|
||||
network: &NetworkCoordinator<'_>,
|
||||
policy: &CanonicalDerivedNetworkPolicy,
|
||||
build_probe_request: BuildProbeRequest,
|
||||
expected_reachability: &ReachabilityExpectation,
|
||||
) -> Result<NetworkReadinessEvidence, SandboxCoordinatorError>
|
||||
where
|
||||
BuildProbeRequest: FnOnce(NamespaceIdentity) -> NetworkProbeRequest,
|
||||
{
|
||||
let staged = self
|
||||
.helper_staged
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"helper ownership was not staged",
|
||||
))?;
|
||||
let control = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?;
|
||||
if control.receive()? != SupervisorEvent::HelperReady(staged) {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor helper readiness mismatched ownership",
|
||||
));
|
||||
}
|
||||
network.commit_helper(staged)?;
|
||||
let attestation = match control.receive()? {
|
||||
SupervisorEvent::NetworkAttestation(attestation) => attestation,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not provide network attestation",
|
||||
));
|
||||
}
|
||||
};
|
||||
let namespace = network.begin_network_intent(
|
||||
attestation.generation_id,
|
||||
policy.digest(),
|
||||
attestation.runtime_supervisor,
|
||||
)?;
|
||||
network.validate_attestation(policy, &attestation)?;
|
||||
let probe_request = build_probe_request(namespace);
|
||||
if probe_request.domain_id != attestation.domain_id
|
||||
|| probe_request.generation_id != attestation.generation_id
|
||||
|| probe_request.enforcement_namespace != attestation.enforcement_namespace
|
||||
{
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"network probe request mismatches attestation",
|
||||
));
|
||||
}
|
||||
control.acknowledge_network_attestation(probe_request)?;
|
||||
let probe = match control.receive()? {
|
||||
SupervisorEvent::NetworkProbe(probe) => probe,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not return the network probe",
|
||||
));
|
||||
}
|
||||
};
|
||||
let evidence =
|
||||
network.commit_readiness(policy, attestation, probe, expected_reachability)?;
|
||||
control.acknowledge_network_readiness()?;
|
||||
self.network_evidence = Some(evidence.clone());
|
||||
Ok(evidence)
|
||||
}
|
||||
|
||||
/// Receives and durably stages the exact RS child without releasing its barrier.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses unexpected, duplicate, or mismatched messages and durable-state failures.
|
||||
pub fn receive_staged(
|
||||
&mut self,
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
|
||||
if self.staged.is_some() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"attached runsc stage was already received",
|
||||
));
|
||||
}
|
||||
if self.helper_staged.is_some() && self.network_evidence.is_none() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"attached runsc cannot start before network attestation",
|
||||
));
|
||||
}
|
||||
if self.git_routing_required && self.git_routing_mount.is_none() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"attached runsc cannot start before Git-routing readiness",
|
||||
));
|
||||
}
|
||||
let event = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?
|
||||
.receive()?;
|
||||
let staged = match event {
|
||||
SupervisorEvent::Staged(staged) => staged,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor sent readiness before staged ownership",
|
||||
));
|
||||
}
|
||||
};
|
||||
let acknowledgment = coordinator.stage_attached_runsc(staged)?;
|
||||
self.staged = Some(acknowledgment.committed());
|
||||
Ok(staged)
|
||||
}
|
||||
|
||||
/// Releases the exact pending runsc barrier with its durable typed acknowledgment.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a different or absent staged tuple or channel failure.
|
||||
pub fn acknowledge(
|
||||
mut self,
|
||||
staged: StagedProcessOwnership,
|
||||
handoff: &SandboxHandoffAuthority<'_>,
|
||||
) -> Result<AwaitingSandboxReadiness, SandboxCoordinatorError> {
|
||||
if self.staged != Some(staged) {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"attached runsc acknowledgment does not match pending stage",
|
||||
));
|
||||
}
|
||||
if staged.domain_id() != handoff.domain_id()
|
||||
|| !handoff.matches_command(&self.runsc_command)
|
||||
{
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"sandbox handoff authority does not match attached runsc",
|
||||
));
|
||||
}
|
||||
handoff
|
||||
.revalidate()
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
self.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?
|
||||
.acknowledge_staged(ProcessOwnershipCommitAck::v1(staged))?;
|
||||
Ok(AwaitingSandboxReadiness {
|
||||
supervisor: self.supervisor.take(),
|
||||
control: self.control.take(),
|
||||
lifetime: self.lifetime.take(),
|
||||
staged,
|
||||
network_evidence: self.network_evidence.take(),
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns an error if the exact RS cannot be terminated and reaped.
|
||||
pub fn abort(mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
self.lifetime.take();
|
||||
cleanup_supervisor(&mut self.supervisor)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AwaitingAttachedRunsc {
|
||||
fn drop(&mut self) {
|
||||
self.lifetime.take();
|
||||
let _ = cleanup_supervisor(&mut self.supervisor);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AwaitingSandboxReadiness {
|
||||
supervisor: Option<RetainedProcess>,
|
||||
control: Option<CoordinatorChannel>,
|
||||
lifetime: Option<std::os::fd::OwnedFd>,
|
||||
staged: StagedProcessOwnership,
|
||||
network_evidence: Option<NetworkReadinessEvidence>,
|
||||
}
|
||||
|
||||
impl AwaitingSandboxReadiness {
|
||||
/// Commits only the exact ready attached runsc after RS capability removal.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a mismatched readiness tuple, durable commit failure, or channel failure.
|
||||
pub fn commit(
|
||||
self,
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
|
||||
if self.network_evidence.is_some() {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"networked sandbox requires a successful sandbox-view probe",
|
||||
));
|
||||
}
|
||||
self.commit_after_probe(coordinator, None)
|
||||
}
|
||||
|
||||
/// Commits the Sandbox only after the enforcement-namespace probe is durably accepted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing attestation, failed/mutated probe evidence, or any runtime mismatch.
|
||||
pub fn commit_networked(
|
||||
mut self,
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
) -> Result<(CommittedSandboxRuntime, NetworkReadinessEvidence), SandboxCoordinatorError> {
|
||||
let evidence =
|
||||
self.network_evidence
|
||||
.take()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"network readiness evidence is absent",
|
||||
))?;
|
||||
let runtime = self.commit_after_probe(coordinator, Some(evidence.clone()))?;
|
||||
Ok((runtime, evidence))
|
||||
}
|
||||
|
||||
fn commit_after_probe(
|
||||
mut self,
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
network_evidence: Option<NetworkReadinessEvidence>,
|
||||
) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
|
||||
let control = self
|
||||
.control
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?;
|
||||
let event = control.receive()?;
|
||||
let staged = match event {
|
||||
SupervisorEvent::Ready { staged, .. } => staged,
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
|
||||
}
|
||||
_ => {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not report sandbox readiness",
|
||||
));
|
||||
}
|
||||
};
|
||||
if staged != self.staged {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"sandbox readiness changed attached-runsc ownership",
|
||||
));
|
||||
}
|
||||
coordinator.commit_ready(staged)?;
|
||||
control.acknowledge_sandbox_committed()?;
|
||||
Ok(CommittedSandboxRuntime {
|
||||
generation_id: staged.generation_id(),
|
||||
supervisor: self.supervisor.take(),
|
||||
control: self
|
||||
.control
|
||||
.take()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor control channel is absent",
|
||||
))?,
|
||||
lifetime: self.lifetime.take(),
|
||||
network_evidence,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AwaitingSandboxReadiness {
|
||||
fn drop(&mut self) {
|
||||
self.lifetime.take();
|
||||
let _ = cleanup_supervisor(&mut self.supervisor);
|
||||
}
|
||||
}
|
||||
|
||||
309
crates/lazaret-host-coordinator/src/runtime/committed.rs
Normal file
309
crates/lazaret-host-coordinator/src/runtime/committed.rs
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
pub struct CommittedSandboxRuntime {
|
||||
generation_id: GenerationId,
|
||||
supervisor: Option<RetainedProcess>,
|
||||
control: CoordinatorChannel,
|
||||
lifetime: Option<std::os::fd::OwnedFd>,
|
||||
network_evidence: Option<NetworkReadinessEvidence>,
|
||||
}
|
||||
|
||||
impl CommittedSandboxRuntime {
|
||||
fn receive_runtime_event(&mut self) -> Result<SupervisorEvent, SandboxCoordinatorError> {
|
||||
loop {
|
||||
let event = self.control.receive()?;
|
||||
if event == SupervisorEvent::HelperExited {
|
||||
self.network_evidence = None;
|
||||
} else {
|
||||
return Ok(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn network_evidence(&self) -> Option<&NetworkReadinessEvidence> {
|
||||
self.network_evidence.as_ref()
|
||||
}
|
||||
|
||||
/// Withdraws live readiness after the RS reports exact helper-only loss.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses any event other than connectivity-helper loss.
|
||||
pub fn wait_for_helper_exit(&mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
if self.control.receive()? != SupervisorEvent::HelperExited {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not report helper-only loss",
|
||||
));
|
||||
}
|
||||
self.network_evidence = None;
|
||||
Ok(())
|
||||
}
|
||||
#[must_use]
|
||||
pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
|
||||
self.supervisor.as_ref().map(RetainedProcess::identity)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns an error if the exact retained RS pidfd action fails.
|
||||
pub fn signal_runtime_supervisor(&self, signal: Signal) -> Result<(), SandboxCoordinatorError> {
|
||||
self.supervisor
|
||||
.as_ref()
|
||||
.ok_or(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor handle is absent",
|
||||
))?
|
||||
.send_signal(signal)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns an error if the RS cannot apply the exact retained-runsc pidfd action.
|
||||
pub fn signal_attached_runsc(
|
||||
&mut self,
|
||||
signal: AttachedRunscSignal,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
self.control.signal_attached_runsc(signal)?;
|
||||
if self.receive_runtime_event()? == SupervisorEvent::SignalApplied(signal) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not confirm the exact runsc action",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops and durably cleans only the exact attached sandbox runtime.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
|
||||
pub fn stop_sandbox(
|
||||
&mut self,
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
coordinator.begin_release(self.generation_id)?;
|
||||
self.control.stop_sandbox()?;
|
||||
if self.receive_runtime_event()? != SupervisorEvent::SandboxStopped {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not stop the sandbox",
|
||||
));
|
||||
}
|
||||
coordinator.complete_clean_after_absence(self.generation_id)
|
||||
}
|
||||
|
||||
/// Stops exact generation-local Git routing after sandbox absence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing release intent, changed mount evidence, or an unexpected RS response.
|
||||
pub fn stop_git_routing(
|
||||
&mut self,
|
||||
coordinator: &GitRoutingCoordinator<'_>,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
let expected = coordinator
|
||||
.begin_release(self.generation_id)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
|
||||
self.control.stop_git_routing()?;
|
||||
match self.receive_runtime_event()? {
|
||||
SupervisorEvent::GitRoutingStopped { mount } if mount == expected => coordinator
|
||||
.complete_clean(self.generation_id, mount)
|
||||
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error))),
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
|
||||
}
|
||||
_ => Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not stop exact Git routing",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops only the RS-owned network resources after sandbox absence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an unexpected RS response or channel failure.
|
||||
pub fn stop_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
self.control.stop_sidecar()?;
|
||||
if self.receive_runtime_event()? == SupervisorEvent::SidecarStopped {
|
||||
self.network_evidence = None;
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not stop the sidecar",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Requests exact RS-owned State Anchor unmount after dependent runtime resources stop.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an unexpected RS response or channel failure.
|
||||
pub fn unmount_state_anchor(
|
||||
&mut self,
|
||||
expected_mount: MountIdentity,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
self.control.unmount_state_anchor()?;
|
||||
match self.receive_runtime_event()? {
|
||||
SupervisorEvent::StateAnchorUnmounted { mount } if mount == expected_mount => Ok(()),
|
||||
_ => Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not confirm the exact State Anchor unmount",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Requests a fresh namespace-local observation of exact State Anchor absence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an RS refusal, echoed mount, mismatched authority, or channel failure.
|
||||
pub fn observe_state_anchor_absence(
|
||||
&mut self,
|
||||
user_namespace: NamespaceIdentity,
|
||||
mount_namespace: NamespaceIdentity,
|
||||
mount: MountIdentity,
|
||||
) -> Result<StateAnchorAbsenceObservation, SandboxCoordinatorError> {
|
||||
self.control
|
||||
.observe_state_anchor_absence(user_namespace, mount_namespace, mount)?;
|
||||
match self.receive_runtime_event()? {
|
||||
SupervisorEvent::StateAnchorAbsenceObserved {
|
||||
user_namespace: observed_user_namespace,
|
||||
mount_namespace: observed_mount_namespace,
|
||||
mount: observed_mount,
|
||||
observed,
|
||||
} if observed_user_namespace == user_namespace
|
||||
&& observed_mount_namespace == mount_namespace
|
||||
&& observed_mount == mount
|
||||
&& observed != mount =>
|
||||
{
|
||||
Ok(StateAnchorAbsenceObservation {
|
||||
user_namespace: observed_user_namespace,
|
||||
mount_namespace: observed_mount_namespace,
|
||||
mount: observed_mount,
|
||||
observed,
|
||||
})
|
||||
}
|
||||
SupervisorEvent::Refused(detail) => {
|
||||
Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
|
||||
}
|
||||
_ => Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not provide exact State Anchor absence evidence",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Gracefully exits and reaps the resource-empty exact Runtime Supervisor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an unexpected response or an RS that does not exit.
|
||||
pub fn graceful_exit(mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
self.control.exit()?;
|
||||
if self.receive_runtime_event()? != SupervisorEvent::Exited {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not acknowledge graceful exit",
|
||||
));
|
||||
}
|
||||
self.lifetime.take();
|
||||
wait_and_reap_supervisor(&mut self.supervisor)
|
||||
}
|
||||
|
||||
/// Waits for an exact attached-runsc loss report and reaps the retained RS.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses any other event or an RS that does not then exit.
|
||||
pub fn wait_for_attached_runsc_exit(mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
if self.receive_runtime_event()? != SupervisorEvent::AttachedRunscExited {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not report attached-runsc loss",
|
||||
));
|
||||
}
|
||||
self.lifetime.take();
|
||||
wait_and_reap_supervisor(&mut self.supervisor)
|
||||
}
|
||||
|
||||
/// Waits for an externally stopped exact RS handle to exit and reaps it.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an RS that remains present beyond the bounded observation.
|
||||
pub fn wait_for_runtime_supervisor_exit(mut self) -> Result<(), SandboxCoordinatorError> {
|
||||
self.lifetime.take();
|
||||
wait_and_reap_supervisor(&mut self.supervisor)
|
||||
}
|
||||
|
||||
/// Performs ordered normal stop and reaps the same retained RS handle.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
|
||||
pub fn stop(
|
||||
mut self,
|
||||
coordinator: &SandboxCoordinator<'_>,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
coordinator.begin_release(self.generation_id)?;
|
||||
self.control.stop()?;
|
||||
if self.receive_runtime_event()? != SupervisorEvent::Stopped {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not complete sandbox stop",
|
||||
));
|
||||
}
|
||||
self.lifetime.take();
|
||||
wait_and_reap_supervisor(&mut self.supervisor)?;
|
||||
coordinator.complete_clean_after_absence(self.generation_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CommittedSandboxRuntime {
|
||||
fn drop(&mut self) {
|
||||
self.lifetime.take();
|
||||
let _ = cleanup_supervisor(&mut self.supervisor);
|
||||
}
|
||||
}
|
||||
|
||||
fn sandbox_participant(
|
||||
generation_id: GenerationId,
|
||||
state: ParticipantState,
|
||||
ownership: SandboxOwnership,
|
||||
) -> Result<Participant, SandboxCoordinatorError> {
|
||||
Participant::snapshot(
|
||||
generation_id,
|
||||
ParticipantRole::Sandbox,
|
||||
state,
|
||||
Some(ParticipantOwnership::Sandbox(ownership)),
|
||||
)
|
||||
.map_err(|_| SandboxCoordinatorError::InvalidState("Sandbox participant shape is invalid"))
|
||||
}
|
||||
|
||||
fn sandbox_role(participants: &[Participant], generation_id: GenerationId) -> Option<&Participant> {
|
||||
participants.iter().find(|participant| {
|
||||
participant.generation_id() == generation_id
|
||||
&& participant.role() == ParticipantRole::Sandbox
|
||||
})
|
||||
}
|
||||
|
||||
fn cleanup_supervisor(
|
||||
supervisor: &mut Option<RetainedProcess>,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
let Some(process) = supervisor.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
if !process.has_exited()? {
|
||||
process.send_signal(Signal::KILL)?;
|
||||
}
|
||||
child_broker().reap(process)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wait_and_reap_supervisor(
|
||||
supervisor: &mut Option<RetainedProcess>,
|
||||
) -> Result<(), SandboxCoordinatorError> {
|
||||
let Some(process) = supervisor.take() else {
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor handle is absent",
|
||||
));
|
||||
};
|
||||
let deadline = Instant::now() + SUPERVISOR_EXIT_TIMEOUT;
|
||||
while !process.has_exited()? && Instant::now() < deadline {
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
if !process.has_exited()? {
|
||||
process.send_signal(Signal::KILL)?;
|
||||
child_broker().reap(process)?;
|
||||
return Err(SandboxCoordinatorError::InvalidState(
|
||||
"Runtime Supervisor did not exit after sandbox stop",
|
||||
));
|
||||
}
|
||||
child_broker().reap(process)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Loading…
Reference in a new issue