Enforce exact recovery ownership before cold start

This commit is contained in:
ms2k 2026-07-17 06:48:49 +09:00
commit 2bc5982a62
14 changed files with 1076 additions and 238 deletions

View file

@ -1033,6 +1033,7 @@ pub struct HelperOwnership {
pub struct SandboxOwnership { pub struct SandboxOwnership {
pub container_id: Option<ContainerId>, pub container_id: Option<ContainerId>,
pub runtime_root: Option<HostPath>, pub runtime_root: Option<HostPath>,
pub runtime_root_identity: Option<MountIdentity>,
pub bundle_digest: Option<BundleDigest>, pub bundle_digest: Option<BundleDigest>,
pub host_coordinator: PersistedHostProcessOwnership, pub host_coordinator: PersistedHostProcessOwnership,
pub runtime_supervisor: PersistedHostProcessOwnership, pub runtime_supervisor: PersistedHostProcessOwnership,
@ -1041,8 +1042,12 @@ pub struct SandboxOwnership {
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct PublicationOwnership { pub struct PublicationOwnership {
pub staged_socket_path: Option<HostPath>,
pub staged_identity_path: Option<HostPath>,
pub socket_path: Option<HostPath>, pub socket_path: Option<HostPath>,
pub identity_path: Option<HostPath>, pub identity_path: Option<HostPath>,
pub socket_identity: Option<MountIdentity>,
pub identity_file_identity: Option<MountIdentity>,
} }
impl PreparedLowerOwnership { impl PreparedLowerOwnership {
@ -1309,6 +1314,7 @@ impl SandboxOwnership {
Self { Self {
container_id: None, container_id: None,
runtime_root: None, runtime_root: None,
runtime_root_identity: None,
bundle_digest: None, bundle_digest: None,
host_coordinator: PersistedHostProcessOwnership::unassigned(), host_coordinator: PersistedHostProcessOwnership::unassigned(),
runtime_supervisor: PersistedHostProcessOwnership::unassigned(), runtime_supervisor: PersistedHostProcessOwnership::unassigned(),
@ -1319,6 +1325,7 @@ impl SandboxOwnership {
fn is_empty(&self) -> bool { fn is_empty(&self) -> bool {
self.container_id.is_none() self.container_id.is_none()
&& self.runtime_root.is_none() && self.runtime_root.is_none()
&& self.runtime_root_identity.is_none()
&& self.bundle_digest.is_none() && self.bundle_digest.is_none()
&& self.host_coordinator.is_unassigned() && self.host_coordinator.is_unassigned()
&& self.runtime_supervisor.is_unassigned() && self.runtime_supervisor.is_unassigned()
@ -1328,6 +1335,7 @@ impl SandboxOwnership {
fn is_committed(&self) -> bool { fn is_committed(&self) -> bool {
self.container_id.is_some() self.container_id.is_some()
&& self.runtime_root.is_some() && self.runtime_root.is_some()
&& self.runtime_root_identity.is_some()
&& self.bundle_digest.is_some() && self.bundle_digest.is_some()
&& self.host_coordinator.is_current_committed() && self.host_coordinator.is_current_committed()
&& self.runtime_supervisor.is_current_committed() && self.runtime_supervisor.is_current_committed()
@ -1337,6 +1345,10 @@ impl SandboxOwnership {
fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> { fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
validate_option_progress(self.container_id.as_ref(), next.container_id.as_ref())?; validate_option_progress(self.container_id.as_ref(), next.container_id.as_ref())?;
validate_option_progress(self.runtime_root.as_ref(), next.runtime_root.as_ref())?; validate_option_progress(self.runtime_root.as_ref(), next.runtime_root.as_ref())?;
validate_option_progress(
self.runtime_root_identity.as_ref(),
next.runtime_root_identity.as_ref(),
)?;
validate_option_progress(self.bundle_digest.as_ref(), next.bundle_digest.as_ref())?; validate_option_progress(self.bundle_digest.as_ref(), next.bundle_digest.as_ref())?;
validate_existing_process_progress(&self.host_coordinator, &next.host_coordinator)?; validate_existing_process_progress(&self.host_coordinator, &next.host_coordinator)?;
validate_process_progress(&self.runtime_supervisor, &next.runtime_supervisor)?; validate_process_progress(&self.runtime_supervisor, &next.runtime_supervisor)?;
@ -1347,22 +1359,49 @@ impl SandboxOwnership {
impl PublicationOwnership { impl PublicationOwnership {
fn empty() -> Self { fn empty() -> Self {
Self { Self {
staged_socket_path: None,
staged_identity_path: None,
socket_path: None, socket_path: None,
identity_path: None, identity_path: None,
socket_identity: None,
identity_file_identity: None,
} }
} }
fn is_empty(&self) -> bool { fn is_empty(&self) -> bool {
self.socket_path.is_none() && self.identity_path.is_none() self.staged_socket_path.is_none()
&& self.staged_identity_path.is_none()
&& self.socket_path.is_none()
&& self.identity_path.is_none()
&& self.socket_identity.is_none()
&& self.identity_file_identity.is_none()
} }
fn is_committed(&self) -> bool { fn is_committed(&self) -> bool {
self.socket_path.is_some() && self.identity_path.is_some() self.staged_socket_path.is_some()
&& self.staged_identity_path.is_some()
&& self.socket_path.is_some()
&& self.identity_path.is_some()
&& self.socket_identity.is_some()
&& self.identity_file_identity.is_some()
} }
fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> { fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
validate_option_progress(
self.staged_socket_path.as_ref(),
next.staged_socket_path.as_ref(),
)?;
validate_option_progress(
self.staged_identity_path.as_ref(),
next.staged_identity_path.as_ref(),
)?;
validate_option_progress(self.socket_path.as_ref(), next.socket_path.as_ref())?; validate_option_progress(self.socket_path.as_ref(), next.socket_path.as_ref())?;
validate_option_progress(self.identity_path.as_ref(), next.identity_path.as_ref()) validate_option_progress(self.identity_path.as_ref(), next.identity_path.as_ref())?;
validate_option_progress(self.socket_identity.as_ref(), next.socket_identity.as_ref())?;
validate_option_progress(
self.identity_file_identity.as_ref(),
next.identity_file_identity.as_ref(),
)
} }
} }

View file

@ -457,14 +457,19 @@ fn complete_ownership(role: ParticipantRole, state: ParticipantState) -> Partici
ParticipantRole::Sandbox => ParticipantOwnership::Sandbox(SandboxOwnership { ParticipantRole::Sandbox => ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))), container_id: Some(ContainerId::new(opaque(29))),
runtime_root: Some(HostPath::new(b"runtime".to_vec())), runtime_root: Some(HostPath::new(b"runtime".to_vec())),
runtime_root_identity: Some(mount()),
bundle_digest: Some(BundleDigest::new(opaque(43))), bundle_digest: Some(BundleDigest::new(opaque(43))),
host_coordinator: owned_process(30), host_coordinator: owned_process(30),
runtime_supervisor: owned_process(21), runtime_supervisor: owned_process(21),
attached_runsc: owned_process(44), attached_runsc: owned_process(44),
}), }),
ParticipantRole::Publication => ParticipantOwnership::Publication(PublicationOwnership { ParticipantRole::Publication => ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: Some(HostPath::new(b"staged-socket".to_vec())),
staged_identity_path: Some(HostPath::new(b"staged-identity".to_vec())),
socket_path: Some(HostPath::new(b"socket".to_vec())), socket_path: Some(HostPath::new(b"socket".to_vec())),
identity_path: Some(HostPath::new(b"identity".to_vec())), identity_path: Some(HostPath::new(b"identity".to_vec())),
socket_identity: Some(mount()),
identity_file_identity: Some(mount()),
}), }),
} }
} }
@ -558,6 +563,7 @@ fn participant_edge_snapshot(
ParticipantOwnership::Sandbox(SandboxOwnership { ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))), container_id: Some(ContainerId::new(opaque(29))),
runtime_root: None, runtime_root: None,
runtime_root_identity: None,
bundle_digest: None, bundle_digest: None,
host_coordinator: process_ownership, host_coordinator: process_ownership,
runtime_supervisor: PersistedHostProcessOwnership::unassigned(), runtime_supervisor: PersistedHostProcessOwnership::unassigned(),
@ -569,6 +575,7 @@ fn participant_edge_snapshot(
ParticipantOwnership::Sandbox(SandboxOwnership { ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))), container_id: Some(ContainerId::new(opaque(29))),
runtime_root: None, runtime_root: None,
runtime_root_identity: None,
bundle_digest: None, bundle_digest: None,
host_coordinator: owned_process(122), host_coordinator: owned_process(122),
runtime_supervisor: process_ownership, runtime_supervisor: process_ownership,
@ -580,6 +587,7 @@ fn participant_edge_snapshot(
ParticipantOwnership::Sandbox(SandboxOwnership { ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))), container_id: Some(ContainerId::new(opaque(29))),
runtime_root: None, runtime_root: None,
runtime_root_identity: None,
bundle_digest: None, bundle_digest: None,
host_coordinator: owned_process(122), host_coordinator: owned_process(122),
runtime_supervisor: owned_process(123), runtime_supervisor: owned_process(123),

View file

@ -375,6 +375,11 @@ fn every_schema_shape_round_trips_and_rejects_malformed_or_unknown_versions() {
SandboxOwnership { SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))), container_id: Some(ContainerId::new(opaque(29))),
runtime_root: Some(HostPath::new(b"runtime".to_vec())), runtime_root: Some(HostPath::new(b"runtime".to_vec())),
runtime_root_identity: Some(MountIdentity {
mount_id: MountId::new(68),
device: DeviceId::new(69),
inode: InodeId::new(70),
}),
bundle_digest: Some(BundleDigest::new(opaque(66))), bundle_digest: Some(BundleDigest::new(opaque(66))),
host_coordinator: owned_process(30), host_coordinator: owned_process(30),
runtime_supervisor: owned_process(21), runtime_supervisor: owned_process(21),
@ -384,8 +389,20 @@ fn every_schema_shape_round_trips_and_rejects_malformed_or_unknown_versions() {
); );
verify_shape( verify_shape(
PublicationOwnership { PublicationOwnership {
staged_socket_path: Some(HostPath::new(b"staged-socket".to_vec())),
staged_identity_path: Some(HostPath::new(b"staged-identity".to_vec())),
socket_path: Some(HostPath::new(b"socket".to_vec())), socket_path: Some(HostPath::new(b"socket".to_vec())),
identity_path: Some(HostPath::new(b"identity".to_vec())), identity_path: Some(HostPath::new(b"identity".to_vec())),
socket_identity: Some(MountIdentity {
mount_id: MountId::new(71),
device: DeviceId::new(72),
inode: InodeId::new(73),
}),
identity_file_identity: Some(MountIdentity {
mount_id: MountId::new(74),
device: DeviceId::new(75),
inode: InodeId::new(76),
}),
}, },
&mut seen, &mut seen,
); );

View file

@ -7,7 +7,10 @@ use lazaret_core::configuration::ResolvedDomainConfiguration;
use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity}; use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity};
use lazaret_core::network::{CanonicalDerivedNetworkPolicy, ReachabilityExpectation}; use lazaret_core::network::{CanonicalDerivedNetworkPolicy, ReachabilityExpectation};
use lazaret_core::records::Session; use lazaret_core::records::Session;
use lazaret_core::records::{Generation, GenerationIdentity, PreparedArtifact}; use lazaret_core::records::{
Generation, GenerationIdentity, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PreparedArtifact,
};
use lazaret_linux::process::current_process_identity; use lazaret_linux::process::current_process_identity;
use lazaret_runtime_supervisor::network::NetworkProbeRequest; use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::SandboxBundle; use lazaret_runtime_supervisor::sandbox::SandboxBundle;
@ -20,7 +23,7 @@ use crate::control::{
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry, ControlError, NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry,
StagedControlFace, TrustAnchor, StagedControlFace, TrustAnchor,
}; };
use crate::domain_store::HostCoordinatorDomainStore; use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::lifecycle::{ use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity,
SessionLifecycleAdapter, SessionLifecycleAdapter,
@ -240,6 +243,10 @@ impl<'a> ProductionLifecycleAdapter<'a> {
self.plan.generation_identity.generation_id, self.plan.generation_identity.generation_id,
self.plan.container_id, self.plan.container_id,
&self.plan.runtime_root, &self.plan.runtime_root,
self.plan
.runtime_staging
.runsc_root_identity()
.map_err(failure)?,
self.plan.bundle_digest, self.plan.bundle_digest,
current_process_identity().map_err(failure)?, current_process_identity().map_err(failure)?,
) )
@ -347,8 +354,14 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.trust_anchor .trust_anchor
.take() .take()
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?; .ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
self.staged_control = Some( let generation_id = self.plan.generation_identity.generation_id;
StagedControlFace::stage_with_executor( self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::bare_intent(generation_id, ParticipantRole::Publication),
)])
.map_err(failure)?;
let staged = StagedControlFace::stage_with_executor(
path_from_host( path_from_host(
self.plan self.plan
.configuration .configuration
@ -361,22 +374,85 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Arc::clone(&self.registry), Arc::clone(&self.registry),
Arc::clone(&self.plan.non_interactive_executor), Arc::clone(&self.plan.non_interactive_executor),
) )
.map_err(failure)?;
let ownership = staged.recovery_ownership().map_err(failure)?;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::Intent,
Some(ParticipantOwnership::Publication(ownership)),
)
.map_err(failure)?, .map_err(failure)?,
); )])
.map_err(failure)?;
self.staged_control = Some(staged);
Ok(()) Ok(())
} }
fn expose_publication(&mut self) -> Result<(), ProductionLifecycleError> { fn expose_publication(&mut self) -> Result<(), ProductionLifecycleError> {
self.published_control = Some(Arc::new( let published = self
self.staged_control .staged_control
.take() .take()
.ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))? .ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))?
.expose() .expose()
.map_err(failure)?;
let ownership = published.recovery_ownership().map_err(failure)?;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::snapshot(
self.plan.generation_identity.generation_id,
ParticipantRole::Publication,
ParticipantState::Committed,
Some(ParticipantOwnership::Publication(ownership)),
)
.map_err(failure)?, .map_err(failure)?,
)); )])
.map_err(failure)?;
self.published_control = Some(Arc::new(published));
Ok(()) Ok(())
} }
fn clean_publication_after_withdrawal(&self) -> Result<(), ProductionLifecycleError> {
let generation_id = self.plan.generation_identity.generation_id;
let current = self
.plan
.store
.snapshot()
.map_err(failure)?
.participants
.into_iter()
.find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::Publication
})
.ok_or_else(|| ProductionLifecycleError("Publication participant is absent".into()))?;
let release = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::ReleaseIntent,
current.ownership().cloned(),
)
.map_err(failure)?;
let clean = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::Clean,
Some(ParticipantOwnership::empty(ParticipantRole::Publication)),
)
.map_err(failure)?;
self.plan
.store
.apply(&[
DomainStoreMutation::PutParticipant(release),
DomainStoreMutation::CleanParticipantAfterAbsence(clean),
])
.map_err(failure)
}
fn stop_sandbox(&mut self) -> Result<(), ProductionLifecycleError> { fn stop_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
let store = Arc::clone(&self.plan.store); let store = Arc::clone(&self.plan.store);
let sandbox = SandboxCoordinator::new(&store); let sandbox = SandboxCoordinator::new(&store);
@ -517,6 +593,7 @@ impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
LifecycleAction::BeginTeardown => { LifecycleAction::BeginTeardown => {
if let Some(control) = self.published_control.as_ref() { if let Some(control) = self.published_control.as_ref() {
control.withdraw().map_err(failure)?; control.withdraw().map_err(failure)?;
self.clean_publication_after_withdrawal()?;
} }
self.published_control.take(); self.published_control.take();
Ok(()) Ok(())

View file

@ -13,7 +13,8 @@ use ed25519_dalek::SigningKey;
use lazaret_core::codec::{CodecError, decode, encode}; use lazaret_core::codec::{CodecError, decode, encode};
use lazaret_core::configuration::ConfinedPathPlan; use lazaret_core::configuration::ConfinedPathPlan;
use lazaret_core::identity::{ use lazaret_core::identity::{
CertificateDigest, ClientId, ContainerId, SessionAnchorId, SessionId, TrustManifestDigest, CertificateDigest, ClientId, ContainerId, DeviceId, HostPath, InodeId, MountIdentity,
SessionAnchorId, SessionId, TrustManifestDigest,
}; };
use lazaret_core::protocol::{ use lazaret_core::protocol::{
ProtocolValidationError, issue_protocol_credential, issue_session_capability, ProtocolValidationError, issue_protocol_credential, issue_session_capability,
@ -23,9 +24,10 @@ use lazaret_core::records::{
ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse, ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse,
DurableDecision, GenerationIdentity, NonInteractiveSessionMessage, DurableDecision, GenerationIdentity, NonInteractiveSessionMessage,
NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole, NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, SandboxCredentialBundle, ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, PublicationOwnership,
Session, SessionBinding, SessionExit, SessionPhase, SessionRegistrationRequest, SandboxCredentialBundle, Session, SessionBinding, SessionExit, SessionPhase,
SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId, TrustManifest, SessionRegistrationRequest, SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId,
TrustManifest,
}; };
use rcgen::{ use rcgen::{
BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer, BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer,
@ -46,6 +48,7 @@ use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinator
use crate::lifecycle::{ use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter, HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter,
}; };
use lazaret_linux::filesystem::mount_identity;
use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker}; use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker};
use lazaret_runtime_supervisor::sandbox::RunscCommand; use lazaret_runtime_supervisor::sandbox::RunscCommand;
@ -53,10 +56,10 @@ pub const CONTROL_ALPN: &[u8] = b"lazaret-control/1";
pub const CONTROL_SERVER_NAME: &str = "coordinator.lazaret.invalid"; pub const CONTROL_SERVER_NAME: &str = "coordinator.lazaret.invalid";
pub const MAX_ENROLLMENT_BYTES: usize = 8_192; pub const MAX_ENROLLMENT_BYTES: usize = 8_192;
const MAX_CONTROL_BYTES: usize = 65_536; const MAX_CONTROL_BYTES: usize = 65_536;
const CONTROL_SOCKET_NAME: &str = "c.sock"; pub(crate) const CONTROL_SOCKET_NAME: &str = "c.sock";
const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard"; pub(crate) const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged"; pub(crate) const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged"; pub(crate) const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
#[derive(Debug)] #[derive(Debug)]
pub enum ControlError { pub enum ControlError {
@ -1009,6 +1012,10 @@ pub struct StagedControlFace {
} }
impl StagedControlFace { impl StagedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Stages an undiscoverable private control face before durable publication. /// Stages an undiscoverable private control face before durable publication.
/// ///
/// # Errors /// # Errors
@ -1160,7 +1167,51 @@ impl Drop for StagedControlFace {
} }
} }
fn publication_recovery_ownership(
paths: &ControlPaths,
) -> Result<PublicationOwnership, ControlError> {
let socket_path = if paths.staged_socket.exists() {
&paths.staged_socket
} else {
&paths.socket
};
let identity_path = if paths.staged_identity.exists() {
&paths.staged_identity
} else {
&paths.identity
};
Ok(PublicationOwnership {
staged_socket_path: Some(HostPath::new(
STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(),
)),
staged_identity_path: Some(HostPath::new(
STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(),
)),
socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())),
identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())),
socket_identity: Some(publication_path_identity(paths, socket_path)?),
identity_file_identity: Some(publication_path_identity(paths, identity_path)?),
})
}
fn publication_path_identity(
paths: &ControlPaths,
path: &Path,
) -> Result<MountIdentity, ControlError> {
let metadata = std::fs::symlink_metadata(path)?;
let root = mount_identity(&paths.root_file)?;
Ok(MountIdentity {
mount_id: root.mount_id,
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
})
}
impl PublishedControlFace { impl PublishedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Selects blocking or polling admission for the publication listener. /// Selects blocking or polling admission for the publication listener.
/// ///
/// # Errors /// # Errors

View file

@ -112,8 +112,12 @@ fn publication_partial() -> Participant {
ParticipantRole::Publication, ParticipantRole::Publication,
ParticipantState::Intent, ParticipantState::Intent,
Some(ParticipantOwnership::Publication(PublicationOwnership { Some(ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: None,
staged_identity_path: None,
socket_path: Some(HostPath::new(b"control.sock".to_vec())), socket_path: Some(HostPath::new(b"control.sock".to_vec())),
identity_path: None, identity_path: None,
socket_identity: None,
identity_file_identity: None,
})), })),
) )
.unwrap() .unwrap()
@ -364,8 +368,12 @@ fn lifecycle_updates_enforce_transition_and_absence_boundaries() {
ParticipantRole::Publication, ParticipantRole::Publication,
ParticipantState::Clean, ParticipantState::Clean,
Some(ParticipantOwnership::Publication(PublicationOwnership { Some(ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: None,
staged_identity_path: None,
socket_path: None, socket_path: None,
identity_path: None, identity_path: None,
socket_identity: None,
identity_file_identity: None,
})), })),
) )
.unwrap(), .unwrap(),

View file

@ -10,12 +10,12 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration}; use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
use lazaret_core::identity::{BundleDigest, ContainerId}; use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity};
use lazaret_core::network::{ReachabilityExpectation, derive_ipv4_policy}; use lazaret_core::network::{ReachabilityExpectation, derive_ipv4_policy};
use lazaret_core::records::{ use lazaret_core::records::{
Generation, GenerationIdentity, HostProcessOwnershipSlot, IpAddress, PreparedArtifact, Session, Generation, GenerationIdentity, HostProcessOwnershipSlot, IpAddress, PreparedArtifact, Session,
}; };
use lazaret_linux::filesystem::namespace_identity; use lazaret_linux::filesystem::{mount_identity, namespace_identity};
use lazaret_linux::process::ProcessOwnershipContext; use lazaret_linux::process::ProcessOwnershipContext;
use lazaret_runtime_supervisor::network::NetworkProbeRequest; use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::{ use lazaret_runtime_supervisor::sandbox::{
@ -371,6 +371,7 @@ fn require_inputs(inputs: &ProductionColdStartInputs<'_>) -> Result<(), Producti
} }
pub(crate) struct RuntimeStaging { pub(crate) struct RuntimeStaging {
runsc_root: OwnedFd,
bundles: OwnedFd, bundles: OwnedFd,
bundle: OwnedFd, bundle: OwnedFd,
controls: OwnedFd, controls: OwnedFd,
@ -384,6 +385,10 @@ pub(crate) struct RuntimeStaging {
} }
impl RuntimeStaging { impl RuntimeStaging {
pub(crate) fn runsc_root_identity(&self) -> Result<MountIdentity, ProductionColdStartError> {
mount_identity(&self.runsc_root).map_err(failure)
}
pub(crate) fn cleanup_overlay(&self) -> Result<(), ProductionColdStartError> { pub(crate) fn cleanup_overlay(&self) -> Result<(), ProductionColdStartError> {
let retained = fstat(&self.overlay).map_err(failure)?; let retained = fstat(&self.overlay).map_err(failure)?;
if !self.overlay_identity.matches(&retained) { if !self.overlay_identity.matches(&retained) {
@ -418,12 +423,7 @@ impl RuntimeStaging {
open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?; open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?;
let overlays = let overlays =
open_or_create_private_directory(&root, "runsc-overlay", invoking_uid, false)?; open_or_create_private_directory(&root, "runsc-overlay", invoking_uid, false)?;
drop(open_or_create_private_directory( let runsc_root = open_or_create_private_directory(&root, "runsc", invoking_uid, false)?;
&root,
"runsc",
invoking_uid,
false,
)?);
let name = OsString::from(hex(container_id.as_bytes())); let name = OsString::from(hex(container_id.as_bytes()));
let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?; let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?;
let bundle_identity = FileIdentity::capture(&bundle)?; let bundle_identity = FileIdentity::capture(&bundle)?;
@ -445,6 +445,7 @@ impl RuntimeStaging {
}; };
let overlay_identity = FileIdentity::capture(&overlay)?; let overlay_identity = FileIdentity::capture(&overlay)?;
let staging = Self { let staging = Self {
runsc_root,
bundles, bundles,
bundle, bundle,
controls, controls,

File diff suppressed because it is too large Load diff

View file

@ -107,6 +107,7 @@ impl<'store> SandboxCoordinator<'store> {
generation_id: GenerationId, generation_id: GenerationId,
container_id: ContainerId, container_id: ContainerId,
runtime_root: &Path, runtime_root: &Path,
runtime_root_identity: MountIdentity,
bundle_digest: BundleDigest, bundle_digest: BundleDigest,
host_coordinator: PidfsProcessIdentityV1, host_coordinator: PidfsProcessIdentityV1,
) -> Result<(), SandboxCoordinatorError> { ) -> Result<(), SandboxCoordinatorError> {
@ -123,6 +124,7 @@ impl<'store> SandboxCoordinator<'store> {
let ownership = SandboxOwnership { let ownership = SandboxOwnership {
container_id: Some(container_id), container_id: Some(container_id),
runtime_root: Some(HostPath::new(runtime_root.as_os_str().as_bytes().to_vec())), runtime_root: Some(HostPath::new(runtime_root.as_os_str().as_bytes().to_vec())),
runtime_root_identity: Some(runtime_root_identity),
bundle_digest: Some(bundle_digest), bundle_digest: Some(bundle_digest),
host_coordinator: PersistedHostProcessOwnership::committed(host_coordinator), host_coordinator: PersistedHostProcessOwnership::committed(host_coordinator),
runtime_supervisor: PersistedHostProcessOwnership::unassigned(), runtime_supervisor: PersistedHostProcessOwnership::unassigned(),

View file

@ -6,7 +6,7 @@ use lazaret_core::identity::{
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks, SandboxId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks, SandboxId,
}; };
use lazaret_core::records::{Domain, DurableDecision, Generation}; use lazaret_core::records::{Domain, DurableDecision, Generation};
use lazaret_linux::filesystem::try_exclusive_flock; use lazaret_linux::filesystem::{current_mount_identity, try_exclusive_flock};
use tempfile::TempDir; use tempfile::TempDir;
use super::*; use super::*;
@ -91,6 +91,7 @@ fn sandbox_staging_readiness_and_release_preserve_exact_ownership() {
generation().generation_id, generation().generation_id,
ContainerId::new(bytes(6)), ContainerId::new(bytes(6)),
directory.path(), directory.path(),
current_mount_identity(directory.path()).unwrap(),
BundleDigest::new(bytes(7)), BundleDigest::new(bytes(7)),
process(100), process(100),
) )
@ -155,6 +156,7 @@ fn mismatched_and_skipped_acquisition_boundaries_refuse_without_state_change() {
generation().generation_id, generation().generation_id,
ContainerId::new(bytes(6)), ContainerId::new(bytes(6)),
directory.path(), directory.path(),
current_mount_identity(directory.path()).unwrap(),
BundleDigest::new(bytes(7)), BundleDigest::new(bytes(7)),
process(100), process(100),
) )

View file

@ -12,7 +12,7 @@ use lazaret_core::records::{
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership, ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
StateAnchorReleaseEvidence, StateAnchorReleaseEvidence,
}; };
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process}; use lazaret_linux::process::RetainedProcess;
use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness}; use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness};
use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount}; use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount};
use lazaret_storage_catalog::{Catalog, CatalogError}; use lazaret_storage_catalog::{Catalog, CatalogError};
@ -80,6 +80,12 @@ pub struct StateAnchorReleaseAuthority {
mount_namespace: NamespaceIdentity, mount_namespace: NamespaceIdentity,
} }
#[derive(Clone, Copy)]
pub(crate) enum StateAnchorRecoveryOwner<'owner> {
Matched(&'owner RetainedProcess),
Absent(&'owner RuntimeOwnersAbsent),
}
impl StateAnchorReleaseAuthority { impl StateAnchorReleaseAuthority {
#[must_use] #[must_use]
pub const fn generation_id(&self) -> GenerationId { pub const fn generation_id(&self) -> GenerationId {
@ -380,33 +386,28 @@ impl<'a> StateAnchorCoordinator<'a> {
/// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper /// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper
/// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only /// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only
/// under the caller-provided ordered runtime-owner absence witness. /// under the caller-provided ordered runtime-owner absence witness.
pub fn recover_after_runtime_owner_loss( pub(crate) fn recover_after_runtime_owner_loss(
&self, &self,
generation_id: GenerationId, generation_id: GenerationId,
layout: &StateAnchorLayout<'_>, layout: &StateAnchorLayout<'_>,
owners_absent: Option<&RuntimeOwnersAbsent>, owner: StateAnchorRecoveryOwner<'_>,
) -> Result< ) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation), (StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError, StateAnchorCoordinatorError,
> { > {
let authority = self.begin_release(generation_id, layout)?; let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) { let observed = match owner {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount( StateAnchorRecoveryOwner::Matched(holder) => recover_private_state_anchor_mount(
layout, layout,
&holder, holder,
authority.user_namespace, authority.user_namespace,
authority.mount_namespace, authority.mount_namespace,
authority.mount, authority.mount,
)?, )?,
Ok(PidfsRecoveryMatch::BootMismatch) => { StateAnchorRecoveryOwner::Absent(owners_absent) => {
self.require_absence_witness(owners_absent, &authority)?; self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)? observe_mount_absence(layout, authority.mount)?
} }
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
}; };
Ok(( Ok((
authority.clone(), authority.clone(),
@ -421,12 +422,9 @@ impl<'a> StateAnchorCoordinator<'a> {
fn require_absence_witness( fn require_absence_witness(
&self, &self,
witness: Option<&RuntimeOwnersAbsent>, witness: &RuntimeOwnersAbsent,
authority: &StateAnchorReleaseAuthority, authority: &StateAnchorReleaseAuthority,
) -> Result<(), StateAnchorCoordinatorError> { ) -> Result<(), StateAnchorCoordinatorError> {
let witness = witness.ok_or(StateAnchorCoordinatorError::InvalidState(
"complete runtime-owner absence proof is required",
))?;
let snapshot = self.domain_store.snapshot()?; let snapshot = self.domain_store.snapshot()?;
let generation = snapshot let generation = snapshot
.generations .generations

View file

@ -1,14 +1,22 @@
use std::fs::{self, File};
use std::io; use std::io;
use std::os::fd::OwnedFd;
use std::os::unix::fs::PermissionsExt;
use std::process::ExitCode; use std::process::ExitCode;
use lazaret_core::identity::{ use lazaret_core::identity::{
ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, NetworkSidecarId, SandboxId, BootId, BundleDigest, ConfigurationFingerprint, ContainerId, CoordinatorId, DomainId,
GenerationId, HostPath, HostPidLocator, MountId, MountIdentity, NetworkSidecarId,
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, PidfsProcessIdentityV1,
ProcessStartTimeTicks, SandboxId,
}; };
use lazaret_core::records::{ use lazaret_core::records::{
DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, StateAnchorOwnership, ParticipantState, PersistedHostProcessOwnership, SandboxOwnership, StateAnchorOwnership,
};
use lazaret_host_coordinator::recovery::{
delete_runsc_container, prove_runtime_owners_absent, runtime_recovery_ownership,
}; };
use lazaret_host_coordinator::recovery::{prove_runtime_owners_absent, runtime_recovery_ownership};
use lazaret_linux::process::{TargetCommand, child_broker, dispatch_private_process_role}; use lazaret_linux::process::{TargetCommand, child_broker, dispatch_private_process_role};
use rustix::process::Signal; use rustix::process::Signal;
@ -31,19 +39,61 @@ fn exit(result: io::Result<()>) -> ExitCode {
} }
fn run() -> io::Result<()> { fn run() -> io::Result<()> {
verify_exact_runsc_delete()?;
verify_exact_action(None)?; verify_exact_action(None)?;
verify_exact_action(Some(Signal::STOP)) verify_exact_action(Some(Signal::STOP))
} }
fn verify_exact_runsc_delete() -> io::Result<()> {
let tree = tempfile::tempdir()?;
let root_path = tree.path().join("runsc");
fs::create_dir(&root_path)?;
fs::create_dir(root_path.join("container"))?;
let program = tree.path().join("runsc-fixture");
fs::write(
&program,
b"#!/bin/sh\nroot=${1#-root=}\n[ \"$2\" = delete ] && [ \"$3\" = --force ] && rmdir \"$root/$4\"\n",
)?;
fs::set_permissions(&program, fs::Permissions::from_mode(0o700))?;
let root: OwnedFd = File::open(&root_path)?.into();
delete_runsc_container(&root, "container", program.as_os_str()).map_err(io::Error::other)?;
if root_path.join("container").exists() {
return Err(io::Error::other("exact runsc container remained"));
}
Ok(())
}
fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> { fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
let captured = let state =
child_broker().run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?; child_broker().run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?;
if let Some(signal) = signal { if let Some(signal) = signal {
captured.process().send_signal(signal)?; state.process().send_signal(signal)?;
} }
let process = captured.into_process(); let state = state.into_process();
let supervisor = child_broker()
.run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?
.into_process();
let generation = generation(); let generation = generation();
let participants = [ let participants = [
Participant::snapshot(
generation.generation_id,
ParticipantRole::Sandbox,
ParticipantState::Intent,
Some(ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new([6; 32])),
runtime_root: Some(HostPath::new(b"/runtime/runsc".to_vec())),
runtime_root_identity: Some(MountIdentity {
mount_id: MountId::new(7),
device: lazaret_core::identity::DeviceId::new(8),
inode: lazaret_core::identity::InodeId::new(9),
}),
bundle_digest: Some(BundleDigest::new([10; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(absent_process()),
runtime_supervisor: PersistedHostProcessOwnership::staged(supervisor.identity()),
attached_runsc: PersistedHostProcessOwnership::unassigned(),
})),
)
.map_err(|_| io::Error::other("Sandbox participant is invalid"))?,
Participant::bare_intent(generation.generation_id, ParticipantRole::PreparedLower), Participant::bare_intent(generation.generation_id, ParticipantRole::PreparedLower),
Participant::snapshot( Participant::snapshot(
generation.generation_id, generation.generation_id,
@ -54,7 +104,7 @@ fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
upper_path: None, upper_path: None,
work_path: None, work_path: None,
merged_path: None, merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process.identity()), daemon: PersistedHostProcessOwnership::staged(state.identity()),
user_namespace: None, user_namespace: None,
mount_namespace: None, mount_namespace: None,
mount: None, mount: None,
@ -65,7 +115,20 @@ fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
let ownership = let ownership =
runtime_recovery_ownership(&participants, &generation).map_err(io::Error::other)?; runtime_recovery_ownership(&participants, &generation).map_err(io::Error::other)?;
prove_runtime_owners_absent(&ownership)?; prove_runtime_owners_absent(&ownership)?;
child_broker().reap(process) child_broker().reap(state)?;
child_broker().reap(supervisor)
}
fn absent_process() -> PidfsProcessIdentityV1 {
PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new([0xff; 16]),
pidfs_st_dev: PidfsDeviceId::new(1),
pidfs_st_ino: PidfsInodeId::new(2),
},
HostPidLocator::new(1),
ProcessStartTimeTicks::new(3),
)
} }
fn generation() -> Generation { fn generation() -> Generation {

View file

@ -33,7 +33,7 @@ use lazaret_host_coordinator::terminal::{
NoActivePrompt, RunscSessionTerminalFactory, SessionTerminalEngine, TerminalAcquisition, NoActivePrompt, RunscSessionTerminalFactory, SessionTerminalEngine, TerminalAcquisition,
TerminalRoute, TerminalRoute,
}; };
use lazaret_linux::filesystem::try_exclusive_flock; use lazaret_linux::filesystem::{current_mount_identity, try_exclusive_flock};
use lazaret_linux::platform::{PlatformObservation, check_platform}; use lazaret_linux::platform::{PlatformObservation, check_platform};
use lazaret_linux::process::{ use lazaret_linux::process::{
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity, ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity,
@ -1023,6 +1023,7 @@ impl Fixture {
GenerationId::new(7), GenerationId::new(7),
container_id, container_id,
&runtime_domain.join("runsc"), &runtime_domain.join("runsc"),
current_mount_identity(&runtime_domain.join("runsc"))?,
BundleDigest::new([7; 32]), BundleDigest::new([7; 32]),
current_process_identity()?, current_process_identity()?,
) )
@ -1306,6 +1307,7 @@ fn network_store(
GenerationId::new(7), GenerationId::new(7),
container_id, container_id,
&runtime_domain.join("runsc"), &runtime_domain.join("runsc"),
current_mount_identity(&runtime_domain.join("runsc"))?,
BundleDigest::new([17; 32]), BundleDigest::new([17; 32]),
current_process_identity()?, current_process_identity()?,
) )

View file

@ -1051,6 +1051,7 @@ classDiagram
<<serialized>> <<serialized>>
optional container_id optional container_id
optional runtime_root optional runtime_root
optional runtime_root_identity
optional bundle_digest optional bundle_digest
host_coordinator host_coordinator
runtime_supervisor runtime_supervisor
@ -1059,8 +1060,12 @@ classDiagram
class PublicationOwnership { class PublicationOwnership {
<<serialized>> <<serialized>>
optional staged_socket_path
optional staged_identity_path
optional socket_path optional socket_path
optional identity_path optional identity_path
optional socket_identity
optional identity_file_identity
} }
ParticipantOwnership --> PreparedLowerOwnership ParticipantOwnership --> PreparedLowerOwnership
@ -1106,8 +1111,8 @@ The only serviceable role/state pairs are:
| `GitRouting` | `Prepared` | three paths, mount, current committed daemon | | `GitRouting` | `Prepared` | three paths, mount, current committed daemon |
| `Network` | `Committed` | policy digest, fingerprint, readiness evidence, namespace, current committed RS | | `Network` | `Committed` | policy digest, fingerprint, readiness evidence, namespace, current committed RS |
| `Helper` | `Committed` | sidecar ID, current committed helper and RS | | `Helper` | `Committed` | sidecar ID, current committed helper and RS |
| `Sandbox` | `Committed` | container, runtime root, bundle digest, current committed HC, RS, and runsc | | `Sandbox` | `Committed` | container, runtime root and exact mount/device/inode identity, bundle digest, current committed HC, RS, and runsc |
| `Publication` | `Committed` | socket and identity paths | | `Publication` | `Committed` | staged and exposed fixed labels plus exact socket and identity-file mount/device/inode identities |
Network readiness evidence must name the enclosing Participant generation and Network readiness evidence must name the enclosing Participant generation and
match its policy digest, fingerprint, namespace, and committed RS identity. match its policy digest, fingerprint, namespace, and committed RS identity.