lazaret/crates/lazaret-host-coordinator/src/control.rs

674 lines
26 KiB
Rust
Raw Normal View History

use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Write};
use std::os::fd::{AsFd, AsRawFd};
use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard};
use ed25519_dalek::SigningKey;
pub use lazaret_control_client::{CONTROL_ALPN, CONTROL_SERVER_NAME, PendingClientEnrollment};
use lazaret_control_client::{ControlClient, ControlClientError};
use lazaret_core::codec::{CodecError, decode, encode};
use lazaret_core::configuration::ConfinedPathPlan;
use lazaret_core::identity::{
CertificateDigest, ClientId, ContainerId, DeviceId, HostPath, InodeId, MountIdentity,
SessionAnchorId, SessionId, TrustManifestDigest,
};
use lazaret_core::protocol::{
ProtocolValidationError, issue_protocol_credential, issue_session_capability,
issue_trust_manifest, verify_tls_credential,
};
use lazaret_core::records::{
ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse,
DurableDecision, GenerationIdentity, InteractiveSessionBatch, InteractiveSessionEvent,
InteractiveSessionMessage, InteractiveSessionRequest, NonInteractiveSessionMessage,
NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, PublicationOwnership,
SandboxCredentialBundle, Session, SessionBinding, SessionExit, SessionPhase,
SessionRegistrationRequest, SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId,
TrustManifest,
};
use rcgen::{
BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer,
DistinguishedName, DnType, DnValue, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
PKCS_ED25519,
};
use rustix::fs::{Mode, OFlags, fchmod, openat};
use rustix::net::sockopt::socket_peercred;
use rustls::pki_types::{
CertificateDer, CertificateSigningRequestDer, PrivateKeyDer, PrivatePkcs8KeyDer,
};
use rustls::server::{NoServerSessionStorage, WebPkiClientVerifier};
use rustls::{ClientConfig, ClientConnection, RootCertStore, ServerConfig, ServerConnection};
use sha2::{Digest, Sha256};
use time::{Duration, OffsetDateTime};
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter,
};
use lazaret_linux::filesystem::mount_identity;
use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker};
use lazaret_runtime_supervisor::sandbox::RunscCommand;
pub const MAX_ENROLLMENT_BYTES: usize = 8_192;
const MAX_CONTROL_BYTES: usize = 65_536;
pub(crate) use lazaret_control_client::{CONTROL_SOCKET_NAME, PUBLICATION_IDENTITY_NAME};
pub(crate) const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
pub(crate) const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
#[derive(Debug)]
pub enum ControlError {
Io(io::Error),
Codec(CodecError),
Certificate(rcgen::Error),
Tls(rustls::Error),
Protocol(ProtocolValidationError),
Client(ControlClientError),
DomainStore(DomainStoreError),
InvalidPath(&'static str),
InvalidPublication(&'static str),
UnauthorizedPeer,
InvalidCertificateProfile,
StaleGeneration,
DuplicateIdentity,
InvalidRequest,
OversizeRequest { length: usize, maximum: usize },
LockPoisoned,
Lifecycle(String),
}
impl fmt::Display for ControlError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(error) => write!(formatter, "control I/O failed: {error}"),
Self::Codec(error) => write!(formatter, "control record failed validation: {error}"),
Self::Certificate(error) => write!(formatter, "certificate operation failed: {error}"),
Self::Tls(error) => write!(formatter, "TLS failed: {error}"),
Self::Protocol(error) => write!(formatter, "protocol validation failed: {error}"),
Self::Client(error) => write!(formatter, "{error}"),
Self::DomainStore(error) => write!(formatter, "session persistence failed: {error}"),
Self::InvalidPath(reason) => write!(formatter, "invalid control path: {reason}"),
Self::InvalidPublication(reason) => {
write!(formatter, "invalid control publication: {reason}")
}
Self::UnauthorizedPeer => formatter.write_str("control peer UID is not authorized"),
Self::InvalidCertificateProfile => {
formatter.write_str("TLS certificate profile is invalid")
}
Self::StaleGeneration => formatter.write_str("control generation is stale"),
Self::DuplicateIdentity => formatter.write_str("session identity is already active"),
Self::InvalidRequest => {
formatter.write_str("operation is not allowed on this connection")
}
Self::OversizeRequest { length, maximum } => {
write!(
formatter,
"control request length {length} exceeds {maximum}"
)
}
Self::LockPoisoned => formatter.write_str("control registry lock is poisoned"),
Self::Lifecycle(detail) => write!(formatter, "control lifecycle failed: {detail}"),
}
}
}
impl std::error::Error for ControlError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Codec(error) => Some(error),
Self::Certificate(error) => Some(error),
Self::Tls(error) => Some(error),
Self::Protocol(error) => Some(error),
Self::Client(error) => Some(error),
Self::DomainStore(error) => Some(error),
_ => None,
}
}
}
macro_rules! from_error {
($source:ty, $variant:ident) => {
impl From<$source> for ControlError {
fn from(error: $source) -> Self {
Self::$variant(error)
}
}
};
}
from_error!(io::Error, Io);
from_error!(CodecError, Codec);
from_error!(rcgen::Error, Certificate);
from_error!(rustls::Error, Tls);
from_error!(ProtocolValidationError, Protocol);
from_error!(ControlClientError, Client);
from_error!(DomainStoreError, DomainStore);
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct FileIdentity {
device: u64,
inode: u64,
}
impl FileIdentity {
fn from_metadata(metadata: &std::fs::Metadata) -> Self {
Self {
device: metadata.dev(),
inode: metadata.ino(),
}
}
}
pub struct TrustAnchor {
generation_identity: GenerationIdentity,
tls_ca: CertifiedIssuer<'static, KeyPair>,
tls_ca_der: CertificateDer<'static>,
coordinator_tls_credential: TlsCredential,
coordinator_tls_private_key: PrivatePkcs8KeyDer<'static>,
trust_manifest: TrustManifest,
trust_manifest_digest: TrustManifestDigest,
coordinator_protocol_credential: ProtocolCredential,
sandbox_protocol_credential: ProtocolCredential,
coordinator_protocol_key: SigningKey,
sandbox_protocol_key: Option<SigningKey>,
}
impl TrustAnchor {
#[must_use]
pub const fn generation_identity(&self) -> &GenerationIdentity {
&self.generation_identity
}
/// Creates one ephemeral generation Trust Anchor with role-separated Ed25519 keys.
///
/// # Errors
/// Returns an error when key generation, certificate issuance, or protocol signing fails.
pub fn create(generation_identity: GenerationIdentity) -> Result<Self, ControlError> {
let now = OffsetDateTime::now_utc();
let mut ca_params = CertificateParams::default();
ca_params.distinguished_name = DistinguishedName::new();
ca_params
.distinguished_name
.push(DnType::CommonName, "Lazaret ephemeral TLS CA");
ca_params.not_before = now - Duration::minutes(1);
ca_params.not_after = now + Duration::hours(24);
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
ca_params.key_usages = vec![
KeyUsagePurpose::DigitalSignature,
KeyUsagePurpose::KeyCertSign,
KeyUsagePurpose::CrlSign,
];
let ca_key = KeyPair::generate_for(&PKCS_ED25519)?;
let tls_ca = CertifiedIssuer::self_signed(ca_params, ca_key)?;
let tls_ca_der = tls_ca.der().clone();
let tls_ca_digest = certificate_digest(&tls_ca_der);
let protocol_issuer = random_signing_key()?;
let (trust_manifest, trust_manifest_digest) =
issue_trust_manifest(generation_identity.clone(), tls_ca_digest, &protocol_issuer)?;
let coordinator_protocol_key = random_signing_key()?;
let sandbox_protocol_key = random_signing_key()?;
let coordinator_protocol_credential = issue_protocol_credential(
generation_identity.clone(),
ProtocolCredentialRole::SessionCoordinator,
ProtocolSubjectId::SessionCoordinator(generation_identity.coordinator_id),
&coordinator_protocol_key.verifying_key(),
&protocol_issuer,
)?;
let sandbox_protocol_credential = issue_protocol_credential(
generation_identity.clone(),
ProtocolCredentialRole::AgentSandbox,
ProtocolSubjectId::AgentSandbox(generation_identity.sandbox_id),
&sandbox_protocol_key.verifying_key(),
&protocol_issuer,
)?;
let coordinator_key = KeyPair::generate_for(&PKCS_ED25519)?;
let coordinator_certificate = leaf_parameters(
CONTROL_SERVER_NAME,
ExtendedKeyUsagePurpose::ServerAuth,
now,
)?
.signed_by(&coordinator_key, &tls_ca)?;
let coordinator_certificate_der = coordinator_certificate.der().to_vec();
let coordinator_tls_credential = TlsCredential::session_coordinator(
generation_identity.clone(),
generation_identity.coordinator_id,
coordinator_certificate_der.clone(),
certificate_digest(coordinator_certificate.der()),
);
Ok(Self {
generation_identity,
tls_ca,
tls_ca_der,
coordinator_tls_credential,
coordinator_tls_private_key: PrivatePkcs8KeyDer::from(coordinator_key.serialize_der()),
trust_manifest,
trust_manifest_digest,
coordinator_protocol_credential,
sandbox_protocol_credential,
coordinator_protocol_key,
sandbox_protocol_key: Some(sandbox_protocol_key),
})
}
#[must_use]
pub fn publication_identity(&self) -> ProtocolPublicationIdentity {
ProtocolPublicationIdentity {
protocol_version: ProtocolVersion::V1,
generation_identity: self.generation_identity.clone(),
tls_ca_certificate_der: self.tls_ca_der.to_vec(),
trust_manifest_sha256: self.trust_manifest_digest,
coordinator_tls_certificate_sha256: self.coordinator_tls_credential.certificate_sha256,
}
}
2026-07-15 19:48:42 +09:00
#[must_use]
pub fn coordinator_protocol_key(&self) -> SigningKey {
self.coordinator_protocol_key.clone()
}
/// Transfers the sandbox credential package once without exposing coordinator or issuer keys.
///
/// # Errors
/// Returns an error after the sandbox private key has already been transferred.
pub fn take_sandbox_credential_bundle(
&mut self,
) -> Result<SandboxCredentialBundle, ControlError> {
let private_key = self
.sandbox_protocol_key
.take()
.ok_or(ControlError::InvalidRequest)?
.to_bytes()
.to_vec();
SandboxCredentialBundle::new(
self.generation_identity.clone(),
self.trust_manifest.clone(),
self.coordinator_protocol_credential.clone(),
self.sandbox_protocol_credential.clone(),
private_key,
)
.map_err(|_| ControlError::InvalidCertificateProfile)
}
fn issue_client_credential(
&self,
request: &ClientEnrollmentRequest,
) -> Result<TlsCredential, ControlError> {
if request.protocol_version != ProtocolVersion::V1
|| request.generation_identity != self.generation_identity
{
return Err(ControlError::StaleGeneration);
}
let csr_der =
CertificateSigningRequestDer::from(request.certificate_signing_request_der.as_slice());
let csr = CertificateSigningRequestParams::from_der(&csr_der)?;
if csr.public_key.algorithm() != &PKCS_ED25519
|| csr.params.distinguished_name.get(&DnType::CommonName)
!= Some(&DnValue::Utf8String(enrollment_binding(
&request.generation_identity,
request.cli_id,
)?))
{
return Err(ControlError::InvalidCertificateProfile);
}
let now = OffsetDateTime::now_utc();
let params = leaf_parameters(
&format!("client-{}", short_identity(request.cli_id.as_bytes())),
ExtendedKeyUsagePurpose::ClientAuth,
now,
)?;
let constrained = CertificateSigningRequestParams {
params,
public_key: csr.public_key,
};
let certificate = constrained.signed_by(&self.tls_ca)?;
let certificate_der = certificate.der().to_vec();
Ok(TlsCredential::client_cli(
self.generation_identity.clone(),
request.cli_id,
certificate_der,
certificate_digest(certificate.der()),
))
}
fn server_config(&self) -> Result<ServerConfig, ControlError> {
let mut roots = RootCertStore::empty();
roots.add(self.tls_ca_der.clone())?;
let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
.allow_unauthenticated()
.build()
.map_err(|_| ControlError::InvalidCertificateProfile)?;
let mut config = ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13])
.with_client_cert_verifier(verifier)
.with_single_cert(
vec![CertificateDer::from(
self.coordinator_tls_credential.certificate_der.clone(),
)],
PrivateKeyDer::Pkcs8(self.coordinator_tls_private_key.clone_key()),
)?;
config.alpn_protocols = vec![CONTROL_ALPN.to_vec()];
config.max_early_data_size = 0;
config.send_tls13_tickets = 0;
config.session_storage = Arc::new(NoServerSessionStorage {});
Ok(config)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RegisteredSession {
pub binding: SessionBinding,
pub session_anchor_id: SessionAnchorId,
}
#[derive(Clone)]
pub struct ControlSessionAdmission {
pub request: SessionRegistrationRequest,
pub credential: TlsCredential,
}
pub type ControlHostCoordinatorLifecycle<'store, A> =
HostCoordinatorLifecycle<'store, A, Arc<SessionRegistry>>;
#[derive(Default)]
struct RegistryState {
enrolled: BTreeMap<CertificateDigest, TlsCredential>,
connected: BTreeMap<SessionId, RegisteredSession>,
}
pub struct SessionRegistry {
state: Mutex<RegistryState>,
store: Arc<HostCoordinatorDomainStore>,
generation_identity: GenerationIdentity,
}
impl SessionRegistry {
#[must_use]
pub fn new(
store: Arc<HostCoordinatorDomainStore>,
generation_identity: GenerationIdentity,
) -> Self {
Self {
state: Mutex::new(RegistryState::default()),
store,
generation_identity,
}
}
/// Returns the exact set of connected, registered sessions.
///
/// # Errors
/// Returns an error if the registry lock is poisoned.
pub fn connected_sessions(&self) -> Result<Vec<RegisteredSession>, ControlError> {
Ok(self.lock()?.connected.values().cloned().collect())
}
fn lock(&self) -> Result<MutexGuard<'_, RegistryState>, ControlError> {
self.state.lock().map_err(|_| ControlError::LockPoisoned)
}
fn enroll(&self, credential: TlsCredential) -> Result<(), ControlError> {
self.lock()?
.enrolled
.insert(credential.certificate_sha256, credential);
Ok(())
}
fn credential_for_leaf(&self, certificate: &[u8]) -> Result<TlsCredential, ControlError> {
self.lock()?
.enrolled
.get(&certificate_digest(certificate))
.cloned()
.ok_or(ControlError::InvalidCertificateProfile)
}
fn register(
&self,
request: SessionRegistrationRequest,
credential: &TlsCredential,
) -> Result<RegisteredSession, ControlError> {
let binding = request.session_binding;
if request.protocol_version != ProtocolVersion::V1
|| binding.generation_identity != self.generation_identity
{
return Err(ControlError::StaleGeneration);
}
let TlsSubjectId::ClientCli(certificate_cli_id) = credential.subject_id() else {
return Err(ControlError::InvalidCertificateProfile);
};
if credential.role() != TlsRole::ClientCli || certificate_cli_id != binding.cli_id {
return Err(ControlError::InvalidCertificateProfile);
}
verify_tls_credential(
credential,
&self.generation_identity,
TlsRole::ClientCli,
TlsSubjectId::ClientCli(binding.cli_id),
)?;
let anchor_bytes: [u8; 32] = binding
.session_anchor_public_key
.as_slice()
.try_into()
.map_err(|_| ControlError::InvalidRequest)?;
ed25519_dalek::VerifyingKey::from_bytes(&anchor_bytes)
.map_err(|_| ControlError::InvalidRequest)?;
let session_anchor_id = SessionAnchorId::new(sha256(&anchor_bytes));
self.require_serviceable_generation()?;
let snapshot = self.store.snapshot()?;
let state = self.lock()?;
let duplicate_in_memory = state.connected.values().any(|registered| {
registered.binding.session_id == binding.session_id
|| registered.binding.cli_id == binding.cli_id
|| registered.session_anchor_id == session_anchor_id
});
let resumable_waiter = snapshot.sessions.iter().any(|session| {
session.generation_id == binding.generation_identity.generation_id
&& session.session_id == binding.session_id
&& session.cli_id == binding.cli_id
&& session.session_anchor_id == session_anchor_id
&& session.session_phase == SessionPhase::Waiting
});
let duplicate_session = snapshot.sessions.iter().any(|session| {
let same_session_allowed = resumable_waiter
&& session.session_id == binding.session_id
&& (session.session_phase == SessionPhase::Closed
|| (session.generation_id == binding.generation_identity.generation_id
&& session.cli_id == binding.cli_id
&& session.session_anchor_id == session_anchor_id
&& session.session_phase == SessionPhase::Waiting));
!same_session_allowed
&& (session.session_id == binding.session_id
|| (is_live_session(session.session_phase)
&& (session.cli_id == binding.cli_id
|| session.session_anchor_id == session_anchor_id)))
});
if duplicate_in_memory || duplicate_session {
return Err(ControlError::DuplicateIdentity);
}
let registering = Session {
generation_id: binding.generation_identity.generation_id,
session_id: binding.session_id,
cli_id: binding.cli_id,
session_anchor_id,
session_phase: SessionPhase::Registering,
};
self.store
.apply(&[DomainStoreMutation::PutSession(registering.clone())])?;
let mut attached = registering;
attached.session_phase = SessionPhase::Attached;
if let Err(error) = self
.store
.apply(&[DomainStoreMutation::PutSession(attached)])
{
let mut failed = Session {
generation_id: binding.generation_identity.generation_id,
session_id: binding.session_id,
cli_id: binding.cli_id,
session_anchor_id,
session_phase: SessionPhase::Registering,
};
failed.session_phase = SessionPhase::Failed;
let _ = self.store.apply(&[DomainStoreMutation::PutSession(failed)]);
return Err(error.into());
}
let registered = RegisteredSession {
binding,
session_anchor_id,
};
drop(snapshot);
drop(state);
self.lock()?
.connected
.insert(registered.binding.session_id, registered.clone());
Ok(registered)
}
fn disconnect(&self, session_id: SessionId) -> Result<(), ControlError> {
let registered = self.lock()?.connected.remove(&session_id);
let Some(registered) = registered else {
return Ok(());
};
self.store
.apply(&[DomainStoreMutation::PutSession(Session {
generation_id: registered.binding.generation_identity.generation_id,
session_id,
cli_id: registered.binding.cli_id,
session_anchor_id: registered.session_anchor_id,
session_phase: SessionPhase::Closed,
})])?;
Ok(())
}
fn require_serviceable_generation(&self) -> Result<(), ControlError> {
let decision = self.require_generation_authority()?;
if decision != DurableDecision::Published {
return Err(ControlError::StaleGeneration);
}
Ok(())
}
fn require_generation_authority(&self) -> Result<DurableDecision, ControlError> {
let snapshot = self.store.snapshot()?;
let generation = snapshot
.generations
.iter()
.find(|generation| generation.generation_id == self.generation_identity.generation_id)
.ok_or(ControlError::StaleGeneration)?;
let is_latest = snapshot
.generations
.iter()
.all(|candidate| candidate.generation_id <= self.generation_identity.generation_id);
if !is_latest
|| snapshot.domain.domain_id != self.generation_identity.domain_id
|| snapshot.domain.profile_name != self.generation_identity.agent_profile
|| generation.domain_id != self.generation_identity.domain_id
|| generation.configuration_fingerprint
!= self.generation_identity.configuration_fingerprint
|| generation.coordinator_id != self.generation_identity.coordinator_id
|| generation.sandbox_id != self.generation_identity.sandbox_id
|| !matches!(
generation.durable_decision,
DurableDecision::Preparing | DurableDecision::Published
)
{
return Err(ControlError::StaleGeneration);
}
Ok(generation.durable_decision)
}
}
impl SessionLifecycleAdapter for Arc<SessionRegistry> {
type Error = ControlError;
type Admission = ControlSessionAdmission;
fn admission_identity(&self, admission: &Self::Admission) -> SessionAdmissionIdentity {
let binding = &admission.request.session_binding;
SessionAdmissionIdentity {
generation_id: binding.generation_identity.generation_id,
session_id: binding.session_id,
configuration_fingerprint: binding.generation_identity.configuration_fingerprint,
}
}
fn register_session(
&self,
_store: &HostCoordinatorDomainStore,
entry_action: LifecycleAction,
admission: Self::Admission,
) -> Result<Session, Self::Error> {
if !matches!(
entry_action,
LifecycleAction::StartReadySession | LifecycleAction::AdvanceWaitingSession
) {
return Err(ControlError::InvalidRequest);
}
let registered = self.register(admission.request, &admission.credential)?;
Ok(Session {
generation_id: registered.binding.generation_identity.generation_id,
session_id: registered.binding.session_id,
cli_id: registered.binding.cli_id,
session_anchor_id: registered.session_anchor_id,
session_phase: SessionPhase::Attached,
})
}
fn disconnect_session(
&self,
_store: &HostCoordinatorDomainStore,
session: &Session,
) -> Result<(), Self::Error> {
let mut state = self.lock()?;
let registered = state.connected.get(&session.session_id);
if let Some(registered) = registered
&& (registered.binding.generation_identity.generation_id != session.generation_id
|| registered.binding.cli_id != session.cli_id
|| registered.session_anchor_id != session.session_anchor_id)
{
return Err(ControlError::InvalidRequest);
}
if session.session_phase == SessionPhase::Attached && registered.is_none() {
return Err(ControlError::InvalidRequest);
}
state.connected.remove(&session.session_id);
drop(state);
self.store
.apply(&[DomainStoreMutation::PutSession(Session {
session_phase: SessionPhase::Closed,
..session.clone()
})])?;
Ok(())
}
}
mod execution;
mod publication;
mod staging;
pub(crate) use execution::AcceptedControlConnection;
use execution::{ControlPaths, ExecutionUnavailable, InteractiveExecutionUnavailable};
pub use execution::{
InteractiveControlSession, InteractiveSessionExecutor, NonInteractiveSessionExecutor,
PublishedControlFace, RunscNonInteractiveExecutor,
};
#[cfg(test)]
use execution::{NonInteractiveCommandRunner, hex_lower};
#[cfg(test)]
use publication::require_peer_uid;
use publication::{
certificate_digest, enrollment_binding, is_live_session, leaf_parameters, random_signing_key,
remove_if_identity_matches, sha256, short_identity,
};
pub use staging::StagedControlFace;
use staging::publication_recovery_ownership;
#[cfg(test)]
mod tests;