ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Serve concurrent terminal sessions with exact teardown

Browse source
This commit is contained in:
ms2k 2026-07-18 02:51:19 +09:00
commit 254bb37764
4 changed files with 278 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

17
crates/lazaret-host-coordinator/src/bootstrap.rs
View file

@ -20,8 +20,9 @@ use lazaret_storage_catalog::Catalog;
use crate::control::ControlSessionAdmission;
use crate::control::{
ControlError, InteractiveSessionExecutor, NonInteractiveSessionExecutor, PublishedControlFace,
SessionRegistry, StagedControlFace, TrustAnchor,
AcceptedControlConnection, ControlError, InteractiveSessionExecutor,
NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry, StagedControlFace,
TrustAnchor,
};
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::git_routing::GitRoutingCoordinator;
@ -682,7 +683,17 @@ pub fn serve_production_control(
.adapter()
.published_control()
.ok_or(ControlError::StaleGeneration)?;
face.serve_next_coordinated(
let connection = face.accept_connection()?;
serve_production_control_connection(lifecycle, &face, connection)
}
pub(crate) fn serve_production_control_connection(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
face: &PublishedControlFace,
connection: AcceptedControlConnection,
) -> Result<(), ControlError> {
face.serve_connection_coordinated(
connection,
|admission| {
lifecycle
.lock()

165
crates/lazaret-host-coordinator/src/client_bootstrap.rs
View file

@ -20,13 +20,15 @@ use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use rustix::fs::{Mode, OFlags, openat};
use crate::bootstrap::{ProductionHostCoordinatorLifecycle, serve_production_control};
use crate::bootstrap::{ProductionHostCoordinatorLifecycle, serve_production_control_connection};
use crate::control::ControlError;
use crate::domain_store::HostCoordinatorDomainStore;
use crate::production::{BootstrapOwnerGuard, ProductionColdStart, ProductionColdStartError};
use crate::production_selector::{InstalledProductPaths, select_production_cold_start};
use crate::waiter::BootstrapWaiterFace;
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
const MAX_CONCURRENT_CONNECTIONS: usize = 32;
#[derive(Debug)]
pub struct PrivateClientBootstrapError(String);
@ -169,47 +171,108 @@ fn serve_published(
.ok_or_else(|| ProductionColdStartError::service("control face is absent"))?
.set_nonblocking(true)
.map_err(ProductionColdStartError::service)?;
loop {
{
let mut lifecycle = lifecycle.lock().map_err(|_| {
ProductionColdStartError::service("production lifecycle lock is poisoned")
})?;
lifecycle
.reconcile_published_bootstrap_owner(
bootstrap_owner_session_id,
bootstrap_owner
.is_live()
.map_err(ProductionColdStartError::service)?,
)
.map_err(ProductionColdStartError::service)?;
if lifecycle.adapter().published_control().is_none() {
return Ok(());
std::thread::scope(|scope| {
let (completed, completions) =
std::sync::mpsc::channel::<(u64, Result<(), ControlError>)>();
let mut active: BTreeMap<u64, std::os::unix::net::UnixStream> = BTreeMap::new();
let mut next_connection = 0_u64;
loop {
while let Ok((connection_id, result)) = completions.try_recv() {
active.remove(&connection_id);
connection_completion(result, false)?;
}
waiters
.borrow_mut()
.reconcile_published(&mut lifecycle)
.map_err(ProductionColdStartError::service)?;
}
match serve_production_control(lifecycle) {
Ok(()) => {}
Err(crate::control::ControlError::Io(error))
if error.kind() == std::io::ErrorKind::WouldBlock =>
{
std::thread::sleep(std::time::Duration::from_millis(10));
let face = {
let mut lifecycle = lifecycle.lock().map_err(|_| {
ProductionColdStartError::service("production lifecycle lock is poisoned")
})?;
let Some(face) = lifecycle.adapter().published_control() else {
for socket in active.values() {
let _ = socket.shutdown(std::net::Shutdown::Both);
}
drop(lifecycle);
while !active.is_empty() {
wait_for_connection_completion(&completions, &mut active, true)?;
}
return Ok(());
};
lifecycle
.reconcile_published_bootstrap_owner(
bootstrap_owner_session_id,
bootstrap_owner
.is_live()
.map_err(ProductionColdStartError::service)?,
)
.map_err(ProductionColdStartError::service)?;
waiters
.borrow_mut()
.reconcile_published(&mut lifecycle)
.map_err(ProductionColdStartError::service)?;
face
};
if active.len() == MAX_CONCURRENT_CONNECTIONS {
wait_for_connection_completion(&completions, &mut active, false)?;
continue;
}
match face.accept_connection() {
Ok(connection) => {
let cancellation = connection
.cancellation_handle()
.map_err(ProductionColdStartError::service)?;
let connection_id = next_connection;
next_connection = next_connection.checked_add(1).ok_or_else(|| {
ProductionColdStartError::service("control connection counter overflow")
})?;
active.insert(connection_id, cancellation);
let completed = completed.clone();
scope.spawn(move || {
let result =
serve_production_control_connection(lifecycle, &face, connection);
let _ = completed.send((connection_id, result));
});
}
Err(crate::control::ControlError::Io(error))
if error.kind() == std::io::ErrorKind::WouldBlock =>
{
std::thread::sleep(std::time::Duration::from_millis(10));
}
Err(error) => return Err(ProductionColdStartError::service(error)),
}
Err(error) => return Err(ProductionColdStartError::service(error)),
}
if lifecycle
.lock()
.map_err(|_| {
ProductionColdStartError::service("production lifecycle lock is poisoned")
})?
.adapter()
.published_control()
.is_none()
})
}
fn wait_for_connection_completion(
completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
cancellation_requested: bool,
) -> Result<(), ProductionColdStartError> {
let (connection_id, result) = completions
.recv()
.map_err(ProductionColdStartError::service)?;
active.remove(&connection_id);
connection_completion(result, cancellation_requested)
}
fn connection_completion(
result: Result<(), ControlError>,
cancellation_requested: bool,
) -> Result<(), ProductionColdStartError> {
match result {
Ok(()) => Ok(()),
Err(ControlError::Io(error))
if cancellation_requested
&& matches!(
error.kind(),
std::io::ErrorKind::UnexpectedEof
| std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::NotConnected
) =>
{
return Ok(());
Ok(())
}
Err(error) => Err(ProductionColdStartError::service(error)),
}
}
@ -463,4 +526,32 @@ mod tests {
.map(OsString::from);
assert!(parse_arguments(&extra).is_err());
}
#[test]
fn withdrawal_suppresses_only_cancelled_socket_io() {
assert!(
connection_completion(
Err(ControlError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"withdrawn socket",
))),
true,
)
.is_ok()
);
assert!(
connection_completion(
Err(ControlError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"ordinary socket failure",
))),
false,
)
.is_err()
);
assert!(
connection_completion(Err(ControlError::Lifecycle("teardown failed".into())), true,)
.is_err()
);
}
}

31
crates/lazaret-host-coordinator/src/control.rs
View file

@ -838,6 +838,16 @@ pub struct PublishedControlFace {
interactive_executor: Arc<dyn InteractiveSessionExecutor>,
}
pub(crate) struct AcceptedControlConnection {
socket: std::os::unix::net::UnixStream,
}
impl AcceptedControlConnection {
pub(crate) fn cancellation_handle(&self) -> std::io::Result<std::os::unix::net::UnixStream> {
self.socket.try_clone()
}
}
pub trait NonInteractiveSessionExecutor: Send + Sync {
/// # Errors
/// Returns a runtime, stream, or exact-session execution failure.
@ -1514,6 +1524,25 @@ impl PublishedControlFace {
/// failure.
pub fn serve_next_coordinated<Admit, Disconnect>(
&self,
admit: Admit,
disconnect: Disconnect,
) -> Result<(), ControlError>
where
Admit: FnMut(ControlSessionAdmission) -> Result<(), ControlError>,
Disconnect: FnMut(SessionId) -> Result<(), ControlError>,
{
let connection = self.accept_connection()?;
self.serve_connection_coordinated(connection, admit, disconnect)
}
pub(crate) fn accept_connection(&self) -> Result<AcceptedControlConnection, ControlError> {
let (socket, _) = self.listener.accept()?;
Ok(AcceptedControlConnection { socket })
}
pub(crate) fn serve_connection_coordinated<Admit, Disconnect>(
&self,
connection: AcceptedControlConnection,
mut admit: Admit,
mut disconnect: Disconnect,
) -> Result<(), ControlError>
@ -1521,7 +1550,7 @@ impl PublishedControlFace {
Admit: FnMut(ControlSessionAdmission) -> Result<(), ControlError>,
Disconnect: FnMut(SessionId) -> Result<(), ControlError>,
{
let (socket, _) = self.listener.accept()?;
let socket = connection.socket;
let peer = socket_peercred(&socket).map_err(io::Error::from)?;
require_peer_uid(peer.uid.as_raw(), self.uid)?;
self.validate_publication()?;

21
crates/lazaret-host-coordinator/src/terminal.rs
View file

@ -1713,15 +1713,18 @@ impl SessionTerminalProcess for RunscSessionTerminal {
}
fn reap_exec(&mut self) -> io::Result<ProcessExit> {
if let Some(exit) = self.exit {
return Ok(exit);
}
let exit = child_broker().reap_status(
self.exec
.take()
.ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
)?;
self.exit = Some(exit);
let exit = if let Some(exit) = self.exit {
exit
} else {
let exit = child_broker().reap_status(
self.exec
.take()
.ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
)?;
self.exit = Some(exit);
exit
};
remove_pid_file(&self.request.internal_pid_file)?;
Ok(exit)
}