Serve concurrent terminal sessions with exact teardown
This commit is contained in:
parent
917409fcc6
commit
254bb37764
1 changed files with 278 additions and 51 deletions
|
|
@ -20,13 +20,15 @@ use lazaret_linux::filesystem::try_exclusive_flock;
|
|||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
|
||||
use crate::bootstrap::{ProductionHostCoordinatorLifecycle, serve_production_control};
|
||||
use crate::bootstrap::{ProductionHostCoordinatorLifecycle, serve_production_control_connection};
|
||||
use crate::control::ControlError;
|
||||
use crate::domain_store::HostCoordinatorDomainStore;
|
||||
use crate::production::{BootstrapOwnerGuard, ProductionColdStart, ProductionColdStartError};
|
||||
use crate::production_selector::{InstalledProductPaths, select_production_cold_start};
|
||||
use crate::waiter::BootstrapWaiterFace;
|
||||
|
||||
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
|
||||
const MAX_CONCURRENT_CONNECTIONS: usize = 32;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PrivateClientBootstrapError(String);
|
||||
|
|
@ -169,47 +171,108 @@ fn serve_published(
|
|||
.ok_or_else(|| ProductionColdStartError::service("control face is absent"))?
|
||||
.set_nonblocking(true)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
loop {
|
||||
{
|
||||
let mut lifecycle = lifecycle.lock().map_err(|_| {
|
||||
ProductionColdStartError::service("production lifecycle lock is poisoned")
|
||||
})?;
|
||||
lifecycle
|
||||
.reconcile_published_bootstrap_owner(
|
||||
bootstrap_owner_session_id,
|
||||
bootstrap_owner
|
||||
.is_live()
|
||||
.map_err(ProductionColdStartError::service)?,
|
||||
)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
if lifecycle.adapter().published_control().is_none() {
|
||||
return Ok(());
|
||||
std::thread::scope(|scope| {
|
||||
let (completed, completions) =
|
||||
std::sync::mpsc::channel::<(u64, Result<(), ControlError>)>();
|
||||
let mut active: BTreeMap<u64, std::os::unix::net::UnixStream> = BTreeMap::new();
|
||||
let mut next_connection = 0_u64;
|
||||
loop {
|
||||
while let Ok((connection_id, result)) = completions.try_recv() {
|
||||
active.remove(&connection_id);
|
||||
connection_completion(result, false)?;
|
||||
}
|
||||
waiters
|
||||
.borrow_mut()
|
||||
.reconcile_published(&mut lifecycle)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
}
|
||||
match serve_production_control(lifecycle) {
|
||||
Ok(()) => {}
|
||||
Err(crate::control::ControlError::Io(error))
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock =>
|
||||
{
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
let face = {
|
||||
let mut lifecycle = lifecycle.lock().map_err(|_| {
|
||||
ProductionColdStartError::service("production lifecycle lock is poisoned")
|
||||
})?;
|
||||
let Some(face) = lifecycle.adapter().published_control() else {
|
||||
for socket in active.values() {
|
||||
let _ = socket.shutdown(std::net::Shutdown::Both);
|
||||
}
|
||||
drop(lifecycle);
|
||||
while !active.is_empty() {
|
||||
wait_for_connection_completion(&completions, &mut active, true)?;
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
lifecycle
|
||||
.reconcile_published_bootstrap_owner(
|
||||
bootstrap_owner_session_id,
|
||||
bootstrap_owner
|
||||
.is_live()
|
||||
.map_err(ProductionColdStartError::service)?,
|
||||
)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
waiters
|
||||
.borrow_mut()
|
||||
.reconcile_published(&mut lifecycle)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
face
|
||||
};
|
||||
if active.len() == MAX_CONCURRENT_CONNECTIONS {
|
||||
wait_for_connection_completion(&completions, &mut active, false)?;
|
||||
continue;
|
||||
}
|
||||
match face.accept_connection() {
|
||||
Ok(connection) => {
|
||||
let cancellation = connection
|
||||
.cancellation_handle()
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
let connection_id = next_connection;
|
||||
next_connection = next_connection.checked_add(1).ok_or_else(|| {
|
||||
ProductionColdStartError::service("control connection counter overflow")
|
||||
})?;
|
||||
active.insert(connection_id, cancellation);
|
||||
let completed = completed.clone();
|
||||
scope.spawn(move || {
|
||||
let result =
|
||||
serve_production_control_connection(lifecycle, &face, connection);
|
||||
let _ = completed.send((connection_id, result));
|
||||
});
|
||||
}
|
||||
Err(crate::control::ControlError::Io(error))
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock =>
|
||||
{
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
Err(error) => return Err(ProductionColdStartError::service(error)),
|
||||
}
|
||||
Err(error) => return Err(ProductionColdStartError::service(error)),
|
||||
}
|
||||
if lifecycle
|
||||
.lock()
|
||||
.map_err(|_| {
|
||||
ProductionColdStartError::service("production lifecycle lock is poisoned")
|
||||
})?
|
||||
.adapter()
|
||||
.published_control()
|
||||
.is_none()
|
||||
})
|
||||
}
|
||||
|
||||
fn wait_for_connection_completion(
|
||||
completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
|
||||
active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
|
||||
cancellation_requested: bool,
|
||||
) -> Result<(), ProductionColdStartError> {
|
||||
let (connection_id, result) = completions
|
||||
.recv()
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
active.remove(&connection_id);
|
||||
connection_completion(result, cancellation_requested)
|
||||
}
|
||||
|
||||
fn connection_completion(
|
||||
result: Result<(), ControlError>,
|
||||
cancellation_requested: bool,
|
||||
) -> Result<(), ProductionColdStartError> {
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(ControlError::Io(error))
|
||||
if cancellation_requested
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::UnexpectedEof
|
||||
| std::io::ErrorKind::ConnectionReset
|
||||
| std::io::ErrorKind::ConnectionAborted
|
||||
| std::io::ErrorKind::BrokenPipe
|
||||
| std::io::ErrorKind::NotConnected
|
||||
) =>
|
||||
{
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => Err(ProductionColdStartError::service(error)),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -463,4 +526,32 @@ mod tests {
|
|||
.map(OsString::from);
|
||||
assert!(parse_arguments(&extra).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn withdrawal_suppresses_only_cancelled_socket_io() {
|
||||
assert!(
|
||||
connection_completion(
|
||||
Err(ControlError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"withdrawn socket",
|
||||
))),
|
||||
true,
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
connection_completion(
|
||||
Err(ControlError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"ordinary socket failure",
|
||||
))),
|
||||
false,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
connection_completion(Err(ControlError::Lifecycle("teardown failed".into())), true,)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue