Harden session terminal release authority

This commit is contained in:
ms2k 2026-07-17 05:59:35 +09:00
commit 4dc5daa57b

View file

@ -14,9 +14,9 @@ use lazaret_core::identity::{
PidfsProcessIdentityV1, SessionId,
};
use lazaret_core::records::{
GuestProcessIdentity, ParticipantOwnership, ParticipantRole, ParticipantState,
PersistedHostProcessOwnership, ProcessOwnershipCommitAck, PtyIdentity, SessionExit,
SessionPhase, SessionTerminal, SessionTerminalEvent, SessionTerminalFields,
DomainStore, DurableDecision, GuestProcessIdentity, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, ProcessOwnershipCommitAck, PtyIdentity,
SessionExit, SessionPhase, SessionTerminal, SessionTerminalEvent, SessionTerminalFields,
SessionTerminalMessage, SessionTerminalState, StagedProcessOwnership, TerminalOutput,
TerminalResize, TerminalSignalKind,
};
@ -72,6 +72,10 @@ pub enum TerminalError {
GuestGroupStillPresent,
InvalidExit,
AuthorityMismatch,
AcquisitionCleanup {
original: Box<TerminalError>,
cleanup: Option<Box<TerminalError>>,
},
}
impl std::fmt::Display for TerminalError {
@ -100,11 +104,37 @@ impl std::fmt::Display for TerminalError {
Self::AuthorityMismatch => {
formatter.write_str("Session Terminal runtime authority does not match")
}
Self::AcquisitionCleanup { original, cleanup } => {
write!(formatter, "Session Terminal acquisition failed: {original}")?;
if let Some(cleanup) = cleanup {
write!(formatter, "; partial terminal cleanup failed: {cleanup}")?;
}
Ok(())
}
}
}
}
impl std::error::Error for TerminalError {}
impl std::error::Error for TerminalError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Store(error) => Some(error),
Self::AcquisitionCleanup { original, .. } => Some(original.as_ref()),
Self::UnknownSession
| Self::DuplicateSession
| Self::StaleGeneration
| Self::InactiveRoute
| Self::Releasing
| Self::OversizedInput
| Self::Backpressure
| Self::GuestIdentityMismatch
| Self::GuestGroupStillPresent
| Self::InvalidExit
| Self::AuthorityMismatch => None,
}
}
}
impl From<io::Error> for TerminalError {
fn from(error: io::Error) -> Self {
@ -168,7 +198,7 @@ pub struct TerminalAcquisition {
}
pub struct HostCoordinatorTerminalRecorder {
store: Arc<HostCoordinatorDomainStore>,
store: Arc<dyn TerminalDomainStore>,
}
impl HostCoordinatorTerminalRecorder {
@ -177,22 +207,57 @@ impl HostCoordinatorTerminalRecorder {
Self { store }
}
#[cfg(test)]
fn with_store(store: Arc<dyn TerminalDomainStore>) -> Self {
Self { store }
}
fn put(&self, terminal: SessionTerminal) -> Result<(), TerminalError> {
self.store
.apply(&[DomainStoreMutation::PutSessionTerminal(terminal)])?;
Ok(())
}
fn require_authority(&self, request: &TerminalAcquisition) -> Result<(), TerminalError> {
fn require_authority(
&self,
request: &TerminalAcquisition,
) -> Result<TerminalRoute, TerminalError> {
let snapshot = self.store.snapshot()?;
let generation = snapshot
.generations
.iter()
.find(|generation| generation.generation_id == request.generation_id);
let current = snapshot
.generations
.iter()
.all(|candidate| candidate.generation_id <= request.generation_id);
if snapshot.domain.domain_id != request.domain_id
|| !snapshot.generations.iter().any(|generation| {
generation.generation_id == request.generation_id
&& generation.coordinator_id == request.coordinator_id
|| !current
|| generation.is_none_or(|generation| {
generation.coordinator_id != request.coordinator_id
|| generation.durable_decision != DurableDecision::Published
})
{
return Err(TerminalError::AuthorityMismatch);
}
let route = snapshot
.sessions
.iter()
.find(|session| {
session.generation_id == request.generation_id
&& session.session_id == request.session_id
})
.map(|session| match session.session_phase {
SessionPhase::Attached => TerminalRoute::Sandbox,
SessionPhase::Prompting => TerminalRoute::Prompt,
SessionPhase::HostExecuting => TerminalRoute::HostProcess,
SessionPhase::New
| SessionPhase::Waiting
| SessionPhase::Registering
| SessionPhase::Closed
| SessionPhase::Failed => TerminalRoute::Detached,
})
.ok_or(TerminalError::AuthorityMismatch)?;
let sandbox = snapshot
.participants
.iter()
@ -211,7 +276,7 @@ impl HostCoordinatorTerminalRecorder {
{
return Err(TerminalError::AuthorityMismatch);
}
Ok(())
Ok(route)
}
fn begin_release(&self, terminal: SessionTerminal) -> Result<(), TerminalError> {
@ -224,11 +289,16 @@ impl HostCoordinatorTerminalRecorder {
&& session.session_id == terminal.session_id()
})
.ok_or(TerminalError::UnknownSession)?;
session.session_phase = SessionPhase::Closed;
self.store.apply(&[
DomainStoreMutation::PutSession(session),
DomainStoreMutation::PutSessionTerminal(terminal),
])?;
let mut mutations = Vec::with_capacity(2);
if !matches!(
session.session_phase,
SessionPhase::Closed | SessionPhase::Failed
) {
session.session_phase = SessionPhase::Closed;
mutations.push(DomainStoreMutation::PutSession(session));
}
mutations.push(DomainStoreMutation::PutSessionTerminal(terminal));
self.store.apply(&mutations)?;
Ok(())
}
@ -241,8 +311,24 @@ impl HostCoordinatorTerminalRecorder {
}
}
trait TerminalDomainStore: Send + Sync {
fn snapshot(&self) -> Result<DomainStore, DomainStoreError>;
fn apply(&self, mutations: &[DomainStoreMutation]) -> Result<(), DomainStoreError>;
}
impl TerminalDomainStore for HostCoordinatorDomainStore {
fn snapshot(&self) -> Result<DomainStore, DomainStoreError> {
self.snapshot()
}
fn apply(&self, mutations: &[DomainStoreMutation]) -> Result<(), DomainStoreError> {
self.apply(mutations)
}
}
struct ActiveTerminal {
generation_id: GenerationId,
request: TerminalAcquisition,
route: TerminalRoute,
record: SessionTerminal,
process: Box<dyn SessionTerminalProcess>,
@ -278,7 +364,9 @@ impl SessionTerminalEngine {
if self.active.contains_key(&request.session_id) {
return Err(TerminalError::DuplicateSession);
}
self.recorder.require_authority(request)?;
if self.recorder.require_authority(request)? != TerminalRoute::Sandbox {
return Err(TerminalError::AuthorityMismatch);
}
let intent = terminal_snapshot(
request,
SessionTerminalState::Intent,
@ -293,6 +381,7 @@ impl SessionTerminalEngine {
request.session_id,
ActiveTerminal {
generation_id: request.generation_id,
request: request.clone(),
route: TerminalRoute::Sandbox,
record,
process,
@ -311,6 +400,10 @@ impl SessionTerminalEngine {
generation_id: GenerationId,
route: TerminalRoute,
) -> Result<(), TerminalError> {
let authoritative = self.refresh_route(session_id, generation_id)?;
if authoritative != route {
return Err(TerminalError::InactiveRoute);
}
let terminal = self.active_mut(session_id, generation_id)?;
if terminal.releasing || route == TerminalRoute::Detached {
return Err(TerminalError::Releasing);
@ -335,6 +428,10 @@ impl SessionTerminalEngine {
if matches!(event, SessionTerminalEvent::Close(_)) {
return self.close(session_id, generation_id, None, true);
}
let authoritative = self.refresh_route(session_id, generation_id)?;
if authoritative != TerminalRoute::Sandbox {
return Err(TerminalError::InactiveRoute);
}
let terminal = self.active_mut(session_id, generation_id)?;
if terminal.releasing {
return Err(TerminalError::Releasing);
@ -378,11 +475,12 @@ impl SessionTerminalEngine {
session_id: SessionId,
generation_id: GenerationId,
) -> Result<Vec<SessionTerminalMessage>, TerminalError> {
let authoritative = self.refresh_route(session_id, generation_id)?;
let terminal = self.active_mut(session_id, generation_id)?;
if terminal.process.exec_exited()? {
return self.close(session_id, generation_id, None, true);
}
if terminal.route != TerminalRoute::Sandbox || terminal.releasing {
if authoritative != TerminalRoute::Sandbox || terminal.releasing {
return Ok(Vec::new());
}
flush_input(terminal)?;
@ -430,6 +528,33 @@ impl SessionTerminalEngine {
self.active.contains_key(&session_id)
}
fn refresh_route(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
) -> Result<TerminalRoute, TerminalError> {
let request = {
let terminal = self.active_mut(session_id, generation_id)?;
terminal.request.clone()
};
match self.recorder.require_authority(&request) {
Ok(TerminalRoute::Detached) | Err(TerminalError::AuthorityMismatch) => {
let cleanup = self.close(
session_id,
generation_id,
Some(TerminalLoss::ExecClientLost),
false,
);
Err(with_cleanup(TerminalError::AuthorityMismatch, cleanup))
}
Err(error) => Err(error),
Ok(route) => {
self.active_mut(session_id, generation_id)?.route = route;
Ok(route)
}
}
}
fn active_mut(
&mut self,
session_id: SessionId,
@ -469,46 +594,12 @@ impl SessionTerminalEngine {
terminal.record = release;
self.cancellation.cancel_for_close(session_id)?;
let mut output = Vec::new();
if !matches!(loss, Some(TerminalLoss::HostCoordinatorLost)) {
for signal in ["HUP", "TERM", "KILL"] {
if terminal.process.group_absent()? {
break;
}
terminal.process.signal_group(signal)?;
wait_until(SIGNAL_GRACE, || terminal.process.group_absent())?;
}
}
if !terminal.process.exec_exited()? {
terminal.process.terminate_exec()?;
}
let exit = terminal.process.reap_exec()?;
if connected {
let deadline = Instant::now() + DRAIN_DEADLINE;
let mut bytes = [0_u8; MAX_OUTPUT_BYTES];
let mut drained = 0;
while Instant::now() < deadline && drained < MAX_DRAIN_BYTES {
match terminal.process.read_output(&mut bytes) {
Ok(0) => break,
Ok(length) => {
let length = length.min(MAX_DRAIN_BYTES - drained);
drained += length;
output.push(SessionTerminalMessage::Output(TerminalOutput {
session_id,
bytes: bytes[..length].to_vec(),
}));
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
std::thread::sleep(POLL_INTERVAL);
}
Err(_) => break,
}
}
}
terminal.process.close_pty();
if !terminal.process.group_absent()? {
return Err(TerminalError::GuestGroupStillPresent);
}
let (mut output, exit) = release_physical(
terminal.process.as_mut(),
session_id,
connected,
!matches!(loss, Some(TerminalLoss::HostCoordinatorLost)),
)?;
let clean = clean_terminal(generation_id, session_id)?;
self.recorder.clean(clean)?;
if connected {
@ -525,6 +616,103 @@ impl SessionTerminalEngine {
}
}
fn release_physical(
process: &mut dyn SessionTerminalProcess,
session_id: SessionId,
connected: bool,
signal_guest: bool,
) -> Result<(Vec<SessionTerminalMessage>, ProcessExit), TerminalError> {
let mut failure = None;
if signal_guest {
for signal in ["HUP", "TERM", "KILL"] {
match process.group_absent() {
Ok(true) => break,
Ok(false) => {}
Err(error) => {
failure.get_or_insert(error);
break;
}
}
if let Err(error) = process.signal_group(signal) {
failure.get_or_insert(error);
break;
}
if let Err(error) = wait_until(SIGNAL_GRACE, || process.group_absent()) {
failure.get_or_insert(error);
break;
}
}
}
match process.exec_exited() {
Ok(false) => {
if let Err(error) = process.terminate_exec() {
failure.get_or_insert(error.into());
}
}
Ok(true) => {}
Err(error) => {
failure.get_or_insert(error.into());
}
}
let exit = match process.reap_exec() {
Ok(exit) => Some(exit),
Err(error) => {
failure.get_or_insert(error.into());
None
}
};
let mut output = Vec::new();
if connected {
let deadline = Instant::now() + DRAIN_DEADLINE;
let mut bytes = [0_u8; MAX_OUTPUT_BYTES];
let mut drained = 0;
while Instant::now() < deadline && drained < MAX_DRAIN_BYTES {
match process.read_output(&mut bytes) {
Ok(0) => break,
Ok(length) => {
let length = length.min(MAX_DRAIN_BYTES - drained);
drained += length;
output.push(SessionTerminalMessage::Output(TerminalOutput {
session_id,
bytes: bytes[..length].to_vec(),
}));
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
std::thread::sleep(POLL_INTERVAL);
}
Err(_) => break,
}
}
}
process.close_pty();
match process.group_absent() {
Ok(true) => {}
Ok(false) => {
failure.get_or_insert(TerminalError::GuestGroupStillPresent);
}
Err(error) => {
failure.get_or_insert(error);
}
}
if let Some(error) = failure {
return Err(error);
}
Ok((
output,
exit.ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
))
}
fn with_cleanup<T>(original: TerminalError, cleanup: Result<T, TerminalError>) -> TerminalError {
match cleanup {
Ok(_) => original,
Err(cleanup) => TerminalError::AcquisitionCleanup {
original: Box::new(original),
cleanup: Some(Box::new(cleanup)),
},
}
}
fn flush_input(terminal: &mut ActiveTerminal) -> Result<(), TerminalError> {
while !terminal.pending_input.is_empty() {
let (first, second) = terminal.pending_input.as_slices();
@ -700,19 +888,89 @@ impl SessionTerminalFactory for RunscSessionTerminalFactory {
PersistedHostProcessOwnership::committed(process.identity()),
Some(guest),
)?;
recorder.put(running.clone())?;
Ok((
running,
Box::new(RunscSessionTerminal {
request: request.clone(),
master: Some(master),
exec: Some(process),
guest,
}),
))
let mut terminal_process = RunscSessionTerminal {
request: request.clone(),
master: Some(master),
exec: Some(process),
exit: None,
guest,
};
let exec_client = terminal_process
.exec
.as_ref()
.expect("new terminal retains its exec client")
.identity();
commit_running_or_cleanup(
recorder,
request,
running.clone(),
pty,
exec_client,
guest,
&mut terminal_process,
)?;
Ok((running, Box::new(terminal_process)))
}
}
fn commit_running_or_cleanup(
recorder: &HostCoordinatorTerminalRecorder,
request: &TerminalAcquisition,
running: SessionTerminal,
pty: PtyIdentity,
exec_client: PidfsProcessIdentityV1,
guest: GuestProcessIdentity,
process: &mut dyn SessionTerminalProcess,
) -> Result<(), TerminalError> {
if let Err(original) = recorder.put(running) {
let cleanup =
cleanup_failed_running_commit(recorder, request, pty, exec_client, guest, process)
.err()
.map(Box::new);
return Err(TerminalError::AcquisitionCleanup {
original: Box::new(original),
cleanup,
});
}
Ok(())
}
fn cleanup_failed_running_commit(
recorder: &HostCoordinatorTerminalRecorder,
request: &TerminalAcquisition,
pty: PtyIdentity,
exec_client: PidfsProcessIdentityV1,
guest: GuestProcessIdentity,
process: &mut dyn SessionTerminalProcess,
) -> Result<(), TerminalError> {
let observed = terminal_snapshot(
request,
SessionTerminalState::Intent,
PersistedHostProcessOwnership::committed(request.host_coordinator),
Some(pty),
PersistedHostProcessOwnership::committed(exec_client),
Some(guest),
)?;
let durable = recorder.put(observed.clone()).and_then(|()| {
recorder.put(copy_terminal_state(
&observed,
SessionTerminalState::ReleaseIntent,
)?)
});
let physical = release_physical(process, request.session_id, false, true).map(|_| ());
match (durable, physical) {
(Err(durable), Err(physical)) => {
return Err(TerminalError::AcquisitionCleanup {
original: Box::new(durable),
cleanup: Some(Box::new(physical)),
});
}
(Err(error), Ok(())) | (Ok(()), Err(error)) => return Err(error),
(Ok(()), Ok(())) => {}
}
recorder.clean(clean_terminal(request.generation_id, request.session_id)?)
}
fn clean_failed_blocked_acquisition(
recorder: &HostCoordinatorTerminalRecorder,
request: &TerminalAcquisition,
@ -744,6 +1002,7 @@ struct RunscSessionTerminal {
request: TerminalAcquisition,
master: Option<File>,
exec: Option<RetainedProcess>,
exit: Option<ProcessExit>,
guest: GuestProcessIdentity,
}
@ -782,8 +1041,7 @@ impl SessionTerminalProcess for RunscSessionTerminal {
fn exec_exited(&self) -> io::Result<bool> {
self.exec
.as_ref()
.ok_or_else(|| io::Error::other("exec client was already reaped"))?
.has_exited()
.map_or(Ok(true), RetainedProcess::has_exited)
}
fn signal_group(&self, signal: &str) -> Result<(), TerminalError> {
@ -803,22 +1061,32 @@ impl SessionTerminalProcess for RunscSessionTerminal {
}
fn group_absent(&self) -> Result<bool, TerminalError> {
Ok(observe_guest_identity(&self.request, self.guest.leader_pid.get())?.is_none())
if observe_guest_identity(&self.request, self.guest.leader_pid.get())?
.is_some_and(|observed| observed != self.guest)
{
return Err(TerminalError::GuestIdentityMismatch);
}
observe_group_absent(&self.request, self.guest.process_group_id.get())
}
fn terminate_exec(&self) -> io::Result<()> {
self.exec
.as_ref()
.ok_or_else(|| io::Error::other("exec client was already reaped"))?
.send_signal(rustix::process::Signal::KILL)
match self.exec.as_ref() {
Some(exec) => exec.send_signal(rustix::process::Signal::KILL),
None => Ok(()),
}
}
fn reap_exec(&mut self) -> io::Result<ProcessExit> {
child_broker().reap_status(
if let Some(exit) = self.exit {
return Ok(exit);
}
let exit = child_broker().reap_status(
self.exec
.take()
.ok_or_else(|| io::Error::other("exec client was already reaped"))?,
)
.ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
)?;
self.exit = Some(exit);
Ok(exit)
}
fn close_pty(&mut self) {
@ -952,31 +1220,8 @@ fn observe_guest_identity(
request: &TerminalAcquisition,
pid: u32,
) -> Result<Option<GuestProcessIdentity>, TerminalError> {
let mut target = runsc_target(request);
target = target
.argument("exec")
.argument(hex_lower(request.container_id.as_bytes()))
.argument(SANDBOX_INIT_PATH)
.argument(PRIVATE_SESSION_IDENTITY_ROLE)
.argument(pid.to_string());
let mut child = child_broker().run_transient_captured(&target)?;
let deadline = Instant::now() + IDENTITY_DEADLINE;
let mut stdout = Vec::new();
let mut stderr = Vec::new();
while Instant::now() < deadline && !child.process().has_exited()? {
read_available(child.streams().0, &mut stdout)?;
read_available(child.streams().1, &mut stderr)?;
std::thread::sleep(POLL_INTERVAL);
}
if !child.process().has_exited()? {
child.process().send_signal(rustix::process::Signal::KILL)?;
child_broker().reap_status(child.into_process())?;
return Err(io::Error::other("guest identity observation deadline expired").into());
}
read_available(child.streams().0, &mut stdout)?;
read_available(child.streams().1, &mut stderr)?;
let exit = child_broker().reap_status(child.into_process())?;
if exit.exit_code != Some(0) || exit.terminating_signal.is_some() {
let stdout = run_guest_identity_query(request, &[OsString::from(pid.to_string())])?;
if stdout == b"absent\n" {
return Ok(None);
}
let fields = std::str::from_utf8(&stdout)
@ -1007,6 +1252,55 @@ fn observe_guest_identity(
}))
}
fn observe_group_absent(request: &TerminalAcquisition, group: u32) -> Result<bool, TerminalError> {
let stdout = run_guest_identity_query(
request,
&[OsString::from("--group"), OsString::from(group.to_string())],
)?;
match stdout.as_slice() {
b"absent\n" => Ok(true),
b"present\n" => Ok(false),
_ => Err(io::Error::other("guest group response is malformed").into()),
}
}
fn run_guest_identity_query(
request: &TerminalAcquisition,
arguments: &[OsString],
) -> Result<Vec<u8>, TerminalError> {
let mut target = runsc_target(request);
target = target
.argument("exec")
.argument(hex_lower(request.container_id.as_bytes()))
.argument(SANDBOX_INIT_PATH)
.argument(PRIVATE_SESSION_IDENTITY_ROLE);
for argument in arguments {
target = target.argument(argument.clone());
}
let mut child = child_broker().run_transient_captured(&target)?;
let deadline = Instant::now() + IDENTITY_DEADLINE;
let mut stdout = Vec::new();
let mut stderr = Vec::new();
while Instant::now() < deadline && !child.process().has_exited()? {
read_available(child.streams().0, &mut stdout)?;
read_available(child.streams().1, &mut stderr)?;
std::thread::sleep(POLL_INTERVAL);
}
if !child.process().has_exited()? {
child.process().send_signal(rustix::process::Signal::KILL)?;
child_broker().reap_status(child.into_process())?;
return Err(io::Error::other("guest identity observation deadline expired").into());
}
read_available(child.streams().0, &mut stdout)?;
read_available(child.streams().1, &mut stderr)?;
let exit = child_broker().reap_status(child.into_process())?;
if exit.exit_code != Some(0) || exit.terminating_signal.is_some() {
let stderr = String::from_utf8_lossy(&stderr);
return Err(io::Error::other(format!("guest identity query failed: {stderr}")).into());
}
Ok(stdout)
}
fn runsc_target(request: &TerminalAcquisition) -> TargetCommand {
request.runsc_global_arguments.iter().cloned().fold(
TargetCommand::new(request.runsc_program.clone()),