Unify client Domain observation
This commit is contained in:
parent
30cd36dcdd
commit
786049534d
1 changed files with 443 additions and 120 deletions
|
|
@ -6,7 +6,6 @@ use std::io::{self, IsTerminal, Write};
|
|||
use std::os::fd::{AsFd, AsRawFd};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::PathBuf;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use lazaret_core::configuration::{
|
||||
|
|
@ -25,6 +24,10 @@ use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
|||
use rustix::fs::{Mode, OFlags, openat};
|
||||
use rustix::pipe::{PipeFlags, pipe_with};
|
||||
|
||||
use crate::domain_observation::{
|
||||
DiscoveryAction, DiscoveryState, DomainObservation, ObservationPath, SystemObservationClock,
|
||||
WaiterAction,
|
||||
};
|
||||
use crate::{
|
||||
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, ClientSessionIdentity,
|
||||
ControlEndpoint, DomainStatus, execute_interactive_over_control_face_as,
|
||||
|
|
@ -176,34 +179,160 @@ impl CoordinatorClientBackend {
|
|||
}
|
||||
}
|
||||
|
||||
fn await_publication(
|
||||
fn observe_domain(
|
||||
&mut self,
|
||||
initial_path: ObservationPath,
|
||||
) -> Result<ControlEndpoint, ClientError<CoordinatorClientError>> {
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||
Discovery::Stale | Discovery::Absent => {}
|
||||
}
|
||||
if self
|
||||
.coordinator
|
||||
.as_ref()
|
||||
.ok_or_else(|| ClientError::Backend(failure("Host Coordinator is absent")))?
|
||||
.has_exited()
|
||||
let waiter = if initial_path == ObservationPath::WaitingClient {
|
||||
let configuration = self.configuration().map_err(ClientError::Backend)?;
|
||||
let runtime_authority = configuration
|
||||
.runtime_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?
|
||||
{
|
||||
let status = self
|
||||
.coordinator
|
||||
.take()
|
||||
.expect("checked coordinator remains present")
|
||||
.reap()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
return Err(ClientError::Backend(failure(format!(
|
||||
"Host Coordinator exited before publication with {status:?}"
|
||||
))));
|
||||
.map_err(ClientError::Backend)?;
|
||||
let socket = PathBuf::from(format!(
|
||||
"/proc/self/fd/{}/{}",
|
||||
runtime_authority.as_raw_fd(),
|
||||
BOOTSTRAP_WAITER_SOCKET_NAME
|
||||
));
|
||||
let request = BootstrapWaitRequest {
|
||||
protocol_version: lazaret_core::records::ProtocolVersion::V1,
|
||||
domain_id: configuration.domain_id(),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
cli_id: self.identity.cli_id(),
|
||||
session_id: self.identity.session_id(),
|
||||
session_anchor_id: self.identity.session_anchor_id(),
|
||||
};
|
||||
Some((runtime_authority, socket, request))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut observer = DomainObservation::new(initial_path, SystemObservationClock::start());
|
||||
let mut waiter_transfer = None;
|
||||
loop {
|
||||
let (state, endpoint) = match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => (DiscoveryState::Ready, Some(*endpoint)),
|
||||
Discovery::Stale => (DiscoveryState::Stale, None),
|
||||
Discovery::Absent => (DiscoveryState::Absent, None),
|
||||
};
|
||||
match observer.discovery(state) {
|
||||
DiscoveryAction::ReturnReady => {
|
||||
let endpoint = endpoint.expect("ready discovery supplies an endpoint");
|
||||
return Ok(match waiter_transfer {
|
||||
Some(stream) => endpoint.retain_waiter_transfer(stream),
|
||||
None => endpoint,
|
||||
});
|
||||
}
|
||||
DiscoveryAction::RefuseStale => return Err(ClientError::StaleCoordinator),
|
||||
DiscoveryAction::CheckCoordinator => {
|
||||
let exited = self
|
||||
.coordinator
|
||||
.as_ref()
|
||||
.ok_or_else(|| ClientError::Backend(failure("Host Coordinator is absent")))?
|
||||
.has_exited()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
if observer.coordinator_exited(exited) {
|
||||
let status = self
|
||||
.coordinator
|
||||
.take()
|
||||
.expect("checked coordinator remains present")
|
||||
.reap()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
return Err(ClientError::Backend(failure(format!(
|
||||
"Host Coordinator exited before publication with {status:?}"
|
||||
))));
|
||||
}
|
||||
}
|
||||
DiscoveryAction::AttemptWaiter => {
|
||||
let (_, socket, request) = waiter
|
||||
.as_ref()
|
||||
.expect("waiting observation retains waiter authority");
|
||||
if let Some(stream) = self.attempt_waiter(&observer, socket, request)? {
|
||||
waiter_transfer = Some(stream);
|
||||
observer.transfer_publication();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
DiscoveryAction::Retry => {}
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
observer
|
||||
.retry()
|
||||
.map_err(|_| observation_deadline(observer.path()))?;
|
||||
}
|
||||
}
|
||||
|
||||
fn attempt_waiter(
|
||||
&self,
|
||||
observer: &DomainObservation<SystemObservationClock>,
|
||||
socket: &std::path::Path,
|
||||
request: &BootstrapWaitRequest,
|
||||
) -> Result<Option<std::os::unix::net::UnixStream>, ClientError<CoordinatorClientError>> {
|
||||
let mut stream = match std::os::unix::net::UnixStream::connect(socket) {
|
||||
Ok(stream) => stream,
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused
|
||||
) =>
|
||||
{
|
||||
if observer.lock_available(
|
||||
self.domain_lock_is_available()
|
||||
.map_err(ClientError::Backend)?,
|
||||
) == Some(WaiterAction::RefuseFailed)
|
||||
{
|
||||
return Err(ClientError::WaitingDomainFailed);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Err(error) => return Err(ClientError::Backend(failure(error))),
|
||||
};
|
||||
stream
|
||||
.set_read_timeout(Some(observation_remaining(observer)?))
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
write_frame(&mut stream, request).map_err(|error| {
|
||||
ClientError::Backend(failure(format!("bootstrap waiter request failed: {error}")))
|
||||
})?;
|
||||
let first: BootstrapWaitOutcome = read_frame(&mut stream).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter admission failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
let mut action = observer.waiter(first, false);
|
||||
if action == WaiterAction::AwaitTerminal {
|
||||
action = match read_frame(&mut stream) {
|
||||
Ok(outcome) => observer.waiter(outcome, true),
|
||||
Err(_) => WaiterAction::RefuseFailed,
|
||||
};
|
||||
}
|
||||
match action {
|
||||
WaiterAction::AwaitPublication => Ok(Some(stream)),
|
||||
WaiterAction::RefuseStale => Err(ClientError::StaleCoordinator),
|
||||
WaiterAction::RefuseFailed => Err(ClientError::WaitingDomainFailed),
|
||||
WaiterAction::RefuseProtocol | WaiterAction::AwaitTerminal => Err(
|
||||
ClientError::Protocol("bootstrap waiter sent an invalid terminal outcome"),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn observation_remaining(
|
||||
observer: &DomainObservation<SystemObservationClock>,
|
||||
) -> Result<Duration, ClientError<CoordinatorClientError>> {
|
||||
observer
|
||||
.remaining()
|
||||
.map_err(|_| observation_deadline(observer.path()))
|
||||
}
|
||||
|
||||
fn observation_deadline(path: ObservationPath) -> ClientError<CoordinatorClientError> {
|
||||
match path {
|
||||
ObservationPath::BootstrapLeader => {
|
||||
ClientError::Backend(failure("Host Coordinator publication deadline expired"))
|
||||
}
|
||||
ObservationPath::WaitingClient | ObservationPath::PublicationTransfer => {
|
||||
ClientError::WaitingDomainFailed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -247,85 +376,11 @@ impl ClientBackend for CoordinatorClientBackend {
|
|||
}
|
||||
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.await_publication()
|
||||
self.observe_domain(ObservationPath::BootstrapLeader)
|
||||
}
|
||||
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
let configuration = self.configuration().map_err(ClientError::Backend)?;
|
||||
let runtime_authority = configuration
|
||||
.runtime_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
let socket = PathBuf::from(format!(
|
||||
"/proc/self/fd/{}/{}",
|
||||
runtime_authority.as_raw_fd(),
|
||||
BOOTSTRAP_WAITER_SOCKET_NAME
|
||||
));
|
||||
let request = BootstrapWaitRequest {
|
||||
protocol_version: lazaret_core::records::ProtocolVersion::V1,
|
||||
domain_id: configuration.domain_id(),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
cli_id: self.identity.cli_id(),
|
||||
session_id: self.identity.session_id(),
|
||||
session_anchor_id: self.identity.session_anchor_id(),
|
||||
};
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||
Discovery::Stale | Discovery::Absent => {}
|
||||
}
|
||||
match std::os::unix::net::UnixStream::connect(&socket) {
|
||||
Ok(mut stream) => {
|
||||
write_frame(&mut stream, &request).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter request failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
let first: BootstrapWaitOutcome = read_frame(&mut stream).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter admission failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
if first != BootstrapWaitOutcome::Waiting {
|
||||
return match first {
|
||||
BootstrapWaitOutcome::Refused => Err(ClientError::StaleCoordinator),
|
||||
BootstrapWaitOutcome::Failed => Err(ClientError::WaitingDomainFailed),
|
||||
BootstrapWaitOutcome::Published => {
|
||||
self.await_ready_publication(Some(stream))
|
||||
}
|
||||
BootstrapWaitOutcome::Waiting => unreachable!(),
|
||||
};
|
||||
}
|
||||
return match read_frame(&mut stream) {
|
||||
Ok(BootstrapWaitOutcome::Published) => {
|
||||
self.await_ready_publication(Some(stream))
|
||||
}
|
||||
Ok(BootstrapWaitOutcome::Failed) | Err(_) => {
|
||||
Err(ClientError::WaitingDomainFailed)
|
||||
}
|
||||
Ok(BootstrapWaitOutcome::Waiting | BootstrapWaitOutcome::Refused) => {
|
||||
Err(ClientError::Protocol(
|
||||
"bootstrap waiter sent an invalid terminal outcome",
|
||||
))
|
||||
}
|
||||
};
|
||||
}
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused
|
||||
) => {}
|
||||
Err(error) => return Err(ClientError::Backend(failure(error))),
|
||||
}
|
||||
if self
|
||||
.domain_lock_is_available()
|
||||
.map_err(ClientError::Backend)?
|
||||
{
|
||||
return Err(ClientError::WaitingDomainFailed);
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
self.observe_domain(ObservationPath::WaitingClient)
|
||||
}
|
||||
|
||||
fn execute(
|
||||
|
|
@ -362,27 +417,6 @@ impl ClientBackend for CoordinatorClientBackend {
|
|||
}
|
||||
}
|
||||
|
||||
impl CoordinatorClientBackend {
|
||||
fn await_ready_publication(
|
||||
&self,
|
||||
waiter: Option<std::os::unix::net::UnixStream>,
|
||||
) -> Result<ControlEndpoint, ClientError<CoordinatorClientError>> {
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => {
|
||||
let endpoint = *endpoint;
|
||||
return Ok(match waiter {
|
||||
Some(stream) => endpoint.retain_waiter_transfer(stream),
|
||||
None => endpoint,
|
||||
});
|
||||
}
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => thread::sleep(Duration::from_millis(1)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn open_private_domain_lock(root: &impl AsFd) -> Result<File, CoordinatorClientError> {
|
||||
let file = File::from(
|
||||
openat(
|
||||
|
|
|
|||
Loading…
Reference in a new issue