Implement PTY-native session terminal control
This commit is contained in:
parent
058e58e816
commit
8cef037fa3
3 changed files with 1488 additions and 382 deletions
|
|
@ -1,8 +1,10 @@
|
|||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Write};
|
||||
use std::net::Shutdown;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::{Mutex, mpsc};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::Duration;
|
||||
|
||||
use ed25519_dalek::SigningKey;
|
||||
|
|
@ -10,8 +12,8 @@ use lazaret_control_client::{ControlClient, ControlClientError, PendingClientEnr
|
|||
use lazaret_core::identity::{ClientId, SessionAnchorId, SessionId};
|
||||
use lazaret_core::records::{
|
||||
ClientEnrollmentResponse, ControlRequest, ControlResponse, DeveloperDecision,
|
||||
DeveloperDecisionKind, InteractiveSessionBatch, InteractiveSessionEvent,
|
||||
InteractiveSessionRequest, NonInteractiveSessionMessage, NonInteractiveSessionRequest,
|
||||
DeveloperDecisionKind, InteractiveSessionBatchV2, InteractiveSessionEventV2,
|
||||
InteractiveSessionRequestV2, NonInteractiveSessionMessage, NonInteractiveSessionRequest,
|
||||
ProtocolPublicationIdentity, ProtocolVersion, SessionBinding, SessionExitOutcome,
|
||||
SessionRegistrationRequest, TlsSubjectId,
|
||||
};
|
||||
|
|
@ -414,9 +416,9 @@ pub fn execute_interactive_over_control_face_as<E, H: HostTerminal>(
|
|||
let cli_id = identity.cli_id;
|
||||
let session_id = identity.session_id;
|
||||
let pending = PendingClientEnrollment::create(generation.clone(), cli_id)?;
|
||||
let anonymous = ControlClient::config(&endpoint.publication, None)?;
|
||||
let anonymous = ControlClient::config_v2(&endpoint.publication, None)?;
|
||||
let mut enrollment =
|
||||
ControlClient::connect(&endpoint.socket_path, anonymous, &endpoint.publication)?;
|
||||
ControlClient::connect_v2(&endpoint.socket_path, anonymous, &endpoint.publication)?;
|
||||
ControlClient::send(
|
||||
&mut enrollment,
|
||||
&ControlRequest::Enroll(pending.request().clone()),
|
||||
|
|
@ -442,12 +444,12 @@ pub fn execute_interactive_over_control_face_as<E, H: HostTerminal>(
|
|||
session_id,
|
||||
session_anchor_public_key: identity.anchor.verifying_key().to_bytes().to_vec(),
|
||||
};
|
||||
let authenticated = ControlClient::config(
|
||||
let authenticated = ControlClient::config_v2(
|
||||
&endpoint.publication,
|
||||
Some((&credential, pending.private_key())),
|
||||
)?;
|
||||
let mut stream =
|
||||
ControlClient::connect(&endpoint.socket_path, authenticated, &endpoint.publication)?;
|
||||
ControlClient::connect_v2(&endpoint.socket_path, authenticated, &endpoint.publication)?;
|
||||
ControlClient::send(
|
||||
&mut stream,
|
||||
&ControlRequest::Register(SessionRegistrationRequest {
|
||||
|
|
@ -466,13 +468,70 @@ pub fn execute_interactive_over_control_face_as<E, H: HostTerminal>(
|
|||
));
|
||||
}
|
||||
endpoint.acknowledge_waiter_transfer()?;
|
||||
let mut transport = ControlFaceInteractiveTransport { stream };
|
||||
let mut transport = ControlFaceInteractiveTransport::new(stream).map_err(widen_client_error)?;
|
||||
run_interactive_session(&mut transport, terminal, identity, &binding, argv)
|
||||
.map_err(widen_client_error)
|
||||
}
|
||||
|
||||
struct ControlFaceInteractiveTransport {
|
||||
stream: rustls::StreamOwned<rustls::ClientConnection, std::os::unix::net::UnixStream>,
|
||||
commands: Option<mpsc::Sender<InteractiveTransportCommand>>,
|
||||
batches:
|
||||
mpsc::Receiver<Result<InteractiveSessionBatchV2, ClientError<std::convert::Infallible>>>,
|
||||
cancellation: std::os::unix::net::UnixStream,
|
||||
worker: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
enum InteractiveTransportCommand {
|
||||
Start(InteractiveSessionRequestV2),
|
||||
Event(InteractiveSessionEventV2),
|
||||
}
|
||||
|
||||
impl ControlFaceInteractiveTransport {
|
||||
fn new(
|
||||
mut stream: rustls::StreamOwned<rustls::ClientConnection, std::os::unix::net::UnixStream>,
|
||||
) -> Result<Self, ClientError<std::convert::Infallible>> {
|
||||
let cancellation = stream.sock.try_clone()?;
|
||||
let (commands, command_rx) = mpsc::channel();
|
||||
let (batch_tx, batches) = mpsc::channel();
|
||||
let worker = std::thread::spawn(move || {
|
||||
while let Ok(command) = command_rx.recv() {
|
||||
let result = match command {
|
||||
InteractiveTransportCommand::Start(request) => ControlClient::send(
|
||||
&mut stream,
|
||||
&ControlRequest::ExecuteInteractiveV2(request),
|
||||
)
|
||||
.map_err(map_control_connection)
|
||||
.and_then(|()| receive_interactive_batch(&mut stream)),
|
||||
InteractiveTransportCommand::Event(event) => {
|
||||
ControlClient::send(&mut stream, &event)
|
||||
.map_err(map_control_connection)
|
||||
.and_then(|()| receive_interactive_batch(&mut stream))
|
||||
}
|
||||
};
|
||||
let failed = result.is_err();
|
||||
if batch_tx.send(result).is_err() || failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(Self {
|
||||
commands: Some(commands),
|
||||
batches,
|
||||
cancellation,
|
||||
worker: Some(worker),
|
||||
})
|
||||
}
|
||||
|
||||
fn submit(
|
||||
&self,
|
||||
command: InteractiveTransportCommand,
|
||||
) -> Result<(), ClientError<std::convert::Infallible>> {
|
||||
self.commands
|
||||
.as_ref()
|
||||
.ok_or(ClientError::LostConnection)?
|
||||
.send(command)
|
||||
.map_err(|_| ClientError::LostConnection)
|
||||
}
|
||||
}
|
||||
|
||||
impl InteractiveSessionTransport for ControlFaceInteractiveTransport {
|
||||
|
|
@ -480,27 +539,43 @@ impl InteractiveSessionTransport for ControlFaceInteractiveTransport {
|
|||
|
||||
fn start(
|
||||
&mut self,
|
||||
request: InteractiveSessionRequest,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<Self::Error>> {
|
||||
ControlClient::send(
|
||||
&mut self.stream,
|
||||
&ControlRequest::ExecuteInteractive(request),
|
||||
)?;
|
||||
receive_interactive_batch(&mut self.stream)
|
||||
request: InteractiveSessionRequestV2,
|
||||
) -> Result<(), ClientError<Self::Error>> {
|
||||
self.submit(InteractiveTransportCommand::Start(request))
|
||||
}
|
||||
|
||||
fn exchange(
|
||||
fn send(&mut self, event: InteractiveSessionEventV2) -> Result<(), ClientError<Self::Error>> {
|
||||
self.submit(InteractiveTransportCommand::Event(event))
|
||||
}
|
||||
|
||||
fn try_receive(
|
||||
&mut self,
|
||||
event: InteractiveSessionEvent,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<Self::Error>> {
|
||||
ControlClient::send(&mut self.stream, &event).map_err(map_control_connection)?;
|
||||
receive_interactive_batch(&mut self.stream)
|
||||
) -> Result<Option<InteractiveSessionBatchV2>, ClientError<Self::Error>> {
|
||||
match self.batches.try_recv() {
|
||||
Ok(batch) => batch.map(Some),
|
||||
Err(mpsc::TryRecvError::Empty) => Ok(None),
|
||||
Err(mpsc::TryRecvError::Disconnected) => Err(ClientError::LostConnection),
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel(&mut self) {
|
||||
self.commands.take();
|
||||
let _ = self.cancellation.shutdown(Shutdown::Both);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ControlFaceInteractiveTransport {
|
||||
fn drop(&mut self) {
|
||||
self.cancel();
|
||||
if let Some(worker) = self.worker.take() {
|
||||
let _ = worker.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn receive_interactive_batch(
|
||||
stream: &mut impl std::io::Read,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<std::convert::Infallible>> {
|
||||
) -> Result<InteractiveSessionBatchV2, ClientError<std::convert::Infallible>> {
|
||||
ControlClient::receive(stream).map_err(map_control_connection)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
use std::collections::VecDeque;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::num::NonZeroU16;
|
||||
use std::time::Duration;
|
||||
|
||||
use lazaret_core::records::{
|
||||
DeveloperDecisionKind, InteractiveSessionBatch, InteractiveSessionEvent,
|
||||
InteractiveSessionMessage, InteractiveSessionRequest, ProtocolVersion, SessionBinding,
|
||||
SessionClose, SessionExitOutcome, SessionTerminalEvent, SessionTerminalMessage, TerminalInput,
|
||||
TerminalPrompt, TerminalResize, TerminalSignal, TerminalSignalKind,
|
||||
DeveloperDecisionKind, InteractiveProtocolVersion, InteractiveSessionBatchV2,
|
||||
InteractiveSessionEventV2, InteractiveSessionMessage, InteractiveSessionRequestV2,
|
||||
SessionBinding, SessionExitOutcome, SessionTerminalEventV2, SessionTerminalMessage,
|
||||
TerminalBreak, TerminalInput, TerminalModesV2, TerminalPrompt, TerminalResize,
|
||||
};
|
||||
use nix::sys::signal::{SigSet, SigmaskHow, Signal};
|
||||
use nix::sys::signalfd::{SfdFlags, SignalFd};
|
||||
use rustix::fs::{OFlags, fcntl_getfl, fcntl_setfl};
|
||||
use rustix::termios::{LocalModes, OptionalActions, Termios, tcgetattr, tcgetwinsize, tcsetattr};
|
||||
use rustix::termios::{
|
||||
OptionalActions, SpecialCodeIndex, Termios, tcgetattr, tcgetwinsize, tcsetattr,
|
||||
};
|
||||
|
||||
use crate::{ClientError, ClientSessionIdentity};
|
||||
|
||||
|
|
@ -23,21 +26,26 @@ pub trait InteractiveSessionTransport {
|
|||
/// Returns an authenticated session or framing failure.
|
||||
fn start(
|
||||
&mut self,
|
||||
request: InteractiveSessionRequest,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<Self::Error>>;
|
||||
request: InteractiveSessionRequestV2,
|
||||
) -> Result<(), ClientError<Self::Error>>;
|
||||
|
||||
/// # Errors
|
||||
/// Returns a lost connection, stale session, or framing failure.
|
||||
fn exchange(
|
||||
fn send(&mut self, event: InteractiveSessionEventV2) -> Result<(), ClientError<Self::Error>>;
|
||||
|
||||
/// # Errors
|
||||
/// Returns a lost connection, stale session, or framing failure.
|
||||
fn try_receive(
|
||||
&mut self,
|
||||
event: InteractiveSessionEvent,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<Self::Error>>;
|
||||
) -> Result<Option<InteractiveSessionBatchV2>, ClientError<Self::Error>>;
|
||||
|
||||
fn cancel(&mut self);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum HostTerminalEvent {
|
||||
Input(Vec<u8>),
|
||||
Signal(TerminalSignalKind),
|
||||
Break,
|
||||
Resize,
|
||||
LocalSuspend,
|
||||
Terminate,
|
||||
|
|
@ -54,6 +62,7 @@ pub trait HostTerminal {
|
|||
/// # Errors
|
||||
/// Returns an error when the current terminal size cannot be observed.
|
||||
fn size(&self, session_id: lazaret_core::identity::SessionId) -> io::Result<TerminalResize>;
|
||||
fn modes(&self) -> TerminalModesV2;
|
||||
/// # Errors
|
||||
/// Returns an error while observing terminal input or signals.
|
||||
fn next_event(&mut self) -> io::Result<HostTerminalEvent>;
|
||||
|
|
@ -80,6 +89,9 @@ pub struct SystemTerminal {
|
|||
original_flags: OFlags,
|
||||
raw: bool,
|
||||
signals: SignalQueue,
|
||||
events: VecDeque<HostTerminalEvent>,
|
||||
line_start: bool,
|
||||
escape_pending: bool,
|
||||
}
|
||||
|
||||
impl SystemTerminal {
|
||||
|
|
@ -95,8 +107,59 @@ impl SystemTerminal {
|
|||
terminal,
|
||||
raw: false,
|
||||
signals: SignalQueue::register()?,
|
||||
events: VecDeque::new(),
|
||||
line_start: true,
|
||||
escape_pending: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn queue_input(&mut self, bytes: &[u8]) {
|
||||
let mut forwarded = Vec::new();
|
||||
let mut events = VecDeque::new();
|
||||
for &byte in bytes {
|
||||
if self.escape_pending {
|
||||
self.escape_pending = false;
|
||||
match byte {
|
||||
b'.' | 0x1a | b'B' => {
|
||||
if !forwarded.is_empty() {
|
||||
events.push_back(HostTerminalEvent::Input(std::mem::take(
|
||||
&mut forwarded,
|
||||
)));
|
||||
}
|
||||
events.push_back(match byte {
|
||||
b'.' => HostTerminalEvent::Terminate,
|
||||
0x1a => HostTerminalEvent::LocalSuspend,
|
||||
b'B' => HostTerminalEvent::Break,
|
||||
_ => unreachable!(),
|
||||
});
|
||||
self.line_start = true;
|
||||
continue;
|
||||
}
|
||||
b'~' => {
|
||||
forwarded.push(b'~');
|
||||
self.line_start = false;
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
forwarded.push(b'~');
|
||||
forwarded.push(byte);
|
||||
}
|
||||
}
|
||||
self.line_start = matches!(byte, b'\r' | b'\n');
|
||||
continue;
|
||||
}
|
||||
if self.line_start && byte == b'~' {
|
||||
self.escape_pending = true;
|
||||
continue;
|
||||
}
|
||||
forwarded.push(byte);
|
||||
self.line_start = matches!(byte, b'\r' | b'\n');
|
||||
}
|
||||
if !forwarded.is_empty() {
|
||||
events.push_back(HostTerminalEvent::Input(forwarded));
|
||||
}
|
||||
self.events.extend(events);
|
||||
}
|
||||
}
|
||||
|
||||
impl HostTerminal for SystemTerminal {
|
||||
|
|
@ -106,7 +169,6 @@ impl HostTerminal for SystemTerminal {
|
|||
}
|
||||
let mut raw = self.original.clone();
|
||||
raw.make_raw();
|
||||
raw.local_modes.insert(LocalModes::ISIG);
|
||||
tcsetattr(&self.terminal, OptionalActions::Flush, &raw)?;
|
||||
if let Err(error) = fcntl_setfl(&self.terminal, self.original_flags | OFlags::NONBLOCK) {
|
||||
let _ = tcsetattr(&self.terminal, OptionalActions::Flush, &self.original);
|
||||
|
|
@ -139,7 +201,14 @@ impl HostTerminal for SystemTerminal {
|
|||
})
|
||||
}
|
||||
|
||||
fn modes(&self) -> TerminalModesV2 {
|
||||
terminal_modes_v2(&self.original)
|
||||
}
|
||||
|
||||
fn next_event(&mut self) -> io::Result<HostTerminalEvent> {
|
||||
if let Some(event) = self.events.pop_front() {
|
||||
return Ok(event);
|
||||
}
|
||||
if let Some(event) = self.signals.next_event()? {
|
||||
return Ok(event);
|
||||
}
|
||||
|
|
@ -148,7 +217,8 @@ impl HostTerminal for SystemTerminal {
|
|||
Ok(0) => Ok(HostTerminalEvent::Terminate),
|
||||
Ok(length) => {
|
||||
bytes.truncate(length);
|
||||
Ok(HostTerminalEvent::Input(bytes))
|
||||
self.queue_input(&bytes);
|
||||
Ok(self.events.pop_front().unwrap_or(HostTerminalEvent::Idle))
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
|
|
@ -196,9 +266,7 @@ impl HostTerminal for SystemTerminal {
|
|||
resumed_size = Some(self.size(prompt.session_id)?);
|
||||
}
|
||||
HostTerminalEvent::Terminate => return Err(io::ErrorKind::Interrupted.into()),
|
||||
HostTerminalEvent::Idle
|
||||
| HostTerminalEvent::Resize
|
||||
| HostTerminalEvent::Signal(_) => {}
|
||||
HostTerminalEvent::Idle | HostTerminalEvent::Resize | HostTerminalEvent::Break => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -259,11 +327,11 @@ impl SignalQueue {
|
|||
};
|
||||
let signal = Signal::try_from(info.ssi_signo.cast_signed()).map_err(errno_to_io)?;
|
||||
Ok(match signal {
|
||||
Signal::SIGINT => Some(HostTerminalEvent::Signal(TerminalSignalKind::Interrupt)),
|
||||
Signal::SIGQUIT => Some(HostTerminalEvent::Signal(TerminalSignalKind::Quit)),
|
||||
Signal::SIGINT | Signal::SIGQUIT | Signal::SIGTERM | Signal::SIGHUP => {
|
||||
Some(HostTerminalEvent::Terminate)
|
||||
}
|
||||
Signal::SIGTSTP => Some(HostTerminalEvent::LocalSuspend),
|
||||
Signal::SIGWINCH => Some(HostTerminalEvent::Resize),
|
||||
Signal::SIGTERM | Signal::SIGHUP => Some(HostTerminalEvent::Terminate),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
|
@ -282,6 +350,35 @@ impl Drop for SignalQueue {
|
|||
}
|
||||
}
|
||||
|
||||
fn terminal_modes_v2(modes: &Termios) -> TerminalModesV2 {
|
||||
TerminalModesV2 {
|
||||
input_modes: modes.input_modes.bits(),
|
||||
output_modes: modes.output_modes.bits(),
|
||||
control_modes: modes.control_modes.bits(),
|
||||
local_modes: modes.local_modes.bits(),
|
||||
line_discipline: modes.line_discipline,
|
||||
input_speed: modes.input_speed(),
|
||||
output_speed: modes.output_speed(),
|
||||
interrupt: modes.special_codes[SpecialCodeIndex::VINTR],
|
||||
quit: modes.special_codes[SpecialCodeIndex::VQUIT],
|
||||
erase: modes.special_codes[SpecialCodeIndex::VERASE],
|
||||
kill: modes.special_codes[SpecialCodeIndex::VKILL],
|
||||
eof: modes.special_codes[SpecialCodeIndex::VEOF],
|
||||
time: modes.special_codes[SpecialCodeIndex::VTIME],
|
||||
minimum: modes.special_codes[SpecialCodeIndex::VMIN],
|
||||
switch: modes.special_codes[SpecialCodeIndex::VSWTC],
|
||||
start: modes.special_codes[SpecialCodeIndex::VSTART],
|
||||
stop: modes.special_codes[SpecialCodeIndex::VSTOP],
|
||||
suspend: modes.special_codes[SpecialCodeIndex::VSUSP],
|
||||
end_of_line: modes.special_codes[SpecialCodeIndex::VEOL],
|
||||
reprint: modes.special_codes[SpecialCodeIndex::VREPRINT],
|
||||
discard: modes.special_codes[SpecialCodeIndex::VDISCARD],
|
||||
word_erase: modes.special_codes[SpecialCodeIndex::VWERASE],
|
||||
literal_next: modes.special_codes[SpecialCodeIndex::VLNEXT],
|
||||
end_of_line_2: modes.special_codes[SpecialCodeIndex::VEOL2],
|
||||
}
|
||||
}
|
||||
|
||||
fn errno_to_io(error: nix::errno::Errno) -> io::Error {
|
||||
io::Error::from_raw_os_error(error as i32)
|
||||
}
|
||||
|
|
@ -301,63 +398,81 @@ pub fn run_interactive_session<T: InteractiveSessionTransport, H: HostTerminal>(
|
|||
let result = (|| {
|
||||
let session_id = identity.session_id();
|
||||
let initial_size = terminal.size(session_id)?;
|
||||
let mut batch = transport.start(InteractiveSessionRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
transport.start(InteractiveSessionRequestV2 {
|
||||
protocol_version: InteractiveProtocolVersion::V2,
|
||||
session_id,
|
||||
argv: argv.to_vec(),
|
||||
initial_size,
|
||||
initial_modes: terminal.modes(),
|
||||
})?;
|
||||
let mut pending = VecDeque::new();
|
||||
loop {
|
||||
if let Some(exit) = handle_batch(transport, terminal, identity, binding, batch)? {
|
||||
return match exit.outcome() {
|
||||
SessionExitOutcome::Exited => exit
|
||||
.exit_code()
|
||||
.ok_or(ClientError::Protocol("Exited omitted its exit code")),
|
||||
SessionExitOutcome::Terminated | SessionExitOutcome::RuntimeLost => {
|
||||
Err(ClientError::LostConnection)
|
||||
if let Some(batch) = transport.try_receive()? {
|
||||
let handled = handle_batch(terminal, identity, binding, batch)?;
|
||||
if let Some(exit) = handled.exit {
|
||||
return match exit.outcome() {
|
||||
SessionExitOutcome::Exited => exit
|
||||
.exit_code()
|
||||
.ok_or(ClientError::Protocol("Exited omitted its exit code")),
|
||||
SessionExitOutcome::Terminated | SessionExitOutcome::RuntimeLost => {
|
||||
Err(ClientError::LostConnection)
|
||||
}
|
||||
};
|
||||
}
|
||||
for event in handled.events {
|
||||
enqueue_event(&mut pending, event)?;
|
||||
}
|
||||
} else {
|
||||
let event = terminal.next_event()?;
|
||||
match host_event(terminal, session_id, event)? {
|
||||
HostEvent::Remote(event) => enqueue_event(&mut pending, event)?,
|
||||
HostEvent::Continue => {}
|
||||
HostEvent::Disconnect => {
|
||||
transport.cancel();
|
||||
return Err(ClientError::LostConnection);
|
||||
}
|
||||
};
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let event =
|
||||
match terminal.next_event()? {
|
||||
HostTerminalEvent::Input(bytes) => InteractiveSessionEvent::Terminal(
|
||||
SessionTerminalEvent::Input(TerminalInput { session_id, bytes }),
|
||||
),
|
||||
HostTerminalEvent::Signal(signal) => InteractiveSessionEvent::Terminal(
|
||||
SessionTerminalEvent::Signal(TerminalSignal { session_id, signal }),
|
||||
),
|
||||
HostTerminalEvent::Resize => InteractiveSessionEvent::Terminal(
|
||||
SessionTerminalEvent::Resize(terminal.size(session_id)?),
|
||||
),
|
||||
HostTerminalEvent::LocalSuspend => {
|
||||
terminal.suspend_and_resume()?;
|
||||
InteractiveSessionEvent::Terminal(SessionTerminalEvent::Resize(
|
||||
terminal.size(session_id)?,
|
||||
))
|
||||
let event = if let Some(event) = pending.pop_front() {
|
||||
event
|
||||
} else {
|
||||
let event = terminal.next_event()?;
|
||||
match host_event(terminal, session_id, event)? {
|
||||
HostEvent::Remote(event) => event,
|
||||
HostEvent::Continue => InteractiveSessionEventV2::Poll { session_id },
|
||||
HostEvent::Disconnect => {
|
||||
transport.cancel();
|
||||
return Err(ClientError::LostConnection);
|
||||
}
|
||||
HostTerminalEvent::Terminate => InteractiveSessionEvent::Terminal(
|
||||
SessionTerminalEvent::Close(SessionClose { session_id }),
|
||||
),
|
||||
HostTerminalEvent::Idle => InteractiveSessionEvent::Poll { session_id },
|
||||
};
|
||||
batch = transport.exchange(event)?;
|
||||
}
|
||||
};
|
||||
transport.send(event)?;
|
||||
}
|
||||
})();
|
||||
if result.is_err() {
|
||||
transport.cancel();
|
||||
}
|
||||
let restoration = terminal.restore().map_err(ClientError::Io);
|
||||
restoration?;
|
||||
result
|
||||
}
|
||||
|
||||
fn handle_batch<T: InteractiveSessionTransport, H: HostTerminal>(
|
||||
transport: &mut T,
|
||||
struct HandledBatch {
|
||||
exit: Option<lazaret_core::records::SessionExit>,
|
||||
events: Vec<InteractiveSessionEventV2>,
|
||||
}
|
||||
|
||||
fn handle_batch<H: HostTerminal, E>(
|
||||
terminal: &mut H,
|
||||
identity: &ClientSessionIdentity,
|
||||
binding: &SessionBinding,
|
||||
batch: InteractiveSessionBatch,
|
||||
) -> Result<Option<lazaret_core::records::SessionExit>, ClientError<T::Error>> {
|
||||
batch: InteractiveSessionBatchV2,
|
||||
) -> Result<HandledBatch, ClientError<E>> {
|
||||
if batch.session_id != identity.session_id() {
|
||||
return Err(ClientError::Protocol("response Session ID differs"));
|
||||
}
|
||||
let mut events = Vec::new();
|
||||
for message in batch.messages {
|
||||
match message {
|
||||
InteractiveSessionMessage::Terminal(SessionTerminalMessage::Output(output)) => {
|
||||
|
|
@ -366,35 +481,92 @@ fn handle_batch<T: InteractiveSessionTransport, H: HostTerminal>(
|
|||
}
|
||||
InteractiveSessionMessage::Terminal(SessionTerminalMessage::Exit(exit)) => {
|
||||
require_session(exit.session_id(), identity.session_id())?;
|
||||
return Ok(Some(exit));
|
||||
return Ok(HandledBatch {
|
||||
exit: Some(exit),
|
||||
events,
|
||||
});
|
||||
}
|
||||
InteractiveSessionMessage::Prompt(prompt) => {
|
||||
require_session(prompt.session_id, identity.session_id())?;
|
||||
let outcome = terminal.prompt(&prompt)?;
|
||||
if let Some(resize) = outcome.resumed_size {
|
||||
let response = transport.exchange(InteractiveSessionEvent::Terminal(
|
||||
SessionTerminalEvent::Resize(resize),
|
||||
))?;
|
||||
if let Some(exit) =
|
||||
handle_batch(transport, terminal, identity, binding, response)?
|
||||
{
|
||||
return Ok(Some(exit));
|
||||
}
|
||||
events.push(InteractiveSessionEventV2::Terminal(
|
||||
SessionTerminalEventV2::Resize(resize),
|
||||
));
|
||||
}
|
||||
let signed_decision =
|
||||
identity.sign_developer_decision(binding.clone(), &prompt, outcome.decision)?;
|
||||
let response = transport.exchange(InteractiveSessionEvent::PromptDecision {
|
||||
events.push(InteractiveSessionEventV2::PromptDecision {
|
||||
session_id: identity.session_id(),
|
||||
signed_decision,
|
||||
})?;
|
||||
if let Some(exit) = handle_batch(transport, terminal, identity, binding, response)?
|
||||
{
|
||||
return Ok(Some(exit));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
Ok(HandledBatch { exit: None, events })
|
||||
}
|
||||
|
||||
enum HostEvent {
|
||||
Remote(InteractiveSessionEventV2),
|
||||
Continue,
|
||||
Disconnect,
|
||||
}
|
||||
|
||||
fn host_event<H: HostTerminal, E>(
|
||||
terminal: &mut H,
|
||||
session_id: lazaret_core::identity::SessionId,
|
||||
event: HostTerminalEvent,
|
||||
) -> Result<HostEvent, ClientError<E>> {
|
||||
Ok(match event {
|
||||
HostTerminalEvent::Input(bytes) => HostEvent::Remote(InteractiveSessionEventV2::Terminal(
|
||||
SessionTerminalEventV2::Input(TerminalInput { session_id, bytes }),
|
||||
)),
|
||||
HostTerminalEvent::Break => HostEvent::Remote(InteractiveSessionEventV2::Terminal(
|
||||
SessionTerminalEventV2::Break(TerminalBreak { session_id }),
|
||||
)),
|
||||
HostTerminalEvent::Resize => HostEvent::Remote(InteractiveSessionEventV2::Terminal(
|
||||
SessionTerminalEventV2::Resize(terminal.size(session_id)?),
|
||||
)),
|
||||
HostTerminalEvent::LocalSuspend => {
|
||||
terminal.suspend_and_resume()?;
|
||||
HostEvent::Remote(InteractiveSessionEventV2::Terminal(
|
||||
SessionTerminalEventV2::Resize(terminal.size(session_id)?),
|
||||
))
|
||||
}
|
||||
HostTerminalEvent::Terminate => HostEvent::Disconnect,
|
||||
HostTerminalEvent::Idle => HostEvent::Continue,
|
||||
})
|
||||
}
|
||||
|
||||
fn enqueue_event<E>(
|
||||
pending: &mut VecDeque<InteractiveSessionEventV2>,
|
||||
event: InteractiveSessionEventV2,
|
||||
) -> Result<(), ClientError<E>> {
|
||||
const MAX_PENDING_EVENTS: usize = 256;
|
||||
const MAX_PENDING_INPUT_BYTES: usize = 64 * 1024;
|
||||
|
||||
let bytes = pending
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
InteractiveSessionEventV2::Terminal(SessionTerminalEventV2::Input(input)) => {
|
||||
Some(input.bytes.len())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.sum::<usize>()
|
||||
+ match &event {
|
||||
InteractiveSessionEventV2::Terminal(SessionTerminalEventV2::Input(input)) => {
|
||||
input.bytes.len()
|
||||
}
|
||||
_ => 0,
|
||||
};
|
||||
if pending.len() >= MAX_PENDING_EVENTS || bytes > MAX_PENDING_INPUT_BYTES {
|
||||
return Err(ClientError::Io(io::Error::other(
|
||||
"interactive terminal transport backpressure",
|
||||
)));
|
||||
}
|
||||
pending.push_back(event);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_session<E>(
|
||||
|
|
@ -426,10 +598,81 @@ fn hex(bytes: &[u8]) -> String {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::io::Write as _;
|
||||
|
||||
use nix::sys::signal::{SigSet, SigmaskHow, Signal, kill};
|
||||
use nix::unistd::Pid;
|
||||
use rustix::fs::fcntl_getfl;
|
||||
use rustix::pty::{OpenptFlags, grantpt, ioctl_tiocgptpeer, openpt, unlockpt};
|
||||
use rustix::termios::{LocalModes, tcgetattr};
|
||||
|
||||
use super::SignalQueue;
|
||||
use super::{HostTerminal, HostTerminalEvent, SignalQueue, SystemTerminal};
|
||||
|
||||
fn terminal() -> (SystemTerminal, File) {
|
||||
let master =
|
||||
openpt(OpenptFlags::RDWR | OpenptFlags::NOCTTY | OpenptFlags::CLOEXEC).unwrap();
|
||||
grantpt(&master).unwrap();
|
||||
unlockpt(&master).unwrap();
|
||||
let slave = File::from(
|
||||
ioctl_tiocgptpeer(
|
||||
&master,
|
||||
OpenptFlags::RDWR | OpenptFlags::NOCTTY | OpenptFlags::CLOEXEC,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let original = tcgetattr(&slave).unwrap();
|
||||
let original_flags = fcntl_getfl(&slave).unwrap();
|
||||
(
|
||||
SystemTerminal {
|
||||
terminal: slave,
|
||||
original,
|
||||
original_flags,
|
||||
raw: false,
|
||||
signals: SignalQueue::register().unwrap(),
|
||||
events: VecDeque::new(),
|
||||
line_start: true,
|
||||
escape_pending: false,
|
||||
},
|
||||
File::from(master),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_mode_forwards_control_bytes_without_local_signals() {
|
||||
let (mut terminal, mut master) = terminal();
|
||||
terminal.enter_raw().unwrap();
|
||||
assert!(
|
||||
!tcgetattr(&terminal.terminal)
|
||||
.unwrap()
|
||||
.local_modes
|
||||
.contains(LocalModes::ISIG)
|
||||
);
|
||||
master.write_all(&[3, 28, 26]).unwrap();
|
||||
assert_eq!(
|
||||
terminal.next_event().unwrap(),
|
||||
HostTerminalEvent::Input(vec![3, 28, 26])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssh_style_escapes_survive_read_boundaries_and_preserve_literals() {
|
||||
let (mut terminal, mut master) = terminal();
|
||||
terminal.enter_raw().unwrap();
|
||||
master.write_all(b"line\n~").unwrap();
|
||||
assert_eq!(
|
||||
terminal.next_event().unwrap(),
|
||||
HostTerminalEvent::Input(b"line\n".to_vec())
|
||||
);
|
||||
master.write_all(b"B~.~~literal\n").unwrap();
|
||||
assert_eq!(terminal.next_event().unwrap(), HostTerminalEvent::Break);
|
||||
assert_eq!(terminal.next_event().unwrap(), HostTerminalEvent::Terminate);
|
||||
assert_eq!(
|
||||
terminal.next_event().unwrap(),
|
||||
HostTerminalEvent::Input(b"~literal\n".to_vec())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ use lazaret_core::identity::{
|
|||
SandboxId, SessionId,
|
||||
};
|
||||
use lazaret_core::records::{
|
||||
DeveloperDecision, DeveloperDecisionKind, GenerationIdentity, InteractiveSessionBatch,
|
||||
InteractiveSessionEvent, InteractiveSessionMessage, InteractiveSessionRequest, SessionBinding,
|
||||
SessionExit, SessionTerminalMessage, SignedProtocolRecord, TerminalOutput, TerminalPrompt,
|
||||
TerminalResize, TerminalSignalKind,
|
||||
DeveloperDecision, DeveloperDecisionKind, GenerationIdentity, InteractiveSessionBatchV2,
|
||||
InteractiveSessionEventV2, InteractiveSessionMessage, InteractiveSessionRequestV2,
|
||||
SessionBinding, SessionExit, SessionTerminalEventV2, SessionTerminalMessage,
|
||||
SignedProtocolRecord, TerminalModesV2, TerminalOutput, TerminalPrompt, TerminalResize,
|
||||
};
|
||||
use rustix::fs::{OFlags, fcntl_getfl, fcntl_setfl};
|
||||
use rustix::pty::{OpenptFlags, grantpt, ioctl_tiocgptpeer, openpt, unlockpt};
|
||||
use rustix::termios::{LocalModes, OptionalActions, Termios, tcgetattr, tcsetattr};
|
||||
use rustix::termios::{OptionalActions, Termios, tcgetattr, tcsetattr};
|
||||
|
||||
struct PtyTerminal {
|
||||
_master: File,
|
||||
|
|
@ -74,7 +74,6 @@ impl HostTerminal for PtyTerminal {
|
|||
fn enter_raw(&mut self) -> io::Result<()> {
|
||||
let mut raw = self.original.clone();
|
||||
raw.make_raw();
|
||||
raw.local_modes.insert(LocalModes::ISIG);
|
||||
tcsetattr(&self.slave, OptionalActions::Now, &raw)?;
|
||||
fcntl_setfl(&self.slave, self.original_flags | OFlags::NONBLOCK)?;
|
||||
self.raw = true;
|
||||
|
|
@ -98,6 +97,10 @@ impl HostTerminal for PtyTerminal {
|
|||
})
|
||||
}
|
||||
|
||||
fn modes(&self) -> TerminalModesV2 {
|
||||
test_modes()
|
||||
}
|
||||
|
||||
fn next_event(&mut self) -> io::Result<HostTerminalEvent> {
|
||||
Ok(self.events.pop_front().unwrap_or(HostTerminalEvent::Idle))
|
||||
}
|
||||
|
|
@ -139,11 +142,14 @@ enum TransportFailure {
|
|||
|
||||
struct FixtureTransport {
|
||||
session_id: SessionId,
|
||||
events: Vec<InteractiveSessionEvent>,
|
||||
events: Vec<InteractiveSessionEventV2>,
|
||||
prompt_sent: bool,
|
||||
failure: Option<TransportFailure>,
|
||||
signed_decision: Option<Vec<u8>>,
|
||||
prompt_after_events: usize,
|
||||
batches: VecDeque<InteractiveSessionBatchV2>,
|
||||
cancelled: bool,
|
||||
blocked: bool,
|
||||
}
|
||||
|
||||
impl FixtureTransport {
|
||||
|
|
@ -155,6 +161,9 @@ impl FixtureTransport {
|
|||
failure: None,
|
||||
signed_decision: None,
|
||||
prompt_after_events: 5,
|
||||
batches: VecDeque::new(),
|
||||
cancelled: false,
|
||||
blocked: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -164,13 +173,17 @@ impl InteractiveSessionTransport for FixtureTransport {
|
|||
|
||||
fn start(
|
||||
&mut self,
|
||||
request: InteractiveSessionRequest,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<Self::Error>> {
|
||||
request: InteractiveSessionRequestV2,
|
||||
) -> Result<(), ClientError<Self::Error>> {
|
||||
if matches!(self.failure, Some(TransportFailure::Stale)) {
|
||||
return Err(ClientError::StaleCoordinator);
|
||||
}
|
||||
assert_eq!(request.session_id, self.session_id);
|
||||
Ok(batch(
|
||||
assert_eq!(
|
||||
request.protocol_version,
|
||||
lazaret_core::records::InteractiveProtocolVersion::V2
|
||||
);
|
||||
self.batches.push_back(batch(
|
||||
self.session_id,
|
||||
vec![InteractiveSessionMessage::Terminal(
|
||||
SessionTerminalMessage::Output(TerminalOutput {
|
||||
|
|
@ -178,32 +191,31 @@ impl InteractiveSessionTransport for FixtureTransport {
|
|||
bytes: b"guest-output".to_vec(),
|
||||
}),
|
||||
)],
|
||||
))
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn exchange(
|
||||
&mut self,
|
||||
event: InteractiveSessionEvent,
|
||||
) -> Result<InteractiveSessionBatch, ClientError<Self::Error>> {
|
||||
fn send(&mut self, event: InteractiveSessionEventV2) -> Result<(), ClientError<Self::Error>> {
|
||||
if matches!(self.failure, Some(TransportFailure::Lost)) {
|
||||
return Err(ClientError::LostConnection);
|
||||
}
|
||||
self.events.push(event.clone());
|
||||
if let InteractiveSessionEvent::PromptDecision {
|
||||
if let InteractiveSessionEventV2::PromptDecision {
|
||||
signed_decision, ..
|
||||
} = event
|
||||
{
|
||||
self.signed_decision = Some(signed_decision);
|
||||
return Ok(batch(
|
||||
self.batches.push_back(batch(
|
||||
self.session_id,
|
||||
vec![InteractiveSessionMessage::Terminal(
|
||||
SessionTerminalMessage::Exit(SessionExit::exited(self.session_id, 23)),
|
||||
)],
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
if !self.prompt_sent && self.events.len() >= self.prompt_after_events {
|
||||
self.prompt_sent = true;
|
||||
return Ok(batch(
|
||||
self.batches.push_back(batch(
|
||||
self.session_id,
|
||||
vec![InteractiveSessionMessage::Prompt(TerminalPrompt {
|
||||
session_id: self.session_id,
|
||||
|
|
@ -214,8 +226,23 @@ impl InteractiveSessionTransport for FixtureTransport {
|
|||
justification: "update host metadata".into(),
|
||||
})],
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
Ok(batch(self.session_id, Vec::new()))
|
||||
self.batches.push_back(batch(self.session_id, Vec::new()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn try_receive(
|
||||
&mut self,
|
||||
) -> Result<Option<InteractiveSessionBatchV2>, ClientError<Self::Error>> {
|
||||
if self.blocked {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(self.batches.pop_front())
|
||||
}
|
||||
|
||||
fn cancel(&mut self) {
|
||||
self.cancelled = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -243,23 +270,21 @@ fn prompt_suspend_sends_resumed_size_before_decision() {
|
|||
assert!(matches!(
|
||||
transport.events.as_slice(),
|
||||
[
|
||||
InteractiveSessionEvent::Poll { .. },
|
||||
InteractiveSessionEvent::Terminal(lazaret_core::records::SessionTerminalEvent::Resize(
|
||||
_
|
||||
)),
|
||||
InteractiveSessionEvent::PromptDecision { .. }
|
||||
InteractiveSessionEventV2::Poll { .. },
|
||||
InteractiveSessionEventV2::Terminal(SessionTerminalEventV2::Resize(_)),
|
||||
InteractiveSessionEventV2::PromptDecision { .. }
|
||||
]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_fixture_routes_resize_signals_suspend_prompt_and_nonzero_exit() {
|
||||
fn terminal_fixture_routes_resize_input_break_suspend_prompt_and_nonzero_exit() {
|
||||
let identity = ClientSessionIdentity::generate::<std::convert::Infallible>().unwrap();
|
||||
let binding = binding(&identity);
|
||||
let mut terminal = PtyTerminal::new([
|
||||
HostTerminalEvent::Resize,
|
||||
HostTerminalEvent::Signal(TerminalSignalKind::Interrupt),
|
||||
HostTerminalEvent::Signal(TerminalSignalKind::Quit),
|
||||
HostTerminalEvent::Input(vec![3]),
|
||||
HostTerminalEvent::Break,
|
||||
HostTerminalEvent::LocalSuspend,
|
||||
HostTerminalEvent::Input(b"input".to_vec()),
|
||||
]);
|
||||
|
|
@ -289,21 +314,16 @@ fn terminal_fixture_routes_resize_signals_suspend_prompt_and_nonzero_exit() {
|
|||
assert_eq!(decision.approved_command_argv, [b"/usr/bin/tool".to_vec()]);
|
||||
assert!(transport.events.iter().any(|event| matches!(
|
||||
event,
|
||||
InteractiveSessionEvent::Terminal(lazaret_core::records::SessionTerminalEvent::Resize(_))
|
||||
InteractiveSessionEventV2::Terminal(SessionTerminalEventV2::Resize(_))
|
||||
)));
|
||||
for signal in [TerminalSignalKind::Interrupt, TerminalSignalKind::Quit] {
|
||||
assert!(transport.events.iter().any(|event| matches!(
|
||||
event,
|
||||
InteractiveSessionEvent::Terminal(
|
||||
lazaret_core::records::SessionTerminalEvent::Signal(sent)
|
||||
) if sent.signal == signal
|
||||
)));
|
||||
}
|
||||
assert!(!transport.events.iter().any(|event| matches!(
|
||||
assert!(transport.events.iter().any(|event| matches!(
|
||||
event,
|
||||
InteractiveSessionEvent::Terminal(
|
||||
lazaret_core::records::SessionTerminalEvent::Signal(sent)
|
||||
) if sent.signal == TerminalSignalKind::Suspend
|
||||
InteractiveSessionEventV2::Terminal(SessionTerminalEventV2::Input(input))
|
||||
if input.bytes == [3]
|
||||
)));
|
||||
assert!(transport.events.iter().any(|event| matches!(
|
||||
event,
|
||||
InteractiveSessionEventV2::Terminal(SessionTerminalEventV2::Break(_))
|
||||
)));
|
||||
}
|
||||
|
||||
|
|
@ -335,6 +355,34 @@ fn terminal_fixture_restores_raw_mode_for_stale_and_lost_coordinator() {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_disconnect_cancels_blocked_transport_and_restores_terminal() {
|
||||
let identity = ClientSessionIdentity::generate::<std::convert::Infallible>().unwrap();
|
||||
let binding = binding(&identity);
|
||||
let mut terminal = PtyTerminal::new([
|
||||
HostTerminalEvent::LocalSuspend,
|
||||
HostTerminalEvent::Terminate,
|
||||
]);
|
||||
let mut transport = FixtureTransport {
|
||||
blocked: true,
|
||||
..FixtureTransport::successful(identity.session_id())
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
run_interactive_session(
|
||||
&mut transport,
|
||||
&mut terminal,
|
||||
&identity,
|
||||
&binding,
|
||||
&[b"agent".to_vec()],
|
||||
),
|
||||
Err(ClientError::LostConnection)
|
||||
));
|
||||
assert!(transport.cancelled);
|
||||
assert_eq!(terminal.suspend_count, 1);
|
||||
assert!(terminal.restored());
|
||||
}
|
||||
|
||||
fn binding(identity: &ClientSessionIdentity) -> SessionBinding {
|
||||
SessionBinding {
|
||||
generation_identity: GenerationIdentity {
|
||||
|
|
@ -354,9 +402,38 @@ fn binding(identity: &ClientSessionIdentity) -> SessionBinding {
|
|||
fn batch(
|
||||
session_id: SessionId,
|
||||
messages: Vec<InteractiveSessionMessage>,
|
||||
) -> InteractiveSessionBatch {
|
||||
InteractiveSessionBatch {
|
||||
) -> InteractiveSessionBatchV2 {
|
||||
InteractiveSessionBatchV2 {
|
||||
session_id,
|
||||
messages,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_modes() -> TerminalModesV2 {
|
||||
TerminalModesV2 {
|
||||
input_modes: 0,
|
||||
output_modes: 0,
|
||||
control_modes: 0,
|
||||
local_modes: 0,
|
||||
line_discipline: 0,
|
||||
input_speed: 38_400,
|
||||
output_speed: 38_400,
|
||||
interrupt: 3,
|
||||
quit: 28,
|
||||
erase: 127,
|
||||
kill: 21,
|
||||
eof: 4,
|
||||
time: 0,
|
||||
minimum: 1,
|
||||
switch: 0,
|
||||
start: 17,
|
||||
stop: 19,
|
||||
suspend: 26,
|
||||
end_of_line: 0,
|
||||
reprint: 18,
|
||||
discard: 15,
|
||||
word_erase: 23,
|
||||
literal_next: 22,
|
||||
end_of_line_2: 0,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue