Decompose oversized Rust files by responsibility
This commit is contained in:
parent
03bfe92ca0
commit
ee9201b057
5 changed files with 18648 additions and 18513 deletions
File diff suppressed because it is too large
Load diff
458
crates/lazaret-state-anchor/src/lib/authority.rs
Normal file
458
crates/lazaret-state-anchor/src/lib/authority.rs
Normal file
|
|
@ -0,0 +1,458 @@
|
|||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct VerifiedStateAnchorReadiness {
|
||||
staged: StagedProcessOwnership,
|
||||
mount: MountIdentity,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct StateAnchorAcquisition<'path> {
|
||||
pub lower: &'path Path,
|
||||
pub expected_lower: (u64, u64),
|
||||
pub expected_parent: lazaret_core::identity::PidfsProcessIdentityV1,
|
||||
pub ownership: ProcessOwnershipContext,
|
||||
}
|
||||
|
||||
pub struct StateAnchorLaunchAuthority {
|
||||
lower: OwnedFd,
|
||||
lower_label: PathBuf,
|
||||
persistent_root: OwnedFd,
|
||||
persistent_anchor: OwnedFd,
|
||||
persistent_anchor_label: PathBuf,
|
||||
runtime_root: OwnedFd,
|
||||
runtime_anchor: OwnedFd,
|
||||
runtime_anchor_label: PathBuf,
|
||||
expected_lower: (u64, u64),
|
||||
persistent_destinations: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
impl StateAnchorLaunchAuthority {
|
||||
/// Captures exact directory authorities for inheritance by the Runtime Supervisor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a changed layout or prepared lower.
|
||||
pub fn capture(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
artifact: &PreparedArtifact,
|
||||
) -> Result<Self, StateAnchorError> {
|
||||
layout.revalidate()?;
|
||||
let expected_lower = (artifact.device.get(), artifact.inode.get());
|
||||
let lower = open_exact_directory_path(&path_from_host(&artifact.path), expected_lower)?;
|
||||
Self::capture_open(
|
||||
layout,
|
||||
expected_lower,
|
||||
path_from_host(&artifact.path),
|
||||
lower,
|
||||
)
|
||||
}
|
||||
|
||||
/// Captures an already-confined prepared-lower authority for inheritance.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a changed layout or prepared-lower descriptor.
|
||||
pub fn capture_retained(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
artifact: &PreparedArtifact,
|
||||
lower_label: &Path,
|
||||
lower: impl AsFd,
|
||||
) -> Result<Self, StateAnchorError> {
|
||||
layout.revalidate()?;
|
||||
let expected_lower = (artifact.device.get(), artifact.inode.get());
|
||||
require_directory_descriptor_identity(&lower, expected_lower)?;
|
||||
Self::capture_open(layout, expected_lower, lower_label.to_owned(), lower)
|
||||
}
|
||||
|
||||
fn capture_open(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
expected_lower: (u64, u64),
|
||||
lower_label: PathBuf,
|
||||
lower: impl AsFd,
|
||||
) -> Result<Self, StateAnchorError> {
|
||||
Ok(Self {
|
||||
lower: inherit_for_exec(lower)?,
|
||||
lower_label,
|
||||
persistent_root: inherit_for_exec(&layout.persistent_root)?,
|
||||
persistent_anchor: inherit_for_exec(&layout.persistent_anchor)?,
|
||||
persistent_anchor_label: layout
|
||||
.upper_label
|
||||
.parent()
|
||||
.ok_or(StateAnchorError::InvalidLayout(
|
||||
"upper label has no State Anchor parent",
|
||||
))?
|
||||
.to_owned(),
|
||||
runtime_root: inherit_for_exec(&layout.runtime_root)?,
|
||||
runtime_anchor: inherit_for_exec(&layout.runtime_anchor)?,
|
||||
runtime_anchor_label: layout
|
||||
.merged_label
|
||||
.parent()
|
||||
.ok_or(StateAnchorError::InvalidLayout(
|
||||
"merged label has no State Anchor parent",
|
||||
))?
|
||||
.to_owned(),
|
||||
expected_lower,
|
||||
persistent_destinations: layout.persistent_destinations.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn lower_fd(&self) -> i32 {
|
||||
self.lower.as_raw_fd()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn persistent_anchor_fd(&self) -> i32 {
|
||||
self.persistent_anchor.as_raw_fd()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn persistent_root_fd(&self) -> i32 {
|
||||
self.persistent_root.as_raw_fd()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn runtime_anchor_fd(&self) -> i32 {
|
||||
self.runtime_anchor.as_raw_fd()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn runtime_root_fd(&self) -> i32 {
|
||||
self.runtime_root.as_raw_fd()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn runtime_anchor_label(&self) -> &Path {
|
||||
&self.runtime_anchor_label
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn lower_label(&self) -> &Path {
|
||||
&self.lower_label
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn persistent_anchor_label(&self) -> &Path {
|
||||
&self.persistent_anchor_label
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn expected_lower(&self) -> (u64, u64) {
|
||||
self.expected_lower
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn persistent_destinations(&self) -> &[PathBuf] {
|
||||
&self.persistent_destinations
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RuntimeStateAnchorAuthority {
|
||||
lower: OwnedFd,
|
||||
persistent_anchor: OwnedFd,
|
||||
runtime_anchor: OwnedFd,
|
||||
merged_label: PathBuf,
|
||||
merged_descriptor: OwnedFd,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct RuntimeStateAnchorInherited<'a> {
|
||||
pub lower_fd: i32,
|
||||
pub lower_label: &'a Path,
|
||||
pub persistent_root_fd: i32,
|
||||
pub persistent_anchor_fd: i32,
|
||||
pub persistent_anchor_label: &'a Path,
|
||||
pub runtime_root_fd: i32,
|
||||
pub runtime_anchor_fd: i32,
|
||||
pub runtime_anchor_label: &'a Path,
|
||||
pub expected_lower: (u64, u64),
|
||||
}
|
||||
|
||||
impl RuntimeStateAnchorAuthority {
|
||||
/// Reopens inherited authorities without falling back to ambient configured paths.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing, replaced, or incorrectly typed descriptors and fixed children.
|
||||
pub fn from_inherited(
|
||||
inherited: RuntimeStateAnchorInherited<'_>,
|
||||
) -> Result<Self, StateAnchorError> {
|
||||
let retained_lower = reopen_inherited_directory(inherited.lower_fd)?;
|
||||
require_directory_descriptor_identity(&retained_lower, inherited.expected_lower)?;
|
||||
if !inherited.lower_label.is_absolute() {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"prepared lower label is not absolute",
|
||||
));
|
||||
}
|
||||
let lower = open_exact_directory_path(inherited.lower_label, inherited.expected_lower)?;
|
||||
let persistent_root = reopen_inherited_directory(inherited.persistent_root_fd)?;
|
||||
let retained_persistent_anchor =
|
||||
reopen_inherited_directory(inherited.persistent_anchor_fd)?;
|
||||
if !inherited.persistent_anchor_label.is_absolute() {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"persistent State Anchor label is not absolute",
|
||||
));
|
||||
}
|
||||
let retained_persistent =
|
||||
rustix::fs::fstat(&retained_persistent_anchor).map_err(io::Error::from)?;
|
||||
let confined_persistent_anchor =
|
||||
open_directory(&persistent_root, Path::new("state-anchor"))?;
|
||||
require_directory_descriptor_identity(
|
||||
&confined_persistent_anchor,
|
||||
(retained_persistent.st_dev, retained_persistent.st_ino),
|
||||
)?;
|
||||
let persistent_anchor = open_exact_directory_path(
|
||||
inherited.persistent_anchor_label,
|
||||
(retained_persistent.st_dev, retained_persistent.st_ino),
|
||||
)?;
|
||||
let runtime_root = reopen_inherited_directory(inherited.runtime_root_fd)?;
|
||||
let retained_runtime_anchor = reopen_inherited_directory(inherited.runtime_anchor_fd)?;
|
||||
if !inherited.runtime_anchor_label.is_absolute() {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"runtime State Anchor label is not absolute",
|
||||
));
|
||||
}
|
||||
let confined_runtime_anchor = open_directory(&runtime_root, Path::new("state-anchor"))?;
|
||||
let retained = rustix::fs::fstat(&retained_runtime_anchor).map_err(io::Error::from)?;
|
||||
require_directory_descriptor_identity(
|
||||
&confined_runtime_anchor,
|
||||
(retained.st_dev, retained.st_ino),
|
||||
)?;
|
||||
let runtime_anchor = open_exact_directory_path(
|
||||
inherited.runtime_anchor_label,
|
||||
(retained.st_dev, retained.st_ino),
|
||||
)?;
|
||||
for name in ["upper", "work"] {
|
||||
drop(open_directory(&persistent_anchor, Path::new(name))?);
|
||||
}
|
||||
let merged_descriptor = open_directory(&runtime_anchor, Path::new("merged"))?;
|
||||
let merged_label = inherited.runtime_anchor_label.join("merged");
|
||||
Ok(Self {
|
||||
lower,
|
||||
persistent_anchor,
|
||||
runtime_anchor,
|
||||
merged_label,
|
||||
merged_descriptor,
|
||||
})
|
||||
}
|
||||
|
||||
/// Mounts the exact inherited prepared lower and anchors under RS child ownership.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses acquisition, acknowledgment, readiness, or authority failures.
|
||||
pub fn acquire<Stage, Acknowledge, Ready>(
|
||||
self,
|
||||
expected_parent: lazaret_core::identity::PidfsProcessIdentityV1,
|
||||
ownership: ProcessOwnershipContext,
|
||||
stage: Stage,
|
||||
acknowledge: Acknowledge,
|
||||
ready: Ready,
|
||||
) -> Result<RuntimeStateAnchorMount, StateAnchorError>
|
||||
where
|
||||
Stage: FnOnce(StagedProcessOwnership) -> io::Result<()>,
|
||||
Acknowledge: FnOnce(StagedProcessOwnership) -> io::Result<ProcessOwnershipCommitAck>,
|
||||
Ready: FnOnce(VerifiedStateAnchorReadiness) -> io::Result<()>,
|
||||
{
|
||||
if ownership.slot != HostProcessOwnershipSlot::StateAnchorDaemon {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"process slot is not State Anchor daemon",
|
||||
));
|
||||
}
|
||||
let baseline = mount_identity(&self.merged_descriptor)?;
|
||||
let lower_launch = inherit_for_exec(&self.lower)?;
|
||||
let persistent_launch = inherit_for_exec(&self.persistent_anchor)?;
|
||||
let target = TargetCommand::new("fuse-overlayfs")
|
||||
.argument("-f")
|
||||
.argument("-o")
|
||||
.argument(overlay_options(
|
||||
&descriptor_path(&lower_launch),
|
||||
&descriptor_child_path(&persistent_launch, "upper"),
|
||||
&descriptor_child_path(&persistent_launch, "work"),
|
||||
))
|
||||
.argument(&self.merged_label);
|
||||
let process =
|
||||
child_broker().acquire(expected_parent, ownership, &target, stage, acknowledge)?;
|
||||
let staged = StagedProcessOwnership::v1(
|
||||
ownership.domain_id,
|
||||
ownership.generation_id,
|
||||
ownership.coordinator_id,
|
||||
ownership.slot,
|
||||
process.identity(),
|
||||
);
|
||||
let deadline = Instant::now() + MOUNT_TIMEOUT;
|
||||
loop {
|
||||
if process.has_exited()? {
|
||||
child_broker().reap(process)?;
|
||||
return Err(StateAnchorError::DaemonExited);
|
||||
}
|
||||
let mounted = open_mounted_directory(&self.runtime_anchor, Path::new("merged"))?;
|
||||
let observed = mount_identity(&mounted)?;
|
||||
if observed.mount_id != baseline.mount_id {
|
||||
let readiness = VerifiedStateAnchorReadiness {
|
||||
staged,
|
||||
mount: observed,
|
||||
};
|
||||
if let Err(error) = ready(readiness) {
|
||||
process.send_signal(rustix::process::Signal::KILL)?;
|
||||
while !process.has_exited()? {
|
||||
thread::yield_now();
|
||||
}
|
||||
let _ = rootless_unmount_path(&self.merged_label);
|
||||
child_broker().reap(process)?;
|
||||
return Err(error.into());
|
||||
}
|
||||
return Ok(RuntimeStateAnchorMount {
|
||||
authority: self,
|
||||
process,
|
||||
mount: observed,
|
||||
merged: mounted,
|
||||
});
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
process.send_signal(rustix::process::Signal::KILL)?;
|
||||
while !process.has_exited()? {
|
||||
thread::yield_now();
|
||||
}
|
||||
let merged = open_mounted_directory(&self.runtime_anchor, Path::new("merged"))?;
|
||||
let observed = mount_identity(&merged)?;
|
||||
if observed.mount_id != baseline.mount_id {
|
||||
let _ = rootless_unmount_path(&self.merged_label);
|
||||
}
|
||||
child_broker().reap(process)?;
|
||||
return Err(StateAnchorError::MountTimeout);
|
||||
}
|
||||
thread::sleep(MOUNT_POLL_INTERVAL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RuntimeStateAnchorMount {
|
||||
authority: RuntimeStateAnchorAuthority,
|
||||
process: RetainedProcess,
|
||||
mount: MountIdentity,
|
||||
merged: OwnedFd,
|
||||
}
|
||||
|
||||
pub struct RuntimeStateAnchorAbsenceAuthority {
|
||||
runtime_anchor: OwnedFd,
|
||||
mount: MountIdentity,
|
||||
}
|
||||
|
||||
impl RuntimeStateAnchorAbsenceAuthority {
|
||||
/// Reopens the exact runtime anchor to produce a fresh post-unmount observation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an unreadable anchor or the continued presence of the retired mount.
|
||||
pub fn observe_absence(
|
||||
&self,
|
||||
expected_mount: MountIdentity,
|
||||
) -> Result<MountIdentity, StateAnchorError> {
|
||||
if expected_mount != self.mount {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
let observed = mount_identity(&open_directory(&self.runtime_anchor, Path::new("merged"))?)?;
|
||||
if observed == self.mount {
|
||||
Err(StateAnchorError::OwnershipMismatch)
|
||||
} else {
|
||||
Ok(observed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RuntimeStateAnchorMount {
|
||||
#[must_use]
|
||||
pub const fn mount_identity(&self) -> MountIdentity {
|
||||
self.mount
|
||||
}
|
||||
|
||||
/// Validates configured persistent subpaths beneath the exact mounted view.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses missing paths, symlinks, nested mounts, or mount replacement.
|
||||
pub fn require_persistent_destinations(
|
||||
&self,
|
||||
destinations: &[PathBuf],
|
||||
) -> Result<(), StateAnchorError> {
|
||||
if mount_identity(&self.merged)? != self.mount {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
for destination in destinations {
|
||||
if destination.as_os_str().is_empty()
|
||||
|| destination
|
||||
.components()
|
||||
.any(|component| !matches!(component, std::path::Component::Normal(_)))
|
||||
{
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"persistent destination is not normalized and relative",
|
||||
));
|
||||
}
|
||||
let descriptor = openat2(
|
||||
&self.merged,
|
||||
destination,
|
||||
OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
ResolveFlags::BENEATH
|
||||
| ResolveFlags::NO_MAGICLINKS
|
||||
| ResolveFlags::NO_SYMLINKS
|
||||
| ResolveFlags::NO_XDEV,
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
if !FileType::from_raw_mode(
|
||||
rustix::fs::fstat(&descriptor)
|
||||
.map_err(io::Error::from)?
|
||||
.st_mode,
|
||||
)
|
||||
.is_dir()
|
||||
{
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"persistent destination is not a directory",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Unmounts and reaps only the retained exact RS-owned State Anchor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses mount replacement, unmount failure, or exact child cleanup failure.
|
||||
pub fn unmount(self) -> Result<RuntimeStateAnchorAbsenceAuthority, StateAnchorError> {
|
||||
let Self {
|
||||
authority,
|
||||
process,
|
||||
mount,
|
||||
merged,
|
||||
} = self;
|
||||
let observed = mount_identity(&merged)?;
|
||||
if observed != mount {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
drop(merged);
|
||||
rootless_unmount_path(&authority.merged_label)?;
|
||||
let after = mount_identity(&open_directory(
|
||||
&authority.runtime_anchor,
|
||||
Path::new("merged"),
|
||||
)?)?;
|
||||
if after == mount {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
child_broker().reap(process)?;
|
||||
Ok(RuntimeStateAnchorAbsenceAuthority {
|
||||
runtime_anchor: authority.runtime_anchor,
|
||||
mount,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl VerifiedStateAnchorReadiness {
|
||||
#[must_use]
|
||||
pub const fn from_parts(staged: StagedProcessOwnership, mount: MountIdentity) -> Self {
|
||||
Self { staged, mount }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn staged(self) -> StagedProcessOwnership {
|
||||
self.staged
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn mount(self) -> MountIdentity {
|
||||
self.mount
|
||||
}
|
||||
}
|
||||
424
crates/lazaret-state-anchor/src/lib/mount.rs
Normal file
424
crates/lazaret-state-anchor/src/lib/mount.rs
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
pub struct FuseOverlayfsMount<'layout, 'configuration> {
|
||||
process: RetainedProcess,
|
||||
mount: MountIdentity,
|
||||
layout: &'layout StateAnchorLayout<'configuration>,
|
||||
}
|
||||
|
||||
impl<'layout, 'configuration> FuseOverlayfsMount<'layout, 'configuration> {
|
||||
/// Starts fuse-overlayfs through causal child acquisition and records staged/committed identity
|
||||
/// through caller-owned durable callbacks.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses changed layout or lower identity, callback failure, daemon loss, and mount timeout.
|
||||
pub fn acquire<Stage, Acknowledge, Ready>(
|
||||
layout: &'layout StateAnchorLayout<'configuration>,
|
||||
acquisition: StateAnchorAcquisition<'_>,
|
||||
stage: Stage,
|
||||
acknowledge: Acknowledge,
|
||||
ready: Ready,
|
||||
) -> Result<Self, StateAnchorError>
|
||||
where
|
||||
Stage: FnOnce(StagedProcessOwnership) -> io::Result<()>,
|
||||
Acknowledge: FnOnce(StagedProcessOwnership) -> io::Result<ProcessOwnershipCommitAck>,
|
||||
Ready: FnOnce(VerifiedStateAnchorReadiness) -> io::Result<()>,
|
||||
{
|
||||
if acquisition.ownership.slot != HostProcessOwnershipSlot::StateAnchorDaemon {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"process slot is not State Anchor daemon",
|
||||
));
|
||||
}
|
||||
layout.revalidate()?;
|
||||
let lower_descriptor =
|
||||
open_exact_directory_path(acquisition.lower, acquisition.expected_lower)?;
|
||||
let lower_launch = inherit_for_exec(&lower_descriptor)?;
|
||||
let persistent_launch = inherit_for_exec(&layout.persistent_anchor)?;
|
||||
let runtime_launch = inherit_for_exec(&layout.runtime_anchor)?;
|
||||
let lower_path = descriptor_path(&lower_launch);
|
||||
let upper_path = descriptor_child_path(&persistent_launch, "upper");
|
||||
let work_path = descriptor_child_path(&persistent_launch, "work");
|
||||
let merged_path = descriptor_child_path(&runtime_launch, "merged");
|
||||
let baseline = layout.current_merged_identity()?;
|
||||
if baseline != mount_identity(&layout.merged_descriptor)? {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
let options = overlay_options(&lower_path, &upper_path, &work_path);
|
||||
let target = TargetCommand::new("fuse-overlayfs")
|
||||
.argument("-f")
|
||||
.argument("-o")
|
||||
.argument(options)
|
||||
.argument(merged_path.as_os_str());
|
||||
let process = child_broker().acquire(
|
||||
acquisition.expected_parent,
|
||||
acquisition.ownership,
|
||||
&target,
|
||||
stage,
|
||||
acknowledge,
|
||||
)?;
|
||||
let staged = StagedProcessOwnership::v1(
|
||||
acquisition.ownership.domain_id,
|
||||
acquisition.ownership.generation_id,
|
||||
acquisition.ownership.coordinator_id,
|
||||
acquisition.ownership.slot,
|
||||
process.identity(),
|
||||
);
|
||||
let deadline = Instant::now() + MOUNT_TIMEOUT;
|
||||
loop {
|
||||
if process.has_exited()? {
|
||||
child_broker().reap(process)?;
|
||||
return Err(StateAnchorError::DaemonExited);
|
||||
}
|
||||
let observed = layout.current_merged_identity()?;
|
||||
if observed.mount_id != baseline.mount_id {
|
||||
let readiness = VerifiedStateAnchorReadiness {
|
||||
staged,
|
||||
mount: observed,
|
||||
};
|
||||
if let Err(error) = ready(readiness) {
|
||||
process.send_signal(rustix::process::Signal::KILL)?;
|
||||
while !process.has_exited()? {
|
||||
thread::yield_now();
|
||||
}
|
||||
rootless_unmount(layout)?;
|
||||
child_broker().reap(process)?;
|
||||
return Err(error.into());
|
||||
}
|
||||
return Ok(Self {
|
||||
process,
|
||||
mount: observed,
|
||||
layout,
|
||||
});
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
process.send_signal(rustix::process::Signal::KILL)?;
|
||||
while !process.has_exited()? {
|
||||
thread::yield_now();
|
||||
}
|
||||
if match layout.current_merged_identity() {
|
||||
Ok(identity) => identity.mount_id != baseline.mount_id,
|
||||
Err(_) => true,
|
||||
} {
|
||||
let _ = rootless_unmount(layout);
|
||||
}
|
||||
child_broker().reap(process)?;
|
||||
return Err(StateAnchorError::MountTimeout);
|
||||
}
|
||||
thread::sleep(MOUNT_POLL_INTERVAL);
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn mount_identity(&self) -> MountIdentity {
|
||||
self.mount
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn process_ownership(&self) -> PersistedHostProcessOwnership {
|
||||
PersistedHostProcessOwnership::committed(self.process.identity())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn process(&self) -> &RetainedProcess {
|
||||
&self.process
|
||||
}
|
||||
|
||||
/// Verifies and unmounts only the exact retained State Anchor, then reaps its daemon.
|
||||
///
|
||||
/// # Errors
|
||||
/// A missing, replaced, renamed, or remounted target is left untouched.
|
||||
pub fn unmount(self) -> Result<(), StateAnchorError> {
|
||||
self.layout.revalidate_authorities()?;
|
||||
require_mount_record(self.layout, self.mount)?;
|
||||
rootless_unmount(self.layout)?;
|
||||
if has_mount_record(self.layout, self.mount)? {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
self.layout.require_unmounted()?;
|
||||
child_broker().reap(self.process)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reaps the retained daemon after an independently verified external unmount.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses while the exact recorded mount remains or when exact reaping fails.
|
||||
pub fn finish_after_external_unmount(self) -> Result<(), StateAnchorError> {
|
||||
self.layout.revalidate_authorities()?;
|
||||
if has_mount_record(self.layout, self.mount)? {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
self.layout.require_unmounted()?;
|
||||
child_broker().reap(self.process)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies and unmounts an exact recorded mount without opening either durable store.
|
||||
///
|
||||
/// # Errors
|
||||
/// A missing or mismatched mount is left untouched.
|
||||
pub fn unmount_after_coordinator_loss(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
expected: MountIdentity,
|
||||
) -> Result<(), StateAnchorError> {
|
||||
layout.revalidate_authorities()?;
|
||||
require_mount_record(layout, expected)?;
|
||||
rootless_unmount(layout)?;
|
||||
if has_mount_record(layout, expected)? {
|
||||
return Err(StateAnchorError::OwnershipMismatch);
|
||||
}
|
||||
layout.require_unmounted()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Enters an exact live holder's private namespaces and unmounts only its recorded State Anchor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses descriptor transfer, pidfs, namespace, path, mount, helper, or acknowledgment drift.
|
||||
pub fn recover_private_state_anchor_mount(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
holder: &RetainedProcess,
|
||||
user_namespace: NamespaceIdentity,
|
||||
mount_namespace: NamespaceIdentity,
|
||||
expected_mount: MountIdentity,
|
||||
) -> Result<MountIdentity, StateAnchorError> {
|
||||
let merged = &layout.merged_label;
|
||||
let (coordinator, helper) = private_seqpacket_pair()?;
|
||||
send_descriptors(
|
||||
&coordinator,
|
||||
&[holder.as_fd(), layout.runtime_anchor.as_fd()],
|
||||
)?;
|
||||
let authority = holder.identity().authority();
|
||||
let target = TargetCommand::new("/proc/self/exe")
|
||||
.argument(PRIVATE_RECOVERY_ROLE)
|
||||
.argument(authority.pidfs_st_dev.get().to_string())
|
||||
.argument(authority.pidfs_st_ino.get().to_string())
|
||||
.argument(user_namespace.device.get().to_string())
|
||||
.argument(user_namespace.inode.get().to_string())
|
||||
.argument(mount_namespace.device.get().to_string())
|
||||
.argument(mount_namespace.inode.get().to_string())
|
||||
.argument(expected_mount.mount_id.get().to_string())
|
||||
.argument(expected_mount.device.get().to_string())
|
||||
.argument(expected_mount.inode.get().to_string())
|
||||
.argument(merged.as_os_str());
|
||||
child_broker().run_transient_with_private_stdout(&target, helper)?;
|
||||
let observed: MountIdentity = receive_packet(coordinator)?;
|
||||
if observed == expected_mount {
|
||||
Err(StateAnchorError::OwnershipMismatch)
|
||||
} else {
|
||||
Ok(observed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatches the bounded private-namespace State Anchor recovery helper.
|
||||
#[must_use]
|
||||
pub fn dispatch_private_state_anchor_recovery(arguments: &[OsString]) -> Option<io::Result<()>> {
|
||||
if arguments
|
||||
.first()
|
||||
.is_none_or(|arg| arg != PRIVATE_RECOVERY_ROLE)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(run_private_state_anchor_recovery(arguments))
|
||||
}
|
||||
|
||||
fn run_private_state_anchor_recovery(arguments: &[OsString]) -> io::Result<()> {
|
||||
if arguments.len() != 11 {
|
||||
return Err(io::Error::other(
|
||||
"private State Anchor recovery arguments are malformed",
|
||||
));
|
||||
}
|
||||
let values = arguments[1..10]
|
||||
.iter()
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.ok_or_else(|| io::Error::other("private recovery integer is not UTF-8"))?
|
||||
.parse::<u64>()
|
||||
.map_err(|_| io::Error::other("private recovery integer is invalid"))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()?;
|
||||
let expected_pidfs = (values[0], values[1]);
|
||||
let expected_user = NamespaceIdentity {
|
||||
device: DeviceId::new(values[2]),
|
||||
inode: InodeId::new(values[3]),
|
||||
};
|
||||
let expected_mount_namespace = NamespaceIdentity {
|
||||
device: DeviceId::new(values[4]),
|
||||
inode: InodeId::new(values[5]),
|
||||
};
|
||||
let expected_mount = MountIdentity {
|
||||
mount_id: lazaret_core::identity::MountId::new(values[6]),
|
||||
device: DeviceId::new(values[7]),
|
||||
inode: InodeId::new(values[8]),
|
||||
};
|
||||
let merged = PathBuf::from(&arguments[10]);
|
||||
let stdout = io::stdout();
|
||||
let mut descriptors = receive_descriptors(&stdout, 2)?;
|
||||
let runtime_anchor = descriptors
|
||||
.pop()
|
||||
.ok_or_else(|| io::Error::other("runtime anchor descriptor is absent"))?;
|
||||
let pidfd = descriptors
|
||||
.pop()
|
||||
.ok_or_else(|| io::Error::other("pidfd descriptor is absent"))?;
|
||||
let mut operations = KernelPrivateRecovery {
|
||||
pidfd,
|
||||
runtime_anchor,
|
||||
expected_pidfs,
|
||||
merged,
|
||||
reopened_parent: None,
|
||||
};
|
||||
let observed = execute_private_recovery(
|
||||
&mut operations,
|
||||
expected_user,
|
||||
expected_mount_namespace,
|
||||
expected_mount,
|
||||
)?;
|
||||
send_packet(stdout, &observed)
|
||||
}
|
||||
|
||||
trait PrivateRecoveryOperations {
|
||||
fn validate_pidfs(&mut self) -> io::Result<()>;
|
||||
fn enter_user_namespace(&mut self) -> io::Result<()>;
|
||||
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity>;
|
||||
fn enter_mount_namespace(&mut self) -> io::Result<()>;
|
||||
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity>;
|
||||
fn verify_mount(&mut self) -> io::Result<MountIdentity>;
|
||||
fn unmount(&mut self) -> io::Result<()>;
|
||||
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity>;
|
||||
}
|
||||
|
||||
fn execute_private_recovery(
|
||||
operations: &mut impl PrivateRecoveryOperations,
|
||||
expected_user: NamespaceIdentity,
|
||||
expected_mount_namespace: NamespaceIdentity,
|
||||
expected_mount: MountIdentity,
|
||||
) -> io::Result<MountIdentity> {
|
||||
operations
|
||||
.validate_pidfs()
|
||||
.map_err(|error| io::Error::other(format!("validate pidfs: {error}")))?;
|
||||
operations
|
||||
.enter_user_namespace()
|
||||
.map_err(|error| io::Error::other(format!("enter user namespace: {error}")))?;
|
||||
if operations
|
||||
.observe_user_namespace()
|
||||
.map_err(|error| io::Error::other(format!("observe user namespace: {error}")))?
|
||||
!= expected_user
|
||||
{
|
||||
return Err(io::Error::other("private recovery user namespace mismatch"));
|
||||
}
|
||||
operations
|
||||
.enter_mount_namespace()
|
||||
.map_err(|error| io::Error::other(format!("enter mount namespace: {error}")))?;
|
||||
if operations
|
||||
.observe_mount_namespace()
|
||||
.map_err(|error| io::Error::other(format!("observe mount namespace: {error}")))?
|
||||
!= expected_mount_namespace
|
||||
{
|
||||
return Err(io::Error::other(
|
||||
"private recovery mount namespace mismatch",
|
||||
));
|
||||
}
|
||||
if operations
|
||||
.verify_mount()
|
||||
.map_err(|error| io::Error::other(format!("verify mount: {error}")))?
|
||||
!= expected_mount
|
||||
{
|
||||
return Err(io::Error::other("private recovery mount identity mismatch"));
|
||||
}
|
||||
operations
|
||||
.unmount()
|
||||
.map_err(|error| io::Error::other(format!("unmount: {error}")))?;
|
||||
let observed = operations
|
||||
.observe_after_unmount()
|
||||
.map_err(|error| io::Error::other(format!("observe after unmount: {error}")))?;
|
||||
if observed == expected_mount {
|
||||
Err(io::Error::other("private recovery mount remains present"))
|
||||
} else {
|
||||
Ok(observed)
|
||||
}
|
||||
}
|
||||
|
||||
struct KernelPrivateRecovery {
|
||||
pidfd: OwnedFd,
|
||||
runtime_anchor: OwnedFd,
|
||||
expected_pidfs: (u64, u64),
|
||||
merged: PathBuf,
|
||||
reopened_parent: Option<OwnedFd>,
|
||||
}
|
||||
|
||||
impl PrivateRecoveryOperations for KernelPrivateRecovery {
|
||||
fn validate_pidfs(&mut self) -> io::Result<()> {
|
||||
require_pidfs_magic(rustix::fs::fstatfs(&self.pidfd)?.f_type as i64)?;
|
||||
let stat = rustix::fs::fstat(&self.pidfd)?;
|
||||
if (stat.st_dev, stat.st_ino) == self.expected_pidfs {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::other(
|
||||
"private recovery pidfs authority mismatch",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn enter_user_namespace(&mut self) -> io::Result<()> {
|
||||
move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::USER)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
|
||||
namespace_identity(Path::new("/proc/self/ns/user"))
|
||||
}
|
||||
|
||||
fn enter_mount_namespace(&mut self) -> io::Result<()> {
|
||||
move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::MOUNT)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
|
||||
namespace_identity(Path::new("/proc/self/ns/mnt"))
|
||||
}
|
||||
|
||||
fn verify_mount(&mut self) -> io::Result<MountIdentity> {
|
||||
let parent_path = self
|
||||
.merged
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("private recovery merged path has no parent"))?;
|
||||
let reopened_parent = openat2(
|
||||
rustix::fs::CWD,
|
||||
parent_path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
|
||||
)?;
|
||||
let retained = rustix::fs::fstat(&self.runtime_anchor)?;
|
||||
let reopened = rustix::fs::fstat(&reopened_parent)?;
|
||||
if (retained.st_dev, retained.st_ino) != (reopened.st_dev, reopened.st_ino) {
|
||||
return Err(io::Error::other("private recovery runtime anchor mismatch"));
|
||||
}
|
||||
let mounted = open_final_mount_directory(&reopened_parent, self.merged_name()?)?;
|
||||
let identity = mount_identity(&mounted)?;
|
||||
self.reopened_parent = Some(reopened_parent);
|
||||
Ok(identity)
|
||||
}
|
||||
|
||||
fn unmount(&mut self) -> io::Result<()> {
|
||||
rustix::mount::unmount(&self.merged, rustix::mount::UnmountFlags::empty())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
|
||||
let parent = self
|
||||
.reopened_parent
|
||||
.as_ref()
|
||||
.ok_or_else(|| io::Error::other("private recovery parent was not verified"))?;
|
||||
mount_identity(&open_final_mount_directory(parent, self.merged_name()?)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl KernelPrivateRecovery {
|
||||
fn merged_name(&self) -> io::Result<&Path> {
|
||||
self.merged
|
||||
.file_name()
|
||||
.map(Path::new)
|
||||
.ok_or_else(|| io::Error::other("private recovery merged path has no name"))
|
||||
}
|
||||
}
|
||||
559
crates/lazaret-state-anchor/src/lib/storage.rs
Normal file
559
crates/lazaret-state-anchor/src/lib/storage.rs
Normal file
|
|
@ -0,0 +1,559 @@
|
|||
/// Independently verifies that no mount is attached at the exact merged path.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses unreadable mount information or an inaccessible merged directory.
|
||||
pub fn require_mount_absent(layout: &StateAnchorLayout<'_>) -> Result<(), StateAnchorError> {
|
||||
layout.require_unmounted()
|
||||
}
|
||||
|
||||
/// Freshly observes absence of one exact retired mount in the caller's current namespace.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses unreadable layout evidence or continued presence of the exact mount.
|
||||
pub fn observe_mount_absence(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
expected: MountIdentity,
|
||||
) -> Result<MountIdentity, StateAnchorError> {
|
||||
let observed = layout.current_merged_identity()?;
|
||||
if observed == expected {
|
||||
Err(StateAnchorError::OwnershipMismatch)
|
||||
} else {
|
||||
Ok(observed)
|
||||
}
|
||||
}
|
||||
|
||||
fn inspect_directory(
|
||||
directory: impl AsFd,
|
||||
prefix: &Path,
|
||||
entries: &mut Vec<StateAnchorEntry>,
|
||||
) -> Result<(), StateAnchorError> {
|
||||
for name in directory_names(&directory)? {
|
||||
let descriptor = openat2(
|
||||
directory.as_fd(),
|
||||
Path::new(&name),
|
||||
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
confined_resolve_flags(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let metadata = rustix::fs::fstat(&descriptor).map_err(io::Error::from)?;
|
||||
let kind = FileType::from_raw_mode(metadata.st_mode);
|
||||
let relative = prefix.join(&name);
|
||||
let entry_kind = if kind.is_dir() {
|
||||
StateAnchorEntryKind::Directory
|
||||
} else if kind.is_file() {
|
||||
StateAnchorEntryKind::RegularFile
|
||||
} else if kind.is_symlink() {
|
||||
StateAnchorEntryKind::Symlink
|
||||
} else if kind.is_char_device()
|
||||
&& rustix::fs::major(metadata.st_rdev) == 0
|
||||
&& rustix::fs::minor(metadata.st_rdev) == 0
|
||||
{
|
||||
StateAnchorEntryKind::Whiteout
|
||||
} else {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"upper layer contains an unsupported entry type",
|
||||
));
|
||||
};
|
||||
entries.push(StateAnchorEntry {
|
||||
path: relative.as_os_str().as_bytes().to_vec(),
|
||||
kind: entry_kind,
|
||||
mode: metadata.st_mode,
|
||||
uid: metadata.st_uid,
|
||||
gid: metadata.st_gid,
|
||||
size: u64::try_from(metadata.st_size).map_err(|_| {
|
||||
StateAnchorError::InvalidLayout("upper-layer entry has a negative size")
|
||||
})?,
|
||||
});
|
||||
if kind.is_dir() {
|
||||
let child = open_directory(&directory, Path::new(&name))?;
|
||||
inspect_directory(child, &relative, entries)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_or_create_quarantine(parent: impl AsFd) -> Result<OwnedFd, StateAnchorError> {
|
||||
match mkdirat(parent.as_fd(), RESET_QUARANTINE, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) => rustix::fs::fsync(parent.as_fd()).map_err(io::Error::from)?,
|
||||
Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(io::Error::from(error).into()),
|
||||
}
|
||||
open_directory(parent, Path::new(RESET_QUARANTINE))
|
||||
}
|
||||
|
||||
fn move_all_entries(upper: impl AsFd, quarantine: impl AsFd) -> Result<(), StateAnchorError> {
|
||||
for name in directory_names(&upper)? {
|
||||
move_if_present(&upper, Path::new(&name), &quarantine, Path::new(&name))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn move_selected_entries(
|
||||
upper: impl AsFd,
|
||||
quarantine: impl AsFd,
|
||||
relative: &Path,
|
||||
) -> Result<(), StateAnchorError> {
|
||||
let leaf = relative.file_name().ok_or(StateAnchorError::InvalidLayout(
|
||||
"reset target has no final component",
|
||||
))?;
|
||||
let parent = open_optional_parent(&upper, relative.parent().unwrap_or(Path::new("")))?;
|
||||
let Some(parent) = parent else {
|
||||
return Ok(());
|
||||
};
|
||||
move_if_present(&parent, Path::new(leaf), &quarantine, Path::new("entry"))?;
|
||||
let mut whiteout = OsString::from(".wh.");
|
||||
whiteout.push(leaf);
|
||||
move_if_present(
|
||||
&parent,
|
||||
Path::new(&whiteout),
|
||||
&quarantine,
|
||||
Path::new("whiteout"),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_optional_parent(root: impl AsFd, path: &Path) -> Result<Option<OwnedFd>, StateAnchorError> {
|
||||
if path.as_os_str().is_empty() {
|
||||
return rustix::io::dup(root.as_fd())
|
||||
.map(Some)
|
||||
.map_err(io::Error::from)
|
||||
.map_err(StateAnchorError::from);
|
||||
}
|
||||
match openat2(
|
||||
root.as_fd(),
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
confined_resolve_flags(),
|
||||
) {
|
||||
Ok(directory) => Ok(Some(directory)),
|
||||
Err(rustix::io::Errno::NOENT) => Ok(None),
|
||||
Err(error) => Err(io::Error::from(error).into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn move_if_present(
|
||||
source_parent: impl AsFd,
|
||||
source: &Path,
|
||||
destination_parent: impl AsFd,
|
||||
destination: &Path,
|
||||
) -> Result<(), StateAnchorError> {
|
||||
let source_exists = entry_exists(source_parent.as_fd(), source)?;
|
||||
let destination_exists = entry_exists(destination_parent.as_fd(), destination)?;
|
||||
match (source_exists, destination_exists) {
|
||||
(false, false | true) => Ok(()),
|
||||
(true, true) => Err(StateAnchorError::InvalidLayout(
|
||||
"reset source and quarantine entry both exist",
|
||||
)),
|
||||
(true, false) => {
|
||||
renameat_with(
|
||||
source_parent.as_fd(),
|
||||
source,
|
||||
destination_parent.as_fd(),
|
||||
destination,
|
||||
RenameFlags::NOREPLACE,
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
rustix::fs::fsync(source_parent.as_fd()).map_err(io::Error::from)?;
|
||||
rustix::fs::fsync(destination_parent.as_fd()).map_err(io::Error::from)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn entry_exists(parent: impl AsFd, path: &Path) -> Result<bool, StateAnchorError> {
|
||||
match statat(parent.as_fd(), path, AtFlags::SYMLINK_NOFOLLOW) {
|
||||
Ok(_) => Ok(true),
|
||||
Err(rustix::io::Errno::NOENT) => Ok(false),
|
||||
Err(error) => Err(io::Error::from(error).into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_directory_contents(directory: impl AsFd) -> Result<(), StateAnchorError> {
|
||||
for name in directory_names(&directory)? {
|
||||
let path = Path::new(&name);
|
||||
let descriptor = openat2(
|
||||
directory.as_fd(),
|
||||
path,
|
||||
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
confined_resolve_flags(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let kind = FileType::from_raw_mode(
|
||||
rustix::fs::fstat(&descriptor)
|
||||
.map_err(io::Error::from)?
|
||||
.st_mode,
|
||||
);
|
||||
if kind.is_dir() {
|
||||
let child = open_directory(&directory, path)?;
|
||||
remove_directory_contents(&child)?;
|
||||
unlinkat(directory.as_fd(), path, AtFlags::REMOVEDIR).map_err(io::Error::from)?;
|
||||
} else {
|
||||
unlinkat(directory.as_fd(), path, AtFlags::empty()).map_err(io::Error::from)?;
|
||||
}
|
||||
rustix::fs::fsync(directory.as_fd()).map_err(io::Error::from)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn directory_names(directory: impl AsFd) -> Result<Vec<OsString>, StateAnchorError> {
|
||||
let descriptor = rustix::io::dup(directory.as_fd()).map_err(io::Error::from)?;
|
||||
let mut buffer = [std::mem::MaybeUninit::uninit(); 8192];
|
||||
let mut raw = RawDir::new(descriptor, &mut buffer);
|
||||
let mut names = Vec::new();
|
||||
while let Some(entry) = raw.next() {
|
||||
let entry = entry.map_err(io::Error::from)?;
|
||||
let bytes = entry.file_name().to_bytes();
|
||||
if bytes != b"." && bytes != b".." {
|
||||
names.push(OsString::from_vec(bytes.to_vec()));
|
||||
}
|
||||
}
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
fn directory_has_entries(directory: impl AsFd) -> Result<bool, StateAnchorError> {
|
||||
Ok(!directory_names(directory)?.is_empty())
|
||||
}
|
||||
|
||||
fn validate_relative_path(path: &Path) -> Result<(), StateAnchorError> {
|
||||
if path.as_os_str().is_empty()
|
||||
|| path
|
||||
.components()
|
||||
.any(|component| !matches!(component, Component::Normal(_)))
|
||||
{
|
||||
Err(StateAnchorError::InvalidLayout(
|
||||
"reset target is not a normalized relative path",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn create_directory(parent: impl AsFd, name: &str) -> Result<(), StateAnchorError> {
|
||||
match mkdirat(parent.as_fd(), name, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(io::Error::from(error).into()),
|
||||
}
|
||||
drop(open_directory(parent, Path::new(name))?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchorError> {
|
||||
let descriptor = openat2(
|
||||
parent.as_fd(),
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
ResolveFlags::BENEATH
|
||||
| ResolveFlags::NO_SYMLINKS
|
||||
| ResolveFlags::NO_MAGICLINKS
|
||||
| ResolveFlags::NO_XDEV,
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let metadata = rustix::fs::fstat(&descriptor).map_err(io::Error::from)?;
|
||||
if !FileType::from_raw_mode(metadata.st_mode).is_dir() {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"fixed path is not a directory",
|
||||
));
|
||||
}
|
||||
Ok(descriptor)
|
||||
}
|
||||
|
||||
fn open_final_mount_directory(parent: impl AsFd, path: &Path) -> io::Result<OwnedFd> {
|
||||
let mut components = path.components();
|
||||
if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() {
|
||||
return Err(io::Error::other(
|
||||
"final mount path is not one normal component",
|
||||
));
|
||||
}
|
||||
open_mounted_directory(parent, path).map_err(io::Error::other)
|
||||
}
|
||||
|
||||
fn open_mounted_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchorError> {
|
||||
let descriptor = openat2(
|
||||
parent.as_fd(),
|
||||
path,
|
||||
OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS | ResolveFlags::NO_MAGICLINKS,
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
Ok(descriptor)
|
||||
}
|
||||
|
||||
fn open_exact_directory_path(
|
||||
path: &Path,
|
||||
expected: (u64, u64),
|
||||
) -> Result<OwnedFd, StateAnchorError> {
|
||||
let descriptor = openat2(
|
||||
rustix::fs::CWD,
|
||||
path,
|
||||
OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
ResolveFlags::NO_SYMLINKS | ResolveFlags::NO_MAGICLINKS,
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
require_directory_descriptor_identity(&descriptor, expected)?;
|
||||
Ok(descriptor)
|
||||
}
|
||||
|
||||
fn require_directory_descriptor_identity(
|
||||
descriptor: impl AsFd,
|
||||
expected: (u64, u64),
|
||||
) -> Result<(), StateAnchorError> {
|
||||
let metadata = rustix::fs::fstat(descriptor).map_err(io::Error::from)?;
|
||||
if FileType::from_raw_mode(metadata.st_mode).is_dir()
|
||||
&& (metadata.st_dev, metadata.st_ino) == expected
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StateAnchorError::OwnershipMismatch)
|
||||
}
|
||||
}
|
||||
|
||||
fn require_same_descriptor(
|
||||
observed: impl AsFd,
|
||||
expected: impl AsFd,
|
||||
) -> Result<(), StateAnchorError> {
|
||||
let observed = mount_identity(observed)?;
|
||||
let expected = mount_identity(expected)?;
|
||||
if observed == expected {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StateAnchorError::OwnershipMismatch)
|
||||
}
|
||||
}
|
||||
|
||||
fn create_regular_file(parent: impl AsFd, name: &str) -> Result<(), StateAnchorError> {
|
||||
match openat2(
|
||||
parent.as_fd(),
|
||||
name,
|
||||
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::from_raw_mode(0o600),
|
||||
confined_resolve_flags(),
|
||||
) {
|
||||
Ok(descriptor) => File::from(descriptor).sync_all()?,
|
||||
Err(rustix::io::Errno::EXIST) => drop(open_regular_file(parent, name)?),
|
||||
Err(error) => return Err(io::Error::from(error).into()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_regular_file(parent: impl AsFd, name: &str) -> Result<OwnedFd, StateAnchorError> {
|
||||
let descriptor = openat2(
|
||||
parent.as_fd(),
|
||||
name,
|
||||
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
confined_resolve_flags(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let metadata = rustix::fs::fstat(&descriptor).map_err(io::Error::from)?;
|
||||
if FileType::from_raw_mode(metadata.st_mode).is_file() {
|
||||
Ok(descriptor)
|
||||
} else {
|
||||
Err(StateAnchorError::InvalidLayout(
|
||||
"protected runtime path is not a regular file",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn atomic_replace_at(parent: impl AsFd, name: &str, bytes: &[u8]) -> Result<(), StateAnchorError> {
|
||||
let temporary_name = loop {
|
||||
let sequence = TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
||||
let candidate =
|
||||
OsString::from(format!(".{name}.lazaret-{}-{sequence}", std::process::id()));
|
||||
match openat2(
|
||||
parent.as_fd(),
|
||||
Path::new(&candidate),
|
||||
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::from_raw_mode(0o600),
|
||||
confined_resolve_flags(),
|
||||
) {
|
||||
Ok(descriptor) => {
|
||||
let mut file = File::from(descriptor);
|
||||
if let Err(error) = file.write_all(bytes).and_then(|()| file.sync_all()) {
|
||||
let _ = unlinkat(parent.as_fd(), Path::new(&candidate), AtFlags::empty());
|
||||
return Err(error.into());
|
||||
}
|
||||
break candidate;
|
||||
}
|
||||
Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(io::Error::from(error).into()),
|
||||
}
|
||||
};
|
||||
let result = renameat(
|
||||
parent.as_fd(),
|
||||
Path::new(&temporary_name),
|
||||
parent.as_fd(),
|
||||
name,
|
||||
)
|
||||
.and_then(|()| rustix::fs::fsync(parent.as_fd()));
|
||||
if result.is_err() {
|
||||
let _ = unlinkat(parent.as_fd(), Path::new(&temporary_name), AtFlags::empty());
|
||||
}
|
||||
result
|
||||
.map_err(io::Error::from)
|
||||
.map_err(StateAnchorError::from)
|
||||
}
|
||||
|
||||
const fn confined_resolve_flags() -> ResolveFlags {
|
||||
ResolveFlags::BENEATH
|
||||
.union(ResolveFlags::NO_SYMLINKS)
|
||||
.union(ResolveFlags::NO_MAGICLINKS)
|
||||
.union(ResolveFlags::NO_XDEV)
|
||||
}
|
||||
|
||||
fn descriptor_path(descriptor: impl AsFd) -> PathBuf {
|
||||
PathBuf::from(format!("/proc/self/fd/{}", descriptor.as_fd().as_raw_fd()))
|
||||
}
|
||||
|
||||
fn descriptor_child_path(descriptor: impl AsFd, child: &str) -> PathBuf {
|
||||
descriptor_path(descriptor).join(child)
|
||||
}
|
||||
|
||||
fn inherit_for_exec(descriptor: impl AsFd) -> Result<OwnedFd, StateAnchorError> {
|
||||
dup(descriptor).map_err(io::Error::from).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn reopen_inherited_directory(descriptor: i32) -> Result<OwnedFd, StateAnchorError> {
|
||||
if descriptor < 0 {
|
||||
return Err(StateAnchorError::InvalidLayout(
|
||||
"inherited directory descriptor is invalid",
|
||||
));
|
||||
}
|
||||
let file = File::open(format!("/proc/self/fd/{descriptor}"))?;
|
||||
let descriptor: OwnedFd = file.into();
|
||||
let metadata = rustix::fs::fstat(&descriptor).map_err(io::Error::from)?;
|
||||
if FileType::from_raw_mode(metadata.st_mode).is_dir() {
|
||||
Ok(descriptor)
|
||||
} else {
|
||||
Err(StateAnchorError::InvalidLayout(
|
||||
"inherited authority is not a directory",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn overlay_options(lower: &Path, upper: &Path, work: &Path) -> OsString {
|
||||
let mut bytes = b"lowerdir=".to_vec();
|
||||
append_fuse_option_path(&mut bytes, lower);
|
||||
bytes.extend_from_slice(b",upperdir=");
|
||||
append_fuse_option_path(&mut bytes, upper);
|
||||
bytes.extend_from_slice(b",workdir=");
|
||||
append_fuse_option_path(&mut bytes, work);
|
||||
OsString::from_vec(bytes)
|
||||
}
|
||||
|
||||
fn append_fuse_option_path(target: &mut Vec<u8>, path: &Path) {
|
||||
for byte in path.as_os_str().as_bytes() {
|
||||
if *byte == b':' {
|
||||
target.push(b'\\');
|
||||
target.push(b'\\');
|
||||
} else if *byte == b',' {
|
||||
target.push(b'\\');
|
||||
}
|
||||
target.push(*byte);
|
||||
}
|
||||
}
|
||||
|
||||
fn rootless_unmount(layout: &StateAnchorLayout<'_>) -> Result<(), StateAnchorError> {
|
||||
rootless_unmount_anchor(&layout.runtime_anchor)
|
||||
}
|
||||
|
||||
fn rootless_unmount_anchor(runtime_anchor: impl AsFd) -> Result<(), StateAnchorError> {
|
||||
let inherited_anchor = inherit_for_exec(runtime_anchor)?;
|
||||
let path = descriptor_child_path(&inherited_anchor, "merged");
|
||||
rootless_unmount_path(&path)
|
||||
}
|
||||
|
||||
fn rootless_unmount_path(path: &Path) -> Result<(), StateAnchorError> {
|
||||
child_broker()
|
||||
.run_transient(
|
||||
&TargetCommand::new("fusermount3")
|
||||
.argument("-u")
|
||||
.argument(path),
|
||||
)
|
||||
.map(drop)
|
||||
.map_err(StateAnchorError::from)
|
||||
}
|
||||
|
||||
fn require_mount_record(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
expected: MountIdentity,
|
||||
) -> Result<(), StateAnchorError> {
|
||||
if has_mount_record(layout, expected)? {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StateAnchorError::OwnershipMismatch)
|
||||
}
|
||||
}
|
||||
|
||||
fn has_mount_record(
|
||||
layout: &StateAnchorLayout<'_>,
|
||||
expected: MountIdentity,
|
||||
) -> Result<bool, StateAnchorError> {
|
||||
let expected_path = std::fs::read_link(descriptor_path(&layout.runtime_anchor))?
|
||||
.join("merged")
|
||||
.into_os_string()
|
||||
.into_vec();
|
||||
for line in std::fs::read("/proc/self/mountinfo")?.split(|byte| *byte == b'\n') {
|
||||
let fields = line.split(|byte| *byte == b' ').take(5).collect::<Vec<_>>();
|
||||
if fields.len() != 5 {
|
||||
continue;
|
||||
}
|
||||
let Some(mount_id) = parse_decimal(fields[0]) else {
|
||||
continue;
|
||||
};
|
||||
let Some((major, minor)) = parse_device(fields[2]) else {
|
||||
continue;
|
||||
};
|
||||
if decode_mount_field(fields[4]) == expected_path
|
||||
&& mount_id == expected.mount_id.get()
|
||||
&& rustix::fs::makedev(major, minor) == expected.device.get()
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn parse_decimal(bytes: &[u8]) -> Option<u64> {
|
||||
std::str::from_utf8(bytes).ok()?.parse().ok()
|
||||
}
|
||||
|
||||
fn parse_device(bytes: &[u8]) -> Option<(u32, u32)> {
|
||||
let separator = bytes.iter().position(|byte| *byte == b':')?;
|
||||
let (major, minor) = (&bytes[..separator], &bytes[separator + 1..]);
|
||||
Some((
|
||||
u32::try_from(parse_decimal(major)?).ok()?,
|
||||
u32::try_from(parse_decimal(minor)?).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
fn decode_mount_field(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut decoded = Vec::with_capacity(bytes.len());
|
||||
let mut index = 0;
|
||||
while index < bytes.len() {
|
||||
if bytes[index] == b'\\' && index + 3 < bytes.len() {
|
||||
let digits = &bytes[index + 1..index + 4];
|
||||
if digits.iter().all(|digit| matches!(digit, b'0'..=b'7')) {
|
||||
decoded.push((digits[0] - b'0') * 64 + (digits[1] - b'0') * 8 + digits[2] - b'0');
|
||||
index += 4;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
decoded.push(bytes[index]);
|
||||
index += 1;
|
||||
}
|
||||
decoded
|
||||
}
|
||||
|
||||
fn path_from_host(path: &HostPath) -> PathBuf {
|
||||
PathBuf::from(OsString::from_vec(path.as_bytes().to_vec()))
|
||||
}
|
||||
|
||||
fn host_path(path: &Path) -> HostPath {
|
||||
HostPath::new(path.as_os_str().as_bytes().to_vec())
|
||||
}
|
||||
196
crates/lazaret-state-anchor/src/lib/tests.rs
Normal file
196
crates/lazaret-state-anchor/src/lib/tests.rs
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
#[cfg(test)]
|
||||
mod recovery_tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
struct FailpointRecovery {
|
||||
fail_at: Option<usize>,
|
||||
boundary: usize,
|
||||
events: Vec<&'static str>,
|
||||
user: NamespaceIdentity,
|
||||
mount_namespace: NamespaceIdentity,
|
||||
mount: MountIdentity,
|
||||
observed: MountIdentity,
|
||||
}
|
||||
|
||||
impl FailpointRecovery {
|
||||
fn boundary(&mut self, name: &'static str) -> io::Result<()> {
|
||||
self.boundary += 1;
|
||||
self.events.push(name);
|
||||
if self.fail_at == Some(self.boundary) {
|
||||
Err(io::Error::other("injected recovery boundary failure"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PrivateRecoveryOperations for FailpointRecovery {
|
||||
fn validate_pidfs(&mut self) -> io::Result<()> {
|
||||
self.boundary("pidfs")
|
||||
}
|
||||
|
||||
fn enter_user_namespace(&mut self) -> io::Result<()> {
|
||||
self.boundary("enter-user")
|
||||
}
|
||||
|
||||
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
|
||||
self.boundary("observe-user")?;
|
||||
Ok(self.user)
|
||||
}
|
||||
|
||||
fn enter_mount_namespace(&mut self) -> io::Result<()> {
|
||||
self.boundary("enter-mount")
|
||||
}
|
||||
|
||||
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
|
||||
self.boundary("observe-mount")?;
|
||||
Ok(self.mount_namespace)
|
||||
}
|
||||
|
||||
fn verify_mount(&mut self) -> io::Result<MountIdentity> {
|
||||
self.boundary("verify-mount")?;
|
||||
Ok(self.mount)
|
||||
}
|
||||
|
||||
fn unmount(&mut self) -> io::Result<()> {
|
||||
self.boundary("unmount")
|
||||
}
|
||||
|
||||
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
|
||||
self.boundary("observe-absence")?;
|
||||
Ok(self.observed)
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(fail_at: Option<usize>) -> FailpointRecovery {
|
||||
FailpointRecovery {
|
||||
fail_at,
|
||||
boundary: 0,
|
||||
events: Vec::new(),
|
||||
user: NamespaceIdentity {
|
||||
device: DeviceId::new(1),
|
||||
inode: InodeId::new(2),
|
||||
},
|
||||
mount_namespace: NamespaceIdentity {
|
||||
device: DeviceId::new(3),
|
||||
inode: InodeId::new(4),
|
||||
},
|
||||
mount: MountIdentity {
|
||||
mount_id: lazaret_core::identity::MountId::new(5),
|
||||
device: DeviceId::new(6),
|
||||
inode: InodeId::new(7),
|
||||
},
|
||||
observed: MountIdentity {
|
||||
mount_id: lazaret_core::identity::MountId::new(8),
|
||||
device: DeviceId::new(6),
|
||||
inode: InodeId::new(7),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_recovery_failpoints_refuse_before_every_later_boundary() {
|
||||
for boundary in 1..=8 {
|
||||
let mut operations = fixture(Some(boundary));
|
||||
let expected_user = operations.user;
|
||||
let expected_mount_namespace = operations.mount_namespace;
|
||||
let expected_mount = operations.mount;
|
||||
let result = execute_private_recovery(
|
||||
&mut operations,
|
||||
expected_user,
|
||||
expected_mount_namespace,
|
||||
expected_mount,
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert_eq!(operations.events.len(), boundary);
|
||||
}
|
||||
let mut operations = fixture(None);
|
||||
let expected_user = operations.user;
|
||||
let expected_mount_namespace = operations.mount_namespace;
|
||||
let expected_mount = operations.mount;
|
||||
assert_eq!(
|
||||
execute_private_recovery(
|
||||
&mut operations,
|
||||
expected_user,
|
||||
expected_mount_namespace,
|
||||
expected_mount,
|
||||
)
|
||||
.unwrap(),
|
||||
operations.observed
|
||||
);
|
||||
assert_eq!(
|
||||
operations.events,
|
||||
[
|
||||
"pidfs",
|
||||
"enter-user",
|
||||
"observe-user",
|
||||
"enter-mount",
|
||||
"observe-mount",
|
||||
"verify-mount",
|
||||
"unmount",
|
||||
"observe-absence",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_mount_open_refuses_intermediate_and_link_traversal() {
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"lazaret-final-mount-open-{}-{}",
|
||||
std::process::id(),
|
||||
TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir_all(root.join("nested/final")).unwrap();
|
||||
symlink("nested", root.join("link")).unwrap();
|
||||
let parent = File::open(&root).unwrap();
|
||||
let descriptor = File::open(root.join("nested")).unwrap();
|
||||
symlink(
|
||||
format!("/proc/self/fd/{}", descriptor.as_raw_fd()),
|
||||
root.join("magic"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(open_final_mount_directory(&parent, Path::new("nested/final")).is_err());
|
||||
assert!(open_final_mount_directory(&parent, Path::new("link")).is_err());
|
||||
assert!(open_final_mount_directory(&parent, Path::new("magic")).is_err());
|
||||
assert!(open_final_mount_directory(&parent, Path::new("nested")).is_ok());
|
||||
|
||||
std::fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_recovery_refuses_wrong_or_absent_final_mount_before_action() {
|
||||
for observed_mount_id in [4, 8] {
|
||||
let mut operations = fixture(None);
|
||||
operations.mount.mount_id = lazaret_core::identity::MountId::new(observed_mount_id);
|
||||
let expected_user = operations.user;
|
||||
let expected_mount_namespace = operations.mount_namespace;
|
||||
let expected_mount = MountIdentity {
|
||||
mount_id: lazaret_core::identity::MountId::new(5),
|
||||
device: DeviceId::new(6),
|
||||
inode: InodeId::new(7),
|
||||
};
|
||||
assert!(
|
||||
execute_private_recovery(
|
||||
&mut operations,
|
||||
expected_user,
|
||||
expected_mount_namespace,
|
||||
expected_mount,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
operations.events,
|
||||
[
|
||||
"pidfs",
|
||||
"enter-user",
|
||||
"observe-user",
|
||||
"enter-mount",
|
||||
"observe-mount",
|
||||
"verify-mount",
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue