Seal ignored-workspace routing at Domain startup
This commit is contained in:
parent
b114e3514b
commit
1ad3b84db2
5 changed files with 720 additions and 285 deletions
|
|
@ -4,9 +4,10 @@ use std::io;
|
|||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bstr::BStr;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::backing::{BackingRoot, validate_relative};
|
||||
|
||||
|
|
@ -40,10 +41,7 @@ impl ClassificationSnapshot {
|
|||
}
|
||||
|
||||
pub struct GitClassifier {
|
||||
workspace: PathBuf,
|
||||
root: BackingRoot,
|
||||
global_excludes: Option<PathBuf>,
|
||||
current: Mutex<Arc<SnapshotInner>>,
|
||||
current: Arc<SnapshotInner>,
|
||||
}
|
||||
|
||||
/// Resolves the root repository's effective common Git directory.
|
||||
|
|
@ -59,29 +57,19 @@ pub fn git_common_directory(workspace: &Path) -> io::Result<PathBuf> {
|
|||
impl GitClassifier {
|
||||
/// # Errors
|
||||
/// Refuses a non-repository workspace or unreadable Git policy input.
|
||||
pub fn open(workspace: &Path, global_excludes: Option<PathBuf>) -> io::Result<Self> {
|
||||
pub fn open(workspace: &Path, global_excludes: Option<&Path>) -> io::Result<Self> {
|
||||
let workspace = workspace.canonicalize()?;
|
||||
let root = BackingRoot::open(&workspace)?;
|
||||
Self::from_root(workspace, root, global_excludes)
|
||||
Self::from_root(&workspace, &root, global_excludes)
|
||||
}
|
||||
|
||||
pub(crate) fn from_root(
|
||||
workspace: PathBuf,
|
||||
root: BackingRoot,
|
||||
global_excludes: Option<PathBuf>,
|
||||
workspace: &Path,
|
||||
root: &BackingRoot,
|
||||
global_excludes: Option<&Path>,
|
||||
) -> io::Result<Self> {
|
||||
let current = Arc::new(SnapshotInner::load(
|
||||
&workspace,
|
||||
&root,
|
||||
global_excludes.as_deref(),
|
||||
1,
|
||||
)?);
|
||||
Ok(Self {
|
||||
workspace,
|
||||
root,
|
||||
global_excludes,
|
||||
current: Mutex::new(current),
|
||||
})
|
||||
let current = Arc::new(SnapshotInner::load(workspace, root, global_excludes, 1)?);
|
||||
Ok(Self { current })
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
|
@ -96,35 +84,51 @@ impl GitClassifier {
|
|||
|
||||
#[must_use]
|
||||
pub(crate) fn snapshot_for_paths(&self, relatives: &[&Path]) -> ClassificationSnapshot {
|
||||
let mut current = self
|
||||
.current
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if relatives
|
||||
.iter()
|
||||
.any(|relative| current.relevant_input_changed(&self.root, relative))
|
||||
{
|
||||
let generation = current.generation.saturating_add(1);
|
||||
if let Ok(next) = SnapshotInner::load(
|
||||
&self.workspace,
|
||||
&self.root,
|
||||
self.global_excludes.as_deref(),
|
||||
generation,
|
||||
) {
|
||||
*current = Arc::new(next);
|
||||
}
|
||||
}
|
||||
let _ = relatives;
|
||||
ClassificationSnapshot {
|
||||
inner: Arc::clone(¤t),
|
||||
inner: Arc::clone(&self.current),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn policy_record(&self) -> PolicyRecord {
|
||||
self.current.record.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot_from_record(
|
||||
workspace: &Path,
|
||||
record: PolicyRecord,
|
||||
) -> io::Result<ClassificationSnapshot> {
|
||||
Ok(ClassificationSnapshot {
|
||||
inner: Arc::new(SnapshotInner::from_record(workspace, record)?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct SnapshotInner {
|
||||
generation: u64,
|
||||
repositories: Vec<RepositoryPolicy>,
|
||||
workspace_inputs: BTreeMap<PathBuf, Option<InputStamp>>,
|
||||
external_inputs: BTreeMap<PathBuf, Option<InputStamp>>,
|
||||
record: PolicyRecord,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub(crate) struct PolicyRecord {
|
||||
schema: u32,
|
||||
repositories: Vec<RepositoryRecord>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
struct RepositoryRecord {
|
||||
prefix: Vec<u8>,
|
||||
tracked: Vec<Vec<u8>>,
|
||||
patterns: Vec<PatternRecord>,
|
||||
ignore_case: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
struct PatternRecord {
|
||||
bytes: Vec<u8>,
|
||||
source: Vec<u8>,
|
||||
root: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl SnapshotInner {
|
||||
|
|
@ -193,11 +197,37 @@ impl SnapshotInner {
|
|||
));
|
||||
}
|
||||
}
|
||||
let record = PolicyRecord {
|
||||
schema: 1,
|
||||
repositories: repositories
|
||||
.iter()
|
||||
.map(|repository| repository.record.clone())
|
||||
.collect(),
|
||||
};
|
||||
Ok(Self {
|
||||
generation,
|
||||
repositories,
|
||||
workspace_inputs,
|
||||
external_inputs,
|
||||
record,
|
||||
})
|
||||
}
|
||||
|
||||
fn from_record(workspace: &Path, record: PolicyRecord) -> io::Result<Self> {
|
||||
if record.schema != 1 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unsupported routing policy record",
|
||||
));
|
||||
}
|
||||
let repositories = record
|
||||
.repositories
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|record| RepositoryPolicy::from_record(workspace, record))
|
||||
.collect::<io::Result<Vec<_>>>()?;
|
||||
Ok(Self {
|
||||
generation: 0,
|
||||
repositories,
|
||||
record,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -227,44 +257,6 @@ impl SnapshotInner {
|
|||
.max_by_key(|repository| repository.prefix.components().count())
|
||||
.is_some_and(|repository| repository.is_tracked(relative))
|
||||
}
|
||||
|
||||
fn relevant_input_changed(&self, root: &BackingRoot, relative: &Path) -> bool {
|
||||
if validate_relative(relative).is_err() {
|
||||
return false;
|
||||
}
|
||||
let mut directory = if root
|
||||
.stat(relative)
|
||||
.is_ok_and(|stat| stat.st_mode & libc::S_IFMT == libc::S_IFDIR)
|
||||
{
|
||||
relative
|
||||
} else {
|
||||
relative.parent().unwrap_or_else(|| Path::new(""))
|
||||
};
|
||||
loop {
|
||||
for name in [OsStr::new(".gitignore"), OsStr::new(".git")] {
|
||||
let path = directory.join(name);
|
||||
let Ok(current) = workspace_input_stamp(root, &path) else {
|
||||
return true;
|
||||
};
|
||||
if self.workspace_inputs.get(&path).copied().unwrap_or(None) != current {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
let Some(parent) = directory.parent() else {
|
||||
break;
|
||||
};
|
||||
if parent == directory {
|
||||
break;
|
||||
}
|
||||
directory = parent;
|
||||
}
|
||||
self.external_inputs
|
||||
.iter()
|
||||
.any(|(path, expected)| match input_stamp(path) {
|
||||
Ok(current) => current != *expected,
|
||||
Err(_) => true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RepositoryPolicy {
|
||||
|
|
@ -273,6 +265,7 @@ struct RepositoryPolicy {
|
|||
tracked_directories: BTreeSet<Vec<u8>>,
|
||||
search: gix_ignore::Search,
|
||||
case: gix_ignore::glob::pattern::Case,
|
||||
record: RepositoryRecord,
|
||||
}
|
||||
|
||||
impl RepositoryPolicy {
|
||||
|
|
@ -301,6 +294,7 @@ impl RepositoryPolicy {
|
|||
}
|
||||
|
||||
let mut search = gix_ignore::Search::default();
|
||||
let mut patterns = Vec::new();
|
||||
if let Some(global) = global_excludes
|
||||
&& let Some(bytes) = read_external_optional(global)?
|
||||
{
|
||||
|
|
@ -310,6 +304,7 @@ impl RepositoryPolicy {
|
|||
Some(&repository),
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
patterns.push(PatternRecord::new(&bytes, global, Some(&repository)));
|
||||
}
|
||||
if let Some(bytes) = read_external_optional(&metadata.info_exclude)? {
|
||||
search.add_patterns_buffer(
|
||||
|
|
@ -318,6 +313,7 @@ impl RepositoryPolicy {
|
|||
None,
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
patterns.push(PatternRecord::new(&bytes, &metadata.info_exclude, None));
|
||||
}
|
||||
ignore_files.sort_by_key(|path| path.components().count());
|
||||
for path in ignore_files {
|
||||
|
|
@ -325,10 +321,11 @@ impl RepositoryPolicy {
|
|||
let source = workspace.join(&path);
|
||||
search.add_patterns_buffer(
|
||||
&bytes,
|
||||
source,
|
||||
&source,
|
||||
Some(&repository),
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
patterns.push(PatternRecord::new(&bytes, &source, Some(&repository)));
|
||||
}
|
||||
let ignore_case = match read_external_optional(&metadata.config)? {
|
||||
Some(bytes) => config_boolean(&bytes, "core", "ignoreCase")?,
|
||||
|
|
@ -339,12 +336,65 @@ impl RepositoryPolicy {
|
|||
} else {
|
||||
gix_ignore::glob::pattern::Case::Sensitive
|
||||
};
|
||||
let record = RepositoryRecord {
|
||||
prefix: path_bytes(&prefix),
|
||||
tracked: tracked.iter().cloned().collect(),
|
||||
patterns,
|
||||
ignore_case,
|
||||
};
|
||||
Ok(Self {
|
||||
prefix,
|
||||
tracked,
|
||||
tracked_directories,
|
||||
search,
|
||||
case,
|
||||
record,
|
||||
})
|
||||
}
|
||||
|
||||
fn from_record(workspace: &Path, record: RepositoryRecord) -> io::Result<Self> {
|
||||
let prefix = path_from_bytes(&record.prefix);
|
||||
validate_relative(&prefix)?;
|
||||
let tracked = record.tracked.iter().cloned().collect::<BTreeSet<_>>();
|
||||
let mut tracked_directories = BTreeSet::new();
|
||||
for path in &tracked {
|
||||
for separator in path
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, byte)| (*byte == b'/').then_some(index))
|
||||
{
|
||||
tracked_directories.insert(path[..separator].to_vec());
|
||||
}
|
||||
}
|
||||
let mut search = gix_ignore::Search::default();
|
||||
for pattern in &record.patterns {
|
||||
let source = path_from_bytes(&pattern.source);
|
||||
let root = pattern.root.as_deref().map(path_from_bytes);
|
||||
search.add_patterns_buffer(
|
||||
&pattern.bytes,
|
||||
&source,
|
||||
root.as_deref(),
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
}
|
||||
let case = if record.ignore_case {
|
||||
gix_ignore::glob::pattern::Case::Fold
|
||||
} else {
|
||||
gix_ignore::glob::pattern::Case::Sensitive
|
||||
};
|
||||
if !prefix.as_os_str().is_empty() && !workspace.join(&prefix).starts_with(workspace) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"routing policy repository escapes the workspace",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
prefix,
|
||||
tracked,
|
||||
tracked_directories,
|
||||
search,
|
||||
case,
|
||||
record,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -394,6 +444,16 @@ impl RepositoryPolicy {
|
|||
}
|
||||
}
|
||||
|
||||
impl PatternRecord {
|
||||
fn new(bytes: &[u8], source: &Path, root: Option<&Path>) -> Self {
|
||||
Self {
|
||||
bytes: bytes.to_vec(),
|
||||
source: path_bytes(source),
|
||||
root: root.map(path_bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RepositoryMetadata {
|
||||
index: PathBuf,
|
||||
config: PathBuf,
|
||||
|
|
@ -707,6 +767,14 @@ fn config_boolean(bytes: &[u8], section: &str, key: &str) -> io::Result<bool> {
|
|||
.map_err(|error| io::Error::other(format!("Git configuration boolean is invalid: {error}")))
|
||||
}
|
||||
|
||||
fn path_bytes(path: &Path) -> Vec<u8> {
|
||||
path.as_os_str().as_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn path_from_bytes(bytes: &[u8]) -> PathBuf {
|
||||
PathBuf::from(OsString::from_vec(bytes.to_vec()))
|
||||
}
|
||||
|
||||
fn input_stamp(path: &Path) -> io::Result<Option<InputStamp>> {
|
||||
let parent = path
|
||||
.parent()
|
||||
|
|
@ -872,7 +940,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn snapshots_route_tracked_ordinary_ignored_and_live_policy_changes() {
|
||||
fn snapshots_route_tracked_ordinary_and_startup_sealed_ignored_paths() {
|
||||
let fixture = Fixture::new("classification");
|
||||
repository(&fixture.workspace, &[b"tracked.cache"]);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"*.cache\n").unwrap();
|
||||
|
|
@ -895,10 +963,10 @@ mod tests {
|
|||
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"*.different\n").unwrap();
|
||||
let second = classifier.snapshot_for(std::path::Path::new("ignored.cache"));
|
||||
assert!(second.generation() > first.generation());
|
||||
assert_eq!(second.generation(), first.generation());
|
||||
assert_eq!(
|
||||
second.classify(std::path::Path::new("ignored.cache"), false),
|
||||
Classification::Host
|
||||
Classification::Isolated
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ impl IgnoredWorkspace {
|
|||
pub fn open(
|
||||
workspace: &Path,
|
||||
isolated_state: &Path,
|
||||
global_excludes: Option<PathBuf>,
|
||||
global_excludes: Option<&Path>,
|
||||
) -> io::Result<Self> {
|
||||
let workspace = workspace.canonicalize()?;
|
||||
let isolated_state = isolated_state.canonicalize()?;
|
||||
|
|
@ -103,16 +103,16 @@ impl IgnoredWorkspace {
|
|||
}
|
||||
let host = BackingRoot::open(&workspace)?;
|
||||
let isolated = BackingRoot::open(&isolated_state)?;
|
||||
Self::from_roots(workspace, host, isolated, global_excludes)
|
||||
Self::from_roots(&workspace, host, isolated, global_excludes)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses aliased or non-directory authorities and unreadable Git policy inputs.
|
||||
pub fn from_authorities(
|
||||
workspace_label: PathBuf,
|
||||
workspace_label: &Path,
|
||||
workspace: File,
|
||||
isolated_state: File,
|
||||
global_excludes: Option<PathBuf>,
|
||||
global_excludes: Option<&Path>,
|
||||
) -> io::Result<Self> {
|
||||
let host = BackingRoot::from_file(workspace)?;
|
||||
let isolated = BackingRoot::from_file(isolated_state)?;
|
||||
|
|
@ -122,10 +122,10 @@ impl IgnoredWorkspace {
|
|||
/// # Errors
|
||||
/// Refuses aliased or non-directory authorities and unreadable Git policy inputs.
|
||||
pub fn from_authorities_with_policy(
|
||||
workspace_label: PathBuf,
|
||||
workspace_label: &Path,
|
||||
workspace: File,
|
||||
isolated_state: File,
|
||||
global_excludes: Option<PathBuf>,
|
||||
global_excludes: Option<&Path>,
|
||||
isolate_ignored: bool,
|
||||
) -> io::Result<Self> {
|
||||
let host = BackingRoot::from_file(workspace)?;
|
||||
|
|
@ -140,19 +140,19 @@ impl IgnoredWorkspace {
|
|||
}
|
||||
|
||||
fn from_roots(
|
||||
workspace_label: PathBuf,
|
||||
workspace_label: &Path,
|
||||
host: BackingRoot,
|
||||
isolated: BackingRoot,
|
||||
global_excludes: Option<PathBuf>,
|
||||
global_excludes: Option<&Path>,
|
||||
) -> io::Result<Self> {
|
||||
Self::from_roots_with_policy(workspace_label, host, isolated, global_excludes, true)
|
||||
}
|
||||
|
||||
fn from_roots_with_policy(
|
||||
workspace_label: PathBuf,
|
||||
workspace_label: &Path,
|
||||
host: BackingRoot,
|
||||
isolated: BackingRoot,
|
||||
global_excludes: Option<PathBuf>,
|
||||
global_excludes: Option<&Path>,
|
||||
isolate_ignored: bool,
|
||||
) -> io::Result<Self> {
|
||||
if host.identity()? == isolated.identity()? {
|
||||
|
|
@ -161,7 +161,10 @@ impl IgnoredWorkspace {
|
|||
crate::migration::initialize(&host, &isolated)?;
|
||||
let classifier_root = BackingRoot::from_file(host.duplicate()?)?;
|
||||
let classifier =
|
||||
GitClassifier::from_root(workspace_label, classifier_root, global_excludes)?;
|
||||
GitClassifier::from_root(workspace_label, &classifier_root, global_excludes)?;
|
||||
if isolate_ignored {
|
||||
crate::migration::admit_policy(workspace_label, &host, &isolated, &classifier)?;
|
||||
}
|
||||
Ok(Self {
|
||||
host,
|
||||
isolated,
|
||||
|
|
@ -183,7 +186,7 @@ impl IgnoredWorkspace {
|
|||
let filesystem = Self::open(
|
||||
&configuration.workspace,
|
||||
&configuration.isolated_state,
|
||||
configuration.global_excludes,
|
||||
configuration.global_excludes.as_deref(),
|
||||
)?;
|
||||
let config = mount_config(configuration.threads);
|
||||
fuser::mount2(filesystem, configuration.mountpoint, &config)
|
||||
|
|
@ -196,10 +199,10 @@ impl IgnoredWorkspace {
|
|||
ready: impl FnOnce() -> io::Result<()>,
|
||||
) -> io::Result<()> {
|
||||
let filesystem = Self::from_authorities_with_policy(
|
||||
configuration.workspace_label,
|
||||
&configuration.workspace_label,
|
||||
configuration.workspace,
|
||||
configuration.isolated_state,
|
||||
configuration.global_excludes,
|
||||
configuration.global_excludes.as_deref(),
|
||||
configuration.isolate_ignored,
|
||||
)?;
|
||||
let session = fuser::spawn_mount2(
|
||||
|
|
|
|||
|
|
@ -49,15 +49,13 @@ impl IgnoredWorkspace {
|
|||
if snapshot.is_tracked(&path) {
|
||||
continue;
|
||||
}
|
||||
let host = self.host.stat(&path).ok();
|
||||
let isolated = self.isolated.stat(&path).ok();
|
||||
if let Some(isolated) = isolated {
|
||||
let compatible = host.is_some_and(|host| {
|
||||
host.st_mode & libc::S_IFMT == isolated.st_mode & libc::S_IFMT
|
||||
});
|
||||
if !compatible {
|
||||
return Err(io::Error::from_raw_os_error(libc::EBUSY));
|
||||
}
|
||||
let directory = self
|
||||
.isolated
|
||||
.stat(&path)
|
||||
.or_else(|_| self.host.stat(&path))
|
||||
.is_ok_and(|stat| stat.st_mode & libc::S_IFMT == libc::S_IFDIR);
|
||||
if snapshot.classify(&path, directory) == Classification::Isolated {
|
||||
return Err(io::Error::from_raw_os_error(libc::EBUSY));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ fn disabled_isolation_routes_ignored_paths_to_the_host_backing() {
|
|||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"ignored/\n").unwrap();
|
||||
let filesystem = IgnoredWorkspace::from_authorities_with_policy(
|
||||
fixture.workspace.clone(),
|
||||
&fixture.workspace,
|
||||
fs::File::open(&fixture.workspace).unwrap(),
|
||||
fs::File::open(&fixture.isolated).unwrap(),
|
||||
None,
|
||||
|
|
@ -82,7 +82,139 @@ fn disabled_isolation_routes_ignored_paths_to_the_host_backing() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn newly_ignored_created_directory_remains_a_safe_traversal_shell() {
|
||||
fn first_admission_does_not_import_ignored_host_material() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"secret\n").unwrap();
|
||||
fs::write(fixture.workspace.join("secret"), b"host secret").unwrap();
|
||||
|
||||
let filesystem = IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap();
|
||||
|
||||
assert!(!fixture.isolated.join("secret").exists());
|
||||
let snapshot = filesystem
|
||||
.classifier
|
||||
.snapshot_for(std::path::Path::new("secret"));
|
||||
assert_eq!(
|
||||
filesystem
|
||||
.resolve(&snapshot, std::path::Path::new("secret"))
|
||||
.err()
|
||||
.unwrap()
|
||||
.raw_os_error(),
|
||||
Some(libc::ENOENT)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_clones_host_paths_that_become_ignored() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join("artifact"), b"host generation").unwrap();
|
||||
fs::write(fixture.isolated.join("artifact"), b"stale isolated").unwrap();
|
||||
drop(IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap());
|
||||
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"artifact\n").unwrap();
|
||||
let filesystem = IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
fs::read(fixture.isolated.join("artifact")).unwrap(),
|
||||
b"host generation"
|
||||
);
|
||||
let snapshot = filesystem
|
||||
.classifier
|
||||
.snapshot_for(std::path::Path::new("artifact"));
|
||||
assert_eq!(
|
||||
filesystem
|
||||
.resolve(&snapshot, std::path::Path::new("artifact"))
|
||||
.unwrap()
|
||||
.backing,
|
||||
Classification::Isolated
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_removes_isolated_paths_that_return_to_host_routing() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"artifact\n").unwrap();
|
||||
fs::write(fixture.workspace.join("artifact"), b"host generation").unwrap();
|
||||
drop(IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap());
|
||||
fs::write(fixture.isolated.join("artifact"), b"isolated generation").unwrap();
|
||||
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"").unwrap();
|
||||
let filesystem = IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap();
|
||||
|
||||
assert!(!fixture.isolated.join("artifact").exists());
|
||||
let snapshot = filesystem
|
||||
.classifier
|
||||
.snapshot_for(std::path::Path::new("artifact"));
|
||||
assert_eq!(
|
||||
filesystem
|
||||
.resolve(&snapshot, std::path::Path::new("artifact"))
|
||||
.unwrap()
|
||||
.backing,
|
||||
Classification::Host
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_preserves_paths_that_remain_ignored() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"artifact\n").unwrap();
|
||||
drop(IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap());
|
||||
fs::write(fixture.isolated.join("artifact"), b"isolated generation").unwrap();
|
||||
|
||||
drop(IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap());
|
||||
|
||||
assert_eq!(
|
||||
fs::read(fixture.isolated.join("artifact")).unwrap(),
|
||||
b"isolated generation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_discards_unpublished_policy_staging() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
drop(IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap());
|
||||
let staging = fixture
|
||||
.isolated
|
||||
.join(".lazaret-ignored-workspace-transactions/routing/stage-interrupted");
|
||||
fs::write(&staging, b"unpublished").unwrap();
|
||||
|
||||
drop(IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap());
|
||||
|
||||
assert!(!staging.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_publication_cannot_change_an_isolated_route_mid_domain() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"artifact\n").unwrap();
|
||||
let filesystem = IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap();
|
||||
fs::write(
|
||||
fixture.workspace.join("candidate-index"),
|
||||
index_with_path(b"artifact"),
|
||||
)
|
||||
.unwrap();
|
||||
let snapshot = filesystem
|
||||
.classifier
|
||||
.snapshot_for(std::path::Path::new("artifact"));
|
||||
|
||||
let error = filesystem
|
||||
.require_index_publication_safe(
|
||||
Classification::Host,
|
||||
std::path::Path::new("candidate-index"),
|
||||
&snapshot,
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(error.raw_os_error(), Some(libc::EBUSY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_created_during_a_domain_does_not_change_its_routes() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
let filesystem = IgnoredWorkspace::open(&fixture.workspace, &fixture.isolated, None).unwrap();
|
||||
|
|
@ -106,7 +238,7 @@ fn newly_ignored_created_directory_remains_a_safe_traversal_shell() {
|
|||
assert_eq!(entry.backing, Classification::Host);
|
||||
assert_eq!(
|
||||
filesystem.classify(&snapshot, &path.join("python3"), false),
|
||||
Classification::Isolated
|
||||
Classification::Host
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -140,7 +272,7 @@ fn ignored_host_directory_requires_current_creation_provenance() {
|
|||
|
||||
#[test]
|
||||
#[ignore = "requires a usable host FUSE device"]
|
||||
fn physical_router_preserves_lock_domains_across_transition() {
|
||||
fn physical_router_keeps_lock_domains_on_the_startup_route() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"ignored/\n").unwrap();
|
||||
|
|
@ -165,42 +297,25 @@ fn physical_router_preserves_lock_domains_across_transition() {
|
|||
.write(true)
|
||||
.open(&mounted_path)
|
||||
.unwrap();
|
||||
let lock = record_lock(0, 0, libc::F_WRLCK).unwrap();
|
||||
fcntl(old.as_fd(), FcntlArg::F_SETLK(&lock)).unwrap();
|
||||
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"different/\n").unwrap();
|
||||
let new = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&mounted_path)
|
||||
.unwrap();
|
||||
assert_eq!(fs::read(&mounted_path).unwrap(), b"host");
|
||||
assert_eq!(fs::read(&mounted_path).unwrap(), b"isolated");
|
||||
|
||||
let isolated = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(fixture.isolated.join("ignored/locked"))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
fcntl(isolated.as_fd(), FcntlArg::F_OFD_SETLK(&lock)).unwrap_err(),
|
||||
nix::errno::Errno::EAGAIN
|
||||
);
|
||||
let host = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(fixture.workspace.join("ignored/locked"))
|
||||
.unwrap();
|
||||
let lock = record_lock(0, 0, libc::F_WRLCK).unwrap();
|
||||
fcntl(host.as_fd(), FcntlArg::F_OFD_SETLK(&lock)).unwrap();
|
||||
let unlock = record_lock(0, 0, libc::F_UNLCK).unwrap();
|
||||
fcntl(host.as_fd(), FcntlArg::F_OFD_SETLK(&unlock)).unwrap();
|
||||
|
||||
fcntl(new.as_fd(), FcntlArg::F_SETLK(&lock)).unwrap();
|
||||
assert_eq!(
|
||||
fcntl(host.as_fd(), FcntlArg::F_OFD_SETLK(&lock)).unwrap_err(),
|
||||
nix::errno::Errno::EAGAIN
|
||||
);
|
||||
drop(old);
|
||||
fcntl(isolated.as_fd(), FcntlArg::F_OFD_SETLK(&lock)).unwrap();
|
||||
drop(new);
|
||||
fcntl(host.as_fd(), FcntlArg::F_OFD_SETLK(&lock)).unwrap();
|
||||
|
||||
|
|
@ -209,10 +324,14 @@ fn physical_router_preserves_lock_domains_across_transition() {
|
|||
|
||||
#[test]
|
||||
#[ignore = "requires a usable host FUSE device"]
|
||||
fn physical_router_preserves_routes_transitions_and_cross_route_atomicity() {
|
||||
fn physical_router_seals_routes_and_preserves_cross_route_atomicity() {
|
||||
let fixture = Fixture::new();
|
||||
repository(&fixture.workspace);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"ignored/\n").unwrap();
|
||||
fs::write(
|
||||
fixture.workspace.join(".gitignore"),
|
||||
b"ignored/\ndestination\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir(fixture.workspace.join("ignored")).unwrap();
|
||||
fs::write(fixture.workspace.join("ignored/sentinel"), b"host").unwrap();
|
||||
fs::create_dir(fixture.isolated.join("ignored")).unwrap();
|
||||
|
|
@ -246,7 +365,7 @@ fn physical_router_preserves_routes_transitions_and_cross_route_atomicity() {
|
|||
fs::write(fixture.workspace.join(".gitignore"), b"different/\n").unwrap();
|
||||
assert_eq!(
|
||||
fs::read(fixture.mountpoint.join("ignored/sentinel")).unwrap(),
|
||||
b"host"
|
||||
b"isolated"
|
||||
);
|
||||
pinned.write_all(b"-pinned").unwrap();
|
||||
pinned.flush().unwrap();
|
||||
|
|
@ -402,6 +521,21 @@ fn repository(root: &std::path::Path) {
|
|||
fs::write(root.join(".git/index"), index).unwrap();
|
||||
}
|
||||
|
||||
fn index_with_path(path: &[u8]) -> Vec<u8> {
|
||||
let mut index = b"DIRC".to_vec();
|
||||
index.extend_from_slice(&2u32.to_be_bytes());
|
||||
index.extend_from_slice(&1u32.to_be_bytes());
|
||||
index.extend_from_slice(&[0; 60]);
|
||||
index.extend_from_slice(&(u16::try_from(path.len()).unwrap()).to_be_bytes());
|
||||
index.extend_from_slice(path);
|
||||
index.push(0);
|
||||
while !(index.len() - 12).is_multiple_of(8) {
|
||||
index.push(0);
|
||||
}
|
||||
index.extend_from_slice(&[0; 20]);
|
||||
index
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
root: std::path::PathBuf,
|
||||
workspace: std::path::PathBuf,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read, Write};
|
||||
|
|
@ -14,8 +14,12 @@ use serde::{Deserialize, Serialize};
|
|||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::backing::BackingRoot;
|
||||
use crate::classifier::{Classification, ClassificationSnapshot, GitClassifier, PolicyRecord};
|
||||
|
||||
const TRANSACTIONS: &str = ".lazaret-ignored-workspace-transactions";
|
||||
const ROUTING: &str = "routing";
|
||||
const ROUTING_POLICY: &str = "policy.json";
|
||||
const ROUTING_POLICY_TEMPORARY: &str = "policy.tmp";
|
||||
const INTENT_SUFFIX: &str = ".json";
|
||||
static TRANSACTION_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
|
|
@ -154,6 +158,44 @@ pub(crate) fn initialize(host: &BackingRoot, isolated: &BackingRoot) -> io::Resu
|
|||
recover(host, isolated)
|
||||
}
|
||||
|
||||
pub(crate) fn admit_policy(
|
||||
workspace: &Path,
|
||||
host: &BackingRoot,
|
||||
isolated: &BackingRoot,
|
||||
classifier: &GitClassifier,
|
||||
) -> io::Result<()> {
|
||||
let routing = Path::new(TRANSACTIONS).join(ROUTING);
|
||||
match isolated.mkdir(&routing, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
isolated.sync_directory(Path::new(TRANSACTIONS))?;
|
||||
clean_policy_staging(isolated)?;
|
||||
let previous = read_policy(isolated)?;
|
||||
if let Some(previous) = previous {
|
||||
let previous = GitClassifier::snapshot_from_record(workspace, previous)?;
|
||||
reconcile_policy(host, isolated, &previous, &classifier.snapshot())?;
|
||||
}
|
||||
persist_policy(isolated, &classifier.policy_record())
|
||||
}
|
||||
|
||||
fn clean_policy_staging(isolated: &BackingRoot) -> io::Result<()> {
|
||||
let directory = Path::new(TRANSACTIONS).join(ROUTING);
|
||||
for (name, kind) in isolated.typed_entries(&directory)? {
|
||||
let path = directory.join(&name);
|
||||
if name == OsStr::new(ROUTING_POLICY) && kind == rustix::fs::FileType::RegularFile {
|
||||
continue;
|
||||
}
|
||||
if name == OsStr::new(ROUTING_POLICY_TEMPORARY) || name.as_bytes().starts_with(b"stage-") {
|
||||
remove_tree(isolated, &path)?;
|
||||
continue;
|
||||
}
|
||||
return Err(io::Error::from_raw_os_error(libc::EIO));
|
||||
}
|
||||
isolated.sync_directory(&directory)
|
||||
}
|
||||
|
||||
pub(crate) fn migrate(
|
||||
host: &BackingRoot,
|
||||
isolated: &BackingRoot,
|
||||
|
|
@ -261,8 +303,227 @@ pub(crate) fn migrate(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn read_policy(isolated: &BackingRoot) -> io::Result<Option<PolicyRecord>> {
|
||||
let path = Path::new(TRANSACTIONS).join(ROUTING).join(ROUTING_POLICY);
|
||||
match isolated.read_file(&path, 64 * 1024 * 1024) {
|
||||
Ok(bytes) => serde_json::from_slice(&bytes)
|
||||
.map(Some)
|
||||
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn persist_policy(isolated: &BackingRoot, policy: &PolicyRecord) -> io::Result<()> {
|
||||
let directory = Path::new(TRANSACTIONS).join(ROUTING);
|
||||
let temporary = directory.join(ROUTING_POLICY_TEMPORARY);
|
||||
let destination = directory.join(ROUTING_POLICY);
|
||||
if stat_if_exists(isolated, &temporary)?.is_some() {
|
||||
isolated.unlink(&temporary, false)?;
|
||||
}
|
||||
let bytes = serde_json::to_vec(policy).map_err(io::Error::other)?;
|
||||
isolated.write_file(&temporary, &bytes, Mode::from_raw_mode(0o600))?;
|
||||
isolated.rename(&temporary, &destination, RenameFlags::empty())?;
|
||||
isolated.sync_directory(&directory)
|
||||
}
|
||||
|
||||
fn reconcile_policy(
|
||||
host: &BackingRoot,
|
||||
isolated: &BackingRoot,
|
||||
previous: &ClassificationSnapshot,
|
||||
current: &ClassificationSnapshot,
|
||||
) -> io::Result<()> {
|
||||
let mut paths = BTreeMap::new();
|
||||
collect_paths(host, Path::new(""), &mut paths)?;
|
||||
collect_paths(isolated, Path::new(""), &mut paths)?;
|
||||
paths.remove(Path::new(TRANSACTIONS));
|
||||
paths.retain(|path, _| !reserved(path));
|
||||
|
||||
let mut transitions = paths
|
||||
.into_iter()
|
||||
.map(|(path, directory)| {
|
||||
let old = previous.classify(&path, directory);
|
||||
let new = current.classify(&path, directory);
|
||||
(path, directory, old, new)
|
||||
})
|
||||
.filter(|(_, _, old, new)| old != new)
|
||||
.collect::<Vec<_>>();
|
||||
let expected_admissions = admission_stamps(host, &transitions)?;
|
||||
|
||||
transitions.sort_by_key(|(path, _, old, new)| {
|
||||
let depth = path.components().count();
|
||||
match (*old, *new) {
|
||||
(Classification::Isolated, Classification::Host) => usize::MAX - depth,
|
||||
_ => depth,
|
||||
}
|
||||
});
|
||||
|
||||
for (path, directory, old, new) in transitions {
|
||||
match (old, new) {
|
||||
(Classification::Host, Classification::Isolated) => {
|
||||
clone_admitted_path(host, isolated, &path, directory)?;
|
||||
}
|
||||
(Classification::Isolated, Classification::Host) => {
|
||||
remove_released_path(isolated, current, &path, directory)?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let mut current_paths = BTreeMap::new();
|
||||
collect_paths(host, Path::new(""), &mut current_paths)?;
|
||||
collect_paths(isolated, Path::new(""), &mut current_paths)?;
|
||||
current_paths.retain(|path, _| !reserved(path));
|
||||
let current_transitions = current_paths
|
||||
.into_iter()
|
||||
.map(|(path, directory)| {
|
||||
let old = previous.classify(&path, directory);
|
||||
let new = current.classify(&path, directory);
|
||||
(path, directory, old, new)
|
||||
})
|
||||
.filter(|(_, _, old, new)| old != new)
|
||||
.collect::<Vec<_>>();
|
||||
if admission_stamps(host, ¤t_transitions)? != expected_admissions {
|
||||
return Err(io::Error::from_raw_os_error(libc::EAGAIN));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn admission_stamps(
|
||||
host: &BackingRoot,
|
||||
transitions: &[(PathBuf, bool, Classification, Classification)],
|
||||
) -> io::Result<BTreeMap<PathBuf, Option<ObjectStamp>>> {
|
||||
transitions
|
||||
.iter()
|
||||
.filter(|(_, _, old, new)| *old == Classification::Host && *new == Classification::Isolated)
|
||||
.map(|(path, _, _, _)| {
|
||||
stat_if_exists(host, path)
|
||||
.map(|stat| (path.clone(), stat.as_ref().map(ObjectStamp::from_stat)))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn collect_paths(
|
||||
root: &BackingRoot,
|
||||
directory: &Path,
|
||||
paths: &mut BTreeMap<PathBuf, bool>,
|
||||
) -> io::Result<()> {
|
||||
for (name, kind) in root.typed_entries(directory)? {
|
||||
let path = directory.join(name);
|
||||
if reserved(&path) {
|
||||
continue;
|
||||
}
|
||||
let is_directory = kind == rustix::fs::FileType::Directory
|
||||
|| (kind == rustix::fs::FileType::Unknown
|
||||
&& root
|
||||
.stat(&path)
|
||||
.is_ok_and(|stat| stat.st_mode & libc::S_IFMT == libc::S_IFDIR));
|
||||
paths
|
||||
.entry(path.clone())
|
||||
.and_modify(|known| *known |= is_directory)
|
||||
.or_insert(is_directory);
|
||||
if is_directory {
|
||||
collect_paths(root, &path, paths)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn clone_admitted_path(
|
||||
host: &BackingRoot,
|
||||
isolated: &BackingRoot,
|
||||
path: &Path,
|
||||
directory: bool,
|
||||
) -> io::Result<()> {
|
||||
let Some(host_stat) = stat_if_exists(host, path)? else {
|
||||
if stat_if_exists(isolated, path)?.is_some() {
|
||||
remove_tree(isolated, path)?;
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let host_directory = host_stat.st_mode & libc::S_IFMT == libc::S_IFDIR;
|
||||
if host_directory != directory {
|
||||
return Err(io::Error::from_raw_os_error(libc::EIO));
|
||||
}
|
||||
if directory {
|
||||
if let Some(stat) = stat_if_exists(isolated, path)?
|
||||
&& stat.st_mode & libc::S_IFMT != libc::S_IFDIR
|
||||
{
|
||||
remove_tree(isolated, path)?;
|
||||
}
|
||||
isolated.ensure_parent_directories(path)?;
|
||||
match isolated.mkdir(path, Mode::from_raw_mode(host_stat.st_mode & 0o7777)) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let output = isolated.open_directory(path)?;
|
||||
set_metadata(&output, &host_stat)?;
|
||||
output.sync_all()?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let sequence = TRANSACTION_SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
||||
let staging = Path::new(TRANSACTIONS)
|
||||
.join(ROUTING)
|
||||
.join(format!("stage-{}-{sequence}", std::process::id()));
|
||||
if stat_if_exists(isolated, &staging)?.is_some() {
|
||||
remove_tree(isolated, &staging)?;
|
||||
}
|
||||
let manifest = scan_tree(host, path)?;
|
||||
copy_tree(host, path, isolated, &staging)?;
|
||||
require_exact_tree(host, path, &manifest)?;
|
||||
require_equivalent_tree(isolated, &staging, &manifest)?;
|
||||
if stat_if_exists(isolated, path)?.is_some() {
|
||||
remove_tree(isolated, path)?;
|
||||
}
|
||||
isolated.ensure_parent_directories(path)?;
|
||||
isolated.rename(&staging, path, RenameFlags::NOREPLACE)?;
|
||||
isolated.sync_directory(path.parent().unwrap_or_else(|| Path::new("")))?;
|
||||
require_exact_tree(host, path, &manifest)?;
|
||||
require_equivalent_tree(isolated, path, &manifest)
|
||||
}
|
||||
|
||||
fn remove_released_path(
|
||||
isolated: &BackingRoot,
|
||||
current: &ClassificationSnapshot,
|
||||
path: &Path,
|
||||
directory: bool,
|
||||
) -> io::Result<()> {
|
||||
if stat_if_exists(isolated, path)?.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if directory && has_isolated_descendant(isolated, current, path)? {
|
||||
return Ok(());
|
||||
}
|
||||
remove_tree(isolated, path)
|
||||
}
|
||||
|
||||
fn has_isolated_descendant(
|
||||
isolated: &BackingRoot,
|
||||
current: &ClassificationSnapshot,
|
||||
directory: &Path,
|
||||
) -> io::Result<bool> {
|
||||
for (name, kind) in isolated.typed_entries(directory)? {
|
||||
let child = directory.join(name);
|
||||
let child_directory = kind == rustix::fs::FileType::Directory
|
||||
|| (kind == rustix::fs::FileType::Unknown
|
||||
&& isolated
|
||||
.stat(&child)
|
||||
.is_ok_and(|stat| stat.st_mode & libc::S_IFMT == libc::S_IFDIR));
|
||||
if current.classify(&child, child_directory) == Classification::Isolated
|
||||
|| (child_directory && has_isolated_descendant(isolated, current, &child)?)
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn recover(host: &BackingRoot, isolated: &BackingRoot) -> io::Result<()> {
|
||||
for (name, kind) in isolated.typed_entries(Path::new(TRANSACTIONS))? {
|
||||
if name == OsStr::new(ROUTING) && kind == rustix::fs::FileType::Directory {
|
||||
continue;
|
||||
}
|
||||
if kind == rustix::fs::FileType::RegularFile && name.as_bytes().ends_with(b".tmp") {
|
||||
isolated.unlink(&Path::new(TRANSACTIONS).join(name), false)?;
|
||||
continue;
|
||||
|
|
|
|||
Loading…
Reference in a new issue