Add identity and target authorization package
OIDC validation, principal mapping, remora:v1 roles, and current-authz filtering with kind acceptance on Keycloak and ZITADEL.
This commit is contained in:
parent
91ac6dee85
commit
bf028161ae
3 changed files with 3153 additions and 21 deletions
289
crates/remora-identity/tests/policy.rs
Normal file
289
crates/remora-identity/tests/policy.rs
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
use remora_domain::{BucketName, OpaqueId, StorageScope, Target, TargetPrefix, TargetStoreId};
|
||||
use remora_identity::{
|
||||
AuthorizedResource, DefaultTarget, IdentityError, ManagementOperation, ResourceVisibility,
|
||||
TargetOverride, can_target, decode_roles, encode_admin_role, encode_storage_role,
|
||||
filter_visible, may_manage, may_request_execution_grant_revocation, parse_default_target,
|
||||
principal_id_for, resolve_authorized_target, resolve_target, resource_visibility,
|
||||
roles_from_names,
|
||||
};
|
||||
|
||||
fn scope(store: &str, bucket: &str, prefix: &str) -> StorageScope {
|
||||
StorageScope {
|
||||
store: TargetStoreId::parse(store).unwrap(),
|
||||
bucket: BucketName::parse(bucket).unwrap(),
|
||||
prefix: TargetPrefix::parse(prefix, ["_remora"]).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn target(store: &str, bucket: &str, prefix: &str) -> Target {
|
||||
Target {
|
||||
store: TargetStoreId::parse(store).unwrap(),
|
||||
bucket: BucketName::parse(bucket).unwrap(),
|
||||
prefix: TargetPrefix::parse(prefix, ["_remora"]).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn principal(
|
||||
label: &str,
|
||||
administrator: bool,
|
||||
scopes: Vec<StorageScope>,
|
||||
) -> remora_identity::Principal {
|
||||
remora_identity::Principal {
|
||||
id: OpaqueId::parse(format!("principal-{label}-aaaa")).unwrap(),
|
||||
issuer: format!("https://issuer.example/{label}"),
|
||||
subject: format!("subject-{label}"),
|
||||
administrator,
|
||||
scopes,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scope_segment_matching_rejects_textual_prefix_coincidence() {
|
||||
let owner = principal(
|
||||
"owner",
|
||||
false,
|
||||
vec![scope("target-main", "target-bucket", "research")],
|
||||
);
|
||||
assert!(can_target(
|
||||
&owner,
|
||||
&target("target-main", "target-bucket", "research/data")
|
||||
));
|
||||
assert!(!can_target(
|
||||
&owner,
|
||||
&target("target-main", "target-bucket", "researchers")
|
||||
));
|
||||
assert!(!can_target(
|
||||
&owner,
|
||||
&target("target-main", "target-bucket", "researc")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_prefix_nfc_canonicalization_and_reserved_segments() {
|
||||
let defaults = parse_default_target("target-main", "target-bucket", "", &["_remora"]).unwrap();
|
||||
let resolved = resolve_target(
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: None,
|
||||
prefix: Some("cafe\u{0301}/models".into()),
|
||||
},
|
||||
&["_remora"],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(resolved.prefix.canonical(), "caf\u{e9}/models");
|
||||
|
||||
let reserved = resolve_target(
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: None,
|
||||
prefix: Some("_remora/private".into()),
|
||||
},
|
||||
&["_remora"],
|
||||
);
|
||||
assert!(reserved.is_err());
|
||||
|
||||
let category_c = resolve_target(
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: None,
|
||||
prefix: Some("private/\u{e000}".into()),
|
||||
},
|
||||
&["_remora"],
|
||||
);
|
||||
assert!(category_c.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_target_applies_bucket_and_prefix_overrides() {
|
||||
let defaults = DefaultTarget {
|
||||
store: TargetStoreId::parse("target-main").unwrap(),
|
||||
bucket: BucketName::parse("default-bucket").unwrap(),
|
||||
prefix: TargetPrefix::parse("defaults", []).unwrap(),
|
||||
};
|
||||
let full_default = resolve_target(&defaults, &TargetOverride::default(), &[]).unwrap();
|
||||
assert_eq!(full_default.bucket.as_str(), "default-bucket");
|
||||
assert_eq!(full_default.prefix.canonical(), "defaults");
|
||||
|
||||
let overridden = resolve_target(
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: Some("other-bucket".into()),
|
||||
prefix: Some("models/current/".into()),
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(overridden.store.as_str(), "target-main");
|
||||
assert_eq!(overridden.bucket.as_str(), "other-bucket");
|
||||
assert_eq!(overridden.prefix.canonical(), "models/current");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_authorized_target_requires_covering_scope() {
|
||||
let defaults = DefaultTarget {
|
||||
store: TargetStoreId::parse("target-main").unwrap(),
|
||||
bucket: BucketName::parse("target-bucket").unwrap(),
|
||||
prefix: TargetPrefix::parse("defaults", []).unwrap(),
|
||||
};
|
||||
let scoped = principal(
|
||||
"scoped",
|
||||
false,
|
||||
vec![scope("target-main", "target-bucket", "models")],
|
||||
);
|
||||
let ok = resolve_authorized_target(
|
||||
&scoped,
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: None,
|
||||
prefix: Some("models/current".into()),
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(ok.prefix.canonical(), "models/current");
|
||||
|
||||
let denied = resolve_authorized_target(
|
||||
&scoped,
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: None,
|
||||
prefix: Some("other".into()),
|
||||
},
|
||||
&[],
|
||||
);
|
||||
assert!(matches!(denied, Err(IdentityError::UnauthorizedTarget)));
|
||||
|
||||
let admin = principal("admin", true, vec![]);
|
||||
let admin_ok = resolve_authorized_target(
|
||||
&admin,
|
||||
&defaults,
|
||||
&TargetOverride {
|
||||
bucket: Some("any-bucket".into()),
|
||||
prefix: Some("anywhere".into()),
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(admin_ok.bucket.as_str(), "any-bucket");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revocation_of_roles_removes_current_management_not_admin_revoke_for_owners() {
|
||||
let target = target("target-main", "target-bucket", "models/current");
|
||||
let resource = AuthorizedResource {
|
||||
owner: OpaqueId::parse("principal-owner-aaaa").unwrap(),
|
||||
target: target.clone(),
|
||||
};
|
||||
let with_scope = principal(
|
||||
"owner",
|
||||
false,
|
||||
vec![scope("target-main", "target-bucket", "models")],
|
||||
);
|
||||
assert!(may_manage(
|
||||
&with_scope,
|
||||
&resource,
|
||||
ManagementOperation::Cancel
|
||||
));
|
||||
assert!(!may_request_execution_grant_revocation(&with_scope));
|
||||
|
||||
let after_role_loss = principal("owner", false, vec![]);
|
||||
assert!(!may_manage(
|
||||
&after_role_loss,
|
||||
&resource,
|
||||
ManagementOperation::Read
|
||||
));
|
||||
assert_eq!(
|
||||
resource_visibility(&after_role_loss, Some(&resource), ManagementOperation::Read),
|
||||
ResourceVisibility::NotFound
|
||||
);
|
||||
|
||||
let admin = principal("admin", true, vec![]);
|
||||
assert!(may_manage(
|
||||
&admin,
|
||||
&resource,
|
||||
ManagementOperation::HardDelete
|
||||
));
|
||||
assert!(may_request_execution_grant_revocation(&admin));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_and_unauthorized_resources_are_indistinguishable() {
|
||||
let owner = principal(
|
||||
"owner",
|
||||
false,
|
||||
vec![scope("target-main", "target-bucket", "")],
|
||||
);
|
||||
let other = principal(
|
||||
"other",
|
||||
false,
|
||||
vec![scope("target-main", "target-bucket", "")],
|
||||
);
|
||||
let resource = AuthorizedResource {
|
||||
owner: owner.id.clone(),
|
||||
target: target("target-main", "target-bucket", "models"),
|
||||
};
|
||||
|
||||
let missing = resource_visibility(&owner, None, ManagementOperation::Read);
|
||||
let unauthorized = resource_visibility(&other, Some(&resource), ManagementOperation::Read);
|
||||
assert_eq!(missing, ResourceVisibility::NotFound);
|
||||
assert_eq!(unauthorized, ResourceVisibility::NotFound);
|
||||
assert_eq!(missing, unauthorized);
|
||||
|
||||
let visible = resource_visibility(&owner, Some(&resource), ManagementOperation::Read);
|
||||
assert_eq!(visible, ResourceVisibility::Visible);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_filtering_hides_unauthorized_existence() {
|
||||
let owner = principal(
|
||||
"owner",
|
||||
false,
|
||||
vec![scope("target-main", "target-bucket", "models")],
|
||||
);
|
||||
let mine = AuthorizedResource {
|
||||
owner: owner.id.clone(),
|
||||
target: target("target-main", "target-bucket", "models/a"),
|
||||
};
|
||||
let outside = AuthorizedResource {
|
||||
owner: owner.id.clone(),
|
||||
target: target("target-main", "target-bucket", "other"),
|
||||
};
|
||||
let peer = AuthorizedResource {
|
||||
owner: OpaqueId::parse("principal-peer-aaaaaa").unwrap(),
|
||||
target: target("target-main", "target-bucket", "models/b"),
|
||||
};
|
||||
let visible = filter_visible(&owner, [&mine, &outside, &peer]);
|
||||
assert_eq!(visible, vec![&mine]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn principal_ids_are_stable_for_issuer_subject_pairs() {
|
||||
let first = principal_id_for("https://issuer.example", "user-1").unwrap();
|
||||
let second = principal_id_for("https://issuer.example", "user-1").unwrap();
|
||||
let other = principal_id_for("https://issuer.example", "user-2").unwrap();
|
||||
assert_eq!(first, second);
|
||||
assert_ne!(first, other);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn role_encoding_round_trip_and_foreign_roles_ignored() {
|
||||
let storage =
|
||||
encode_storage_role(&scope("target-main", "target-bucket", "models/current")).unwrap();
|
||||
let decoded = decode_roles(
|
||||
[encode_admin_role().as_str(), "test-user", storage.as_str()],
|
||||
&["_remora"],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(decoded.administrator);
|
||||
assert_eq!(decoded.scopes.len(), 1);
|
||||
assert!(decoded.scopes[0].covers(&target(
|
||||
"target-main",
|
||||
"target-bucket",
|
||||
"models/current/file"
|
||||
)));
|
||||
|
||||
let roles = roles_from_names(&[remora_identity::ADMIN_ROLE], &[]).unwrap();
|
||||
assert!(roles.administrator);
|
||||
let bare = roles_from_names(&["admin"], &[]).unwrap();
|
||||
assert!(!bare.administrator);
|
||||
}
|
||||
668
crates/remora-identity/tests/substrate.rs
Normal file
668
crates/remora-identity/tests/substrate.rs
Normal file
|
|
@ -0,0 +1,668 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
|
||||
use remora_durable::migrate as durable_migrate;
|
||||
use remora_durable::{PoolConfig, connect};
|
||||
use remora_identity::{
|
||||
OidcConfig, OidcIdentityProvider, migrate as identity_migrate, resolve_principal,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::PgPool;
|
||||
|
||||
fn database_url() -> Option<String> {
|
||||
std::env::var("DATABASE_URL")
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn oidc_issuer() -> Option<String> {
|
||||
std::env::var("REMORA_OIDC_ISSUER")
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn require_env() -> bool {
|
||||
std::env::var("REMORA_IDENTITY_REQUIRE").is_ok()
|
||||
}
|
||||
|
||||
fn env_required(name: &str) -> String {
|
||||
std::env::var(name)
|
||||
.unwrap_or_else(|_| panic!("{name} is required when REMORA_IDENTITY_REQUIRE is set"))
|
||||
}
|
||||
|
||||
async fn test_pool() -> Option<PgPool> {
|
||||
let Some(url) = database_url() else {
|
||||
if require_env() {
|
||||
panic!("DATABASE_URL is required when REMORA_IDENTITY_REQUIRE is set");
|
||||
}
|
||||
eprintln!("skipping identity durable test: DATABASE_URL is unset");
|
||||
return None;
|
||||
};
|
||||
let pool = connect(
|
||||
&url,
|
||||
PoolConfig {
|
||||
max_connections: 8,
|
||||
acquire_timeout: Duration::from_secs(10),
|
||||
idle_timeout: Some(Duration::from_secs(30)),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("connect postgres");
|
||||
durable_migrate(&pool).await.expect("durable migrate");
|
||||
identity_migrate(&pool).await.expect("identity migrate");
|
||||
Some(pool)
|
||||
}
|
||||
|
||||
fn oidc_provider() -> OidcIdentityProvider {
|
||||
let issuer = env_required("REMORA_OIDC_ISSUER");
|
||||
let provider_kind = std::env::var("REMORA_OIDC_PROVIDER").unwrap_or_default();
|
||||
// Keycloak ROPC tokens use the OIDC client id as `aud` (via audience mapper).
|
||||
// ZITADEL JWT-profile access tokens assert the project resource as `aud`.
|
||||
let audience = if provider_kind == "zitadel" {
|
||||
env_required("REMORA_OIDC_PROJECT_ID")
|
||||
} else {
|
||||
env_required("REMORA_OIDC_CLIENT_ID")
|
||||
};
|
||||
let mut config = OidcConfig::new(issuer, audience).expect("oidc config");
|
||||
if provider_kind == "zitadel" {
|
||||
let project_id = env_required("REMORA_OIDC_PROJECT_ID");
|
||||
config = config
|
||||
.with_zitadel_project_roles()
|
||||
// Preferred project-scoped claim shape (ZITADEL docs).
|
||||
.with_project_role_claim(format!("urn:zitadel:iam:org:project:{project_id}:roles"))
|
||||
.with_resource_access_client(env_required("REMORA_OIDC_CLIENT_ID"));
|
||||
}
|
||||
OidcIdentityProvider::new(config).expect("provider")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn principal_mapping_is_stable_and_idempotent() {
|
||||
let Some(pool) = test_pool().await else {
|
||||
return;
|
||||
};
|
||||
let external = remora_identity::ExternalIdentity {
|
||||
issuer: "https://issuer.example/lane".into(),
|
||||
subject: "subject-stable-1".into(),
|
||||
roles: remora_identity::DecodedRoles {
|
||||
administrator: false,
|
||||
scopes: vec![],
|
||||
},
|
||||
};
|
||||
let first = resolve_principal(&pool, external.clone())
|
||||
.await
|
||||
.expect("first resolve");
|
||||
let second = resolve_principal(&pool, external)
|
||||
.await
|
||||
.expect("second resolve");
|
||||
assert_eq!(first.id, second.id);
|
||||
assert_eq!(first.issuer, "https://issuer.example/lane");
|
||||
assert_eq!(first.subject, "subject-stable-1");
|
||||
|
||||
let loaded = remora_identity::load_by_id(&pool, &first.id)
|
||||
.await
|
||||
.expect("load")
|
||||
.expect("present");
|
||||
assert_eq!(loaded.subject, "subject-stable-1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_oidc_jwks_rejects_invalid_tokens() {
|
||||
if oidc_issuer().is_none() {
|
||||
if require_env() {
|
||||
panic!("REMORA_OIDC_ISSUER is required when REMORA_IDENTITY_REQUIRE is set");
|
||||
}
|
||||
eprintln!("skipping oidc invalid-token test: REMORA_OIDC_ISSUER is unset");
|
||||
return;
|
||||
}
|
||||
let provider = oidc_provider();
|
||||
let minted = mint_unsigned_looking_token();
|
||||
let cases = [
|
||||
"",
|
||||
"not-a-jwt",
|
||||
"Bearer ",
|
||||
"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.e30.signature",
|
||||
minted.as_str(),
|
||||
];
|
||||
for token in cases {
|
||||
let result = provider.authenticate_token(token).await;
|
||||
assert!(
|
||||
matches!(result, Err(remora_identity::IdentityError::InvalidToken)),
|
||||
"expected InvalidToken for {token:?}, got {result:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Sequential mint then revoke: one fixture principal, one nextest process.
|
||||
#[tokio::test]
|
||||
async fn provider_issued_token_and_grant_loss() {
|
||||
if oidc_issuer().is_none() {
|
||||
if require_env() {
|
||||
panic!("REMORA_OIDC_ISSUER is required when REMORA_IDENTITY_REQUIRE is set");
|
||||
}
|
||||
eprintln!("skipping provider-issued token test: REMORA_OIDC_ISSUER is unset");
|
||||
return;
|
||||
}
|
||||
ensure_remora_grants()
|
||||
.await
|
||||
.expect("ensure remora grants before mint");
|
||||
let provider = oidc_provider();
|
||||
let before = mint_provider_token().await.expect("mint with grants");
|
||||
let with_grants = match provider.authenticate_token(&before).await {
|
||||
Ok(identity) => identity,
|
||||
Err(error) => panic!(
|
||||
"authenticate provider token: {error}; claims={}",
|
||||
jwt_payload_preview(&before)
|
||||
),
|
||||
};
|
||||
assert!(!with_grants.subject.is_empty());
|
||||
assert_eq!(with_grants.issuer, env_required("REMORA_OIDC_ISSUER"));
|
||||
assert!(
|
||||
with_grants.roles.administrator || !with_grants.roles.scopes.is_empty(),
|
||||
"expected remora roles on provider token, got {with_grants:?}; claims={}",
|
||||
jwt_payload_preview(&before)
|
||||
);
|
||||
let storage_role = std::env::var("REMORA_OIDC_STORAGE_ROLE").unwrap_or_default();
|
||||
if !storage_role.is_empty() {
|
||||
assert!(
|
||||
!with_grants.roles.scopes.is_empty(),
|
||||
"expected storage scope from fixture role {storage_role}"
|
||||
);
|
||||
assert_eq!(with_grants.roles.scopes[0].store.as_str(), "target-main");
|
||||
assert_eq!(with_grants.roles.scopes[0].bucket.as_str(), "target-bucket");
|
||||
assert_eq!(with_grants.roles.scopes[0].prefix.canonical(), "models");
|
||||
}
|
||||
|
||||
revoke_remora_grants().await.expect("revoke remora grants");
|
||||
|
||||
let after = mint_provider_token().await.expect("mint after revoke");
|
||||
let without_grants = provider
|
||||
.authenticate_token(&after)
|
||||
.await
|
||||
.expect("auth after revoke");
|
||||
assert!(
|
||||
!without_grants.roles.administrator,
|
||||
"new token must not be administrator after grant loss"
|
||||
);
|
||||
assert!(
|
||||
without_grants.roles.scopes.is_empty(),
|
||||
"new token must not carry storage scopes after grant loss"
|
||||
);
|
||||
}
|
||||
|
||||
async fn mint_provider_token() -> Result<String, String> {
|
||||
match env_required("REMORA_OIDC_PROVIDER").as_str() {
|
||||
"keycloak" => mint_keycloak_password_token().await,
|
||||
"zitadel" => mint_zitadel_jwt_bearer_token().await,
|
||||
other => Err(format!("unsupported REMORA_OIDC_PROVIDER={other}")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn mint_keycloak_password_token() -> Result<String, String> {
|
||||
let issuer = env_required("REMORA_OIDC_ISSUER");
|
||||
let client_id = env_required("REMORA_OIDC_CLIENT_ID");
|
||||
let username = env_required("REMORA_OIDC_USERNAME");
|
||||
let password = env_required("REMORA_OIDC_PASSWORD");
|
||||
let token_url = format!(
|
||||
"{}/protocol/openid-connect/token",
|
||||
issuer.trim_end_matches('/')
|
||||
);
|
||||
let client = reqwest::Client::new();
|
||||
let response = client
|
||||
.post(token_url)
|
||||
.form(&[
|
||||
("grant_type", "password"),
|
||||
("client_id", client_id.as_str()),
|
||||
("username", username.as_str()),
|
||||
("password", password.as_str()),
|
||||
("scope", "openid"),
|
||||
])
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!(
|
||||
"keycloak token endpoint: {} {}",
|
||||
response.status(),
|
||||
response.text().await.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
let body: Value = response.json().await.map_err(|error| error.to_string())?;
|
||||
body.get("access_token")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
.ok_or_else(|| "keycloak token response missing access_token".into())
|
||||
}
|
||||
|
||||
/// Mint a provider-issued ZITADEL access token via JWT Profile (user application key).
|
||||
async fn mint_zitadel_jwt_bearer_token() -> Result<String, String> {
|
||||
let issuer = env_required("REMORA_OIDC_ISSUER");
|
||||
let user_id = env_required("REMORA_OIDC_USER_ID");
|
||||
let project_id = env_required("REMORA_OIDC_PROJECT_ID");
|
||||
let key_b64 = env_required("REMORA_OIDC_USER_KEY_B64");
|
||||
let key_json = URL_SAFE_NO_PAD
|
||||
.decode(key_b64.as_bytes())
|
||||
.or_else(|_| base64::engine::general_purpose::STANDARD.decode(key_b64.as_bytes()))
|
||||
.map_err(|error| format!("decode user key: {error}"))?;
|
||||
let key: Value =
|
||||
serde_json::from_slice(&key_json).map_err(|error| format!("parse user key: {error}"))?;
|
||||
let key_id = key
|
||||
.get("keyId")
|
||||
.or_else(|| key.get("key_id"))
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("user key missing keyId")?
|
||||
.to_owned();
|
||||
let private_key = key
|
||||
.get("key")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("user key missing private key PEM")?
|
||||
.replace("\\n", "\n");
|
||||
let issuer = issuer.trim_end_matches('/').to_owned();
|
||||
let token_url = format!("{issuer}/oauth/v2/token");
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_err(|error| error.to_string())?
|
||||
.as_secs() as i64;
|
||||
// Prefer userId from the key document when present.
|
||||
let subject = key
|
||||
.get("userId")
|
||||
.or_else(|| key.get("userID"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or(user_id.as_str())
|
||||
.to_owned();
|
||||
let encoding = EncodingKey::from_rsa_pem(private_key.as_bytes()).map_err(|error| {
|
||||
format!(
|
||||
"user key pem: {error}; fields={:?}",
|
||||
key.as_object().map(|o| o.keys().collect::<Vec<_>>())
|
||||
)
|
||||
})?;
|
||||
// RFC 7523 assertion aud is the token endpoint; some builds also accept issuer.
|
||||
let audiences = [token_url.clone(), issuer.clone()];
|
||||
// Non-default port must appear in Host so issued iss matches REMORA_OIDC_ISSUER.
|
||||
let host = issuer
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| issuer.strip_prefix("https://"))
|
||||
.unwrap_or(issuer.as_str());
|
||||
let scope = format!(
|
||||
"openid profile \
|
||||
urn:zitadel:iam:org:project:id:{project_id}:aud \
|
||||
urn:zitadel:iam:org:project:roles \
|
||||
urn:zitadel:iam:org:projects:roles \
|
||||
urn:zitadel:iam:org:project:{project_id}:roles"
|
||||
);
|
||||
let client = reqwest::Client::new();
|
||||
let mut last_error = String::new();
|
||||
for audience in audiences {
|
||||
let mut header = Header::new(Algorithm::RS256);
|
||||
header.kid = Some(key_id.clone());
|
||||
let assertion = match encode(
|
||||
&header,
|
||||
&json!({
|
||||
"iss": subject,
|
||||
"sub": subject,
|
||||
"aud": audience,
|
||||
"iat": now,
|
||||
"exp": now + 3600,
|
||||
}),
|
||||
&encoding,
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
last_error = format!("sign assertion: {error}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let response = client
|
||||
.post(&token_url)
|
||||
.header("Host", host)
|
||||
.form(&[
|
||||
("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
|
||||
("assertion", assertion.as_str()),
|
||||
("scope", scope.as_str()),
|
||||
])
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if response.status().is_success() {
|
||||
let body: Value = response.json().await.map_err(|error| error.to_string())?;
|
||||
// Prefer whichever JWT carries remora role keys; fall back to access_token.
|
||||
let access = body.get("access_token").and_then(Value::as_str);
|
||||
let id_token = body.get("id_token").and_then(Value::as_str);
|
||||
for token in [access, id_token].into_iter().flatten() {
|
||||
if jwt_payload_preview(token).contains("remora:v1:") {
|
||||
return Ok(token.to_owned());
|
||||
}
|
||||
}
|
||||
if let Some(token) = access.or(id_token) {
|
||||
return Ok(token.to_owned());
|
||||
}
|
||||
return Err(format!(
|
||||
"zitadel token response missing tokens; body_keys={:?}",
|
||||
body.as_object().map(|o| o.keys().collect::<Vec<_>>())
|
||||
));
|
||||
}
|
||||
last_error = format!(
|
||||
"aud={audience}: {} {}",
|
||||
response.status(),
|
||||
response.text().await.unwrap_or_default()
|
||||
);
|
||||
}
|
||||
Err(format!("zitadel jwt-bearer token failed: {last_error}"))
|
||||
}
|
||||
|
||||
async fn ensure_remora_grants() -> Result<(), String> {
|
||||
match env_required("REMORA_OIDC_PROVIDER").as_str() {
|
||||
"keycloak" => ensure_keycloak_remora_client_roles().await,
|
||||
"zitadel" => {
|
||||
let storage_role = env_required("REMORA_OIDC_STORAGE_ROLE");
|
||||
set_zitadel_user_grant_roles(&["remora:v1:admin", storage_role.as_str()]).await
|
||||
}
|
||||
other => Err(format!("unsupported REMORA_OIDC_PROVIDER={other}")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn revoke_remora_grants() -> Result<(), String> {
|
||||
match env_required("REMORA_OIDC_PROVIDER").as_str() {
|
||||
"keycloak" => revoke_keycloak_remora_client_roles().await,
|
||||
"zitadel" => set_zitadel_user_grant_roles(&["test-user"]).await,
|
||||
other => Err(format!("unsupported REMORA_OIDC_PROVIDER={other}")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn revoke_keycloak_remora_client_roles() -> Result<(), String> {
|
||||
let (admin_token, realm, user_id, client_uuid, storage_role) =
|
||||
keycloak_admin_role_context().await?;
|
||||
let client = reqwest::Client::new();
|
||||
let roles_url = format!(
|
||||
"http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
|
||||
);
|
||||
let mapped: Value = client
|
||||
.get(&roles_url)
|
||||
.bearer_auth(&admin_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let remora_roles: Vec<Value> = mapped
|
||||
.as_array()
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|role| {
|
||||
role.get("name")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|name| {
|
||||
name == "remora:v1:admin"
|
||||
|| name == storage_role
|
||||
|| name.starts_with("remora:v1:")
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
if remora_roles.is_empty() {
|
||||
return Err("keycloak user has no remora client roles to revoke".into());
|
||||
}
|
||||
let delete = client
|
||||
.delete(&roles_url)
|
||||
.bearer_auth(&admin_token)
|
||||
.json(&remora_roles)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !delete.status().is_success() {
|
||||
return Err(format!(
|
||||
"keycloak delete client roles: {} {}",
|
||||
delete.status(),
|
||||
delete.text().await.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set_zitadel_user_grant_roles(role_keys: &[&str]) -> Result<(), String> {
|
||||
let issuer = env_required("REMORA_OIDC_ISSUER");
|
||||
let org_id = env_required("REMORA_OIDC_TENANT");
|
||||
let user_id = env_required("REMORA_OIDC_USER_ID");
|
||||
let project_id = env_required("REMORA_OIDC_PROJECT_ID");
|
||||
let mgmt_token = env_required("REMORA_OIDC_MGMT_TOKEN");
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let search_url = format!(
|
||||
"{}/management/v1/users/grants/_search",
|
||||
issuer.trim_end_matches('/')
|
||||
);
|
||||
let host = issuer
|
||||
.trim_end_matches('/')
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| issuer.trim_end_matches('/').strip_prefix("https://"))
|
||||
.unwrap_or(issuer.trim_end_matches('/'));
|
||||
let search = client
|
||||
.post(search_url)
|
||||
.header("Host", host)
|
||||
.header("Authorization", format!("Bearer {mgmt_token}"))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("x-zitadel-orgid", &org_id)
|
||||
.json(&json!({
|
||||
"queries": [{
|
||||
"userIdQuery": { "userId": user_id }
|
||||
}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !search.status().is_success() {
|
||||
return Err(format!(
|
||||
"zitadel search grants: {} {}",
|
||||
search.status(),
|
||||
search.text().await.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
let body: Value = search.json().await.map_err(|error| error.to_string())?;
|
||||
let grants = body
|
||||
.get("result")
|
||||
.and_then(Value::as_array)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
let grant = grants
|
||||
.iter()
|
||||
.find(|row| {
|
||||
row.get("projectId")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|id| id == project_id)
|
||||
})
|
||||
.ok_or("zitadel user project grant not found")?;
|
||||
let grant_id = grant
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("zitadel grant missing id")?;
|
||||
|
||||
let update_url = format!(
|
||||
"{}/management/v1/users/{user_id}/grants/{grant_id}",
|
||||
issuer.trim_end_matches('/')
|
||||
);
|
||||
let update = client
|
||||
.put(update_url)
|
||||
.header("Host", host)
|
||||
.header("Authorization", format!("Bearer {mgmt_token}"))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("x-zitadel-orgid", &org_id)
|
||||
.json(&json!({ "roleKeys": role_keys }))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !update.status().is_success() {
|
||||
let status = update.status();
|
||||
let body = update.text().await.unwrap_or_default();
|
||||
// Idempotent ensure: already-correct grants are rejected as unchanged.
|
||||
if status.as_u16() == 400 && body.contains("User grant has not been changed") {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(format!("zitadel update grant: {status} {body}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_keycloak_remora_client_roles() -> Result<(), String> {
|
||||
let (admin_token, realm, user_id, client_uuid, storage_role) =
|
||||
keycloak_admin_role_context().await?;
|
||||
let client = reqwest::Client::new();
|
||||
let available_url = format!(
|
||||
"http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}/available"
|
||||
);
|
||||
let available: Value = client
|
||||
.get(available_url)
|
||||
.bearer_auth(&admin_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let wanted: Vec<Value> = available
|
||||
.as_array()
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|role| {
|
||||
role.get("name")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|name| name == "remora:v1:admin" || name == storage_role)
|
||||
})
|
||||
.collect();
|
||||
if wanted.is_empty() {
|
||||
// Roles already mapped (or missing from the client entirely).
|
||||
return Ok(());
|
||||
}
|
||||
let roles_url = format!(
|
||||
"http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
|
||||
);
|
||||
let post = client
|
||||
.post(roles_url)
|
||||
.bearer_auth(&admin_token)
|
||||
.json(&wanted)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !post.status().is_success() {
|
||||
return Err(format!(
|
||||
"keycloak assign client roles: {} {}",
|
||||
post.status(),
|
||||
post.text().await.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn keycloak_admin_role_context() -> Result<(String, String, String, String, String), String> {
|
||||
let realm = env_required("REMORA_OIDC_TENANT");
|
||||
let client_id = env_required("REMORA_OIDC_CLIENT_ID");
|
||||
let username = env_required("REMORA_OIDC_USERNAME");
|
||||
let storage_role = env_required("REMORA_OIDC_STORAGE_ROLE");
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let admin_token_url =
|
||||
"http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/token";
|
||||
let admin_token_response = client
|
||||
.post(admin_token_url)
|
||||
.form(&[
|
||||
("grant_type", "password"),
|
||||
("client_id", "admin-cli"),
|
||||
("username", "remora-admin"),
|
||||
("password", "synthetic-keycloak"),
|
||||
])
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !admin_token_response.status().is_success() {
|
||||
return Err(format!(
|
||||
"keycloak admin token: {} {}",
|
||||
admin_token_response.status(),
|
||||
admin_token_response.text().await.unwrap_or_default()
|
||||
));
|
||||
}
|
||||
let admin_body: Value = admin_token_response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let admin_token = admin_body
|
||||
.get("access_token")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("keycloak admin token missing access_token")?
|
||||
.to_owned();
|
||||
|
||||
let users_url = format!(
|
||||
"http://keycloak.remora.test:8080/admin/realms/{realm}/users?username={username}&exact=true"
|
||||
);
|
||||
let users: Value = client
|
||||
.get(users_url)
|
||||
.bearer_auth(&admin_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let user_id = users
|
||||
.as_array()
|
||||
.and_then(|rows| rows.first())
|
||||
.and_then(|row| row.get("id"))
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("keycloak user not found")?
|
||||
.to_owned();
|
||||
|
||||
let clients_url = format!(
|
||||
"http://keycloak.remora.test:8080/admin/realms/{realm}/clients?clientId={client_id}"
|
||||
);
|
||||
let clients: Value = client
|
||||
.get(clients_url)
|
||||
.bearer_auth(&admin_token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let client_uuid = clients
|
||||
.as_array()
|
||||
.and_then(|rows| rows.first())
|
||||
.and_then(|row| row.get("id"))
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("keycloak client not found")?
|
||||
.to_owned();
|
||||
|
||||
Ok((admin_token, realm, user_id, client_uuid, storage_role))
|
||||
}
|
||||
|
||||
fn jwt_payload_preview(token: &str) -> String {
|
||||
let parts: Vec<&str> = token.split('.').collect();
|
||||
if parts.len() != 3 {
|
||||
return format!("<not-a-jwt parts={} len={}>", parts.len(), token.len());
|
||||
}
|
||||
let Ok(bytes) = URL_SAFE_NO_PAD
|
||||
.decode(parts[1].as_bytes())
|
||||
.or_else(|_| base64::engine::general_purpose::STANDARD.decode(parts[1].as_bytes()))
|
||||
else {
|
||||
return "<payload-decode-failed>".into();
|
||||
};
|
||||
match String::from_utf8(bytes) {
|
||||
Ok(text) => text,
|
||||
Err(_) => "<payload-not-utf8>".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn mint_unsigned_looking_token() -> String {
|
||||
let header = Header::new(Algorithm::HS256);
|
||||
let claims = json!({
|
||||
"iss": "https://attacker.example",
|
||||
"sub": "attacker",
|
||||
"aud": "remora-client",
|
||||
"exp": 4_102_444_800_i64
|
||||
});
|
||||
encode(
|
||||
&header,
|
||||
&claims,
|
||||
&EncodingKey::from_secret(b"not-the-fixture-secret"),
|
||||
)
|
||||
.expect("mint")
|
||||
}
|
||||
163
crates/remora-identity/tests/token_validation.rs
Normal file
163
crates/remora-identity/tests/token_validation.rs
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, encode};
|
||||
use remora_domain::{BucketName, StorageScope, TargetPrefix, TargetStoreId};
|
||||
use remora_identity::{
|
||||
OidcConfig, OidcIdentityProvider, ZITADEL_PROJECT_ROLES_CLAIM, encode_admin_role,
|
||||
encode_storage_role,
|
||||
};
|
||||
use rsa::RsaPrivateKey;
|
||||
use rsa::pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding};
|
||||
use serde_json::json;
|
||||
|
||||
struct EphemeralRsa {
|
||||
encoding: EncodingKey,
|
||||
decoding: DecodingKey,
|
||||
}
|
||||
|
||||
static EPHEMERAL_RSA: LazyLock<EphemeralRsa> = LazyLock::new(|| {
|
||||
let mut rng = rand::thread_rng();
|
||||
let private = RsaPrivateKey::new(&mut rng, 2048).expect("generate ephemeral RSA key");
|
||||
let private_pem = private
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("encode private pem");
|
||||
let public_pem = private
|
||||
.to_public_key()
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("encode public pem");
|
||||
EphemeralRsa {
|
||||
encoding: EncodingKey::from_rsa_pem(private_pem.as_bytes()).expect("encoding key"),
|
||||
decoding: DecodingKey::from_rsa_pem(public_pem.as_bytes()).expect("decoding key"),
|
||||
}
|
||||
});
|
||||
|
||||
fn storage_scope() -> StorageScope {
|
||||
StorageScope {
|
||||
store: TargetStoreId::parse("target-main").unwrap(),
|
||||
bucket: BucketName::parse("target-bucket").unwrap(),
|
||||
prefix: TargetPrefix::parse("models", []).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn provider(with_zitadel_claim: bool) -> OidcIdentityProvider {
|
||||
let mut keys = HashMap::new();
|
||||
keys.insert("test-kid".into(), EPHEMERAL_RSA.decoding.clone());
|
||||
let mut config = OidcConfig::new("https://issuer.example", "remora-client").expect("config");
|
||||
if with_zitadel_claim {
|
||||
config = config.with_zitadel_project_roles();
|
||||
}
|
||||
OidcIdentityProvider::with_static_keys(config, keys).expect("provider")
|
||||
}
|
||||
|
||||
fn mint(claims: serde_json::Value) -> String {
|
||||
let mut header = Header::new(Algorithm::RS256);
|
||||
header.kid = Some("test-kid".into());
|
||||
encode(&header, &claims, &EPHEMERAL_RSA.encoding).expect("encode")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn valid_token_maps_roles_from_configured_zitadel_claim() {
|
||||
let storage = encode_storage_role(&storage_scope()).unwrap();
|
||||
let admin = encode_admin_role();
|
||||
let mut project_roles = serde_json::Map::new();
|
||||
project_roles.insert(admin, json!({ "org-1": "example.local" }));
|
||||
project_roles.insert(storage, json!({ "org-1": "example.local" }));
|
||||
let token = mint(json!({
|
||||
"iss": "https://issuer.example",
|
||||
"sub": "user-12345678",
|
||||
"aud": "remora-client",
|
||||
"exp": 4_102_444_800_i64,
|
||||
ZITADEL_PROJECT_ROLES_CLAIM: project_roles,
|
||||
// Untrusted sources must not grant authority.
|
||||
"realm_access": { "roles": ["admin", "remora:v1:admin"] },
|
||||
"roles": ["remora:v1:admin"],
|
||||
}));
|
||||
let identity = provider(true)
|
||||
.authenticate_token(&token)
|
||||
.await
|
||||
.expect("valid token");
|
||||
assert_eq!(identity.subject, "user-12345678");
|
||||
assert!(identity.roles.administrator);
|
||||
assert_eq!(identity.roles.scopes.len(), 1);
|
||||
assert_eq!(identity.roles.scopes[0].prefix.canonical(), "models");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unconfigured_project_claim_and_bare_admin_do_not_grant() {
|
||||
let mut project_roles = serde_json::Map::new();
|
||||
project_roles.insert(encode_admin_role(), json!({ "org-1": "example.local" }));
|
||||
let token = mint(json!({
|
||||
"iss": "https://issuer.example",
|
||||
"sub": "user-12345678",
|
||||
"aud": "remora-client",
|
||||
"exp": 4_102_444_800_i64,
|
||||
"urn:zitadel:iam:org:project:other:roles": project_roles,
|
||||
"resource_access": {
|
||||
"evil-client": { "roles": ["remora:v1:admin"] }
|
||||
},
|
||||
"realm_access": { "roles": ["admin"] },
|
||||
}));
|
||||
let identity = provider(true)
|
||||
.authenticate_token(&token)
|
||||
.await
|
||||
.expect("valid token without remora grants");
|
||||
assert!(!identity.roles.administrator);
|
||||
assert!(identity.roles.scopes.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn keycloak_resource_access_client_roles_decode() {
|
||||
let token = mint(json!({
|
||||
"iss": "https://issuer.example",
|
||||
"sub": "kc-user-0001",
|
||||
"aud": ["remora-client", "account"],
|
||||
"exp": 4_102_444_800_i64,
|
||||
"resource_access": {
|
||||
"remora-client": { "roles": [encode_admin_role()] },
|
||||
"account": { "roles": ["manage-account"] }
|
||||
},
|
||||
}));
|
||||
let identity = provider(false)
|
||||
.authenticate_token(&token)
|
||||
.await
|
||||
.expect("valid keycloak-shaped token");
|
||||
assert!(identity.roles.administrator);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_issuer_audience_exp_and_signature_fail_closed() {
|
||||
let good_base = json!({
|
||||
"iss": "https://issuer.example",
|
||||
"sub": "user-12345678",
|
||||
"aud": "remora-client",
|
||||
"exp": 4_102_444_800_i64,
|
||||
"resource_access": {
|
||||
"remora-client": { "roles": [encode_admin_role()] }
|
||||
},
|
||||
});
|
||||
|
||||
let wrong_issuer = {
|
||||
let mut claims = good_base.clone();
|
||||
claims["iss"] = json!("https://evil.example");
|
||||
mint(claims)
|
||||
};
|
||||
let wrong_aud = {
|
||||
let mut claims = good_base.clone();
|
||||
claims["aud"] = json!("other-client");
|
||||
mint(claims)
|
||||
};
|
||||
let expired = {
|
||||
let mut claims = good_base.clone();
|
||||
claims["exp"] = json!(1_000_000_i64);
|
||||
mint(claims)
|
||||
};
|
||||
|
||||
for token in [wrong_issuer, wrong_aud, expired, "totally-not-a-jwt".into()] {
|
||||
let result = provider(false).authenticate_token(&token).await;
|
||||
assert!(
|
||||
matches!(result, Err(remora_identity::IdentityError::InvalidToken)),
|
||||
"token should be rejected: {result:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue