Add Redis-compatible coordination substrate

Cargo.lock

@@ -31,6 +31,15 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 
+[[package]]
+name = "arc-swap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "atoi"
 version = "2.0.0"
@@ -46,6 +55,15 @@ version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
+[[package]]
+name = "backon"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
+dependencies = [
+ "fastrand",
+]
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -157,6 +175,20 @@ version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
 
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "2.5.0"
@@ -358,6 +390,12 @@ dependencies = [
  "regex-syntax",
 ]
 
+[[package]]
+name = "fastrand"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+
 [[package]]
 name = "find-msvc-tools"
 version = "0.1.9"
@@ -1045,6 +1083,31 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
 
+[[package]]
+name = "redis"
+version = "0.32.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44"
+dependencies = [
+ "arc-swap",
+ "backon",
+ "bytes",
+ "cfg-if",
+ "combine",
+ "futures-channel",
+ "futures-util",
+ "itoa",
+ "num-bigint",
+ "percent-encoding",
+ "pin-project-lite",
+ "ryu",
+ "sha1_smol",
+ "socket2",
+ "tokio",
+ "tokio-util",
+ "url",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"
@@ -1136,6 +1199,17 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "remora-coordination"
+version = "0.0.0"
+dependencies = [
+ "redis",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "tokio",
+]
+
 [[package]]
 name = "remora-domain"
 version = "0.0.0"
@@ -1318,6 +1392,12 @@ dependencies = [
  "digest 0.11.3",
 ]
 
+[[package]]
+name = "sha1_smol"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
+
 [[package]]
 name = "sha2"
 version = "0.10.9"
@@ -1714,6 +1794,19 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-util"
+version = "0.7.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tracing"
 version = "0.1.44"

Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "crates/remora-api",
     "crates/remora-contracts",
+    "crates/remora-coordination",
     "crates/remora-domain",
     "crates/remora-durable",
     "crates/remora-worker",

Dockerfile.test

@@ -62,6 +62,26 @@ COPY crates crates
 COPY --from=kind-durable-build /tmp/remora-durable-tests.tar.zst /tests/remora-durable-tests.tar.zst
 CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && cargo-nextest nextest run --archive-file /tests/remora-durable-tests.tar.zst --workspace-remap /workspace"]
 
+FROM rust-tools AS kind-coordination-build
+WORKDIR /workspace
+COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY .config .config
+COPY contracts contracts
+COPY crates crates
+COPY docs/api docs/api
+RUN cargo nextest archive \
+    --package remora-coordination \
+    --build-jobs 4 \
+    --locked \
+    --archive-file /tmp/remora-coordination-tests.tar.zst
+
+FROM rust-tools AS kind-coordination-runner
+WORKDIR /workspace
+COPY Cargo.toml ./
+COPY crates crates
+COPY --from=kind-coordination-build /tmp/remora-coordination-tests.tar.zst /tests/remora-coordination-tests.tar.zst
+CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$REMORA_COORDINATION_URL\" && export REMORA_COORDINATION_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-coordination-tests.tar.zst --workspace-remap /workspace"]
+
 FROM ${NODE_IMAGE} AS node-runtime
 
 FROM ${PLAYWRIGHT_IMAGE} AS web-dependencies

Makefile

@@ -1,4 +1,4 @@
-.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
+.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
 
 CONTAINER_ENGINE ?= nerdctl
 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@@ -58,3 +58,6 @@ check-kind-sample:
 
 check-durable:
 	./tests/kind/run-durable-job.sh
+
+check-coordination:
+	./tests/kind/run-coordination-job.sh

crates/remora-coordination/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "remora-coordination"
+edition.workspace = true
+publish = false
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+redis = { version = "0.32.7", default-features = false, features = [
+    "aio",
+    "connection-manager",
+    "script",
+    "tokio-comp",
+] }
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }

crates/remora-coordination/src/error.rs

@@ -0,0 +1,28 @@
+use std::fmt;
+
+#[derive(Debug, thiserror::Error)]
+pub enum CoordinationError {
+    #[error("coordination store error: {0}")]
+    Redis(#[from] redis::RedisError),
+    #[error("invalid coordination response: {0}")]
+    Protocol(String),
+    #[error("configuration error: {0}")]
+    Config(String),
+    #[error("local permit capacity exhausted")]
+    LocalFull,
+    /// Value exceeds the IEEE-754 exact integer range used by Lua `tonumber`.
+    #[error("value {value} exceeds max safe integer {max}")]
+    UnsafeInteger { value: u64, max: u64 },
+    #[error("json error: {0}")]
+    Json(#[from] serde_json::Error),
+}
+
+impl CoordinationError {
+    pub fn protocol(message: impl fmt::Display) -> Self {
+        Self::Protocol(message.to_string())
+    }
+
+    pub fn config(message: impl fmt::Display) -> Self {
+        Self::Config(message.to_string())
+    }
+}

crates/remora-coordination/src/keys.rs

@@ -0,0 +1,54 @@
+//! Namespaced keys for queue, reservation, lease, progress, cancellation, and permits.
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct KeySpace {
+    prefix: String,
+}
+
+impl KeySpace {
+    pub fn new(prefix: impl Into<String>) -> Self {
+        Self {
+            prefix: prefix.into(),
+        }
+    }
+
+    #[must_use]
+    pub fn prefix(&self) -> &str {
+        &self.prefix
+    }
+
+    #[must_use]
+    pub fn queue(&self) -> String {
+        format!("{}queue", self.prefix)
+    }
+
+    #[must_use]
+    pub fn queue_members(&self) -> String {
+        format!("{}queue:members", self.prefix)
+    }
+
+    #[must_use]
+    pub fn reserve(&self, job_id: &str) -> String {
+        format!("{}reserve:{job_id}", self.prefix)
+    }
+
+    #[must_use]
+    pub fn lease(&self, job_id: &str) -> String {
+        format!("{}lease:{job_id}", self.prefix)
+    }
+
+    #[must_use]
+    pub fn progress(&self, job_id: &str) -> String {
+        format!("{}progress:{job_id}", self.prefix)
+    }
+
+    #[must_use]
+    pub fn cancel(&self, job_id: &str) -> String {
+        format!("{}cancel:{job_id}", self.prefix)
+    }
+
+    #[must_use]
+    pub fn permit_slots(&self) -> String {
+        format!("{}permit:slots", self.prefix)
+    }
+}

crates/remora-coordination/src/lib.rs

@@ -0,0 +1,18 @@
+//! Dragonfly/Redis coordination primitives with G-A3 global slots and L-B1
+//! process-local permits; feature packages own protocol orchestration.
+
+mod error;
+mod keys;
+mod local;
+mod scripts;
+mod store;
+mod types;
+
+pub use error::CoordinationError;
+pub use keys::KeySpace;
+pub use local::LocalPermitGate;
+pub use store::{CoordinationStore, StoreConfig};
+pub use types::{
+    ActivateOptions, LeaseOwner, LeaseRecord, LiveLease, MAX_SAFE_INTEGER, OpCode, OpResult,
+    ProgressHint, Reservation, ensure_safe_integer,
+};

crates/remora-coordination/src/local.rs

@@ -0,0 +1,101 @@
+//! IF-1 L-B1 permit accounting holds local capacity until cooperative stop or
+//! process death, independently of lease expiry.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use crate::error::CoordinationError;
+
+#[derive(Clone, Debug)]
+pub struct LocalPermitGate {
+    inner: Arc<Mutex<LocalState>>,
+    limit: u64,
+}
+
+#[derive(Debug, Default)]
+struct LocalState {
+    /// Units held by this incarnation (reserves + executing tasks).
+    holds: HashMap<String, u64>,
+    used: u64,
+}
+
+impl LocalPermitGate {
+    #[must_use]
+    pub fn new(limit: u64) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(LocalState::default())),
+            limit,
+        }
+    }
+
+    #[must_use]
+    pub fn limit(&self) -> u64 {
+        self.limit
+    }
+
+    pub fn used(&self) -> u64 {
+        self.inner.lock().expect("local permit lock").used
+    }
+
+    /// Nested acquisition increments a refcount without increasing occupancy;
+    /// each acquire needs `release`, or cooperative stop needs `release_all`.
+    pub fn try_acquire(&self, hold_id: impl Into<String>) -> Result<bool, CoordinationError> {
+        let hold_id = hold_id.into();
+        let mut state = self.inner.lock().expect("local permit lock");
+        if let Some(count) = state.holds.get_mut(&hold_id) {
+            *count = count.saturating_add(1);
+            return Ok(true);
+        }
+        if state.used >= self.limit {
+            return Ok(false);
+        }
+        state.holds.insert(hold_id, 1);
+        state.used = state.used.saturating_add(1);
+        Ok(true)
+    }
+
+    /// Acquire or return [`CoordinationError::LocalFull`].
+    pub fn acquire(&self, hold_id: impl Into<String>) -> Result<(), CoordinationError> {
+        if self.try_acquire(hold_id)? {
+            Ok(())
+        } else {
+            Err(CoordinationError::LocalFull)
+        }
+    }
+
+    /// Release one nested hold. When the last nested unit for `hold_id` is
+    /// released, global local occupancy decreases by one.
+    pub fn release(&self, hold_id: &str) -> bool {
+        let mut state = self.inner.lock().expect("local permit lock");
+        let Some(count) = state.holds.get_mut(hold_id) else {
+            return false;
+        };
+        *count = count.saturating_sub(1);
+        if *count == 0 {
+            state.holds.remove(hold_id);
+            state.used = state.used.saturating_sub(1);
+        }
+        true
+    }
+
+    /// Drop every hold for `hold_id` (cooperative stop of one task).
+    pub fn release_all(&self, hold_id: &str) -> bool {
+        let mut state = self.inner.lock().expect("local permit lock");
+        if state.holds.remove(hold_id).is_some() {
+            state.used = state.used.saturating_sub(1);
+            true
+        } else {
+            false
+        }
+    }
+
+    /// True while `hold_id` still occupies local capacity (including after
+    /// lease expiry while the attempt is still executing).
+    pub fn holds(&self, hold_id: &str) -> bool {
+        self.inner
+            .lock()
+            .expect("local permit lock")
+            .holds
+            .contains_key(hold_id)
+    }
+}

crates/remora-coordination/src/lua/activate.lua

@@ -0,0 +1,116 @@
+-- KEYS: lease, reserve, progress, slots. ARGV: attempt_id, fence, worker_id,
+-- lease_ttl_ms, seed_progress, require_reserve, job_id, global_limit, progress_ttl_ms.
+local function now_ms()
+  local t = redis.call('TIME')
+  return (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
+end
+
+local function fence_str(f)
+  if type(f) == 'string' then
+    return f
+  end
+  return string.format('%.0f', tonumber(f))
+end
+
+local function owner_eq(rec, attempt_id, fence, worker_id)
+  return rec.attempt_id == attempt_id
+    and fence_str(rec.fence) == fence_str(fence)
+    and rec.worker_id == worker_id
+end
+
+local function seed_progress_hint(progress_key, attempt_id, fence, progress_ttl_ms)
+  local prog = {
+    attempt_id = attempt_id,
+    fence = fence_str(fence),
+    bytes = '0'
+  }
+  redis.call('SET', progress_key, cjson.encode(prog))
+  redis.call('PEXPIRE', progress_key, progress_ttl_ms)
+end
+
+local lease_key = KEYS[1]
+local reserve_key = KEYS[2]
+local progress_key = KEYS[3]
+local slots_key = KEYS[4]
+local attempt_id = ARGV[1]
+local fence = ARGV[2]
+local worker_id = ARGV[3]
+local lease_ttl_ms = tonumber(ARGV[4])
+local seed_progress = ARGV[5] == '1'
+local require_reserve = ARGV[6] == '1'
+local job_id = ARGV[7]
+local global_limit = tonumber(ARGV[8])
+local progress_ttl_ms = tonumber(ARGV[9]) or lease_ttl_ms
+
+local now = now_ms()
+local raw = redis.call('GET', lease_key)
+local rec = nil
+if raw then
+  rec = cjson.decode(raw)
+end
+
+if rec and tonumber(rec.expires_at) > now then
+  if owner_eq(rec, attempt_id, fence, worker_id) then
+    -- At-least-once activate: refresh lease only; never reset progress bytes.
+    rec.expires_at = now + lease_ttl_ms
+    redis.call('SET', lease_key, cjson.encode(rec))
+    redis.call('PEXPIRE', lease_key, lease_ttl_ms)
+    redis.call('SADD', slots_key, job_id)
+    return {
+      'OK_IDEMPOTENT',
+      attempt_id,
+      fence_str(fence),
+      worker_id,
+      string.format('%.0f', tonumber(rec.expires_at))
+    }
+  end
+  return {
+    'FAIL_HELD',
+    rec.attempt_id,
+    fence_str(rec.fence),
+    rec.worker_id,
+    string.format('%.0f', tonumber(rec.expires_at))
+  }
+end
+
+local res_raw = redis.call('GET', reserve_key)
+if res_raw then
+  local res = cjson.decode(res_raw)
+  if res and res.worker_id and res.worker_id ~= worker_id then
+    return {'FAIL_RESERVE', res.worker_id}
+  end
+elseif require_reserve then
+  return {'FAIL_RESERVE', 'missing'}
+end
+
+if redis.call('SISMEMBER', slots_key, job_id) == 0 then
+  local used = redis.call('SCARD', slots_key)
+  if used >= global_limit then
+    return {'FAIL_FULL', tostring(used), tostring(global_limit)}
+  end
+  redis.call('SADD', slots_key, job_id)
+end
+
+local new_rec = {
+  attempt_id = attempt_id,
+  fence = fence_str(fence),
+  worker_id = worker_id,
+  expires_at = now + lease_ttl_ms
+}
+redis.call('SET', lease_key, cjson.encode(new_rec))
+redis.call('PEXPIRE', lease_key, lease_ttl_ms)
+if res_raw then
+  redis.call('DEL', reserve_key)
+end
+if seed_progress then
+  seed_progress_hint(progress_key, attempt_id, fence, progress_ttl_ms)
+else
+  redis.call('DEL', progress_key)
+end
+return {
+  'OK',
+  attempt_id,
+  fence_str(fence),
+  worker_id,
+  string.format('%.0f', tonumber(new_rec.expires_at))
+}

crates/remora-coordination/src/lua/clean_terminal.lua

@@ -0,0 +1,22 @@
+-- KEYS: queue, members, reserve, progress, cancel, slots. ARGV[1]=job_id.
+-- Terminal cleanup preserves live leases and frees only reservation occupancy.
+local queue_key = KEYS[1]
+local members_key = KEYS[2]
+local reserve_key = KEYS[3]
+local progress_key = KEYS[4]
+local cancel_key = KEYS[5]
+local slots_key = KEYS[6]
+local job_id = ARGV[1]
+
+redis.call('LREM', queue_key, 0, job_id)
+redis.call('SREM', members_key, job_id)
+
+local had_reserve = redis.call('DEL', reserve_key)
+redis.call('DEL', progress_key)
+redis.call('DEL', cancel_key)
+
+if had_reserve > 0 then
+  redis.call('SREM', slots_key, job_id)
+end
+
+return {'OK', tostring(had_reserve)}

crates/remora-coordination/src/lua/clear_stale_progress.lua

@@ -0,0 +1,38 @@
+-- KEYS[1]=lease KEYS[2]=progress
+local function now_ms()
+  local t = redis.call('TIME')
+  return (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
+end
+
+local function fence_str(f)
+  if type(f) == 'string' then
+    return f
+  end
+  return string.format('%.0f', tonumber(f))
+end
+
+local lease_key = KEYS[1]
+local progress_key = KEYS[2]
+local praw = redis.call('GET', progress_key)
+if not praw then
+  return {'OK_GONE'}
+end
+local prog = cjson.decode(praw)
+local lraw = redis.call('GET', lease_key)
+if not lraw then
+  redis.call('DEL', progress_key)
+  return {'OK', 'no_lease'}
+end
+local rec = cjson.decode(lraw)
+local now = now_ms()
+if tonumber(rec.expires_at) <= now then
+  redis.call('DEL', progress_key)
+  return {'OK', 'expired_lease'}
+end
+if rec.attempt_id == prog.attempt_id
+  and fence_str(rec.fence) == fence_str(prog.fence)
+then
+  return {'OK_IDEMPOTENT', 'live'}
+end
+redis.call('DEL', progress_key)
+return {'OK', 'mismatched'}

crates/remora-coordination/src/lua/dequeue.lua

@@ -0,0 +1,9 @@
+-- KEYS[1]=queue list KEYS[2]=queue members
+local queue_key = KEYS[1]
+local members_key = KEYS[2]
+local job_id = redis.call('LPOP', queue_key)
+if not job_id then
+  return {'NONE'}
+end
+redis.call('SREM', members_key, job_id)
+return {'OK', job_id}

crates/remora-coordination/src/lua/enqueue.lua

@@ -0,0 +1,10 @@
+-- KEYS[1]=queue list KEYS[2]=queue members
+-- ARGV[1]=job_id — idempotent projection of Queued dispatch.
+local queue_key = KEYS[1]
+local members_key = KEYS[2]
+local job_id = ARGV[1]
+if redis.call('SADD', members_key, job_id) == 0 then
+  return {'OK_IDEMPOTENT', job_id}
+end
+redis.call('RPUSH', queue_key, job_id)
+return {'OK', job_id}

crates/remora-coordination/src/lua/expire.lua

@@ -0,0 +1,34 @@
+-- KEYS: lease, slots, reserve. ARGV[1]=job_id.
+-- C2 expiry preserves G-A3 occupancy while a reservation remains.
+local function now_ms()
+  local t = redis.call('TIME')
+  return (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
+end
+
+local function free_slot_if_unreserved(slots_key, reserve_key, job_id)
+  if redis.call('EXISTS', reserve_key) == 0 then
+    redis.call('SREM', slots_key, job_id)
+  end
+end
+
+local lease_key = KEYS[1]
+local slots_key = KEYS[2]
+local reserve_key = KEYS[3]
+local job_id = ARGV[1]
+local raw = redis.call('GET', lease_key)
+if not raw then
+  free_slot_if_unreserved(slots_key, reserve_key, job_id)
+  return {'OK_GONE'}
+end
+local rec = cjson.decode(raw)
+local now = now_ms()
+if tonumber(rec.expires_at) <= now then
+  redis.call('DEL', lease_key)
+  free_slot_if_unreserved(slots_key, reserve_key, job_id)
+  return {'OK', 'expired'}
+end
+return {
+  'FAIL_HELD',
+  string.format('%.0f', tonumber(rec.expires_at)),
+  string.format('%.0f', now)
+}

crates/remora-coordination/src/lua/free_reserve.lua

@@ -0,0 +1,39 @@
+-- KEYS: reserve, slots, lease. ARGV: worker_id (empty = any), job_id.
+-- Preserve the global slot while a live lease remains.
+local function now_ms()
+  local t = redis.call('TIME')
+  return (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
+end
+
+local function live_lease(lease_key)
+  local lraw = redis.call('GET', lease_key)
+  if not lraw then
+    return false
+  end
+  local rec = cjson.decode(lraw)
+  return tonumber(rec.expires_at) > now_ms()
+end
+
+local reserve_key = KEYS[1]
+local slots_key = KEYS[2]
+local lease_key = KEYS[3]
+local worker_id = ARGV[1]
+local job_id = ARGV[2]
+local raw = redis.call('GET', reserve_key)
+if not raw then
+  if not live_lease(lease_key) then
+    redis.call('SREM', slots_key, job_id)
+  end
+  return {'OK_GONE'}
+end
+if worker_id and worker_id ~= '' then
+  local res = cjson.decode(raw)
+  if res.worker_id ~= worker_id then
+    return {'FAIL_RESERVE', res.worker_id}
+  end
+end
+redis.call('DEL', reserve_key)
+if not live_lease(lease_key) then
+  redis.call('SREM', slots_key, job_id)
+end
+return {'OK'}

crates/remora-coordination/src/lua/get_live.lua

@@ -0,0 +1,29 @@
+-- KEYS[1]=lease
+local function now_ms()
+  local t = redis.call('TIME')
+  return (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
+end
+
+local function fence_str(f)
+  if type(f) == 'string' then
+    return f
+  end
+  return string.format('%.0f', tonumber(f))
+end
+
+local raw = redis.call('GET', KEYS[1])
+if not raw then
+  return {'NONE'}
+end
+local rec = cjson.decode(raw)
+local now = now_ms()
+local live = tonumber(rec.expires_at) > now and '1' or '0'
+return {
+  'REC',
+  live,
+  rec.attempt_id,
+  fence_str(rec.fence),
+  rec.worker_id,
+  string.format('%.0f', tonumber(rec.expires_at)),
+  string.format('%.0f', now)
+}

crates/remora-coordination/src/lua/publish_cancel.lua

@@ -0,0 +1,14 @@
+-- KEYS[1]=cancel ARGV[1]=ttl_ms (0 = no TTL)
+local cancel_key = KEYS[1]
+local ttl_ms = tonumber(ARGV[1])
+if redis.call('EXISTS', cancel_key) == 1 then
+  if ttl_ms > 0 then
+    redis.call('PEXPIRE', cancel_key, ttl_ms)
+  end
+  return {'OK_IDEMPOTENT'}
+end
+redis.call('SET', cancel_key, '1')
+if ttl_ms > 0 then
+  redis.call('PEXPIRE', cancel_key, ttl_ms)
+end
+return {'OK'}

crates/remora-coordination/src/lua/release.lua

@@ -0,0 +1,52 @@
+-- KEYS[1]=lease KEYS[2]=slots KEYS[3]=reserve
+-- ARGV[1]=attempt_id ARGV[2]=fence ARGV[3]=worker_id ARGV[4]=job_id
+local function now_ms()
+  local t = redis.call('TIME')
+  return (tonumber(t[1]) * 1000) + math.floor(tonumber(t[2]) / 1000)
+end
+
+local function fence_str(f)
+  if type(f) == 'string' then
+    return f
+  end
+  return string.format('%.0f', tonumber(f))
+end
+
+local function owner_eq(rec, attempt_id, fence, worker_id)
+  return rec.attempt_id == attempt_id
+    and fence_str(rec.fence) == fence_str(fence)
+    and rec.worker_id == worker_id
+end
+
+local function free_slot_if_unreserved(slots_key, reserve_key, job_id)
+  if redis.call('EXISTS', reserve_key) == 0 then
+    redis.call('SREM', slots_key, job_id)
+  end
+end
+
+local lease_key = KEYS[1]
+local slots_key = KEYS[2]
+local reserve_key = KEYS[3]
+local attempt_id = ARGV[1]
+local fence = ARGV[2]
+local worker_id = ARGV[3]
+local job_id = ARGV[4]
+
+local raw = redis.call('GET', lease_key)
+if not raw then
+  free_slot_if_unreserved(slots_key, reserve_key, job_id)
+  return {'OK_GONE'}
+end
+local rec = cjson.decode(raw)
+local now = now_ms()
+if not owner_eq(rec, attempt_id, fence, worker_id) then
+  if tonumber(rec.expires_at) > now then
+    return {'FAIL_STOLEN', rec.attempt_id, fence_str(rec.fence), rec.worker_id}
+  end
+  redis.call('DEL', lease_key)
+  free_slot_if_unreserved(slots_key, reserve_key, job_id)
+  return {'OK_GONE'}
+end
+redis.call('DEL', lease_key)
+free_slot_if_unreserved(slots_key, reserve_key, job_id)
+return {'OK'}

crates/remora-coordination/src/lua/remove_queue.lua

@@ -0,0 +1,10 @@
+-- KEYS[1]=queue list KEYS[2]=queue members ARGV[1]=job_id
+local queue_key = KEYS[1]
+local members_key = KEYS[2]
+local job_id = ARGV[1]
+redis.call('LREM', queue_key, 0, job_id)
+local removed = redis.call('SREM', members_key, job_id)
+if removed == 0 then
+  return {'OK_GONE', job_id}
+end
+return {'OK', job_id}