ms2k/remora
SHA256
Watch
0
Code Releases Activity

Harden WebBFF and deployment release behavior

Browse source 
Model fail-closed cross-replica session renewal and implement owner-fenced refresh, transient observer handling, logout fencing, and graceful shutdown. Remove chart-owned Ingress while retaining qualification-owned external routing.
This commit is contained in:
ms2k 2026-07-20 17:09:26 +09:00
commit 414b380516
8 changed files with 980 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

120
web/server.mjs
View file

@ -4,10 +4,12 @@ import { extname, resolve, sep } from 'node:path'
import { pathToFileURL } from 'node:url'
import { loadConfig } from './server/config.mjs'
import { OAuthClient } from './server/oauth.mjs'
import { OAuthClient, SessionRenewalPendingError } from './server/oauth.mjs'
import { ApiProxy } from './server/proxy.mjs'
import { SessionStore } from './server/sessions.mjs'
const SHUTDOWN_TIMEOUT_MILLISECONDS = 25_000
export function createWebServer(config, dependencies = {}) {
const store = dependencies.store ?? new SessionStore(config)
const oauth =
@ -17,7 +19,9 @@ export function createWebServer(config, dependencies = {}) {
dependencies.proxy ??
new ApiProxy(config, oauth, dependencies.fetchImplementation)
const startedAt = Date.now()
const runtime = { accepting: true }
let requests = 0
let shutdownPromise
const server = createServer(async (request, response) => {
requests += 1
@ -35,6 +39,7 @@ export function createWebServer(config, dependencies = {}) {
status: response.statusCode,
})}\n`,
)
if (!runtime.accepting) server.closeIdleConnections()
})
try {
await route(
@ -46,21 +51,39 @@ export function createWebServer(config, dependencies = {}) {
proxy,
startedAt,
requests,
runtime,
)
} catch {
} catch (error) {
if (!response.headersSent)
response.writeHead(502, { 'Cache-Control': 'no-store' })
response.writeHead(
error instanceof SessionRenewalPendingError ? 503 : 502,
{ 'Cache-Control': 'no-store' },
)
response.end()
}
})
server.on('upgrade', async (request, socket, head) => {
try {
if (!(await proxy.upgrade(request, socket, head))) socket.destroy()
} catch {
socket.destroy()
} catch (error) {
if (error instanceof SessionRenewalPendingError)
socket.end(
'HTTP/1.1 503 Service Unavailable\r\nCache-Control: no-store\r\nConnection: close\r\nContent-Length: 0\r\n\r\n',
)
else socket.destroy()
}
})
return { server, store }
const shutdown = () => {
shutdownPromise ??= shutdownWebServer(
server,
store,
proxy,
runtime,
dependencies.shutdownTimeoutMilliseconds ?? SHUTDOWN_TIMEOUT_MILLISECONDS,
)
return shutdownPromise
}
return { server, store, shutdown }
}
async function route(
@ -72,6 +95,7 @@ async function route(
proxy,
startedAt,
requests,
runtime,
) {
const url = new URL(request.url ?? '/', config.publicOrigin)
if (request.method === 'POST' && url.pathname === '/client-errors') {
@ -92,8 +116,10 @@ async function route(
if (url.pathname === '/health/ready') {
json(
response,
store.ready ? 200 : 503,
{ status: store.ready ? 'ready' : 'unavailable' },
runtime.accepting && store.ready ? 200 : 503,
{
status: runtime.accepting && store.ready ? 'ready' : 'unavailable',
},
request.method,
)
return
@ -299,19 +325,30 @@ function traceIdentifier(traceparent) {
async function main() {
const config = loadConfig()
const { server, store } = createWebServer(config)
const { server, store, shutdown } = createWebServer(config)
const stopping = terminationSignal()
await store.connect()
server.listen(config.port, '0.0.0.0', () => {
process.stdout.write(
`${JSON.stringify({
timestamp: new Date().toISOString(),
level: 'info',
component: 'remora-web',
message: 'web application ready',
port: config.port,
})}\n`,
)
})
await listen(server, config.port)
process.stdout.write(
`${JSON.stringify({
timestamp: new Date().toISOString(),
level: 'info',
component: 'remora-web',
message: 'web application ready',
port: config.port,
})}\n`,
)
const signal = await stopping
process.stdout.write(
`${JSON.stringify({
timestamp: new Date().toISOString(),
level: 'info',
component: 'remora-web',
message: 'web application stopping',
signal,
})}\n`,
)
await shutdown()
}
if (import.meta.url === pathToFileURL(process.argv[1] || '').href)
@ -326,3 +363,46 @@ if (import.meta.url === pathToFileURL(process.argv[1] || '').href)
)
process.exitCode = 1
})
async function shutdownWebServer(
server,
store,
proxy,
runtime,
timeoutMilliseconds,
) {
runtime.accepting = false
proxy.close?.()
server.closeIdleConnections()
await new Promise((resolve, reject) => {
const timeout = setTimeout(
() => server.closeAllConnections(),
timeoutMilliseconds,
)
timeout.unref()
server.close((error) => {
clearTimeout(timeout)
if (error && error.code !== 'ERR_SERVER_NOT_RUNNING') reject(error)
else resolve()
})
})
await store.close?.()
}
function listen(server, port) {
return new Promise((resolve, reject) => {
const failed = (error) => reject(error)
server.once('error', failed)
server.listen(port, '0.0.0.0', () => {
server.off('error', failed)
resolve()
})
})
}
function terminationSignal() {
return new Promise((resolve) => {
process.once('SIGINT', () => resolve('SIGINT'))
process.once('SIGTERM', () => resolve('SIGTERM'))
})
}

132
web/server.test.mjs
View file

@ -1,10 +1,12 @@
import assert from 'node:assert/strict'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { connect } from 'node:net'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { after, before, test } from 'node:test'
import { createWebServer } from './server.mjs'
import { SessionRenewalPendingError } from './server/oauth.mjs'
let base
let root
@ -103,3 +105,133 @@ test('limits the SPA fallback to extensionless HTML navigation', async () => {
})
assert.equal(traversal.status, 404)
})
test('graceful shutdown drains active requests and closes dependencies', async () => {
const requestStarted = deferred()
const requestReleased = deferred()
let storeClosed = false
let proxyClosed = false
const created = createWebServer(
{
root,
publicOrigin: 'http://127.0.0.1',
},
{
store: {
ready: true,
close: async () => {
storeClosed = true
},
},
oauth: { route: async () => false },
proxy: {
route: async (_request, response, url) => {
if (url.pathname !== '/bff/api/blocked') return false
requestStarted.resolve()
await requestReleased.promise
response.writeHead(204)
response.end()
return true
},
upgrade: async () => false,
close: () => {
proxyClosed = true
},
},
shutdownTimeoutMilliseconds: 1_000,
},
)
await new Promise((resolve) => created.server.listen(0, '127.0.0.1', resolve))
const address = created.server.address()
const request = fetch(`http://127.0.0.1:${address.port}/bff/api/blocked`)
await requestStarted.promise
let stopped = false
const stopping = created.shutdown().then(() => {
stopped = true
})
await new Promise((resolve) => setTimeout(resolve, 10))
assert.equal(stopped, false)
assert.equal(proxyClosed, true)
requestReleased.resolve()
assert.equal((await request).status, 204)
await stopping
assert.equal(storeClosed, true)
})
test('preserves the session cookie while renewal is pending', async () => {
const created = createWebServer(
{
root,
publicOrigin: 'http://127.0.0.1',
},
{
store: { ready: true },
oauth: {
route: async () => {
throw new SessionRenewalPendingError()
},
},
proxy: {
route: async () => false,
upgrade: async () => false,
},
},
)
await new Promise((resolve) => created.server.listen(0, '127.0.0.1', resolve))
const address = created.server.address()
const response = await fetch(`http://127.0.0.1:${address.port}/auth/session`)
assert.equal(response.status, 503)
assert.equal(response.headers.get('set-cookie'), null)
await created.shutdown()
})
test('returns transient unavailability for a pending WebSocket renewal', async () => {
const created = createWebServer(
{
root,
publicOrigin: 'http://127.0.0.1',
},
{
store: { ready: true },
oauth: { route: async () => false },
proxy: {
route: async () => false,
upgrade: async () => {
throw new SessionRenewalPendingError()
},
},
},
)
await new Promise((resolve) => created.server.listen(0, '127.0.0.1', resolve))
const address = created.server.address()
const response = await new Promise((resolve, reject) => {
const socket = connect(address.port, '127.0.0.1')
let received = ''
socket.setEncoding('utf8')
socket.once('connect', () => {
socket.write(
'GET /bff/events HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n',
)
})
socket.on('data', (chunk) => {
received += chunk
})
socket.once('end', () => resolve(received))
socket.once('error', reject)
})
assert.match(response, /^HTTP\/1\.1 503 Service Unavailable\r\n/)
await created.shutdown()
})
function deferred() {
let resolve
const promise = new Promise((complete) => {
resolve = complete
})
return { promise, resolve }
}

83
web/server/oauth.mjs
View file

@ -4,6 +4,7 @@ import {
equalSecret,
expiredSessionCookie,
randomValue,
SESSION_RENEWAL_PENDING,
sessionCookie,
sessionIdentifier,
} from './sessions.mjs'
@ -11,6 +12,8 @@ import {
const CLOCK_SKEW_SECONDS = 30
const OIDC_TIMEOUT_MILLISECONDS = 15_000
export class SessionRenewalPendingError extends Error {}
export class OAuthClient {
constructor(config, store, fetchImplementation = fetch) {
this.config = config
@ -127,23 +130,16 @@ export class OAuthClient {
)
return true
}
json(
response,
200,
{
kind: 'authenticated',
identity: loaded.session.identity,
csrfToken: loaded.session.csrfToken,
},
loaded.rotated
? { 'Set-Cookie': sessionCookie(loaded.identifier, this.config) }
: {},
)
json(response, 200, {
kind: 'authenticated',
identity: loaded.session.identity,
csrfToken: loaded.session.csrfToken,
})
return true
}
async logout(request, response) {
const loaded = await this.authenticated(request, false)
const loaded = await this.storedSession(request)
if (
loaded === null ||
!validUnsafeRequest(request, this.config, loaded.session.csrfToken)
@ -173,29 +169,58 @@ export class OAuthClient {
}
async authenticated(request, refresh = true) {
const loaded = await this.storedSession(request)
if (loaded === null) return null
const { identifier } = loaded
let { session } = loaded
if (session.accessExpiresAt <= now() + CLOCK_SKEW_SECONDS) {
if (!refresh) {
if (session.accessExpiresAt <= now()) {
session = await this.store.renew(
identifier,
(current) =>
validAuthenticatedSession(current) &&
current.accessExpiresAt <= now(),
async () => null,
)
if (session === SESSION_RENEWAL_PENDING)
throw new SessionRenewalPendingError()
if (!validAuthenticatedSession(session)) return null
}
} else {
const renewed = await this.store.renew(
identifier,
(current) =>
validAuthenticatedSession(current) &&
current.accessExpiresAt <= now() + CLOCK_SKEW_SECONDS,
(current) => this.refresh(current).catch(() => null),
)
if (renewed === SESSION_RENEWAL_PENDING) {
if (session.accessExpiresAt <= now())
throw new SessionRenewalPendingError()
} else {
session = renewed
}
if (!validAuthenticatedSession(session)) {
if (session?.kind === 'authenticated')
await this.store.destroy(identifier)
return null
}
}
}
return { identifier, session }
}
async storedSession(request) {
const identifier = sessionIdentifier(request)
if (identifier === null) return null
let session = await this.store.read(identifier)
const session = await this.store.read(identifier)
if (!validAuthenticatedSession(session)) {
if (session?.kind === 'authenticated')
await this.store.destroy(identifier)
return null
}
let rotated = false
if (session.accessExpiresAt <= now() + CLOCK_SKEW_SECONDS) {
if (!refresh) {
await this.store.destroy(identifier)
return null
}
session = await this.refresh(session).catch(() => null)
if (session === null) {
await this.store.destroy(identifier)
return null
}
identifier = await this.store.replace(identifier, session)
rotated = true
}
return { identifier, session, rotated }
return { identifier, session }
}
async refresh(session) {

96
web/server/oauth.test.mjs
View file

@ -2,8 +2,8 @@ import assert from 'node:assert/strict'
import { createSign, generateKeyPairSync } from 'node:crypto'
import { test } from 'node:test'
import { OAuthClient } from './oauth.mjs'
import { SESSION_COOKIE } from './sessions.mjs'
import { OAuthClient, SessionRenewalPendingError } from './oauth.mjs'
import { SESSION_COOKIE, SESSION_RENEWAL_PENDING } from './sessions.mjs'
const identifier = 'a'.repeat(43)
const replacement = 'b'.repeat(43)
@ -163,9 +163,88 @@ test('expired and malformed authenticated sessions are destroyed', async () => {
assert.deepEqual(store.destroyed, [identifier, identifier])
})
test('token renewal preserves the session identifier and CSRF value', async () => {
const store = new MemoryStore()
store.value = {
...validSession(),
refreshToken: 'refresh-token',
accessExpiresAt: now + 20,
}
const oauth = new OAuthClient(config, store)
oauth.refresh = async (session) => ({
...session,
accessToken: 'renewed-access-token',
accessExpiresAt: now + 300,
})
const loaded = await oauth.authenticated({
headers: { cookie: `${SESSION_COOKIE}=${identifier}` },
})
assert.equal(loaded.identifier, identifier)
assert.equal(loaded.session.accessToken, 'renewed-access-token')
assert.equal(loaded.session.csrfToken, 'csrf-value')
assert.equal(store.replaced, 0)
})
test('concurrent renewal uses only an unexpired access token', async () => {
const store = new MemoryStore()
store.pending = true
store.value = {
...validSession(),
refreshToken: 'refresh-token',
accessExpiresAt: now + 20,
}
const oauth = new OAuthClient(config, store)
const request = {
headers: { cookie: `${SESSION_COOKIE}=${identifier}` },
}
assert.equal(
(await oauth.authenticated(request)).session.accessToken,
'access-token',
)
store.value.accessExpiresAt = now - 1
await assert.rejects(
oauth.authenticated(request, false),
SessionRenewalPendingError,
)
assert.deepEqual(store.destroyed, [])
})
test('logout destroys an expired session during owned renewal', async () => {
const store = new MemoryStore()
store.pending = true
store.value = {
...validSession(),
accessExpiresAt: now - 1,
}
const oauth = new OAuthClient(config, store)
oauth.metadata = async () => ({})
const response = new TestResponse()
await oauth.logout(
{
headers: {
cookie: `${SESSION_COOKIE}=${identifier}`,
origin: config.publicOrigin,
'sec-fetch-site': 'same-origin',
'x-remora-csrf': 'csrf-value',
},
},
response,
)
assert.equal(response.status, 200)
assert.deepEqual(store.destroyed, [identifier])
assert.match(response.headers['Set-Cookie'], /Max-Age=0/)
})
class MemoryStore {
value = null
destroyed = []
replaced = 0
pending = false
async create(value) {
this.value = value
@ -177,10 +256,23 @@ class MemoryStore {
}
async replace(_identifier, value) {
this.replaced += 1
this.value = value
return replacement
}
async renew(identifier, shouldRenew, renewal) {
if (!shouldRenew(this.value)) return this.value
if (this.pending) return SESSION_RENEWAL_PENDING
const renewed = await renewal(this.value)
if (renewed === null) {
await this.destroy(identifier)
return null
}
this.value = renewed
return this.value
}
async destroy(identifier) {
this.destroyed.push(identifier)
this.value = null

19
web/server/proxy.mjs
View file

@ -1,7 +1,7 @@
import { WebSocket, WebSocketServer } from 'ws'
import { validBrowserContext, validUnsafeRequest } from './oauth.mjs'
import { equalSecret, sessionCookie } from './sessions.mjs'
import { equalSecret } from './sessions.mjs'
const MAX_REQUEST_BYTES = 1024 * 1024
const MAX_EVENT_BYTES = 1024 * 1024
@ -90,11 +90,6 @@ export class ApiProxy {
signal: AbortSignal.timeout(API_TIMEOUT_MILLISECONDS),
})
const responseHeaders = { 'Cache-Control': 'no-store' }
if (loaded.rotated)
responseHeaders['Set-Cookie'] = sessionCookie(
loaded.identifier,
this.config,
)
for (const name of ['content-type', 'traceparent'])
if (upstreamResponse.headers.has(name))
responseHeaders[name] = upstreamResponse.headers.get(name)
@ -136,6 +131,17 @@ export class ApiProxy {
})
return true
}
close() {
for (const browser of this.webSockets.clients) {
browser.close(1012, 'service restart')
const timeout = setTimeout(() => {
if (browser.readyState !== WebSocket.CLOSED) browser.terminate()
}, 1_000)
timeout.unref()
}
this.webSockets.close()
}
}
function copyHeader(request, headers, name) {
@ -169,6 +175,7 @@ function relay(browser, upstream) {
}
const closeUpstream = (code = 1011, reason = '') => {
if (upstream.readyState === WebSocket.OPEN) upstream.close(code, reason)
else if (upstream.readyState === WebSocket.CONNECTING) upstream.terminate()
}
upstream.on('close', closeBrowser)
browser.on('close', closeUpstream)

1
web/server/security.test.mjs
View file

@ -114,7 +114,6 @@ test('event ticket requests use the validated public origin upstream', async ()
},
{
authenticated: async () => ({
rotated: false,
session: {
accessToken: 'server-token',
csrfToken: 'csrf-value',

59
web/server/sessions.mjs
View file

@ -3,6 +3,7 @@ import { randomBytes, timingSafeEqual } from 'node:crypto'
import { createClient } from 'redis'
export const SESSION_COOKIE = '__Host-remora-session'
export const SESSION_RENEWAL_PENDING = Symbol('session renewal pending')
export class SessionStore {
constructor(config, client = createClient({ url: config.sessionUrl })) {
@ -45,6 +46,56 @@ export class SessionStore {
return replacement
}
async renew(identifier, shouldRenew, renewal) {
if (!validIdentifier(identifier)) return null
const lock = this.lockKey(identifier)
const owner = randomValue()
const acquired = await this.client.eval(
'local ttl = redis.call("pttl", KEYS[2]) if ttl <= 0 then return 0 end if redis.call("set", KEYS[1], ARGV[1], "NX", "PX", ttl) then return 1 else return 0 end',
{
keys: [lock, this.key(identifier)],
arguments: [owner],
},
)
if (acquired !== 1) {
const current = await this.read(identifier)
if (current === null || !shouldRenew(current)) return current
return SESSION_RENEWAL_PENDING
}
let releaseOwnership = true
try {
const current = await this.read(identifier)
if (current === null || !shouldRenew(current)) return current
releaseOwnership = false
const renewed = await renewal(current)
if (renewed === null) {
await this.destroy(identifier)
releaseOwnership = true
return null
}
const stored = await this.client.eval(
'if redis.call("get", KEYS[1]) == ARGV[1] and redis.call("exists", KEYS[2]) == 1 then redis.call("set", KEYS[2], ARGV[2], "EX", ARGV[3]) return 1 else return 0 end',
{
keys: [lock, this.key(identifier)],
arguments: [
owner,
JSON.stringify(renewed),
String(this.config.sessionTtlSeconds),
],
},
)
if (stored !== 1) return null
releaseOwnership = true
return renewed
} finally {
if (releaseOwnership)
await this.client.eval(
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end',
{ keys: [lock], arguments: [owner] },
)
}
}
async destroy(identifier) {
if (validIdentifier(identifier)) await this.client.del(this.key(identifier))
}
@ -52,6 +103,14 @@ export class SessionStore {
key(identifier) {
return `${this.config.sessionNamespace}${identifier}`
}
lockKey(identifier) {
return `${this.key(identifier)}:renew`
}
async close() {
if (this.client.isOpen) await this.client.quit()
}
}
export function sessionIdentifier(request) {

186
web/server/sessions.test.mjs Normal file
View file

@ -0,0 +1,186 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { SESSION_RENEWAL_PENDING, SessionStore } from './sessions.mjs'
const identifier = 'a'.repeat(43)
const config = {
sessionNamespace: 'remora:test:',
sessionTtlSeconds: 300,
}
test('serializes renewal across session store users', async () => {
const client = new FakeRedis()
const store = new SessionStore(config, client)
const original = { kind: 'authenticated', accessToken: 'old' }
await client.set(store.key(identifier), JSON.stringify(original))
const blocked = deferred()
let renewals = 0
const renewal = async () => {
renewals += 1
await blocked.promise
return { kind: 'authenticated', accessToken: 'new' }
}
const shouldRenew = (session) => session.accessToken === 'old'
const first = store.renew(identifier, shouldRenew, renewal)
await waitFor(() => renewals === 1)
const second = await store.renew(identifier, shouldRenew, renewal)
assert.equal(second, SESSION_RENEWAL_PENDING)
blocked.resolve()
const left = await first
assert.deepEqual(left, { kind: 'authenticated', accessToken: 'new' })
assert.equal(renewals, 1)
assert.deepEqual(await store.read(identifier), left)
assert.deepEqual(await store.renew(identifier, shouldRenew, renewal), left)
assert.equal(renewals, 1)
})
test('logout fences an in-flight renewal from restoring the session', async () => {
const client = new FakeRedis()
const store = new SessionStore(config, client)
await client.set(
store.key(identifier),
JSON.stringify({ kind: 'authenticated', accessToken: 'old' }),
)
const blocked = deferred()
let started = false
const renewing = store.renew(
identifier,
() => true,
async () => {
started = true
await blocked.promise
return { kind: 'authenticated', accessToken: 'new' }
},
)
await waitFor(() => started)
await store.destroy(identifier)
blocked.resolve()
assert.equal(await renewing, null)
assert.equal(await store.read(identifier), null)
})
test('ambiguous commit retains renewal ownership', async () => {
const client = new FakeRedis()
const store = new SessionStore(config, client)
await client.set(
store.key(identifier),
JSON.stringify({ accessToken: 'old' }),
)
client.failCommit = true
await assert.rejects(
store.renew(
identifier,
() => true,
async () => ({ accessToken: 'new' }),
),
/commit unavailable/,
)
assert.equal(client.values.has(store.lockKey(identifier)), true)
assert.equal(
await store.renew(
identifier,
() => true,
async () => {
throw new Error('duplicate provider refresh')
},
),
SESSION_RENEWAL_PENDING,
)
})
test('ambiguous destruction retains renewal ownership', async () => {
const client = new FakeRedis()
const store = new SessionStore(config, client)
await client.set(
store.key(identifier),
JSON.stringify({ accessToken: 'old' }),
)
client.failDeleteKey = store.key(identifier)
await assert.rejects(
store.renew(
identifier,
() => true,
async () => null,
),
/deletion unavailable/,
)
assert.equal(client.values.has(store.lockKey(identifier)), true)
assert.equal(
await store.renew(
identifier,
() => true,
async () => {
throw new Error('duplicate provider refresh')
},
),
SESSION_RENEWAL_PENDING,
)
})
class FakeRedis {
values = new Map()
isOpen = true
failCommit = false
failDeleteKey = null
async get(key) {
return this.values.get(key) ?? null
}
async set(key, value, options = {}) {
if (options.NX && this.values.has(key)) return null
this.values.set(key, value)
return 'OK'
}
async del(...keys) {
if (keys.includes(this.failDeleteKey))
throw new Error('deletion unavailable')
let deleted = 0
for (const key of keys) deleted += Number(this.values.delete(key))
return deleted
}
async eval(_script, { keys, arguments: values }) {
if (keys.length === 2 && values.length === 1) {
if (!this.values.has(keys[1]) || this.values.has(keys[0])) return 0
this.values.set(keys[0], values[0])
return 1
}
if (keys.length === 2) {
if (this.failCommit) throw new Error('commit unavailable')
if (this.values.get(keys[0]) !== values[0] || !this.values.has(keys[1]))
return 0
this.values.set(keys[1], values[1])
return 1
}
if (this.values.get(keys[0]) !== values[0]) return 0
this.values.delete(keys[0])
return 1
}
async quit() {
this.isOpen = false
}
}
function deferred() {
let resolve
const promise = new Promise((complete) => {
resolve = complete
})
return { promise, resolve }
}
async function waitFor(predicate) {
while (!predicate()) await new Promise((resolve) => setTimeout(resolve, 1))
}