Harden WebBFF and deployment release behavior

Model fail-closed cross-replica session renewal and implement owner-fenced refresh, transient observer handling, logout fencing, and graceful shutdown. Remove chart-owned Ingress while retaining qualification-owned external routing.

web/server.mjs

@@ -4,10 +4,12 @@ import { extname, resolve, sep } from 'node:path'
 import { pathToFileURL } from 'node:url'
 
 import { loadConfig } from './server/config.mjs'
-import { OAuthClient } from './server/oauth.mjs'
+import { OAuthClient, SessionRenewalPendingError } from './server/oauth.mjs'
 import { ApiProxy } from './server/proxy.mjs'
 import { SessionStore } from './server/sessions.mjs'
 
+const SHUTDOWN_TIMEOUT_MILLISECONDS = 25_000
+
 export function createWebServer(config, dependencies = {}) {
   const store = dependencies.store ?? new SessionStore(config)
   const oauth =
@@ -17,7 +19,9 @@ export function createWebServer(config, dependencies = {}) {
     dependencies.proxy ??
     new ApiProxy(config, oauth, dependencies.fetchImplementation)
   const startedAt = Date.now()
+  const runtime = { accepting: true }
   let requests = 0
+  let shutdownPromise
 
   const server = createServer(async (request, response) => {
     requests += 1
@@ -35,6 +39,7 @@ export function createWebServer(config, dependencies = {}) {
           status: response.statusCode,
         })}\n`,
       )
+      if (!runtime.accepting) server.closeIdleConnections()
     })
     try {
       await route(
@@ -46,21 +51,39 @@ export function createWebServer(config, dependencies = {}) {
         proxy,
         startedAt,
         requests,
+        runtime,
       )
-    } catch {
+    } catch (error) {
       if (!response.headersSent)
-        response.writeHead(502, { 'Cache-Control': 'no-store' })
+        response.writeHead(
+          error instanceof SessionRenewalPendingError ? 503 : 502,
+          { 'Cache-Control': 'no-store' },
+        )
       response.end()
     }
   })
   server.on('upgrade', async (request, socket, head) => {
     try {
       if (!(await proxy.upgrade(request, socket, head))) socket.destroy()
-    } catch {
-      socket.destroy()
+    } catch (error) {
+      if (error instanceof SessionRenewalPendingError)
+        socket.end(
+          'HTTP/1.1 503 Service Unavailable\r\nCache-Control: no-store\r\nConnection: close\r\nContent-Length: 0\r\n\r\n',
+        )
+      else socket.destroy()
     }
   })
-  return { server, store }
+  const shutdown = () => {
+    shutdownPromise ??= shutdownWebServer(
+      server,
+      store,
+      proxy,
+      runtime,
+      dependencies.shutdownTimeoutMilliseconds ?? SHUTDOWN_TIMEOUT_MILLISECONDS,
+    )
+    return shutdownPromise
+  }
+  return { server, store, shutdown }
 }
 
 async function route(
@@ -72,6 +95,7 @@ async function route(
   proxy,
   startedAt,
   requests,
+  runtime,
 ) {
   const url = new URL(request.url ?? '/', config.publicOrigin)
   if (request.method === 'POST' && url.pathname === '/client-errors') {
@@ -92,8 +116,10 @@ async function route(
   if (url.pathname === '/health/ready') {
     json(
       response,
-      store.ready ? 200 : 503,
-      { status: store.ready ? 'ready' : 'unavailable' },
+      runtime.accepting && store.ready ? 200 : 503,
+      {
+        status: runtime.accepting && store.ready ? 'ready' : 'unavailable',
+      },
       request.method,
     )
     return
@@ -299,19 +325,30 @@ function traceIdentifier(traceparent) {
 
 async function main() {
   const config = loadConfig()
-  const { server, store } = createWebServer(config)
+  const { server, store, shutdown } = createWebServer(config)
+  const stopping = terminationSignal()
   await store.connect()
-  server.listen(config.port, '0.0.0.0', () => {
-    process.stdout.write(
-      `${JSON.stringify({
-        timestamp: new Date().toISOString(),
-        level: 'info',
-        component: 'remora-web',
-        message: 'web application ready',
-        port: config.port,
-      })}\n`,
-    )
-  })
+  await listen(server, config.port)
+  process.stdout.write(
+    `${JSON.stringify({
+      timestamp: new Date().toISOString(),
+      level: 'info',
+      component: 'remora-web',
+      message: 'web application ready',
+      port: config.port,
+    })}\n`,
+  )
+  const signal = await stopping
+  process.stdout.write(
+    `${JSON.stringify({
+      timestamp: new Date().toISOString(),
+      level: 'info',
+      component: 'remora-web',
+      message: 'web application stopping',
+      signal,
+    })}\n`,
+  )
+  await shutdown()
 }
 
 if (import.meta.url === pathToFileURL(process.argv[1] || '').href)
@@ -326,3 +363,46 @@ if (import.meta.url === pathToFileURL(process.argv[1] || '').href)
     )
     process.exitCode = 1
   })
+
+async function shutdownWebServer(
+  server,
+  store,
+  proxy,
+  runtime,
+  timeoutMilliseconds,
+) {
+  runtime.accepting = false
+  proxy.close?.()
+  server.closeIdleConnections()
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(
+      () => server.closeAllConnections(),
+      timeoutMilliseconds,
+    )
+    timeout.unref()
+    server.close((error) => {
+      clearTimeout(timeout)
+      if (error && error.code !== 'ERR_SERVER_NOT_RUNNING') reject(error)
+      else resolve()
+    })
+  })
+  await store.close?.()
+}
+
+function listen(server, port) {
+  return new Promise((resolve, reject) => {
+    const failed = (error) => reject(error)
+    server.once('error', failed)
+    server.listen(port, '0.0.0.0', () => {
+      server.off('error', failed)
+      resolve()
+    })
+  })
+}
+
+function terminationSignal() {
+  return new Promise((resolve) => {
+    process.once('SIGINT', () => resolve('SIGINT'))
+    process.once('SIGTERM', () => resolve('SIGTERM'))
+  })
+}