ce7cb702218693d08ea6d83464d2457c9250027c

services/stalwart/cmd/controller/app_password.go

@@ -0,0 +1,427 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var appPasswordPattern = regexp.MustCompile(`app_[A-Za-z0-9]+`)
+
+type vaultClient struct {
+	address string
+	token   string
+	client  *http.Client
+}
+
+func newVaultHTTPClient(caFile string) (*http.Client, error) {
+	certificate, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, err
+	}
+	roots, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, err
+	}
+	if !roots.AppendCertsFromPEM(certificate) {
+		return nil, errors.New("vault CA file contains no certificates")
+	}
+	transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}}
+	return &http.Client{Timeout: 15 * time.Second, Transport: transport}, nil
+}
+
+func vaultLogin(ctx context.Context, value options) (vaultClient, error) {
+	client, err := newVaultHTTPClient(value.vaultCACertFile)
+	if err != nil {
+		return vaultClient{}, err
+	}
+	jwt, err := readSecret(value.vaultTokenFile)
+	if err != nil {
+		return vaultClient{}, err
+	}
+	payload, err := json.Marshal(map[string]string{"role": value.vaultRole, "jwt": jwt})
+	if err != nil {
+		return vaultClient{}, err
+	}
+	target := strings.TrimRight(value.vaultAddress, "/") + "/v1/auth/kubernetes/login"
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(payload))
+	if err != nil {
+		return vaultClient{}, err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	response, err := client.Do(request)
+	if err != nil {
+		return vaultClient{}, err
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode != http.StatusOK {
+		return vaultClient{}, responseStatusError("vault Kubernetes login", response, jwt)
+	}
+	var result struct {
+		Auth struct {
+			ClientToken string `json:"client_token"`
+		} `json:"auth"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
+		return vaultClient{}, err
+	}
+	if result.Auth.ClientToken == "" {
+		return vaultClient{}, errors.New("vault login response has no client token")
+	}
+	return vaultClient{address: strings.TrimRight(value.vaultAddress, "/"), token: result.Auth.ClientToken, client: client}, nil
+}
+
+func (client vaultClient) request(ctx context.Context, method, path string, payload []byte) (*http.Response, error) {
+	request, err := http.NewRequestWithContext(ctx, method, client.address+"/v1/"+path, bytes.NewReader(payload))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("X-Vault-Token", client.token)
+	if len(payload) != 0 {
+		request.Header.Set("Content-Type", "application/json")
+	}
+	return client.client.Do(request)
+}
+
+type storedAppPassword struct {
+	ID       string
+	Password string
+}
+
+func (client vaultClient) appPassword(ctx context.Context, path, key string) (storedAppPassword, bool, error) {
+	response, err := client.request(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return storedAppPassword{}, false, fmt.Errorf("read app password from vault: %w", err)
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode == http.StatusNotFound {
+		return storedAppPassword{}, false, nil
+	}
+	if response.StatusCode != http.StatusOK {
+		return storedAppPassword{}, false, fmt.Errorf("read app password from vault: HTTP %d", response.StatusCode)
+	}
+	var result struct {
+		Data struct {
+			Data map[string]string `json:"data"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
+		return storedAppPassword{}, false, fmt.Errorf("decode app password from vault: %w", err)
+	}
+	credential := storedAppPassword{
+		ID:       result.Data.Data["stalwart_app_password_id"],
+		Password: result.Data.Data[key],
+	}
+	return credential, credential.Password != "", nil
+}
+
+func (client vaultClient) writeAppPassword(ctx context.Context, path, key string, credential storedAppPassword) error {
+	payload, err := json.Marshal(map[string]any{"data": map[string]string{
+		key:                        credential.Password,
+		"stalwart_app_password_id": credential.ID,
+	}})
+	if err != nil {
+		return fmt.Errorf("encode app password for vault: %w", err)
+	}
+	response, err := client.request(ctx, http.MethodPost, path, payload)
+	if err != nil {
+		return fmt.Errorf("write app password to vault: %w", err)
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("write app password to vault: HTTP %d", response.StatusCode)
+	}
+	return nil
+}
+
+type appPassword struct {
+	ID          string `json:"id"`
+	Description string `json:"description"`
+}
+
+func appPasswords(output []byte) ([]appPassword, error) {
+	decoder := json.NewDecoder(bytes.NewReader(output))
+	var items []appPassword
+	for {
+		var item appPassword
+		err := decoder.Decode(&item)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if item.ID == "" {
+			return nil, errors.New("AppPassword query returned an empty id")
+		}
+		items = append(items, item)
+	}
+	return items, nil
+}
+
+func createdAppPassword(output []byte) (string, error) {
+	matches := appPasswordPattern.FindAll(output, -1)
+	if len(matches) != 1 {
+		return "", errors.New("AppPassword creation did not return one credential")
+	}
+	return string(matches[0]), nil
+}
+
+func appPasswordToken(ctx context.Context, value options, clientIDFile, clientSecretFile string) (string, error) {
+	return oidcTokenFromFiles(ctx, value.oidcIssuerFile, value.oidcAudienceFile, clientIDFile, clientSecretFile)
+}
+
+func appPasswordWorks(ctx context.Context, baseURL, username, password string) (bool, error) {
+	target, err := url.Parse(baseURL)
+	if err != nil || target.Scheme == "" || target.Host == "" {
+		return false, errors.New("stalwart URL is invalid")
+	}
+	target.Path = "/jmap/session"
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil)
+	if err != nil {
+		return false, fmt.Errorf("build app-password verification request: %w", err)
+	}
+	request.SetBasicAuth(username, password)
+	client := &http.Client{Timeout: 15 * time.Second}
+	response, err := client.Do(request)
+	if err != nil {
+		return false, fmt.Errorf("verify app password: %w", err)
+	}
+	defer func() { _ = response.Body.Close() }()
+	switch response.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return false, nil
+	default:
+		return false, fmt.Errorf("verify app password: HTTP %d", response.StatusCode)
+	}
+}
+
+type appPasswordIdentity struct {
+	ClientIDFile     string          `json:"clientIdFile"`
+	ClientSecretFile string          `json:"clientSecretFile"`
+	Username         string          `json:"username"`
+	Description      string          `json:"description"`
+	VaultPath        string          `json:"vaultPath"`
+	VaultKey         string          `json:"vaultKey"`
+	Permissions      json.RawMessage `json:"permissions"`
+}
+
+func loadAppPasswordIdentities(path string) ([]appPasswordIdentity, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var identities []appPasswordIdentity
+	if err := json.Unmarshal(content, &identities); err != nil {
+		return nil, err
+	}
+	for index, identity := range identities {
+		if identity.ClientIDFile == "" || identity.ClientSecretFile == "" || identity.Username == "" ||
+			identity.Description == "" || identity.VaultPath == "" || identity.VaultKey == "" || len(identity.Permissions) == 0 {
+			return nil, fmt.Errorf("app-password identity %d is incomplete", index)
+		}
+		var permissions map[string]any
+		if err := json.Unmarshal(identity.Permissions, &permissions); err != nil || len(permissions) == 0 {
+			return nil, fmt.Errorf("app-password identity %d has invalid permissions", index)
+		}
+	}
+	return identities, nil
+}
+
+type appPasswordOperations struct {
+	query  func() ([]appPassword, error)
+	create func([]appPassword) (storedAppPassword, error)
+	delete func([]string) error
+	verify func(string) (bool, error)
+	read   func() (storedAppPassword, bool, error)
+	write  func(storedAppPassword) error
+}
+
+func queryAppPasswords(ctx context.Context, value options, environment []string) ([]appPassword, error) {
+	output, err := runCLI(ctx, value.cli, environment, true, "query", "AppPassword", "--fields", "id,description", "--json", "--no-color")
+	if err != nil {
+		return nil, err
+	}
+	items, err := appPasswords(output)
+	if err != nil {
+		return nil, fmt.Errorf("decode app-password inventory: %w", err)
+	}
+	return items, nil
+}
+
+func appPasswordIDs(items []appPassword, except string) []string {
+	ids := make([]string, 0, len(items))
+	for _, item := range items {
+		if item.ID != except {
+			ids = append(ids, item.ID)
+		}
+	}
+	return ids
+}
+
+func deleteAppPasswords(ctx context.Context, value options, environment []string, ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	_, err := runCLI(ctx, value.cli, environment, false, "delete", "AppPassword", "--ids", strings.Join(ids, ","), "--no-color")
+	return err
+}
+
+func createAppPassword(
+	ctx context.Context,
+	value options,
+	environment []string,
+	identity appPasswordIdentity,
+	existing []appPassword,
+) (storedAppPassword, error) {
+	output, err := runCLI(
+		ctx,
+		value.cli,
+		environment,
+		true,
+		"create",
+		"AppPassword",
+		"--field",
+		"description="+identity.Description,
+		"--field",
+		"permissions="+string(identity.Permissions),
+		"--field",
+		"allowedIps={}",
+		"--no-color",
+	)
+	if err != nil {
+		return storedAppPassword{}, err
+	}
+	password, err := createdAppPassword(output)
+	if err != nil {
+		return storedAppPassword{}, fmt.Errorf("decode created app password: %w", err)
+	}
+	existingIDs := make(map[string]struct{}, len(existing))
+	for _, item := range existing {
+		existingIDs[item.ID] = struct{}{}
+	}
+	current, err := queryAppPasswords(ctx, value, environment)
+	if err != nil {
+		return storedAppPassword{}, err
+	}
+	var created []appPassword
+	for _, item := range current {
+		if _, existed := existingIDs[item.ID]; !existed && item.Description == identity.Description {
+			created = append(created, item)
+		}
+	}
+	if len(created) != 1 {
+		return storedAppPassword{}, errors.New("created app password is not uniquely identifiable")
+	}
+	return storedAppPassword{ID: created[0].ID, Password: password}, nil
+}
+
+func convergeAppPassword(description string, operations appPasswordOperations) error {
+	items, err := operations.query()
+	if err != nil {
+		return err
+	}
+	credential, present, err := operations.read()
+	if err != nil {
+		return err
+	}
+	if present {
+		valid, verifyErr := operations.verify(credential.Password)
+		if verifyErr != nil {
+			return verifyErr
+		}
+		if valid {
+			if credential.ID != "" {
+				for _, item := range items {
+					if item.ID == credential.ID && item.Description == description {
+						return operations.delete(appPasswordIDs(items, credential.ID))
+					}
+				}
+			}
+			if len(items) == 1 && items[0].Description == description {
+				credential.ID = items[0].ID
+				return operations.write(credential)
+			}
+		}
+	}
+	created, err := operations.create(items)
+	if err != nil {
+		return err
+	}
+	valid, verifyErr := operations.verify(created.Password)
+	if verifyErr != nil {
+		cleanupErr := operations.delete([]string{created.ID})
+		return errors.Join(verifyErr, cleanupErr)
+	}
+	if !valid {
+		cleanupErr := operations.delete([]string{created.ID})
+		return errors.Join(errors.New("new app password failed authentication"), cleanupErr)
+	}
+	if err := operations.write(created); err != nil {
+		cleanupErr := operations.delete([]string{created.ID})
+		return errors.Join(err, cleanupErr)
+	}
+	return operations.delete(appPasswordIDs(items, created.ID))
+}
+
+func reconcileAppPassword(ctx context.Context, value options, vault vaultClient, identity appPasswordIdentity) error {
+	token, err := appPasswordToken(ctx, value, identity.ClientIDFile, identity.ClientSecretFile)
+	if err != nil {
+		return fmt.Errorf("issue app-password token: %w", err)
+	}
+	environment := []string{"STALWART_TOKEN=" + token, "STALWART_URL=" + value.stalwartURL}
+	operations := appPasswordOperations{
+		query: func() ([]appPassword, error) {
+			return queryAppPasswords(ctx, value, environment)
+		},
+		create: func(existing []appPassword) (storedAppPassword, error) {
+			return createAppPassword(ctx, value, environment, identity, existing)
+		},
+		delete: func(ids []string) error {
+			return deleteAppPasswords(ctx, value, environment, ids)
+		},
+		verify: func(password string) (bool, error) {
+			return appPasswordWorks(ctx, value.stalwartURL, identity.Username, password)
+		},
+		read: func() (storedAppPassword, bool, error) {
+			return vault.appPassword(ctx, identity.VaultPath, identity.VaultKey)
+		},
+		write: func(credential storedAppPassword) error {
+			return vault.writeAppPassword(ctx, identity.VaultPath, identity.VaultKey, credential)
+		},
+	}
+	return convergeAppPassword(identity.Description, operations)
+}
+
+func reconcileAppPasswords(ctx context.Context, value options) error {
+	if value.appPasswords == "" {
+		return nil
+	}
+	identities, err := loadAppPasswordIdentities(value.appPasswords)
+	if err != nil {
+		return fmt.Errorf("load app-password identities: %w", err)
+	}
+	vault, err := vaultLogin(ctx, value)
+	if err != nil {
+		return fmt.Errorf("log in to vault: %w", err)
+	}
+	for _, identity := range identities {
+		if err := reconcileAppPassword(ctx, value, vault, identity); err != nil {
+			return fmt.Errorf("reconcile app-password identity %q: %w", identity.Description, err)
+		}
+	}
+	return nil
+}

services/stalwart/cmd/controller/app_password_test.go

@@ -0,0 +1,309 @@
+package main
+
+import (
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestConvergeAppPasswordPreservesCredentialWhenVerificationIsUnavailable(t *testing.T) {
+	unavailable := errors.New("stalwart unavailable")
+	operations := appPasswordOperations{
+		query: func() ([]appPassword, error) {
+			return []appPassword{{ID: "existing-id", Description: "alert-submit-v1"}}, nil
+		},
+		read: func() (storedAppPassword, bool, error) {
+			return storedAppPassword{ID: "existing-id", Password: "app_existing"}, true, nil
+		},
+		verify: func(string) (bool, error) {
+			return false, unavailable
+		},
+		create: func([]appPassword) (storedAppPassword, error) {
+			t.Fatal("created a credential while verification was unavailable")
+			return storedAppPassword{}, nil
+		},
+		delete: func([]string) error {
+			t.Fatal("deleted a credential while verification was unavailable")
+			return nil
+		},
+		write: func(storedAppPassword) error {
+			t.Fatal("wrote a credential while verification was unavailable")
+			return nil
+		},
+	}
+	if err := convergeAppPassword("alert-submit-v1", operations); !errors.Is(err, unavailable) {
+		t.Fatalf("error = %v, want %v", err, unavailable)
+	}
+}
+
+func TestConvergeAppPasswordStoresReplacementBeforeDeletingOld(t *testing.T) {
+	var calls []string
+	operations := appPasswordOperations{
+		query: func() ([]appPassword, error) {
+			calls = append(calls, "query")
+			return []appPassword{{ID: "old-id", Description: "alert-submit-v1"}}, nil
+		},
+		read: func() (storedAppPassword, bool, error) {
+			calls = append(calls, "read")
+			return storedAppPassword{ID: "old-id", Password: "app_old"}, true, nil
+		},
+		verify: func(password string) (bool, error) {
+			calls = append(calls, "verify-"+password)
+			return password == "app_new", nil
+		},
+		create: func(existing []appPassword) (storedAppPassword, error) {
+			calls = append(calls, "create")
+			if len(existing) != 1 || existing[0].ID != "old-id" {
+				t.Fatalf("existing inventory = %#v", existing)
+			}
+			return storedAppPassword{ID: "new-id", Password: "app_new"}, nil
+		},
+		write: func(credential storedAppPassword) error {
+			calls = append(calls, "write")
+			if credential.ID != "new-id" || credential.Password != "app_new" {
+				t.Fatalf("written credential = %#v", credential)
+			}
+			return nil
+		},
+		delete: func(ids []string) error {
+			calls = append(calls, "delete")
+			if !slices.Equal(ids, []string{"old-id"}) {
+				t.Fatalf("deleted IDs = %q", ids)
+			}
+			return nil
+		},
+	}
+	if err := convergeAppPassword("alert-submit-v1", operations); err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"query", "read", "verify-app_old", "create", "verify-app_new", "write", "delete"}
+	if !slices.Equal(calls, want) {
+		t.Fatalf("calls = %q, want %q", calls, want)
+	}
+}
+
+func TestAppPasswords(t *testing.T) {
+	items, err := appPasswords([]byte("{\"id\":\"one\",\"description\":\"first\"}\n{\"id\":\"two\",\"description\":\"second\"}\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(items) != 2 || items[0].ID != "one" || items[0].Description != "first" || items[1].ID != "two" || items[1].Description != "second" {
+		t.Fatalf("unexpected AppPasswords: %v", items)
+	}
+	if _, err := appPasswords([]byte("{\"id\":\"\"}\n")); err == nil {
+		t.Fatal("empty id accepted")
+	}
+}
+
+func TestCreatedAppPasswordRequiresOneCredential(t *testing.T) {
+	password, err := createdAppPassword([]byte("Created app_synthetic123\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if password != "app_synthetic123" {
+		t.Fatalf("unexpected credential: %s", password)
+	}
+	if _, err := createdAppPassword([]byte("none")); err == nil {
+		t.Fatal("missing credential accepted")
+	}
+	if _, err := createdAppPassword([]byte("app_one app_two")); err == nil {
+		t.Fatal("multiple credentials accepted")
+	}
+}
+
+func TestLoadAppPasswordIdentities(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "app-passwords.json")
+	content := `[{
+  "clientIdFile": "/run/control/client-id",
+  "clientSecretFile": "/run/control/client-secret",
+  "username": "[email protected]",
+  "description": "synthetic-client-v1",
+  "vaultPath": "secret/data/example/client",
+  "vaultKey": "password",
+  "permissions": {"@type":"Replace","permissions":{"authenticate":true}}
+}]`
+	if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	identities, err := loadAppPasswordIdentities(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(identities) != 1 || identities[0].Username != "[email protected]" || identities[0].VaultKey != "password" {
+		t.Fatalf("identities = %#v", identities)
+	}
+}
+
+func TestReconcileSubmitterAppPasswordCreatesThenRetains(t *testing.T) {
+	const generated = "app_synthetic123"
+	const vaultPath = "secret/data/stalwart/alert-identity"
+	var stored string
+	var storedID string
+	var vaultMu sync.Mutex
+	vaultServer := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		switch request.URL.Path {
+		case "/v1/auth/kubernetes/login":
+			_ = json.NewEncoder(response).Encode(map[string]any{"auth": map[string]string{"client_token": "synthetic-vault-token"}})
+		case "/v1/" + vaultPath:
+			if request.Header.Get("X-Vault-Token") != "synthetic-vault-token" {
+				response.WriteHeader(http.StatusForbidden)
+				return
+			}
+			vaultMu.Lock()
+			defer vaultMu.Unlock()
+			if request.Method == http.MethodGet {
+				if stored == "" {
+					response.WriteHeader(http.StatusNotFound)
+					return
+				}
+				_ = json.NewEncoder(response).Encode(map[string]any{"data": map[string]any{"data": map[string]string{
+					"smtp_password":            stored,
+					"stalwart_app_password_id": storedID,
+				}}})
+				return
+			}
+			var payload struct {
+				Data map[string]string `json:"data"`
+			}
+			if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+				t.Fatal(err)
+			}
+			stored = payload.Data["smtp_password"]
+			storedID = payload.Data["stalwart_app_password_id"]
+			response.WriteHeader(http.StatusNoContent)
+		default:
+			response.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer vaultServer.Close()
+
+	var stalwartStatus atomic.Int32
+	stalwartStatus.Store(http.StatusOK)
+	stalwartServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		if status := int(stalwartStatus.Load()); status != http.StatusOK {
+			response.WriteHeader(status)
+			return
+		}
+		username, password, ok := request.BasicAuth()
+		if request.URL.Path != "/jmap/session" || !ok || username != "[email protected]" || password != generated {
+			response.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		response.WriteHeader(http.StatusOK)
+	}))
+	defer stalwartServer.Close()
+	var oidcServer *httptest.Server
+	oidcServer = httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		switch request.URL.Path {
+		case "/.well-known/openid-configuration":
+			_ = json.NewEncoder(response).Encode(map[string]string{
+				"issuer":         oidcServer.URL,
+				"jwks_uri":       oidcServer.URL + "/keys",
+				"token_endpoint": oidcServer.URL + "/token",
+			})
+		case "/token":
+			_ = json.NewEncoder(response).Encode(map[string]string{"access_token": "synthetic-submitter-token"})
+		default:
+			response.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer oidcServer.Close()
+
+	directory := t.TempDir()
+	writeTestFile := func(name, content string) string {
+		path := filepath.Join(directory, name)
+		if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		return path
+	}
+	state := filepath.Join(directory, "state")
+	count := filepath.Join(directory, "count")
+	cli := writeTestFile("stalwart-cli", fmt.Sprintf(`#!/bin/sh
+if [ "$1 $2" = "query AppPassword" ]; then
+  if [ -f %q ]; then printf '{"id":"synthetic-id","description":"alert-submit-v1"}\n'; fi
+elif [ "$1 $2" = "create AppPassword" ]; then
+  touch %q
+  current=0
+  if [ -f %q ]; then current=$(tr -cd '0-9' < %q); fi
+  current=$((current + 1))
+  printf '%%s' "$current" > %q
+  printf 'Created %s\n'
+elif [ "$1 $2" = "delete AppPassword" ]; then
+  rm -f %q
+else
+  exit 1
+fi
+`, state, state, count, count, count, generated, state))
+	if err := os.Chmod(cli, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	certificate := vaultServer.Certificate()
+	caPath := filepath.Join(directory, "vault-ca.pem")
+	if err := os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := x509.ParseCertificate(certificate.Raw); err != nil {
+		t.Fatal(err)
+	}
+	clientIDFile := writeTestFile("client-id", "synthetic-client")
+	clientSecretFile := writeTestFile("client-secret", "synthetic-client-secret")
+	identities, err := json.Marshal([]appPasswordIdentity{{
+		ClientIDFile:     clientIDFile,
+		ClientSecretFile: clientSecretFile,
+		Username:         "[email protected]",
+		Description:      "alert-submit-v1",
+		VaultPath:        vaultPath,
+		VaultKey:         "smtp_password",
+		Permissions:      json.RawMessage(`{"@type":"Replace","permissions":{"authenticate":true,"emailSend":true}}`),
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	value := options{
+		cli:              cli,
+		stalwartURL:      stalwartServer.URL,
+		oidcIssuerFile:   writeTestFile("issuer", oidcServer.URL),
+		oidcAudienceFile: writeTestFile("audience", "synthetic-project"),
+		appPasswords:     writeTestFile("app-passwords.json", string(identities)),
+		vaultAddress:     vaultServer.URL,
+		vaultRole:        "synthetic-role",
+		vaultTokenFile:   writeTestFile("vault-token", "synthetic-service-account-token"),
+		vaultCACertFile:  caPath,
+	}
+	if err := reconcileAppPasswords(t.Context(), value); err != nil {
+		t.Fatal(err)
+	}
+	if err := reconcileAppPasswords(t.Context(), value); err != nil {
+		t.Fatal(err)
+	}
+	stalwartStatus.Store(http.StatusServiceUnavailable)
+	if err := reconcileAppPasswords(t.Context(), value); err == nil {
+		t.Fatal("unavailable Stalwart did not stop reconciliation")
+	}
+	created, err := os.ReadFile(count)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(created) != "1" {
+		t.Fatalf("credential created %s times", created)
+	}
+	vaultMu.Lock()
+	defer vaultMu.Unlock()
+	if stored != generated {
+		t.Fatal("generated credential was not stored")
+	}
+	if storedID != "synthetic-id" {
+		t.Fatalf("stored app-password ID = %q", storedID)
+	}
+}

services/stalwart/cmd/controller/controller.go

@@ -0,0 +1,838 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+const ownershipLockKey int64 = 31987176956605541
+
+const certificatePath = "/run/tls/tls.crt"
+const equivalentCertificatePath = "/run/tls/./tls.crt"
+const privateKeyPath = "/run/tls/tls.key"
+
+var ownershipSessionSettings = []string{
+	"set application_name = 'stalwart-controller'",
+	"set tcp_keepalives_idle = 15",
+	"set tcp_keepalives_interval = 5",
+	"set tcp_keepalives_count = 3",
+}
+
+var errOwnerExists = errors.New("another controller owns reconciliation")
+var errLeaderUnhealthy = errors.New("leader failed reconciliation repeatedly")
+var buildRevision = "unknown"
+
+type controllerStatus struct {
+	Leader              bool   `json:"leader"`
+	Ready               bool   `json:"ready"`
+	PersistentFailure   bool   `json:"persistentFailure"`
+	ConsecutiveFailures int    `json:"consecutiveFailures"`
+	PlanRevision        string `json:"planRevision"`
+	BuildRevision       string `json:"buildRevision"`
+	LastAttempt         string `json:"lastAttempt,omitempty"`
+	LastSuccess         string `json:"lastSuccess,omitempty"`
+	NextAttempt         string `json:"nextAttempt,omitempty"`
+	LastResult          string `json:"lastResult"`
+}
+
+type statusStore struct {
+	mu     sync.RWMutex
+	status controllerStatus
+}
+
+type ownershipConnection interface {
+	Close(context.Context) error
+	Ping(context.Context) error
+}
+
+type statusServer interface {
+	Shutdown(context.Context) error
+}
+
+type controllerRuntime struct {
+	acquireOwnership func(context.Context, string) (ownershipConnection, error)
+	planRevision     func(string) (string, error)
+	reconcile        func(context.Context, options) error
+}
+
+type certificateInventoryResult struct {
+	ids     []string
+	matches []string
+}
+
+func productionControllerRuntime() controllerRuntime {
+	return controllerRuntime{
+		acquireOwnership: func(ctx context.Context, dsnFile string) (ownershipConnection, error) {
+			return acquireOwnership(ctx, dsnFile)
+		},
+		planRevision: planRevision,
+		reconcile:    reconcileOnce,
+	}
+}
+
+func closeOwnership(ownership ownershipConnection) {
+	ownershipContext, cancelOwnership := context.WithTimeout(context.Background(), 5*time.Second)
+	_ = ownership.Close(ownershipContext)
+	cancelOwnership()
+}
+
+func shutdownStatusServer(server statusServer) {
+	serverContext, cancelServer := context.WithTimeout(context.Background(), 5*time.Second)
+	_ = server.Shutdown(serverContext)
+	cancelServer()
+}
+
+func (s *statusStore) update(change func(*controllerStatus)) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	change(&s.status)
+}
+
+func (s *statusStore) snapshot() controllerStatus {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.status
+}
+
+func planRevision(path string) (string, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	sum := sha256.Sum256(content)
+	return hex.EncodeToString(sum[:]), nil
+}
+
+func nextBackoff(current, maximum time.Duration) time.Duration {
+	if current >= maximum/2 {
+		return maximum
+	}
+	return current * 2
+}
+
+func metricBoolean(value bool) int {
+	if value {
+		return 1
+	}
+	return 0
+}
+
+func recordLeaderFailure(store *statusStore, threshold int) bool {
+	relinquish := false
+	store.update(func(status *controllerStatus) {
+		status.ConsecutiveFailures++
+		status.PersistentFailure = status.ConsecutiveFailures >= threshold
+		status.Ready = status.LastSuccess != "" && !status.PersistentFailure
+		status.LastResult = "failure"
+		relinquish = status.PersistentFailure
+	})
+	return relinquish
+}
+
+func configureOwnershipSession(ctx context.Context, execute func(context.Context, string) error) error {
+	for _, statement := range ownershipSessionSettings {
+		if err := execute(ctx, statement); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func acquireOwnership(ctx context.Context, dsnFile string) (*pgx.Conn, error) {
+	dsn, err := os.ReadFile(dsnFile)
+	if err != nil {
+		return nil, err
+	}
+	conn, err := pgx.Connect(ctx, strings.TrimSpace(string(dsn)))
+	if err != nil {
+		return nil, err
+	}
+	if err := configureOwnershipSession(ctx, func(ctx context.Context, statement string) error {
+		_, err := conn.Exec(ctx, statement)
+		return err
+	}); err != nil {
+		_ = conn.Close(ctx)
+		return nil, err
+	}
+	var acquired bool
+	err = conn.QueryRow(ctx, "select pg_try_advisory_lock($1)", ownershipLockKey).Scan(&acquired)
+	if err != nil {
+		_ = conn.Close(ctx)
+		return nil, err
+	}
+	if !acquired {
+		_ = conn.Close(ctx)
+		return nil, errOwnerExists
+	}
+	return conn, nil
+}
+
+func secretPath(root, name string) (string, error) {
+	if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name {
+		return "", errors.New("invalid secret file name")
+	}
+	target := filepath.Join(root, name)
+	relative, err := filepath.Rel(root, target)
+	if err != nil || relative == ".." || strings.HasPrefix(relative, "../") {
+		return "", errors.New("secret file escapes root")
+	}
+	return target, nil
+}
+
+func injectSecrets(value any, root string) (any, error) {
+	switch typed := value.(type) {
+	case []any:
+		for index, item := range typed {
+			replaced, err := injectSecrets(item, root)
+			if err != nil {
+				return nil, err
+			}
+			typed[index] = replaced
+		}
+		return typed, nil
+	case map[string]any:
+		if len(typed) == 1 {
+			name, found := typed["$secretFile"].(string)
+			asSet := false
+			if !found {
+				name, found = typed["$secretSet"].(string)
+				asSet = found
+			}
+			if found {
+				path, err := secretPath(root, name)
+				if err != nil {
+					return nil, err
+				}
+				content, err := os.ReadFile(path)
+				if err != nil {
+					return nil, err
+				}
+				resolved := strings.TrimSuffix(string(content), "\n")
+				if asSet {
+					return map[string]bool{resolved: true}, nil
+				}
+				return resolved, nil
+			}
+		}
+		for key, item := range typed {
+			replaced, err := injectSecrets(item, root)
+			if err != nil {
+				return nil, err
+			}
+			typed[key] = replaced
+		}
+		return typed, nil
+	default:
+		return value, nil
+	}
+}
+
+func renderPlan(source, target, secretRoot string) error {
+	content, err := os.ReadFile(source)
+	if err != nil {
+		return err
+	}
+	var operations []any
+	if err := json.Unmarshal(content, &operations); err != nil {
+		return err
+	}
+	replaced, err := injectSecrets(operations, secretRoot)
+	if err != nil {
+		return err
+	}
+	operations = replaced.([]any)
+	file, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	if err := file.Chmod(0o600); err != nil {
+		_ = file.Close()
+		return err
+	}
+	encoder := json.NewEncoder(file)
+	encoder.SetEscapeHTML(false)
+	for _, operation := range operations {
+		if err := encoder.Encode(operation); err != nil {
+			_ = file.Close()
+			return err
+		}
+	}
+	return file.Close()
+}
+
+func runCLI(
+	ctx context.Context,
+	path string,
+	environment []string,
+	capture bool,
+	arguments ...string,
+) ([]byte, error) {
+	command := exec.CommandContext(ctx, path, arguments...)
+	if len(environment) != 0 {
+		command.Env = mergedEnvironment(os.Environ(), environment)
+	}
+	var stdout bytes.Buffer
+	if capture {
+		command.Stdout = &stdout
+	} else {
+		command.Stdout = io.Discard
+	}
+	var stderr bytes.Buffer
+	command.Stderr = &stderr
+	if err := command.Run(); err != nil {
+		action := strings.Join(arguments[:min(2, len(arguments))], " ")
+		if ctx.Err() != nil {
+			return nil, fmt.Errorf("stalwart CLI %s: %w", action, ctx.Err())
+		}
+		diagnostic := redactDiagnostic(stderr.String(), environment)
+		if diagnostic == "" {
+			return nil, fmt.Errorf("stalwart CLI %s: %w", action, err)
+		}
+		return nil, fmt.Errorf("stalwart CLI %s: %w: %s", action, err, diagnostic)
+	}
+	return stdout.Bytes(), nil
+}
+
+func redactDiagnostic(value string, environment []string) string {
+	value = strings.TrimSpace(value)
+	for _, item := range environment {
+		_, secret, found := strings.Cut(item, "=")
+		if found && secret != "" {
+			value = strings.ReplaceAll(value, secret, "[redacted]")
+		}
+	}
+	const maximum = 4096
+	if len(value) > maximum {
+		value = value[:maximum] + "..."
+	}
+	return value
+}
+
+func mergedEnvironment(base, overrides []string) []string {
+	keys := make(map[string]struct{}, len(overrides))
+	for _, item := range overrides {
+		key, _, _ := strings.Cut(item, "=")
+		keys[key] = struct{}{}
+	}
+	result := make([]string, 0, len(base)+len(overrides))
+	for _, item := range base {
+		key, _, _ := strings.Cut(item, "=")
+		if _, replaced := keys[key]; !replaced {
+			result = append(result, item)
+		}
+	}
+	return append(result, overrides...)
+}
+
+func bootstrapEnvironment(value options) ([]string, error) {
+	if value.bootstrapUser == "" {
+		return nil, nil
+	}
+	password, err := os.ReadFile(value.bootstrapPasswordFile)
+	if err != nil {
+		return nil, err
+	}
+	return []string{
+		"STALWART_USER=" + value.bootstrapUser,
+		"STALWART_PASSWORD=" + strings.TrimSuffix(string(password), "\n"),
+	}, nil
+}
+
+func credentialEnvironment(ctx context.Context, value options) ([]string, error) {
+	environment, err := oidcEnvironment(ctx, value)
+	if err != nil {
+		return nil, fmt.Errorf("load OIDC credentials: %w", err)
+	}
+	if len(environment) == 0 {
+		environment, err = bootstrapEnvironment(value)
+		if err != nil {
+			return nil, fmt.Errorf("load bootstrap credentials: %w", err)
+		}
+	}
+	return append(environment, "STALWART_URL="+value.stalwartURL), nil
+}
+
+func certificateCoversHostname(subjectNames map[string]bool, hostname string) bool {
+	certificate := x509.Certificate{}
+	for name, enabled := range subjectNames {
+		if enabled {
+			certificate.DNSNames = append(certificate.DNSNames, name)
+		}
+	}
+	return certificate.VerifyHostname(hostname) == nil
+}
+
+func certificateInventory(output []byte, san string) (certificateInventoryResult, error) {
+	decoder := json.NewDecoder(bytes.NewReader(output))
+	var result certificateInventoryResult
+	for {
+		var item struct {
+			ID                      string          `json:"id"`
+			SubjectAlternativeNames map[string]bool `json:"subjectAlternativeNames"`
+		}
+		err := decoder.Decode(&item)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return certificateInventoryResult{}, err
+		}
+		if item.ID == "" {
+			return certificateInventoryResult{}, errors.New("certificate query returned an empty id")
+		}
+		result.ids = append(result.ids, item.ID)
+		if certificateCoversHostname(item.SubjectAlternativeNames, san) {
+			result.matches = append(result.matches, item.ID)
+		}
+	}
+	return result, nil
+}
+
+func queryCertificates(ctx context.Context, value options, environment []string) (certificateInventoryResult, error) {
+	output, err := runCLI(
+		ctx,
+		value.cli,
+		environment,
+		true,
+		"query",
+		"Certificate",
+		"--fields",
+		"id,subjectAlternativeNames",
+		"--json",
+		"--no-color",
+	)
+	if err != nil {
+		return certificateInventoryResult{}, err
+	}
+	return certificateInventory(output, value.certificateSAN)
+}
+
+func waitForCertificateSAN(ctx context.Context, value options, environment []string, id string) ([]string, error) {
+	waitContext, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	for {
+		inventory, err := queryCertificates(waitContext, value, environment)
+		if err != nil {
+			return nil, err
+		}
+		if slices.Contains(inventory.matches, id) {
+			return inventory.ids, nil
+		}
+		select {
+		case <-waitContext.Done():
+			return nil, errors.New("updated certificate did not expose the declared SAN before timeout")
+		case <-ticker.C:
+		}
+	}
+}
+
+func fileReferenceField(name, path string) string {
+	return fmt.Sprintf(`%s={"@type":"File","filePath":%q}`, name, path)
+}
+
+func updateCertificate(ctx context.Context, value options, environment []string, id, path string) error {
+	_, err := runCLI(
+		ctx,
+		value.cli,
+		environment,
+		false,
+		"update",
+		"Certificate",
+		id,
+		"--field",
+		fileReferenceField("certificate", path),
+		"--field",
+		fileReferenceField("privateKey", privateKeyPath),
+		"--no-color",
+	)
+	return err
+}
+
+func reconcileCertificate(ctx context.Context, value options, environment []string) error {
+	inventory, err := queryCertificates(ctx, value, environment)
+	if err != nil {
+		return err
+	}
+	var id string
+	if len(inventory.matches) == 0 {
+		existing := make(map[string]struct{}, len(inventory.ids))
+		for _, candidate := range inventory.ids {
+			existing[candidate] = struct{}{}
+		}
+		if _, err := runCLI(
+			ctx,
+			value.cli,
+			environment,
+			false,
+			"create",
+			"Certificate",
+			"--field",
+			fileReferenceField("certificate", certificatePath),
+			"--field",
+			fileReferenceField("privateKey", privateKeyPath),
+			"--no-color",
+		); err != nil {
+			return err
+		}
+		inventory, err = queryCertificates(ctx, value, environment)
+		if err != nil {
+			return err
+		}
+		if len(inventory.matches) == 0 {
+			for _, candidate := range inventory.ids {
+				if _, found := existing[candidate]; !found {
+					inventory.matches = append(inventory.matches, candidate)
+				}
+			}
+			if len(inventory.matches) != 1 {
+				return errors.New("created certificate is not uniquely identifiable")
+			}
+		}
+	}
+	slices.Sort(inventory.matches)
+	id = inventory.matches[0]
+	// Stalwart derives metadata only when the certificate reference changes.
+	// Round-tripping equivalent paths exposes rotations at a projected path.
+	for _, path := range []string{equivalentCertificatePath, certificatePath} {
+		if err := updateCertificate(ctx, value, environment, id, path); err != nil {
+			return err
+		}
+	}
+	if _, err := runCLI(ctx, value.cli, environment, false, "create", "Action/ReloadTlsCertificates", "--json", "{}", "--no-color"); err != nil {
+		return err
+	}
+	ids, err := waitForCertificateSAN(ctx, value, environment, id)
+	if err != nil {
+		return err
+	}
+	if _, err := runCLI(
+		ctx,
+		value.cli,
+		environment,
+		false,
+		"update",
+		"SystemSettings",
+		"--field",
+		"defaultCertificateId="+id,
+		"--no-color",
+	); err != nil {
+		return err
+	}
+	var obsolete []string
+	for _, candidate := range ids {
+		if candidate != id {
+			obsolete = append(obsolete, candidate)
+		}
+	}
+	slices.Sort(obsolete)
+	if len(obsolete) != 0 {
+		if _, err := runCLI(ctx, value.cli, environment, false, "delete", "Certificate", "--ids", strings.Join(obsolete, ","), "--no-color"); err != nil {
+			return err
+		}
+		_, err = runCLI(ctx, value.cli, environment, false, "create", "Action/ReloadTlsCertificates", "--json", "{}", "--no-color")
+		return err
+	}
+	return nil
+}
+
+func reconcileOnce(ctx context.Context, value options) error {
+	if err := renderPlan(value.plan, value.renderedPlan, value.secretRoot); err != nil {
+		return fmt.Errorf("render plan: %w", err)
+	}
+	defer func() {
+		_ = os.Remove(value.renderedPlan)
+	}()
+	authenticationEnvironment, err := credentialEnvironment(ctx, value)
+	if err != nil {
+		return err
+	}
+	if _, err := runCLI(
+		ctx,
+		value.cli,
+		authenticationEnvironment,
+		false,
+		"apply",
+		"--file",
+		value.renderedPlan,
+		"--no-color",
+	); err != nil {
+		return err
+	}
+	if _, err := runCLI(
+		ctx,
+		value.cli,
+		authenticationEnvironment,
+		false,
+		"create",
+		"Action/ReloadSettings",
+		"--json",
+		"{}",
+		"--no-color",
+	); err != nil {
+		return err
+	}
+	if value.updateApps {
+		if _, err := runCLI(
+			ctx,
+			value.cli,
+			authenticationEnvironment,
+			false,
+			"create",
+			"Action/UpdateApps",
+			"--json",
+			"{}",
+			"--no-color",
+		); err != nil {
+			return err
+		}
+	}
+	if err := reconcileAppPasswords(ctx, value); err != nil {
+		return err
+	}
+	if value.certificateSAN == "" {
+		return nil
+	}
+	return reconcileCertificate(ctx, value, authenticationEnvironment)
+}
+
+func statusHandler(store *statusStore) http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/status", func(response http.ResponseWriter, _ *http.Request) {
+		response.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(response).Encode(store.snapshot())
+	})
+	mux.HandleFunc("/readyz", func(response http.ResponseWriter, _ *http.Request) {
+		status := store.snapshot()
+		if !status.Ready || status.PersistentFailure {
+			response.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		response.WriteHeader(http.StatusOK)
+	})
+	mux.HandleFunc("/healthz", func(response http.ResponseWriter, _ *http.Request) {
+		response.WriteHeader(http.StatusOK)
+	})
+	mux.HandleFunc("/metrics", func(response http.ResponseWriter, _ *http.Request) {
+		status := store.snapshot()
+		response.Header().Set("Content-Type", "text/plain; version=0.0.4")
+		_, _ = fmt.Fprintf(
+			response,
+			"stalwart_controller_leader %d\n"+
+				"stalwart_controller_ready %d\n"+
+				"stalwart_controller_persistent_failure %d\n"+
+				"stalwart_controller_consecutive_failures %d\n",
+			metricBoolean(status.Leader),
+			metricBoolean(status.Ready),
+			metricBoolean(status.PersistentFailure),
+			status.ConsecutiveFailures,
+		)
+	})
+	return mux
+}
+
+func waitForRetry(ctx context.Context, serverErrors <-chan error, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-serverErrors:
+		return err
+	case <-timer.C:
+		return nil
+	}
+}
+
+func runLeader(
+	ctx context.Context,
+	value options,
+	store *statusStore,
+	serverErrors <-chan error,
+	ownership ownershipConnection,
+	runtime controllerRuntime,
+) error {
+	defer closeOwnership(ownership)
+	backoff := value.initialBackoff
+	for {
+		select {
+		case err := <-serverErrors:
+			return err
+		default:
+		}
+
+		if err := ownership.Ping(ctx); err != nil {
+			store.update(func(status *controllerStatus) {
+				status.Leader = false
+				status.Ready = false
+				status.PersistentFailure = false
+				status.ConsecutiveFailures = 0
+				status.LastResult = "ownership-lost"
+			})
+			return errOwnerExists
+		}
+		revision, err := runtime.planRevision(value.plan)
+		if err != nil {
+			return err
+		}
+		attemptedAt := time.Now().UTC()
+		store.update(func(status *controllerStatus) {
+			status.PlanRevision = revision
+			status.LastAttempt = attemptedAt.Format(time.RFC3339Nano)
+			status.NextAttempt = ""
+		})
+		reconcileContext, cancel := context.WithTimeout(ctx, value.reconcileTimeout)
+		err = runtime.reconcile(reconcileContext, value)
+		cancel()
+		if ctx.Err() != nil {
+			return nil
+		}
+
+		var delay time.Duration
+		if err == nil {
+			succeededAt := time.Now().UTC()
+			store.update(func(status *controllerStatus) {
+				status.Ready = true
+				status.PersistentFailure = false
+				status.ConsecutiveFailures = 0
+				status.LastSuccess = succeededAt.Format(time.RFC3339Nano)
+				status.LastResult = "success"
+			})
+			backoff = value.initialBackoff
+			delay = value.interval
+		} else {
+			fmt.Fprintln(os.Stderr, "reconciliation failed:", err)
+			if recordLeaderFailure(store, value.persistentFailureAfter) {
+				return errLeaderUnhealthy
+			}
+			delay = backoff
+			backoff = nextBackoff(backoff, value.maxBackoff)
+		}
+
+		next := time.Now().UTC().Add(delay)
+		store.update(func(status *controllerStatus) {
+			status.NextAttempt = next.Format(time.RFC3339Nano)
+		})
+		if err := waitForRetry(ctx, serverErrors, delay); err != nil {
+			return err
+		}
+	}
+}
+
+func runController(ctx context.Context, value options) error {
+	return runControllerWithRuntime(ctx, value, productionControllerRuntime())
+}
+
+func runControllerWithRuntime(ctx context.Context, value options, runtime controllerRuntime) error {
+	if err := validateOptions(value); err != nil {
+		return err
+	}
+	if value.once {
+		reconcileContext, cancel := context.WithTimeout(ctx, value.reconcileTimeout)
+		defer cancel()
+		return runtime.reconcile(reconcileContext, value)
+	}
+	revision, err := runtime.planRevision(value.plan)
+	if err != nil {
+		return err
+	}
+	store := &statusStore{
+		status: controllerStatus{
+			PlanRevision:  revision,
+			BuildRevision: buildRevision,
+			LastResult:    "standby",
+		},
+	}
+	server := &http.Server{
+		Addr:              value.statusAddress,
+		Handler:           statusHandler(store),
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+	defer shutdownStatusServer(server)
+	serverErrors := make(chan error, 1)
+	go func() {
+		errorValue := server.ListenAndServe()
+		if !errors.Is(errorValue, http.ErrServerClosed) {
+			serverErrors <- errorValue
+		}
+	}()
+	backoff := value.initialBackoff
+	for {
+		standbyReady := false
+		coolingDown := false
+		ownershipContext, cancel := context.WithTimeout(ctx, value.reconcileTimeout)
+		ownership, err := runtime.acquireOwnership(ownershipContext, value.dsnFile)
+		cancel()
+		if ctx.Err() != nil {
+			return nil
+		}
+		if err == nil {
+			store.update(func(status *controllerStatus) {
+				status.Leader = true
+				status.Ready = false
+				status.PersistentFailure = false
+				status.ConsecutiveFailures = 0
+				status.LastResult = "not-attempted"
+				status.NextAttempt = ""
+			})
+			err = runLeader(ctx, value, store, serverErrors, ownership, runtime)
+			if ctx.Err() != nil {
+				return nil
+			}
+			if errors.Is(err, errLeaderUnhealthy) {
+				backoff = value.maxBackoff
+				coolingDown = true
+			} else if err != nil && !errors.Is(err, errOwnerExists) {
+				return err
+			} else {
+				backoff = value.initialBackoff
+			}
+		} else if !errors.Is(err, errOwnerExists) {
+			fmt.Fprintln(os.Stderr, "ownership acquisition failed:", err)
+			backoff = nextBackoff(backoff, value.maxBackoff)
+		} else {
+			backoff = value.initialBackoff
+			standbyReady = value.standbyReady
+		}
+		delay := backoff
+		next := time.Now().UTC().Add(delay)
+		store.update(func(status *controllerStatus) {
+			status.Leader = false
+			if coolingDown {
+				status.Ready = false
+				status.PersistentFailure = true
+				status.LastResult = "cooldown"
+			} else {
+				status.Ready = standbyReady
+				status.PersistentFailure = false
+				status.ConsecutiveFailures = 0
+				status.LastResult = "standby"
+			}
+			status.NextAttempt = next.Format(time.RFC3339Nano)
+		})
+		if err := waitForRetry(ctx, serverErrors, delay); err != nil {
+			if errors.Is(err, context.Canceled) {
+				return nil
+			}
+			return err
+		}
+	}
+}

services/stalwart/cmd/controller/http_error.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+func responseStatusError(action string, response *http.Response, secrets ...string) error {
+	body, err := io.ReadAll(io.LimitReader(response.Body, 4096))
+	if err != nil {
+		return fmt.Errorf("%s: HTTP %d", action, response.StatusCode)
+	}
+	detail := strings.TrimSpace(string(body))
+	for _, secret := range secrets {
+		if secret != "" {
+			detail = strings.ReplaceAll(detail, secret, "[redacted]")
+		}
+	}
+	if detail == "" {
+		return fmt.Errorf("%s: HTTP %d", action, response.StatusCode)
+	}
+	return fmt.Errorf("%s: HTTP %d: %q", action, response.StatusCode, detail)
+}

services/stalwart/cmd/controller/http_error_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestResponseStatusErrorIncludesBoundedRedactedDetail(t *testing.T) {
+	secret := "synthetic-secret"
+	response := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Body:       io.NopCloser(strings.NewReader(`{"errors":["invalid synthetic-secret"]}` + strings.Repeat("x", 8192))),
+	}
+	err := responseStatusError("synthetic request", response, secret)
+	if err == nil || !strings.Contains(err.Error(), "HTTP 400") || !strings.Contains(err.Error(), "[redacted]") {
+		t.Fatalf("error = %v", err)
+	}
+	if strings.Contains(err.Error(), secret) || len(err.Error()) > 4200 {
+		t.Fatalf("error is unbounded or contains a secret: %v", err)
+	}
+}

services/stalwart/cmd/controller/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+func runMain(ctx context.Context, arguments []string) error {
+	if len(arguments) == 2 && arguments[0] == "wait-oidc" {
+		return waitForOIDC(ctx, arguments[1], 2*time.Second)
+	}
+	value, err := parseOptions(arguments)
+	if err != nil {
+		return err
+	}
+	return runController(ctx, value)
+}
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	err := runMain(ctx, os.Args[1:])
+	stop()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "controller stopped:", err)
+		os.Exit(1)
+	}
+}

services/stalwart/cmd/controller/oidc.go

@@ -0,0 +1,174 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+type oidcDiscoveryDocument struct {
+	Issuer        string `json:"issuer"`
+	JWKSURI       string `json:"jwks_uri"`
+	TokenEndpoint string `json:"token_endpoint"`
+}
+
+func readSecret(path string) (string, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	value := strings.TrimSpace(string(content))
+	if value == "" {
+		return "", errors.New("secret file is empty")
+	}
+	return value, nil
+}
+
+func fetchJSON(ctx context.Context, client *http.Client, target string, destination any) error {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
+	if err != nil {
+		return err
+	}
+	response, err := client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
+		return responseStatusError("OIDC endpoint", response)
+	}
+	return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(destination)
+}
+
+func discoverOIDC(ctx context.Context, client *http.Client, issuer string) (oidcDiscoveryDocument, error) {
+	issuerURL, err := url.Parse(issuer)
+	if err != nil || issuerURL.Scheme == "" || issuerURL.Host == "" {
+		return oidcDiscoveryDocument{}, errors.New("OIDC issuer is invalid")
+	}
+	target := strings.TrimRight(issuer, "/") + "/.well-known/openid-configuration"
+	var document oidcDiscoveryDocument
+	if err := fetchJSON(ctx, client, target, &document); err != nil {
+		return oidcDiscoveryDocument{}, err
+	}
+	if strings.TrimRight(document.Issuer, "/") != strings.TrimRight(issuer, "/") {
+		return oidcDiscoveryDocument{}, errors.New("OIDC discovery issuer does not match")
+	}
+	for _, endpoint := range []string{document.JWKSURI, document.TokenEndpoint} {
+		endpointURL, err := url.Parse(endpoint)
+		if err != nil || endpointURL.Scheme != issuerURL.Scheme || endpointURL.Host != issuerURL.Host {
+			return oidcDiscoveryDocument{}, errors.New("OIDC endpoint origin does not match issuer")
+		}
+	}
+	return document, nil
+}
+
+func oidcAccessToken(ctx context.Context, issuer, audience, clientID, clientSecret string) (string, error) {
+	client := &http.Client{Timeout: 15 * time.Second}
+	discovery, err := discoverOIDC(ctx, client, issuer)
+	if err != nil {
+		return "", err
+	}
+	form := url.Values{
+		"grant_type": {"client_credentials"},
+		"scope":      {"openid urn:zitadel:iam:org:project:id:" + audience + ":aud"},
+	}
+	request, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		discovery.TokenEndpoint,
+		strings.NewReader(form.Encode()),
+	)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	request.SetBasicAuth(clientID, clientSecret)
+	response, err := client.Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode != http.StatusOK {
+		return "", responseStatusError("OIDC token endpoint", response, clientSecret)
+	}
+	var tokenResponse struct {
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&tokenResponse); err != nil {
+		return "", err
+	}
+	if tokenResponse.AccessToken == "" {
+		return "", errors.New("OIDC token response has no access token")
+	}
+	return tokenResponse.AccessToken, nil
+}
+
+func oidcTokenFromFiles(ctx context.Context, issuerFile, audienceFile, clientIDFile, clientSecretFile string) (string, error) {
+	issuer, err := readSecret(issuerFile)
+	if err != nil {
+		return "", fmt.Errorf("read OIDC issuer: %w", err)
+	}
+	audience, err := readSecret(audienceFile)
+	if err != nil {
+		return "", fmt.Errorf("read OIDC audience: %w", err)
+	}
+	clientID, err := readSecret(clientIDFile)
+	if err != nil {
+		return "", fmt.Errorf("read OIDC client ID: %w", err)
+	}
+	clientSecret, err := readSecret(clientSecretFile)
+	if err != nil {
+		return "", fmt.Errorf("read OIDC client secret: %w", err)
+	}
+	return oidcAccessToken(ctx, issuer, audience, clientID, clientSecret)
+}
+
+func oidcEnvironment(ctx context.Context, value options) ([]string, error) {
+	if value.oidcIssuerFile == "" {
+		return nil, nil
+	}
+	token, err := oidcTokenFromFiles(
+		ctx,
+		value.oidcIssuerFile,
+		value.oidcAudienceFile,
+		value.oidcClientIDFile,
+		value.oidcClientSecretFile,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return []string{"STALWART_TOKEN=" + token}, nil
+}
+
+func waitForOIDC(ctx context.Context, issuerFile string, interval time.Duration) error {
+	issuer, err := readSecret(issuerFile)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{Timeout: 15 * time.Second}
+	for {
+		discovery, err := discoverOIDC(ctx, client, issuer)
+		if err == nil {
+			var keys struct {
+				Keys []json.RawMessage `json:"keys"`
+			}
+			if fetchJSON(ctx, client, discovery.JWKSURI, &keys) == nil && len(keys.Keys) != 0 {
+				return nil
+			}
+		}
+		timer := time.NewTimer(interval)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+}

services/stalwart/cmd/controller/options.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"time"
+)
+
+type options struct {
+	dsnFile                string
+	plan                   string
+	renderedPlan           string
+	secretRoot             string
+	cli                    string
+	stalwartURL            string
+	certificateSAN         string
+	updateApps             bool
+	oidcIssuerFile         string
+	oidcAudienceFile       string
+	oidcClientIDFile       string
+	oidcClientSecretFile   string
+	appPasswords           string
+	vaultAddress           string
+	vaultRole              string
+	vaultTokenFile         string
+	vaultCACertFile        string
+	bootstrapUser          string
+	bootstrapPasswordFile  string
+	statusAddress          string
+	interval               time.Duration
+	initialBackoff         time.Duration
+	maxBackoff             time.Duration
+	reconcileTimeout       time.Duration
+	persistentFailureAfter int
+	standbyReady           bool
+	once                   bool
+}
+
+func parseOptions(arguments []string) (options, error) {
+	var value options
+	flags := flag.NewFlagSet("stalwart-controller", flag.ContinueOnError)
+	flags.StringVar(&value.dsnFile, "dsn-file", "", "PostgreSQL DSN file used for controller ownership")
+	flags.StringVar(&value.plan, "plan", "", "Stalwart desired-state plan")
+	flags.StringVar(&value.renderedPlan, "rendered-plan", "/tmp/stalwart-plan.ndjson", "Rendered plan destination")
+	flags.StringVar(&value.secretRoot, "secret-root", "/run/control", "Root for plan secret-file references")
+	flags.StringVar(&value.cli, "cli", "/usr/local/bin/stalwart-cli", "Stalwart CLI executable")
+	flags.StringVar(&value.stalwartURL, "stalwart-url", "", "Stalwart management URL")
+	flags.StringVar(&value.certificateSAN, "certificate-san", "", "SAN used to select the default certificate")
+	flags.BoolVar(&value.updateApps, "update-apps", false, "Refresh embedded web applications")
+	flags.StringVar(&value.oidcIssuerFile, "oidc-issuer-file", "", "OIDC issuer URL file")
+	flags.StringVar(&value.oidcAudienceFile, "oidc-audience-file", "", "OIDC audience file")
+	flags.StringVar(&value.oidcClientIDFile, "oidc-client-id-file", "", "OIDC client ID file")
+	flags.StringVar(&value.oidcClientSecretFile, "oidc-client-secret-file", "", "OIDC client secret file")
+	flags.StringVar(&value.appPasswords, "app-passwords", "", "Managed app-password identity file")
+	flags.StringVar(&value.vaultAddress, "vault-address", "", "Vault API address")
+	flags.StringVar(&value.vaultRole, "vault-role", "", "Vault Kubernetes authentication role")
+	flags.StringVar(&value.vaultTokenFile, "vault-token-file", "", "Vault service-account token file")
+	flags.StringVar(&value.vaultCACertFile, "vault-ca-cert-file", "", "Vault CA certificate file")
+	flags.StringVar(&value.bootstrapUser, "bootstrap-user", "", "Temporary recovery username")
+	flags.StringVar(&value.bootstrapPasswordFile, "bootstrap-password-file", "", "Temporary recovery password file")
+	flags.StringVar(&value.statusAddress, "status-address", "127.0.0.1:9090", "Status server listen address")
+	flags.DurationVar(&value.interval, "interval", 5*time.Minute, "Successful reconciliation interval")
+	flags.DurationVar(&value.initialBackoff, "initial-backoff", 5*time.Second, "Initial failure retry delay")
+	flags.DurationVar(&value.maxBackoff, "max-backoff", 5*time.Minute, "Maximum failure retry delay")
+	flags.DurationVar(&value.reconcileTimeout, "reconcile-timeout", 2*time.Minute, "Per-reconciliation timeout")
+	flags.IntVar(&value.persistentFailureAfter, "persistent-failure-after", 5, "Failures before relinquishing ownership")
+	flags.BoolVar(&value.standbyReady, "standby-ready", false, "Report a healthy lock contender as ready")
+	flags.BoolVar(&value.once, "once", false, "Reconcile once without ownership or status serving")
+	if err := flags.Parse(arguments); err != nil {
+		return options{}, err
+	}
+	return value, nil
+}
+
+func allOrNone(name string, fields ...string) (bool, error) {
+	configured := 0
+	for _, field := range fields {
+		if field != "" {
+			configured++
+		}
+	}
+	if configured != 0 && configured != len(fields) {
+		return false, fmt.Errorf("%s fields must be set together", name)
+	}
+	return configured != 0, nil
+}
+
+func validateOptions(value options) error {
+	if value.plan == "" {
+		return errors.New("plan is required")
+	}
+	if value.stalwartURL == "" {
+		return errors.New("stalwart-url is required")
+	}
+	if !value.once && value.dsnFile == "" {
+		return errors.New("dsn-file is required in controller mode")
+	}
+	oidcConfigured, err := allOrNone(
+		"OIDC",
+		value.oidcIssuerFile,
+		value.oidcAudienceFile,
+		value.oidcClientIDFile,
+		value.oidcClientSecretFile,
+	)
+	if err != nil {
+		return err
+	}
+	vaultConfigured, err := allOrNone(
+		"vault",
+		value.vaultAddress,
+		value.vaultRole,
+		value.vaultTokenFile,
+		value.vaultCACertFile,
+	)
+	if err != nil {
+		return err
+	}
+	if value.appPasswords != "" && !vaultConfigured {
+		return errors.New("app-password identities require vault")
+	}
+	if value.appPasswords != "" && !oidcConfigured {
+		return errors.New("app-password identities require OIDC issuer and audience")
+	}
+	bootstrapConfigured, err := allOrNone("bootstrap", value.bootstrapUser, value.bootstrapPasswordFile)
+	if err != nil {
+		return err
+	}
+	if oidcConfigured && bootstrapConfigured {
+		return errors.New("OIDC and bootstrap credentials are mutually exclusive")
+	}
+	if !value.once && !oidcConfigured && !bootstrapConfigured {
+		return errors.New("controller mode requires OIDC or bootstrap credentials")
+	}
+	if value.interval <= 0 || value.initialBackoff <= 0 || value.maxBackoff <= 0 {
+		return errors.New("interval and backoff durations must be positive")
+	}
+	if value.initialBackoff > value.maxBackoff {
+		return errors.New("initial backoff exceeds maximum backoff")
+	}
+	if value.reconcileTimeout <= 0 || value.persistentFailureAfter <= 0 {
+		return errors.New("reconcile timeout and failure threshold must be positive")
+	}
+	return nil
+}

services/stalwart/cmd/controller/public_test.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"slices"
+	"strings"
+	"testing"
+	"text/template"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestAccountStaticFrontend(t *testing.T) {
+	deployment := serviceDeployment(t, "k8s/normal/server/deploy.yaml")
+	for _, container := range deployment.Spec.Template.Spec.Containers {
+		if container.Name != "account-web" {
+			continue
+		}
+		prefix, digest, pinned := strings.Cut(container.Image, "@sha256:")
+		if !pinned || prefix != "registry.quorate.internal/quorate/mail-keys" || !hexValue(digest, 64) {
+			t.Fatal("account frontend does not pin the custom UI image")
+		}
+		if len(container.Command) != 0 || len(container.Args) != 0 {
+			t.Fatal("account frontend overrides the image entrypoint")
+		}
+		for _, mount := range container.VolumeMounts {
+			if mount.MountPath == "/etc/nginx/nginx.conf" {
+				t.Fatal("account frontend overrides the static UI routing")
+			}
+		}
+		return
+	}
+	t.Fatal("account frontend is absent")
+}
+
+func TestAdminAPIKeepsIngress(t *testing.T) {
+	var route struct {
+		Spec struct {
+			Hostnames []string `yaml:"hostnames"`
+			Rules     []struct {
+				BackendRefs []struct {
+					Name string `yaml:"name"`
+					Port int    `yaml:"port"`
+				} `yaml:"backendRefs"`
+			} `yaml:"rules"`
+		} `yaml:"spec"`
+	}
+	if err := yaml.Unmarshal(serviceFiles(t, "k8s/normal/gateway/admin.yaml"), &route); err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(route.Spec.Hostnames, []string{"mail.quorate.internal"}) || len(route.Spec.Rules) != 1 {
+		t.Fatal("internal administration route is absent")
+	}
+	backends := route.Spec.Rules[0].BackendRefs
+	if len(backends) != 1 || backends[0].Name != "stalwart-admin" || backends[0].Port != 8080 {
+		t.Fatal("internal administration route does not target the management API")
+	}
+	var operations []struct {
+		Object string          `json:"object"`
+		Value  json.RawMessage `json:"value"`
+	}
+	if err := json.Unmarshal(serviceFiles(t, "k8s/base/plan.json"), &operations); err != nil {
+		t.Fatal(err)
+	}
+	for _, operation := range operations {
+		if operation.Object != "Application" {
+			continue
+		}
+		var apps map[string]struct {
+			Enabled bool `json:"enabled"`
+		}
+		if err := json.Unmarshal(operation.Value, &apps); err != nil {
+			t.Fatal(err)
+		}
+		if app, present := apps["bundled-webui"]; !present || len(apps) != 1 || app.Enabled {
+			t.Fatal("built-in WebUI default is not explicitly disabled")
+		}
+		return
+	}
+	t.Fatal("account application declaration is absent")
+}
+
+func TestAccountPublicConfig(t *testing.T) {
+	decoder := yaml.NewDecoder(bytes.NewReader(serviceFiles(t, "k8s/base/vault/identity.yaml")))
+	for {
+		var secret struct {
+			Metadata struct {
+				Name string `yaml:"name"`
+			} `yaml:"metadata"`
+			Spec struct {
+				Destination struct {
+					Transformation struct {
+						ExcludeRaw bool     `yaml:"excludeRaw"`
+						Excludes   []string `yaml:"excludes"`
+						Templates  map[string]struct {
+							Text string `yaml:"text"`
+						} `yaml:"templates"`
+					} `yaml:"transformation"`
+				} `yaml:"destination"`
+			} `yaml:"spec"`
+		}
+		if err := decoder.Decode(&secret); err == io.EOF {
+			t.Fatal("public account config projection is absent")
+		} else if err != nil {
+			t.Fatal(err)
+		}
+		if secret.Metadata.Name != "mail-web" {
+			continue
+		}
+		transform := secret.Spec.Destination.Transformation
+		if !transform.ExcludeRaw || len(transform.Excludes) != 1 || transform.Excludes[0] != ".*" || len(transform.Templates) != 1 {
+			t.Fatal("account config projection includes raw identity metadata")
+		}
+		functions := template.FuncMap{
+			"get": func(values map[string]string, key string) string { return values[key] },
+			"dict": func(values ...string) map[string]string {
+				result := make(map[string]string)
+				for index := 0; index < len(values); index += 2 {
+					result[values[index]] = values[index+1]
+				}
+				return result
+			},
+			"toJson": func(value any) (string, error) {
+				encoded, err := json.Marshal(value)
+				return string(encoded), err
+			},
+		}
+		parsed, err := template.New("config").Funcs(functions).Parse(transform.Templates["config.json"].Text)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var output bytes.Buffer
+		if err := parsed.Execute(&output, map[string]any{"Secrets": map[string]string{
+			"issuer_url": "https://identity.example.invalid", "account_client_id": "synthetic-account",
+			"audience": "synthetic-project", "client_secret": "synthetic-machine-secret",
+		}}); err != nil {
+			t.Fatal(err)
+		}
+		var config map[string]string
+		if err := json.Unmarshal(output.Bytes(), &config); err != nil {
+			t.Fatal(err)
+		}
+		if len(config) != 3 || config["issuer"] != "https://identity.example.invalid" || config["clientId"] != "synthetic-account" || config["scope"] != "openid profile urn:zitadel:iam:org:project:id:synthetic-project:aud" {
+			t.Fatalf("public account config = %#v", config)
+		}
+		break
+	}
+	var deployment struct {
+		Spec struct {
+			Template struct {
+				Spec struct {
+					Volumes []struct {
+						Name   string `yaml:"name"`
+						Secret *struct {
+							SecretName string `yaml:"secretName"`
+							Items      []struct {
+								Key  string `yaml:"key"`
+								Path string `yaml:"path"`
+							} `yaml:"items"`
+						} `yaml:"secret"`
+					} `yaml:"volumes"`
+					Containers []struct {
+						Name         string `yaml:"name"`
+						VolumeMounts []struct {
+							Name      string `yaml:"name"`
+							MountPath string `yaml:"mountPath"`
+							SubPath   string `yaml:"subPath"`
+							ReadOnly  bool   `yaml:"readOnly"`
+						} `yaml:"volumeMounts"`
+					} `yaml:"containers"`
+				} `yaml:"spec"`
+			} `yaml:"template"`
+		} `yaml:"spec"`
+	}
+	if err := yaml.Unmarshal(serviceFiles(t, "k8s/normal/server/deploy.yaml"), &deployment); err != nil {
+		t.Fatal(err)
+	}
+	secrets := make(map[string]string)
+	configFound := false
+	for _, volume := range deployment.Spec.Template.Spec.Volumes {
+		if volume.Secret == nil {
+			continue
+		}
+		secret := volume.Secret
+		secrets[volume.Name] = secret.SecretName
+		if volume.Name != "account-config" {
+			continue
+		}
+		configFound = true
+		if secret.SecretName != "mail-web" || len(secret.Items) != 1 || secret.Items[0].Key != "config.json" || secret.Items[0].Path != "config.json" {
+			t.Fatal("account config volume projects identity material beyond its public config")
+		}
+	}
+	if !configFound {
+		t.Fatal("account config volume is absent")
+	}
+	for _, container := range deployment.Spec.Template.Spec.Containers {
+		if container.Name != "account-web" {
+			continue
+		}
+		mounted := false
+		for _, mount := range container.VolumeMounts {
+			secret := secrets[mount.Name]
+			if secret != "" && secret != "mail-web" && secret != "stalwart-tls" {
+				t.Fatal("account frontend mounts an automation identity")
+			}
+			if mount.Name == "account-config" {
+				mounted = mount.MountPath == "/run/config" && mount.SubPath == "" && mount.ReadOnly
+			}
+		}
+		if !mounted {
+			t.Fatal("account frontend lacks a read-only public config projection")
+		}
+		return
+	}
+	t.Fatal("account frontend is absent")
+}