Harden NordVPN reconciliation and WebGUI testing

src/opnsense/mvc/app/controllers/OPNsense/NordVPN/Api/ServiceController.php

@@ -7,14 +7,11 @@ namespace OPNsense\NordVPN\Api;
 
 use OPNsense\Base\ApiControllerBase;
 use OPNsense\Core\Backend;
+use OPNsense\NordVPN\Bootstrapper;
 use OPNsense\NordVPN\EnrollmentInspector;
-use OPNsense\NordVPN\ManagedConfiguration;
 use OPNsense\NordVPN\NordApi;
-use OPNsense\NordVPN\ReconciliationLock;
-use OPNsense\NordVPN\Reconciler;
 use OPNsense\NordVPN\Settings;
 use OPNsense\NordVPN\StatusStore;
-use OPNsense\NordVPN\SystemRunner;
 
 class ServiceController extends ApiControllerBase
 {
@@ -24,41 +21,9 @@ class ServiceController extends ApiControllerBase
             return ['status' => 'failed', 'message' => 'POST is required.'];
         }
         $token = trim((string)($this->request->getPost('token') ?? ''));
-        $lock = new ReconciliationLock();
+        $result = (new Bootstrapper())->run($token);
-        try {
+        unset($token);
-            $lock->acquire();
+        return $result;
-            $api = new NordApi();
-            $settings = new Settings();
-            $countries = array_values(array_filter(explode(',', (string)$settings->general->countries)));
-            $candidates = $api->recommendations($countries, 20);
-            if (empty($candidates)) {
-                throw new \RuntimeException('NordVPN returned no eligible servers.');
-            }
-            $configuration = new ManagedConfiguration($settings);
-            $configuration->provision($api->credentials($token), $candidates[0]);
-            unset($token);
-            $runner = new SystemRunner();
-            (new StatusStore())->write([
-                'available' => false,
-                'state' => 'reconciling',
-                'server_hostname' => $candidates[0]['hostname'],
-                'server_country' => $candidates[0]['country'],
-                'message' => 'Waiting for the managed tunnel.',
-            ]);
-            $runner->applyFirewall();
-            $runner->applyWireGuard();
-            $configuration->repairGatewayInterface();
-            $runner->applyRouting();
-            $runner->applyFirewall();
-            sleep(5);
-            $result = (new Reconciler($api, $configuration, null, null, $runner))->run();
-            return ['status' => 'ok', 'tunnel' => $result];
-        } catch (\Throwable $exception) {
-            unset($token);
-            return ['status' => 'failed', 'message' => 'Bootstrap failed. Check the token and plugin configuration.'];
-        } finally {
-            $lock->release();
-        }
     }
 
     public function countriesAction(): array

src/opnsense/mvc/app/library/OPNsense/NordVPN/Bootstrapper.php

@@ -0,0 +1,79 @@
+<?php
+
+// SPDX-License-Identifier: BSD-2-Clause
+// SPDX-FileCopyrightText: 2026 ms2k
+
+namespace OPNsense\NordVPN;
+
+class Bootstrapper
+{
+    private NordApi $api;
+    private ManagedConfiguration $configuration;
+    private StatusStore $status;
+    private SystemRunner $runner;
+    private EndpointStore $endpoints;
+    private ReconciliationLock $lock;
+    private $enqueue;
+
+    public function __construct(
+        ?NordApi $api = null,
+        ?ManagedConfiguration $configuration = null,
+        ?StatusStore $status = null,
+        ?SystemRunner $runner = null,
+        ?EndpointStore $endpoints = null,
+        ?ReconciliationLock $lock = null,
+        ?callable $enqueue = null
+    ) {
+        $this->api = $api ?? new NordApi();
+        $this->configuration = $configuration ?? new ManagedConfiguration();
+        $this->status = $status ?? new StatusStore();
+        $this->runner = $runner ?? new SystemRunner();
+        $this->endpoints = $endpoints ?? new EndpointStore();
+        $this->lock = $lock ?? new ReconciliationLock();
+        $this->enqueue = $enqueue ?? static function (): void {
+            (new \OPNsense\Core\Backend())->configdRun('nordvpn reconcile', true);
+        };
+    }
+
+    public function run(string $token): array
+    {
+        try {
+            $this->lock->acquire();
+            $countries = array_values(array_filter(explode(',', (string)$this->configuration->settings()->general->countries)));
+            $candidates = $this->api->recommendations($countries, 20);
+            if (empty($candidates)) {
+                throw new \RuntimeException('NordVPN returned no eligible servers.');
+            }
+            $privateKey = $this->api->credentials($token);
+            unset($token);
+            $this->configuration->provision($privateKey, $candidates[0]);
+            unset($privateKey);
+            if ((string)($this->configuration->settings()->general->exempt_wireguard ?? '0') === '1') {
+                $this->endpoints->ensure($candidates[0]['address']);
+            }
+            $this->runner->applyFirewall();
+            $this->runner->applyWireGuard();
+            $this->configuration->repairGatewayInterface();
+            $this->runner->applyRouting();
+            $this->runner->applyFirewall();
+            $this->status->write([
+                'available' => false,
+                'state' => 'reconciling',
+                'server_hostname' => $candidates[0]['hostname'],
+                'server_country' => $candidates[0]['country'],
+                'message' => 'Waiting for the managed tunnel.',
+            ]);
+        } catch (\Throwable) {
+            return ['status' => 'failed', 'message' => 'Bootstrap failed. Check the token and plugin configuration.'];
+        } finally {
+            $this->lock->release();
+        }
+
+        try {
+            ($this->enqueue)();
+        } catch (\Throwable) {
+            return ['status' => 'failed', 'message' => 'Bootstrap completed but reconciliation could not start.'];
+        }
+        return ['status' => 'ok', 'tunnel' => $this->status->read()];
+    }
+}

src/opnsense/mvc/app/library/OPNsense/NordVPN/EndpointStore.php

@@ -10,12 +10,17 @@ class EndpointStore
     public const PATH = '/var/db/nordvpn/endpoints.txt';
     public const TABLE = 'nordvpn_endpoints';
     private const MAXIMUM_AGE = 21600;
+    private const RETRY_DELAY = 300;
     private string $path;
+    private string $freshPath;
+    private string $retryPath;
     private $clock;
 
     public function __construct(string $path = self::PATH, ?callable $clock = null)
     {
         $this->path = $path;
+        $this->freshPath = $path . '.refreshed';
+        $this->retryPath = $path . '.retry';
         $this->clock = $clock ?? 'time';
     }
 
@@ -26,14 +31,63 @@ class EndpointStore
 
     public function refreshIfStale(NordApi $api): bool
     {
-        $modified = is_file($this->path) ? filemtime($this->path) : false;
+        $modified = is_file($this->freshPath) ? filemtime($this->freshPath) : false;
         if ($modified !== false && ($this->clock)() - $modified < self::MAXIMUM_AGE) {
             return false;
         }
-        $this->write($api->endpointAddresses());
+        if ($this->retryPending()) {
+            return false;
+        }
+        try {
+            $this->write($api->endpointAddresses());
+            touch($this->freshPath);
+            if (is_file($this->retryPath)) {
+                unlink($this->retryPath);
+            }
+            return true;
+        } catch (\Throwable $exception) {
+            $directory = dirname($this->retryPath);
+            if (is_dir($directory) || mkdir($directory, 0755, true) || is_dir($directory)) {
+                touch($this->retryPath);
+            }
+            throw $exception;
+        }
+    }
+
+    public function retryPending(): bool
+    {
+        $retried = is_file($this->retryPath) ? filemtime($this->retryPath) : false;
+        return $retried !== false && ($this->clock)() - $retried < self::RETRY_DELAY;
+    }
+
+    public function ensure(string $address): bool
+    {
+        if (!filter_var($address, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
+            throw new \InvalidArgumentException('Invalid NordVPN endpoint address.');
+        }
+        $addresses = $this->addresses();
+        if (isset($addresses[$address])) {
+            return false;
+        }
+        $addresses[$address] = true;
+        $this->write(array_keys($addresses));
         return true;
     }
 
+    private function addresses(): array
+    {
+        if (!is_file($this->path)) {
+            return [];
+        }
+        $addresses = [];
+        foreach (file($this->path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $address) {
+            if (filter_var($address, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
+                $addresses[$address] = true;
+            }
+        }
+        return $addresses;
+    }
+
     private function write(array $addresses): void
     {
         $directory = dirname($this->path);
@@ -45,6 +99,7 @@ class EndpointStore
             throw new \RuntimeException('Unable to create the NordVPN endpoint file.');
         }
         try {
+            sort($addresses, SORT_NATURAL);
             $content = implode("\n", $addresses) . "\n";
             if (file_put_contents($temporary, $content, LOCK_EX) === false ||
                 !chmod($temporary, 0644) ||

src/opnsense/mvc/app/library/OPNsense/NordVPN/NordApi.php

@@ -8,7 +8,7 @@ namespace OPNsense\NordVPN;
 class NordApi
 {
     private const BASE_URL = 'https://api.nordvpn.com';
-    private const ENDPOINT_PAGE_LIMIT = 500;
+    private const ENDPOINT_PAGE_LIMIT = 1000;
     private const ENDPOINT_PAGE_MAXIMUM = 40;
     private const WIREGUARD_TECHNOLOGY_ID = 35;
     private $transport;
@@ -69,11 +69,14 @@ class NordApi
         }
 
         $result = [];
-        foreach ($queries as $query) {
+        foreach ($queries as &$query) {
             $query['filters[servers.status]'] = 'online';
             $query['filters[servers_technologies]'] = self::WIREGUARD_TECHNOLOGY_ID;
             $query['filters[servers_technologies][pivot][status]'] = 'online';
-            foreach ($this->request('/v1/servers/recommendations', $query) as $server) {
+        }
+        unset($query);
+        foreach ($this->recommendationResponses($queries) as $servers) {
+            foreach ($servers as $server) {
                 $candidate = $this->candidate($server);
                 if ($candidate !== null) {
                     $result[$candidate['hostname']] = $candidate;
@@ -81,7 +84,7 @@ class NordApi
             }
         }
 
-        return array_slice(array_values($result), 0, $limit);
+        return array_values($result);
     }
 
     public function endpointAddresses(): array
@@ -168,6 +171,48 @@ class NordApi
         }
     }
 
+    private function recommendationResponses(array $queries): array
+    {
+        if ($this->transport !== null) {
+            return array_map(fn($query) => $this->request('/v1/servers/recommendations', $query), $queries);
+        }
+
+        $multi = curl_multi_init();
+        $handles = [];
+        try {
+            foreach ($queries as $index => $query) {
+                $handle = $this->requestHandle('/v1/servers/recommendations', $query);
+                $handles[$index] = $handle;
+                curl_multi_add_handle($multi, $handle);
+            }
+            do {
+                $result = curl_multi_exec($multi, $running);
+                if ($result !== CURLM_OK) {
+                    throw new \RuntimeException('Unable to query NordVPN recommendations.');
+                }
+                if ($running > 0 && curl_multi_select($multi, 1.0) === -1) {
+                    usleep(10_000);
+                }
+            } while ($running > 0);
+
+            $responses = [];
+            foreach ($handles as $index => $handle) {
+                $responses[$index] = $this->decodeResponse(
+                    curl_multi_getcontent($handle),
+                    curl_getinfo($handle, CURLINFO_RESPONSE_CODE),
+                    curl_error($handle)
+                );
+            }
+            return $responses;
+        } finally {
+            foreach ($handles as $handle) {
+                curl_multi_remove_handle($multi, $handle);
+                curl_close($handle);
+            }
+            curl_multi_close($multi);
+        }
+    }
+
     private function request(string $path, array $query = [], ?string $authorization = null): array
     {
         if ($this->transport !== null) {
@@ -178,6 +223,16 @@ class NordApi
             return $response;
         }
 
+        $handle = $this->requestHandle($path, $query, $authorization);
+        $body = curl_exec($handle);
+        $status = curl_getinfo($handle, CURLINFO_RESPONSE_CODE);
+        $error = curl_error($handle);
+        curl_close($handle);
+        return $this->decodeResponse($body, $status, $error);
+    }
+
+    private function requestHandle(string $path, array $query = [], ?string $authorization = null)
+    {
         $url = self::BASE_URL . $path;
         if (!empty($query)) {
             $url .= '?' . http_build_query($query);
@@ -197,10 +252,11 @@ class NordApi
             CURLOPT_SSL_VERIFYPEER => true,
             CURLOPT_SSL_VERIFYHOST => 2,
         ]);
-        $body = curl_exec($handle);
+        return $handle;
-        $status = curl_getinfo($handle, CURLINFO_RESPONSE_CODE);
+    }
-        $error = curl_error($handle);
+
-        curl_close($handle);
+    private function decodeResponse($body, int $status, string $error): array
+    {
         if ($body === false || $status < 200 || $status >= 300) {
             throw new \RuntimeException($error !== '' ? $error : sprintf('NordVPN API returned HTTP %d.', $status));
         }

src/opnsense/mvc/app/library/OPNsense/NordVPN/Reconciler.php

@@ -49,24 +49,17 @@ class Reconciler
             $this->runner->applyFirewall();
         }
 
-        $endpointsChanged = false;
-        if ((string)($settings->general->exempt_wireguard ?? '0') === '1') {
-            try {
-                $endpointsChanged = $this->endpoints->refreshIfStale($this->api);
-            } catch (\Throwable) {
-            }
-        }
-
         $countryCodes = $this->values((string)$settings->general->countries);
         $currentCountry = (string)$settings->managed->server_country;
         $health = $this->health->check($interface);
         if ($health['available'] && (empty($countryCodes) || in_array($currentCountry, $countryCodes, true))) {
-            return $this->publish(array_merge($health, [
+            $this->publish(array_merge($health, [
                 'state' => 'available',
                 'server_hostname' => (string)$settings->managed->server_hostname,
                 'server_country' => $currentCountry,
                 'message' => '',
-            ]), $endpointsChanged);
+            ]));
+            return $this->complete($settings);
         }
 
         $this->publish([
@@ -76,7 +69,7 @@ class Reconciler
             'server_country' => $currentCountry,
             'last_handshake' => $health['last_handshake'],
             'message' => 'Trying eligible NordVPN servers.',
-        ], $endpointsChanged);
+        ]);
 
         try {
             $currentHostname = (string)$settings->managed->server_hostname;
@@ -85,6 +78,9 @@ class Reconciler
                 if ($candidate['hostname'] === $currentHostname) {
                     continue;
                 }
+                if ($this->exemptWireGuard($settings) && $this->endpoints->ensure($candidate['address'])) {
+                    $this->runner->applyFirewall();
+                }
                 $this->configuration->updateCandidate($candidate);
                 $this->runner->applyWireGuard();
                 $this->configuration->repairGatewayInterface();
@@ -107,7 +103,31 @@ class Reconciler
         } catch (\Throwable $exception) {
             $next = $this->failureStatus('error', $exception->getMessage());
         }
-        return $this->publish($next);
+        $this->publish($next);
+        return $this->complete($settings);
+    }
+
+    private function complete(Settings $settings): array
+    {
+        if (!$this->exemptWireGuard($settings)) {
+            $this->status->write(['endpoint_message' => '']);
+            return $this->status->read();
+        }
+        try {
+            if ($this->endpoints->refreshIfStale($this->api)) {
+                $this->publish([], true);
+            }
+            if (!$this->endpoints->retryPending()) {
+                $this->status->write(['endpoint_message' => '']);
+            }
+        } catch (\Throwable) {
+            $this->status->write([
+                'endpoint_message' => $this->endpoints->available()
+                    ? 'NordVPN endpoint refresh failed; the existing endpoint list is retained.'
+                    : 'NordVPN endpoint refresh failed; the WireGuard exemption is inactive.',
+            ]);
+        }
+        return $this->status->read();
     }
 
     private function publish(array $next, bool $firewallChanged = false): array
@@ -131,6 +151,11 @@ class Reconciler
         ))));
     }
 
+    private function exemptWireGuard(Settings $settings): bool
+    {
+        return (string)($settings->general->exempt_wireguard ?? '0') === '1';
+    }
+
     private function failureStatus(string $state, string $message): array
     {
         $settings = $this->configuration->settings();

src/opnsense/mvc/app/library/OPNsense/NordVPN/StatusStore.php

@@ -53,6 +53,7 @@ class StatusStore
             'server_country' => '',
             'last_handshake' => 0,
             'message' => '',
+            'endpoint_message' => '',
             'firewall_pending' => false,
             'updated_at' => 0,
         ];

src/opnsense/mvc/app/library/OPNsense/NordVPN/SystemRunner.php

@@ -8,11 +8,17 @@ namespace OPNsense\NordVPN;
 class SystemRunner
 {
     public const FIREWALL_LABEL = 'nordvpn-egress';
+    private const COMMAND_TIMEOUT_SECONDS = 60;
     private $executor;
+    private int $timeout;
 
-    public function __construct(?callable $executor = null)
+    public function __construct(?callable $executor = null, int $timeout = self::COMMAND_TIMEOUT_SECONDS)
     {
+        if ($timeout < 1) {
+            throw new \InvalidArgumentException('System command timeout must be positive.');
+        }
         $this->executor = $executor;
+        $this->timeout = $timeout;
     }
 
     public function run(array $command): string
@@ -26,11 +32,65 @@ class SystemRunner
         if (!is_resource($process)) {
             throw new \RuntimeException('Unable to start system command.');
         }
-        $stdout = stream_get_contents($pipes[1]);
+        foreach ($pipes as $pipe) {
-        $stderr = stream_get_contents($pipes[2]);
+            stream_set_blocking($pipe, false);
-        fclose($pipes[1]);
+        }
-        fclose($pipes[2]);
+
-        $status = proc_close($process);
+        $stdoutPipe = $pipes[1];
+        $stdout = '';
+        $stderr = '';
+        $deadline = hrtime(true) + $this->timeout * 1_000_000_000;
+        $timedOut = false;
+        $status = -1;
+        try {
+            while (true) {
+                $read = array_values($pipes);
+                if (!empty($read)) {
+                    $write = null;
+                    $except = null;
+                    $ready = stream_select($read, $write, $except, 0, 100_000);
+                    if ($ready === false) {
+                        throw new \RuntimeException('Unable to read system command output.');
+                    }
+                    foreach ($read as $pipe) {
+                        $output = stream_get_contents($pipe);
+                        if ($pipe === $stdoutPipe) {
+                            $stdout .= $output;
+                        } else {
+                            $stderr .= $output;
+                        }
+                    }
+                }
+
+                foreach ($pipes as $index => $pipe) {
+                    if (feof($pipe)) {
+                        fclose($pipe);
+                        unset($pipes[$index]);
+                    }
+                }
+                $processStatus = proc_get_status($process);
+                if (!$processStatus['running'] && empty($pipes)) {
+                    break;
+                }
+                if (hrtime(true) >= $deadline && !$timedOut) {
+                    proc_terminate($process, 9);
+                    $timedOut = true;
+                }
+            }
+        } finally {
+            foreach ($pipes as $pipe) {
+                fclose($pipe);
+            }
+            $processStatus = proc_get_status($process);
+            if ($processStatus['running']) {
+                proc_terminate($process, 9);
+            }
+            $status = proc_close($process);
+        }
+
+        if ($timedOut) {
+            throw new \RuntimeException('System command timed out.');
+        }
         if ($status !== 0) {
             throw new \RuntimeException(trim($stderr) ?: sprintf('System command exited with status %d.', $status));
         }

src/opnsense/mvc/app/views/OPNsense/NordVPN/index.volt

@@ -55,6 +55,8 @@
                 $('#tunnelCountry').text(tunnel.server_country || '-');
                 $('#tunnelMessage').text(tunnel.message || '');
                 $('#rfc1918Warning').toggleClass('hidden', !data.rfc1918_warning);
+                $('#endpointWarning').text(tunnel.endpoint_message || '');
+                $('#endpointWarning').toggleClass('hidden', !tunnel.endpoint_message);
             });
         }
 
@@ -82,7 +84,7 @@
             ajaxCall('/api/nordvpn/service/bootstrap', {token: token}, function(data) {
                 $('#bootstrapMsg').removeClass('hidden alert-success alert-danger');
                 $('#bootstrapMsg').addClass(data.status === 'ok' ? 'alert-success' : 'alert-danger');
-                $('#bootstrapMsg').text(data.status === 'ok' ? '{{ lang._("Bootstrap completed") }}' : data.message);
+                $('#bootstrapMsg').text(data.status === 'ok' ? '{{ lang._("Bootstrap started") }}' : data.message);
                 $('#bootstrapAct').removeClass('disabled').prop('disabled', false);
                 $('#bootstrap_progress').removeClass('fa fa-spinner fa-pulse');
                 refreshStatus();
@@ -104,6 +106,7 @@
 <div class="alert alert-warning hidden" role="alert" id="rfc1918Warning">
     {{ lang._('NordVPN egress enrollment can include an RFC 1918 destination address.') }}
 </div>
+<div class="alert alert-warning hidden" role="alert" id="endpointWarning"></div>
 
 <div class="content-box">
     <h2>{{ lang._('NordVPN egress enrollment') }}</h2>

tests/ui/README.md

@@ -2,23 +2,26 @@
 
 These Playwright tests use a disposable OPNsense deployment. They create and delete only entries containing the synthetic `UI_TEST_NETWORK` alias and require `OPNSENSE_UI_TEST_ALLOW_MUTATION=1`.
 
-Start a QEMU fixture after building a package. The launcher needs a compatible OPNsense base image and transfer disk, neither of which is stored in this repository.
+Run a QEMU fixture after building a package. The runner generates a disposable root WebGUI password, stores it only in `tests/ui/.runtime/credentials.env`, and removes it with the QEMU overlay when it exits. It needs a compatible OPNsense base image and transfer disk, neither of which is stored in this repository.
 
 ```sh
 NORDVPN_PACKAGE_PATH=/path/to/os-nordvpn.pkg \
 OPNSENSE_VM_IMAGE=/path/to/opnsense.qcow2 \
 OPNSENSE_TRANSFER_IMAGE=/path/to/transfer.img \
-tests/ui/start-opnsense.sh
+tests/ui/run-opnsense.sh
 ```
 
-In another terminal, run the browser suite against the disposable WebGUI:
+The runner retains a failed VM for inspection. Stop it with:
 
 ```sh
-OPNSENSE_UI_BASE_URL=https://127.0.0.1:18443 \
+tests/ui/stop-opnsense.sh
-OPNSENSE_UI_USERNAME=root \
+```
-OPNSENSE_UI_PASSWORD=... \
+
-OPNSENSE_UI_TEST_ALLOW_MUTATION=1 \
+To keep the VM running for interactive inspection, use `start-opnsense.sh` instead, then load its ignored credential file before running the suite:
-sh -c 'cd tests/ui && pnpm install --frozen-lockfile && pnpm test'
+
+```sh
+. tests/ui/.runtime/credentials.env
+pnpm --dir tests/ui test
 ```
 
 The suite mocks only the countries API response with synthetic values. It covers populated country options, failure-policy labels, native alias selectors, Add/Edit/Clone state restoration, and the selector menu/footer layout.

tests/ui/guest-setup.php

@@ -1,11 +1,43 @@
 <?php
 
 require_once '/usr/local/etc/inc/config.inc';
+require_once '/usr/local/etc/inc/auth.inc';
 
 use OPNsense\Core\Config;
 use OPNsense\Firewall\Alias;
 
+$password = trim((string)file_get_contents('/mnt/ui-password'));
+if (!preg_match('/^[a-f0-9]{64}$/', $password)) {
+    throw new RuntimeException('The UI test password is invalid.');
+}
+$root =& getUserEntry('root');
+if ($root === false) {
+    throw new RuntimeException('The root user is unavailable.');
+}
+local_user_set_password($root, $password);
+local_user_set($root, true);
+write_config('Set disposable WebGUI test password');
+
+$users = Config::getInstance()->object()->system->user;
+foreach ($users as $user) {
+    if ((string)$user->name === 'root' && password_verify($password, (string)$user->password)) {
+        break;
+    }
+}
+if (!isset($user) || (string)$user->name !== 'root' || !password_verify($password, (string)$user->password)) {
+    throw new RuntimeException('The disposable WebGUI password was not saved.');
+}
+exec('/usr/local/sbin/configctl webgui restart', $output, $status);
+if ($status !== 0) {
+    throw new RuntimeException('The WebGUI did not restart.');
+}
+
 $aliases = new Alias();
+foreach ($aliases->aliases->alias->iterateItems() as $alias) {
+    if ((string)$alias->name === 'UI_TEST_NETWORK') {
+        return;
+    }
+}
 $alias = $aliases->aliases->alias->add();
 $alias->setNodes([
     'enabled' => '1',

tests/ui/run-opnsense.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+set -eu
+umask 077
+
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+runtime_dir="$script_dir/.runtime"
+launcher_file="$runtime_dir/launcher.pid"
+mkdir -p "$runtime_dir"
+
+if [ -r "$launcher_file" ]; then
+    read -r existing_launcher < "$launcher_file"
+    case "$existing_launcher" in
+        *[!0-9]* | '') ;;
+        *)
+            if kill -0 "$existing_launcher" 2>/dev/null; then
+                echo "An OPNsense UI fixture is already running." >&2
+                exit 1
+            fi
+            ;;
+    esac
+fi
+
+cleanup()
+{
+    status=$?
+    set +e
+    if [ "$status" -ne 0 ] && [ "${OPNSENSE_UI_KEEP_VM_ON_FAILURE:-1}" = '1' ]; then
+        printf '%s\n' "OPNSENSE_UI_VM_RETAINED=$runtime_dir"
+        trap - EXIT
+        exit "$status"
+    fi
+    [ -n "${launcher_pid:-}" ] && kill "$launcher_pid" 2>/dev/null
+    [ -n "${launcher_pid:-}" ] && wait "$launcher_pid" 2>/dev/null
+    rm -f "$runtime_dir/launcher.log"
+}
+trap cleanup EXIT INT TERM
+
+"$script_dir/start-opnsense.sh" > "$runtime_dir/launcher.log" 2>&1 &
+launcher_pid=$!
+
+attempts=240
+while [ ! -f "$runtime_dir/ready" ]; do
+    if ! kill -0 "$launcher_pid" 2>/dev/null; then
+        cat "$runtime_dir/launcher.log" >&2
+        exit 1
+    fi
+    attempts=$((attempts - 1))
+    [ "$attempts" -gt 0 ] || {
+        cat "$runtime_dir/launcher.log" >&2
+        exit 1
+    }
+    sleep 1
+done
+
+set -a
+. "$runtime_dir/credentials.env"
+set +a
+
+attempts=60
+while ! curl --fail --insecure --silent --connect-timeout 2 "$OPNSENSE_UI_BASE_URL/" > /dev/null; do
+    if ! kill -0 "$launcher_pid" 2>/dev/null; then
+        cat "$runtime_dir/launcher.log" >&2
+        exit 1
+    fi
+    attempts=$((attempts - 1))
+    [ "$attempts" -gt 0 ] || {
+        cat "$runtime_dir/launcher.log" >&2
+        exit 1
+    }
+    sleep 1
+done
+
+pnpm --dir "$script_dir" install --frozen-lockfile
+pnpm --dir "$script_dir" test

tests/ui/specs/enrollment.spec.js

@@ -3,6 +3,18 @@ const { test, expect } = require("@playwright/test");
 const username = process.env.OPNSENSE_UI_USERNAME;
 const password = process.env.OPNSENSE_UI_PASSWORD;
 
+async function closeDialog(dialog) {
+  const hidden = dialog.evaluate(
+    (element) =>
+      new Promise((resolve) => {
+        window.jQuery(element).one("hidden.bs.modal", resolve);
+      }),
+  );
+  await dialog.getByRole("button", { name: "Close" }).click();
+  await hidden;
+  await expect(dialog).toBeHidden();
+}
+
 if (!username || !password) {
   throw new Error(
     "Set OPNSENSE_UI_USERNAME and OPNSENSE_UI_PASSWORD for the disposable OPNsense deployment.",
@@ -57,21 +69,12 @@ test("renders and preserves NordVPN enrollment controls", async ({ page }) => {
   await page.locator("#usernamefld").fill(username);
   await page.locator("#passwordfld").fill(password);
   await page.getByRole("button", { name: "Login" }).click();
+  await expect(page.locator("#usernamefld")).toBeHidden();
   await page.goto("/ui/nordvpn");
   await expect(
     page.getByRole("heading", { name: "NordVPN egress enrollment" }),
   ).toBeVisible();
 
-  const accessToken = page.locator("#accessToken");
-  const bootstrap = page.getByRole("button", { name: "Bootstrap NordLynx" });
-  await accessToken.fill("synthetic-token");
-  await bootstrap.click();
-  await bootstrapStarted;
-  await expect(accessToken).toHaveValue("");
-  await expect(bootstrap).toBeDisabled();
-  finishBootstrap();
-  await expect(bootstrap).toBeEnabled();
-
   const countries = page.locator('[id="settings.general.countries"]');
   await expect(countries.locator('option[value="XA"]')).toHaveText(
     "Synthetic A",
@@ -150,8 +153,7 @@ test("renders and preserves NordVPN enrollment controls", async ({ page }) => {
   await expect(
     dialog.locator('select[for="enrollment.destination"]'),
   ).toHaveValues(["any"]);
-  await dialog.getByRole("button", { name: "Close" }).click();
+  await closeDialog(dialog);
-  await expect(dialog).toBeHidden();
 
   await row.getByRole("button", { name: "Clone" }).click();
   await expect(dialog).toBeVisible();
@@ -161,8 +163,7 @@ test("renders and preserves NordVPN enrollment controls", async ({ page }) => {
   await expect(
     dialog.locator('select[for="enrollment.destination"]'),
   ).toHaveValues(["any"]);
-  await dialog.getByRole("button", { name: "Close" }).click();
+  await closeDialog(dialog);
-  await expect(dialog).toBeHidden();
 
   const beforeDelete = await testRows.count();
   await row.getByRole("button", { name: "Delete" }).click();
@@ -171,5 +172,16 @@ test("renders and preserves NordVPN enrollment controls", async ({ page }) => {
   await removal.getByRole("button", { name: "Yes" }).click();
   await expect(testRows).toHaveCount(beforeDelete - 1);
 
+  const accessToken = page.locator("#accessToken");
+  const bootstrap = page.getByRole("button", { name: "Bootstrap NordLynx" });
+  await accessToken.fill("synthetic-token");
+  await bootstrap.click();
+  await bootstrapStarted;
+  await expect(accessToken).toHaveValue("");
+  await expect(bootstrap).toBeDisabled();
+  finishBootstrap();
+  await expect(bootstrap).toBeEnabled();
+  await expect(page.locator("#bootstrapMsg")).toHaveText("Bootstrap started");
+
   expect(errors).toEqual([]);
 });

tests/ui/start-opnsense.sh

@@ -10,24 +10,52 @@ umask 077
 script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 runtime_dir="$script_dir/.runtime"
 ui_port=${OPNSENSE_UI_PORT:-18443}
+credentials_file="$runtime_dir/credentials.env"
+launcher_file="$runtime_dir/launcher.pid"
 
 test -r "$NORDVPN_PACKAGE_PATH"
 test -r "$OPNSENSE_VM_IMAGE"
 test -r "$OPNSENSE_TRANSFER_IMAGE"
 
 mkdir -p "$runtime_dir"
+if [ -r "$launcher_file" ]; then
+    read -r previous_launcher < "$launcher_file"
+    case "$previous_launcher" in
+        *[!0-9]* | '') ;;
+        *)
+            if kill -0 "$previous_launcher" 2>/dev/null; then
+                echo "An OPNsense UI fixture is already running." >&2
+                exit 1
+            fi
+            ;;
+    esac
+fi
 rm -f "$runtime_dir/opnsense.qcow2" "$runtime_dir/serial.in" "$runtime_dir/serial.out" \
-    "$runtime_dir/serial.log" "$runtime_dir/transfer.img"
+    "$runtime_dir/serial.log" "$runtime_dir/transfer.img" "$runtime_dir/ready" \
+    "$credentials_file" "$runtime_dir/ui-password"
+password=$(openssl rand -hex 32)
+printf '%s\n' "$password" > "$runtime_dir/ui-password"
+printf '%s\n' "OPNSENSE_UI_BASE_URL=https://127.0.0.1:$ui_port" > "$credentials_file"
+printf '%s\n' 'OPNSENSE_UI_USERNAME=root' >> "$credentials_file"
+printf '%s\n' "OPNSENSE_UI_PASSWORD=$password" >> "$credentials_file"
+printf '%s\n' 'OPNSENSE_UI_TEST_ALLOW_MUTATION=1' >> "$credentials_file"
+chmod 600 "$credentials_file" "$runtime_dir/ui-password"
+printf '%s\n' "$$" > "$launcher_file"
 mkfifo "$runtime_dir/serial.in" "$runtime_dir/serial.out"
 qemu-img create -q -f qcow2 -F qcow2 -b "$OPNSENSE_VM_IMAGE" "$runtime_dir/opnsense.qcow2"
 cp "$OPNSENSE_TRANSFER_IMAGE" "$runtime_dir/transfer.img"
 mcopy -o -i "$runtime_dir/transfer.img" "$NORDVPN_PACKAGE_PATH" ::os-nordvpn.pkg
 mcopy -o -i "$runtime_dir/transfer.img" "$script_dir/guest-setup.php" ::guest-setup.php
+mcopy -o -i "$runtime_dir/transfer.img" "$runtime_dir/ui-password" ::ui-password
 
 cleanup()
 {
     set +e
     [ -n "${qemu_pid:-}" ] && kill "$qemu_pid" 2>/dev/null
+    [ -n "${serial_log_pid:-}" ] && kill "$serial_log_pid" 2>/dev/null
+    rm -f "$runtime_dir/credentials.env" "$runtime_dir/ui-password" "$runtime_dir/ready" \
+        "$runtime_dir/transfer.img" "$runtime_dir/opnsense.qcow2" "$launcher_file" \
+        "$runtime_dir/serial.in" "$runtime_dir/serial.out" "$runtime_dir/serial.log"
 }
 trap cleanup EXIT INT TERM
 
@@ -46,6 +74,7 @@ qemu-system-x86_64 \
 qemu_pid=$!
 
 cat "$runtime_dir/serial.out" >> "$runtime_dir/serial.log" &
+serial_log_pid=$!
 
 wait_for()
 {
@@ -77,7 +106,8 @@ send "mount -uw /\n"
 sleep 2
 send "sh /etc/rc\n"
 wait_for 'root@OPNsense:/ #' 300
-send "mount -o ro -t msdosfs /dev/vtbd1 /mnt && pkg add -f /mnt/os-nordvpn.pkg && php /mnt/guest-setup.php && umount /mnt && pfctl -d && echo UI_VM_READY\n"
+send "mount -o ro -t msdosfs /dev/vtbd1 /mnt && pkg add -f /mnt/os-nordvpn.pkg && php /mnt/guest-setup.php && umount /mnt && (pfctl -d >/dev/null 2>&1 || true) && echo UI_VM_READY\n"
 wait_for 'UI_VM_READY' 120
-printf '%s\n' "OPNSENSE_UI_BASE_URL=https://127.0.0.1:$ui_port"
+touch "$runtime_dir/ready"
+printf '%s\n' "OPNSENSE_UI_CREDENTIALS_FILE=$credentials_file"
 wait "$qemu_pid"

tests/ui/stop-opnsense.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -eu
+
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+pid_file="$script_dir/.runtime/launcher.pid"
+
+if [ ! -r "$pid_file" ]; then
+    rm -f "$script_dir/.runtime/launcher.log"
+    exit 0
+fi
+read -r pid < "$pid_file"
+case "$pid" in
+    '' | *[!0-9]*) exit 1 ;;
+esac
+command=$(ps -p "$pid" -o command= 2>/dev/null || true)
+case "$command" in
+    *"$script_dir/start-opnsense.sh"*)
+        kill "$pid"
+        rm -f "$script_dir/.runtime/launcher.log"
+        ;;
+    *) exit 1 ;;
+esac

tests/unit/BootstrapperTest.php

@@ -0,0 +1,126 @@
+<?php
+
+use OPNsense\NordVPN\Bootstrapper;
+use OPNsense\NordVPN\EndpointStore;
+use OPNsense\NordVPN\ManagedConfiguration;
+use OPNsense\NordVPN\NordApi;
+use OPNsense\NordVPN\ReconciliationLock;
+use OPNsense\NordVPN\Settings;
+use OPNsense\NordVPN\StatusStore;
+use OPNsense\NordVPN\SystemRunner;
+use PHPUnit\Framework\TestCase;
+
+class BootstrapperTestApi extends NordApi
+{
+    public string $token = '';
+    private string $key;
+
+    public function __construct(string $key)
+    {
+        $this->key = $key;
+    }
+
+    public function recommendations(array $countryCodes = [], int $limit = 5): array
+    {
+        return [[
+            'hostname' => 'nl.example.invalid',
+            'address' => '198.51.100.10',
+            'country' => 'NL',
+            'public_key' => $this->key,
+        ]];
+    }
+
+    public function credentials(string $token): string
+    {
+        $this->token = $token;
+        return $this->key;
+    }
+}
+
+class BootstrapperTestConfiguration extends ManagedConfiguration
+{
+    private Settings $settings;
+    public array $provisioned = [];
+
+    public function __construct()
+    {
+        $this->settings = new Settings();
+    }
+
+    public function settings(): Settings
+    {
+        return $this->settings;
+    }
+
+    public function provision(string $privateKey, array $candidate): void
+    {
+        $this->provisioned = [$privateKey, $candidate];
+    }
+
+    public function repairGatewayInterface(): bool
+    {
+        return false;
+    }
+}
+
+class BootstrapperTestRunner extends SystemRunner
+{
+    public array $operations = [];
+
+    public function applyWireGuard(): void
+    {
+        $this->operations[] = 'wireguard';
+    }
+
+    public function applyRouting(): void
+    {
+        $this->operations[] = 'routing';
+    }
+
+    public function applyFirewall(): void
+    {
+        $this->operations[] = 'firewall';
+    }
+}
+
+final class BootstrapperTest extends TestCase
+{
+    public function testBootstrapQueuesReconciliationAfterApplyingInitialTunnel(): void
+    {
+        $key = base64_encode(str_repeat("\0", 32));
+        $api = new BootstrapperTestApi($key);
+        $configuration = new BootstrapperTestConfiguration();
+        $runner = new BootstrapperTestRunner();
+        $statusPath = sys_get_temp_dir() . '/nordvpn-bootstrap-' . bin2hex(random_bytes(8)) . '.json';
+        $lockPath = sys_get_temp_dir() . '/nordvpn-bootstrap-' . bin2hex(random_bytes(8)) . '.lock';
+        $endpointDirectory = sys_get_temp_dir() . '/nordvpn-bootstrap-' . bin2hex(random_bytes(8));
+        $endpointPath = $endpointDirectory . '/endpoints.txt';
+        $configuration->settings()->general->exempt_wireguard = '1';
+        $queued = 0;
+
+        $result = (new Bootstrapper(
+            $api,
+            $configuration,
+            new StatusStore($statusPath),
+            $runner,
+            new EndpointStore($endpointPath),
+            new ReconciliationLock($lockPath),
+            static function () use (&$queued): void {
+                $queued++;
+            }
+        ))->run('synthetic-token');
+
+        self::assertSame('ok', $result['status']);
+        self::assertSame('synthetic-token', $api->token);
+        self::assertSame($key, $configuration->provisioned[0]);
+        self::assertSame(['firewall', 'wireguard', 'routing', 'firewall'], $runner->operations);
+        self::assertSame("198.51.100.10\n", file_get_contents($endpointPath));
+        self::assertSame('reconciling', $result['tunnel']['state']);
+        self::assertSame(1, $queued);
+
+        unlink($statusPath);
+        unlink($lockPath);
+        unlink($endpointPath);
+        rmdir($endpointDirectory);
+    }
+}

tests/unit/NordApiTest.php

@@ -64,6 +64,37 @@ final class NordApiTest extends TestCase
         $this->assertSame([], $api->recommendations([]));
     }
 
+    public function testRecommendationsRequestEverySelectedCountry(): void
+    {
+        $requests = [];
+        $api = new NordApi(function ($path, $query) use (&$requests): array {
+            $requests[] = [$path, $query];
+            if ($path === '/v1/servers/countries') {
+                return [
+                    ['id' => 1, 'code' => 'nl', 'name' => 'Netherlands'],
+                    ['id' => 2, 'code' => 'kr', 'name' => 'Korea'],
+                ];
+            }
+            $country = $query['filters[country_id]'] === 1 ? 'NL' : 'KR';
+            return [[
+                'hostname' => strtolower($country) . '.example.invalid',
+                'station' => $country === 'NL' ? '198.51.100.10' : '198.51.100.11',
+                'locations' => [['country' => ['code' => $country]]],
+                'technologies' => [[
+                    'id' => 35,
+                    'pivot' => ['status' => 'online'],
+                    'metadata' => [['name' => 'public_key', 'value' => $this->key]],
+                ]],
+            ]];
+        });
+
+        $servers = $api->recommendations(['NL', 'KR'], 1);
+
+        self::assertSame(1, $requests[1][1]['filters[country_id]']);
+        self::assertSame(2, $requests[2][1]['filters[country_id]']);
+        self::assertSame(['NL', 'KR'], array_column($servers, 'country'));
+    }
+
     public function testEndpointDiscoveryPaginatesCompleteNordLynxAddressSet(): void
     {
         $requests = [];
@@ -72,7 +103,7 @@ final class NordApiTest extends TestCase
             if ($query['offset'] === 0) {
                 return array_map(
                     static fn($index) => ['station' => sprintf('198.51.%d.%d', intdiv($index, 250), $index % 250 + 1)],
-                    range(0, 499)
+                    range(0, 999)
                 );
             }
             return [['station' => '203.0.113.10'], ['station' => 'invalid']];
@@ -80,10 +111,10 @@ final class NordApiTest extends TestCase
 
         $addresses = $api->endpointAddresses();
 
-        $this->assertCount(501, $addresses);
+        $this->assertCount(1001, $addresses);
         $this->assertSame('/v1/servers', $requests[0][0]);
-        $this->assertSame(500, $requests[0][1]['limit']);
+        $this->assertSame(1000, $requests[0][1]['limit']);
-        $this->assertSame(500, $requests[1][1]['offset']);
+        $this->assertSame(1000, $requests[1][1]['offset']);
         $this->assertSame(35, $requests[0][1]['filters[servers_technologies]']);
     }
 }

tests/unit/ReconcilerTest.php

@@ -1,6 +1,7 @@
 <?php
 
 use OPNsense\NordVPN\ManagedConfiguration;
+use OPNsense\NordVPN\EndpointStore;
 use OPNsense\NordVPN\NordApi;
 use OPNsense\NordVPN\Reconciler;
 use OPNsense\NordVPN\Settings;
@@ -104,6 +105,19 @@ class UnavailableReconcilerHealth extends TunnelHealth
     }
 }
 
+class FailingReconcilerEndpointStore extends EndpointStore
+{
+    public function refreshIfStale(NordApi $api): bool
+    {
+        throw new RuntimeException('Synthetic endpoint failure.');
+    }
+
+    public function available(): bool
+    {
+        return false;
+    }
+}
+
 class FailOnceReconcilerRunner extends SystemRunner
 {
     public int $firewallCalls = 0;
@@ -252,6 +266,31 @@ final class ReconcilerTest extends TestCase
         unlink($path);
     }
 
+    public function testEndpointRefreshFailureDoesNotDelayAvailableStatus(): void
+    {
+        $path = $this->temporaryStatusPath();
+        $configuration = new ReconcilerTestConfiguration();
+        $configuration->settings()->general->exempt_wireguard = '1';
+
+        $result = (new Reconciler(
+            new NordApi(static fn() => []),
+            $configuration,
+            new AvailableReconcilerHealth(),
+            new StatusStore($path),
+            new RepairReconcilerRunner(),
+            null,
+            new FailingReconcilerEndpointStore()
+        ))->run();
+
+        $this->assertTrue($result['available']);
+        $this->assertSame('available', $result['state']);
+        $this->assertSame(
+            'NordVPN endpoint refresh failed; the WireGuard exemption is inactive.',
+            $result['endpoint_message']
+        );
+        unlink($path);
+    }
+
     private function temporaryStatusPath(): string
     {
         return sys_get_temp_dir() . '/nordvpn-reconciler-' . bin2hex(random_bytes(8)) . '.json';

tests/unit/StoreAndInspectorTest.php

@@ -27,6 +27,44 @@ final class StoreAndInspectorTest extends TestCase
         $this->assertSame(1, $requests);
 
         unlink($path);
+        unlink($path . '.refreshed');
+        rmdir($directory);
+    }
+
+    public function testEndpointRefreshDefersAfterFailure(): void
+    {
+        $directory = sys_get_temp_dir() . '/nordvpn-endpoints-' . bin2hex(random_bytes(8));
+        $path = $directory . '/endpoints.txt';
+        $now = time();
+        $requests = 0;
+        $api = new NordApi(static function () use (&$requests): array {
+            $requests++;
+            throw new RuntimeException('Synthetic endpoint failure.');
+        });
+        $store = new EndpointStore($path, static function () use (&$now): int {
+            return $now;
+        });
+
+        try {
+            $store->refreshIfStale($api);
+            $this->fail('Expected endpoint refresh to fail.');
+        } catch (RuntimeException $exception) {
+            $this->assertSame('Synthetic endpoint failure.', $exception->getMessage());
+        }
+        $this->assertTrue($store->retryPending());
+        $this->assertFalse($store->refreshIfStale($api));
+        $this->assertSame(1, $requests);
+
+        $now += 301;
+        try {
+            $store->refreshIfStale($api);
+            $this->fail('Expected endpoint refresh retry to fail.');
+        } catch (RuntimeException $exception) {
+            $this->assertSame('Synthetic endpoint failure.', $exception->getMessage());
+        }
+        $this->assertSame(2, $requests);
+
+        unlink($path . '.retry');
         rmdir($directory);
     }
 

tests/unit/SystemRunnerAndHealthTest.php

@@ -6,6 +6,33 @@ use PHPUnit\Framework\TestCase;
 
 final class SystemRunnerAndHealthTest extends TestCase
 {
+    public function testRunDrainsStderr(): void
+    {
+        $runner = new SystemRunner(null, 2);
+        $output = $runner->run([
+            PHP_BINARY,
+            '-r',
+            'fwrite(STDERR, str_repeat("x", 262144)); echo "done";',
+        ]);
+
+        self::assertSame('done', $output);
+    }
+
+    public function testRunTimesOut(): void
+    {
+        $runner = new SystemRunner(null, 1);
+        $started = hrtime(true);
+
+        try {
+            $runner->run([PHP_BINARY, '-r', 'while (true) { usleep(100000); }']);
+            self::fail('Expected the system command to time out.');
+        } catch (\RuntimeException $exception) {
+            self::assertSame('System command timed out.', $exception->getMessage());
+        }
+
+        self::assertLessThan(3, (hrtime(true) - $started) / 1_000_000_000);
+    }
+
     public function testTunnelHealthRequiresRecentHandshake(): void
     {
         $runner = new SystemRunner(static fn() => "public-key\t900\n");