fix: prevent Bitcoin Core switch from hanging the Hub UI

CHANGELOG.md

@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [Unreleased]
+
+### Fixed
+- Fix Bitcoin Knots → Bitcoin Core switch hanging in the Hub UI
+  - Sessions are now persisted to `/var/lib/secrets/hub-sessions.json` so the
+    browser login survives the Hub service restart that `nixos-rebuild switch`
+    performs during activation. Previously the in-memory session store was
+    wiped by that restart, the `/api/rebuild/status` poll started returning
+    401, and the rebuild modal spun forever showing "Applying changes…" while
+    the switch result was never displayed.
+  - Rebuild and update modals now bail out and reload the page after sustained
+    polling failures instead of hanging indefinitely.
+  - Rebuild/update scripts stream `nixos-rebuild` output into the live log
+    (it was buffered until completion, making long rebuilds look frozen).
+
+---
+
 ## [1.1.0] - 2026-08-11
 
 ### Added

app/sovran_systemsos_web/security_helpers.py

@@ -11,7 +11,11 @@ from __future__ import annotations
 
 import base64
 import ipaddress
+import json
+import os
 import re
+import tempfile
+import time
 import urllib.parse
 
 # ── Nix string escaping ────────────────────────────────────────────────────────
@@ -239,3 +243,71 @@ def _validate_ssh_pubkey(key: str) -> str:
     if len(decoded) < 20:
         raise ValueError("SSH public key payload is too short")
     return key
+
+
+# ── Persistent Hub session store ─────────────────────────────────────────────
+
+def load_session_store(path: str) -> dict[str, float]:
+    """Load persisted Hub sessions from *path*.
+
+    Returns a mapping of session token → expiry timestamp (epoch seconds).
+    Expired entries are discarded.  A missing, unreadable or malformed file
+    yields an empty mapping — losing sessions is a UX inconvenience (the user
+    must log in again), never a fatal error.
+
+    Persistence exists so that authenticated sessions survive a restart of
+    the Hub service itself.  ``nixos-rebuild switch`` restarts
+    ``sovran-hub-web.service`` during activation (its unit definition changes
+    with every feature toggle), and without persistence the in-progress
+    rebuild/update status polling loses authentication and the UI hangs.
+    """
+    try:
+        with open(path, "r") as f:
+            data = json.load(f)
+    except (OSError, ValueError):
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    now = time.time()
+    sessions: dict[str, float] = {}
+    for token, expiry in data.items():
+        if not isinstance(token, str) or not token:
+            continue
+        if isinstance(expiry, bool) or not isinstance(expiry, (int, float)):
+            continue
+        if expiry > now:
+            sessions[token] = float(expiry)
+    return sessions
+
+
+def save_session_store(path: str, sessions: dict[str, float]) -> bool:
+    """Atomically persist *sessions* (token → expiry) to *path* with mode 0600.
+
+    Writes to a temp file in the same directory and renames it into place so
+    the store is never left partially written.  Returns True on success,
+    False otherwise (persistence is best-effort).
+    """
+    directory = os.path.dirname(path) or "."
+    fd = None
+    tmp_path = None
+    try:
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(dir=directory, prefix=".hub_sessions_tmp")
+        with os.fdopen(fd, "w") as f:
+            fd = None  # os.fdopen takes ownership of the descriptor
+            json.dump(sessions, f)
+        os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, path)
+        return True
+    except OSError:
+        if fd is not None:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+        if tmp_path is not None:
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+        return False

app/sovran_systemsos_web/server.py

@@ -52,6 +52,8 @@ from .security_helpers import (
     _SSH_PUBKEY_ALGORITHMS,
     _bech32_decode,
     _bech32_convertbits_decode,
+    load_session_store,
+    save_session_store,
 )
 
 logger = logging.getLogger(__name__)
@@ -144,8 +146,27 @@ HUB_SESSION_SECRET_FILE = "/var/lib/secrets/hub-session-secret"
 SESSION_COOKIE_NAME     = "hub_session"
 SESSION_MAX_AGE         = 86400  # 24 hours
 
-# In-memory session store: token → expiry timestamp (float)
+# Sessions are persisted here so logins survive a restart of the Hub service.
+# nixos-rebuild switch restarts sovran-hub-web.service during activation (its
+# unit definition changes with every feature toggle — e.g. the Bitcoin
+# Knots → Core switch changes services.bitcoind.package, which is on the Hub's
+# PATH).  Without persistence the browser session dies mid-rebuild, the
+# /api/rebuild/status polling starts receiving 401s and the rebuild modal
+# hangs forever showing "Applying changes…".  The file lives in
+# /var/lib/secrets so a security reset wipes it and forces a re-login, like
+# the session secret itself.
+SESSIONS_FILE = "/var/lib/secrets/hub-sessions.json"
+
+# Session store: token → expiry timestamp (float).  Loaded lazily from
+# SESSIONS_FILE on first use and written back on every meaningful change.
 _sessions: dict[str, float] = {}
+_sessions_loaded = False
+_sessions_lock = Lock()
+
+# Sliding the expiry on every authenticated request would rewrite the store on
+# every poll, so persist slide-only updates at most this often.
+_SESSION_PERSIST_MIN_INTERVAL = 30.0  # seconds
+_sessions_last_persist = 0.0
 
 # Failed login tracking: ip → list of failure timestamps
 _login_failures: dict[str, list[float]] = {}
@@ -606,38 +627,74 @@ def _get_or_create_session_secret() -> bytes:
     return token_hex
 
 
+def _load_sessions_once() -> None:
+    """Lazily load the persisted session store on first use (idempotent)."""
+    global _sessions_loaded
+    with _sessions_lock:
+        if _sessions_loaded:
+            return
+        _sessions.update(load_session_store(SESSIONS_FILE))
+        _sessions_loaded = True
+
+
+def _persist_sessions(force: bool = False) -> None:
+    """Write the session store to SESSIONS_FILE (best-effort).
+
+    Expiry slides happen on every authenticated request, so non-forced
+    persists are throttled; create/destroy/purge pass ``force=True``.
+    """
+    global _sessions_last_persist
+    now = time.time()
+    with _sessions_lock:
+        if not force and (now - _sessions_last_persist) < _SESSION_PERSIST_MIN_INTERVAL:
+            return
+        snapshot = dict(_sessions)
+        _sessions_last_persist = now
+    save_session_store(SESSIONS_FILE, snapshot)
+
+
 def _create_session() -> str:
     """Create a new opaque session token and register it in the store."""
+    _load_sessions_once()
     _purge_expired_sessions()
     token = secrets.token_hex(32)
     _sessions[token] = time.time() + SESSION_MAX_AGE
+    _persist_sessions(force=True)
     return token
 
 
 def _destroy_session(token: str) -> None:
     """Remove a session token from the store."""
-    _sessions.pop(token, None)
+    _load_sessions_once()
+    if _sessions.pop(token, None) is not None:
+        _persist_sessions(force=True)
 
 
 def _purge_expired_sessions() -> None:
-    """Remove all expired sessions from the in-memory store."""
+    """Remove all expired sessions from the store."""
+    _load_sessions_once()
     now = time.time()
     expired = [tok for tok, exp in _sessions.items() if exp <= now]
     for tok in expired:
         del _sessions[tok]
+    if expired:
+        _persist_sessions(force=True)
 
 
 def _is_authenticated(request: Request) -> bool:
     """Return True if the request carries a valid, unexpired session cookie."""
+    _load_sessions_once()
     token = request.cookies.get(SESSION_COOKIE_NAME)
     if not token:
         return False
     expiry = _sessions.get(token)
     if expiry is None or time.time() >= expiry:
-        _sessions.pop(token, None)
+        if _sessions.pop(token, None) is not None:
+            _persist_sessions(force=True)
         return False
     # Slide the expiry window on activity
     _sessions[token] = time.time() + SESSION_MAX_AGE
+    _persist_sessions()  # throttled — don't rewrite the store on every poll
     return True
 
 
@@ -6152,6 +6209,9 @@ async def _startup_session_secret():
     """Ensure the session secret exists on disk at startup."""
     loop = asyncio.get_event_loop()
     await loop.run_in_executor(None, _get_or_create_session_secret)
+    # Preload persisted sessions so browser logins survive this restart (the
+    # Hub service is restarted by nixos-rebuild switch during every rebuild).
+    await loop.run_in_executor(None, _load_sessions_once)
 
 
 # ── Startup: recover stale RUNNING status files ──────────────────

app/sovran_systemsos_web/static/js/constants.js

@@ -5,6 +5,10 @@
 const POLL_INTERVAL_SERVICES    = 5000;
 const POLL_INTERVAL_UPDATES     = 1800000;
 const UPDATE_POLL_INTERVAL      = 2000;
+// Max consecutive failed rebuild/update status polls before the page gives up
+// waiting and reloads to re-sync (2s interval → ~2 minutes of failures).
+// A brief Hub restart during activation only causes a handful of failures.
+const STATUS_POLL_MAX_FAILURES  = 60;
 const REBOOT_CHECK_INTERVAL     = 5000;
 const REBOOT_FETCH_TIMEOUT      = 12000;
 const REBOOT_REQUEST_TIMEOUT    = 4000;

app/sovran_systemsos_web/static/js/rebuild.js

@@ -8,6 +8,7 @@ function openRebuildModal() {
   _rebuildLogOffset = 0;
   _rebuildServerDown = false;
   _rebuildFinished = false;
+  _rebuildPollFailures = 0;
   if ($rebuildLog) { $rebuildLog.textContent = ""; $rebuildLog.style.display = "none"; }
   var action = _rebuildIsEnabling ? "Enabling" : "Disabling";
   var label = _rebuildFeatureName || "feature";
@@ -45,6 +46,7 @@ async function pollRebuildStatus() {
   if (_rebuildFinished) return;
   try {
     var data = await apiFetch("/api/rebuild/status?offset=" + _rebuildLogOffset);
+    _rebuildPollFailures = 0;
     if (_rebuildServerDown) { _rebuildServerDown = false; }
     if (data.log) appendRebuildLog(data.log);
     _rebuildLogOffset = data.offset;
@@ -57,6 +59,18 @@ async function pollRebuildStatus() {
       onRebuildDone(data.result === "success");
     }
   } catch (err) {
+    _rebuildPollFailures += 1;
+    // The Hub restarts itself during activation, which briefly drops this poll.
+    // If polling stays broken long past a normal restart, the page's session
+    // almost certainly no longer matches the server (e.g. the Hub restarted
+    // and sessions were not recovered).  Reload to re-authenticate and show
+    // the real feature state instead of spinning forever.
+    if (_rebuildPollFailures >= STATUS_POLL_MAX_FAILURES) {
+      _rebuildFinished = true;
+      stopRebuildPoll();
+      window.location.reload();
+      return;
+    }
     if (!_rebuildServerDown) { _rebuildServerDown = true; if ($rebuildStatus) $rebuildStatus.textContent = "Applying changes…"; }
   }
 }

app/sovran_systemsos_web/static/js/state.js

@@ -9,6 +9,7 @@ let _updatePollTimer  = null;
 let _updateLogOffset  = 0;
 let _serverWasDown    = false;
 let _updateFinished   = false;
+let _updatePollFailures = 0; // consecutive failed update-status polls
 let _supportTimerInt      = null;
 let _supportEnabledAt     = null;
 let _supportStatus        = null;   // last fetched /api/support/status payload
@@ -25,6 +26,7 @@ let _rebuildLogOffset     = 0;
 let _rebuildPollTimer     = null;
 let _rebuildFinished      = false;
 let _rebuildServerDown    = false;
+let _rebuildPollFailures  = 0; // consecutive failed rebuild-status polls
 let _pendingToggle        = null; // {feature, extra} waiting for domain/confirm
 let _rebuildFeatureName   = "";
 let _rebuildIsEnabling    = true;

app/sovran_systemsos_web/static/js/update.js

@@ -33,6 +33,7 @@ function _doOpenUpdateModal() {
   _updateLogOffset = 0;
   _serverWasDown = false;
   _updateFinished = false;
+  _updatePollFailures = 0;
   if ($modalLog) $modalLog.textContent = "";
   if ($modalStatus) $modalStatus.textContent = "Starting update…";
   if ($modalSpinner) $modalSpinner.classList.add("spinning");
@@ -94,6 +95,7 @@ async function pollUpdateStatus() {
   if (_updateFinished) return;
   try {
     var data = await apiFetch("/api/updates/status?offset=" + _updateLogOffset);
+    _updatePollFailures = 0;
     if (_serverWasDown) {
       _serverWasDown = false;
       if (!data.running) {
@@ -143,6 +145,16 @@ async function pollUpdateStatus() {
       onUpdateDone(false);
     }
   } catch (err) {
+    _updatePollFailures += 1;
+    // Same guard as the rebuild modal: if polling stays broken long past a
+    // normal Hub restart, reload to re-authenticate and show the real state
+    // instead of spinning forever.
+    if (_updatePollFailures >= STATUS_POLL_MAX_FAILURES) {
+      _updateFinished = true;
+      stopUpdatePoll();
+      window.location.reload();
+      return;
+    }
     if (!_serverWasDown) { _serverWasDown = true; appendLog("\n[Server restarting — waiting for it to come back…]\n"); if ($modalStatus) $modalStatus.textContent = "Server restarting…"; }
   }
 }

modules/core/sovran-hub.nix

@@ -185,13 +185,14 @@ let
 
     if [ "$RC" -eq 0 ]; then
       echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──"
-      BOOT_OUT=$(nixos-rebuild boot --flake /etc/nixos --print-build-logs \
+      # Stream output straight into $LOG (see rebuild-script) so the Hub UI
+      # shows live progress instead of an empty log during long builds.
+      nixos-rebuild boot --flake /etc/nixos --print-build-logs \
         --option connect-timeout 10 \
         --option stalled-download-timeout 90 \
         --option download-attempts 7 \
-        --option fallback true 2>&1)
+        --option fallback true
       BOOT_RC=$?
-      echo "$BOOT_OUT"
       if [ "$BOOT_RC" -ne 0 ]; then
         echo "[ERROR] nixos-rebuild boot failed"
         RC=1
@@ -240,20 +241,23 @@ let
     echo "══════════════════════════════════════════════════"
     echo ""
     echo "── Rebuilding system configuration ──────────────"
-    SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+    # Stream output straight into $LOG (tee'd by the exec redirect above) so
+    # the Hub UI shows live progress.  Capturing the output in a variable
+    # kept the log empty for the entire build+activation, which made long
+    # rebuilds (e.g. the Bitcoin Knots → Core switch) look like a hang.
+    nixos-rebuild switch --flake /etc/nixos --print-build-logs \
       --option connect-timeout 10 \
       --option stalled-download-timeout 90 \
       --option download-attempts 7 \
-      --option fallback true 2>&1)
+      --option fallback true
     SWITCH_RC=$?
-    echo "$SWITCH_OUT"
     if [ "$SWITCH_RC" -eq 0 ]; then
       echo ""
       echo "══════════════════════════════════════════════════"
       echo "  ✓ Rebuild completed successfully"
       echo "══════════════════════════════════════════════════"
       echo "SUCCESS" > "$STATUS"
-    elif echo "$SWITCH_OUT" | grep -q "switchInhibitors\|Pre-switch checks failed"; then
+    elif grep -q "switchInhibitors\|Pre-switch checks failed" "$LOG"; then
       echo ""
       echo "  ✓ Build succeeded — a reboot is required to apply this rebuild"
       echo "  (Critical system components changed; running nixos-rebuild boot instead)"

tests/test_security.py

@@ -14,6 +14,7 @@ import json
 import os
 import sys
 import tempfile
+import time
 import unittest
 
 # Add the app package to the path so we can import without the full FastAPI tree.
@@ -30,6 +31,8 @@ from sovran_systemsos_web.security_helpers import (  # noqa: E402
     _validate_ssh_pubkey,
     _DDNS_ALLOWED_HOSTNAMES,
     _bech32_decode,
+    load_session_store,
+    save_session_store,
 )
 from sovran_systemsos_web import support_ops  # noqa: E402
 
@@ -362,6 +365,96 @@ class TestAuthExemptPaths(unittest.TestCase):
         self.assertIn("/api/ping", self._get_exempt_paths())
 
 
+# ---------------------------------------------------------------------------
+# Persistent session store
+# ---------------------------------------------------------------------------
+
+class TestSessionStore(unittest.TestCase):
+    """Sessions must persist across Hub restarts so rebuild/update polling
+    keeps working after nixos-rebuild switch restarts the Hub service."""
+
+    def _store_path(self, tmpdir, name="hub-sessions.json"):
+        return os.path.join(tmpdir, name)
+
+    def test_roundtrip(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            future = time.time() + 3600
+            sessions = {"token-a": future, "token-b": future + 10}
+            self.assertTrue(save_session_store(path, sessions))
+            self.assertEqual(load_session_store(path), sessions)
+
+    def test_missing_file_returns_empty(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            self.assertEqual(load_session_store(self._store_path(tmpdir)), {})
+
+    def test_malformed_json_returns_empty(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            with open(path, "w") as f:
+                f.write("{not json")
+            self.assertEqual(load_session_store(path), {})
+
+    def test_non_dict_json_returns_empty(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            with open(path, "w") as f:
+                json.dump(["token"], f)
+            self.assertEqual(load_session_store(path), {})
+
+    def test_expired_sessions_discarded(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            now = time.time()
+            save_session_store(path, {"alive": now + 3600, "dead": now - 1})
+            loaded = load_session_store(path)
+            self.assertIn("alive", loaded)
+            self.assertNotIn("dead", loaded)
+
+    def test_invalid_entries_skipped(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            now = time.time()
+            with open(path, "w") as f:
+                json.dump({
+                    "good": now + 3600,
+                    "": now + 3600,              # empty token
+                    "bool-expiry": True,         # bool is not a valid expiry
+                    "str-expiry": "soon",        # non-numeric expiry
+                    "none-expiry": None,
+                }, f)
+            loaded = load_session_store(path)
+            self.assertEqual(list(loaded.keys()), ["good"])
+
+    def test_file_mode_is_0600(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            save_session_store(path, {"token": time.time() + 60})
+            mode = os.stat(path).st_mode & 0o777
+            self.assertEqual(mode, 0o600)
+
+    def test_save_overwrites_existing_store(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            future = time.time() + 3600
+            save_session_store(path, {"old": future})
+            save_session_store(path, {"new": future})
+            self.assertEqual(load_session_store(path), {"new": future})
+
+    def test_empty_store_roundtrip(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            self.assertTrue(save_session_store(path, {}))
+            self.assertEqual(load_session_store(path), {})
+
+    def test_no_leftover_temp_files(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = self._store_path(tmpdir)
+            save_session_store(path, {"token": time.time() + 60})
+            leftovers = [n for n in os.listdir(tmpdir) if n.startswith(".hub_sessions_tmp")]
+            self.assertEqual(leftovers, [])
+
+
 # ---------------------------------------------------------------------------
 # tech-support.nix validation
 # ---------------------------------------------------------------------------