fix: prevent Bitcoin Core switch from hanging the Hub UI

app/sovran_systemsos_web/security_helpers.py

@@ -11,7 +11,11 @@ from __future__ import annotations
 
 import base64
 import ipaddress
+import json
+import os
 import re
+import tempfile
+import time
 import urllib.parse
 
 # ── Nix string escaping ────────────────────────────────────────────────────────
@@ -239,3 +243,71 @@ def _validate_ssh_pubkey(key: str) -> str:
     if len(decoded) < 20:
         raise ValueError("SSH public key payload is too short")
     return key
+
+
+# ── Persistent Hub session store ─────────────────────────────────────────────
+
+def load_session_store(path: str) -> dict[str, float]:
+    """Load persisted Hub sessions from *path*.
+
+    Returns a mapping of session token → expiry timestamp (epoch seconds).
+    Expired entries are discarded.  A missing, unreadable or malformed file
+    yields an empty mapping — losing sessions is a UX inconvenience (the user
+    must log in again), never a fatal error.
+
+    Persistence exists so that authenticated sessions survive a restart of
+    the Hub service itself.  ``nixos-rebuild switch`` restarts
+    ``sovran-hub-web.service`` during activation (its unit definition changes
+    with every feature toggle), and without persistence the in-progress
+    rebuild/update status polling loses authentication and the UI hangs.
+    """
+    try:
+        with open(path, "r") as f:
+            data = json.load(f)
+    except (OSError, ValueError):
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    now = time.time()
+    sessions: dict[str, float] = {}
+    for token, expiry in data.items():
+        if not isinstance(token, str) or not token:
+            continue
+        if isinstance(expiry, bool) or not isinstance(expiry, (int, float)):
+            continue
+        if expiry > now:
+            sessions[token] = float(expiry)
+    return sessions
+
+
+def save_session_store(path: str, sessions: dict[str, float]) -> bool:
+    """Atomically persist *sessions* (token → expiry) to *path* with mode 0600.
+
+    Writes to a temp file in the same directory and renames it into place so
+    the store is never left partially written.  Returns True on success,
+    False otherwise (persistence is best-effort).
+    """
+    directory = os.path.dirname(path) or "."
+    fd = None
+    tmp_path = None
+    try:
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(dir=directory, prefix=".hub_sessions_tmp")
+        with os.fdopen(fd, "w") as f:
+            fd = None  # os.fdopen takes ownership of the descriptor
+            json.dump(sessions, f)
+        os.chmod(tmp_path, 0o600)
+        os.replace(tmp_path, path)
+        return True
+    except OSError:
+        if fd is not None:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+        if tmp_path is not None:
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+        return False

app/sovran_systemsos_web/server.py

@@ -52,6 +52,8 @@ from .security_helpers import (
     _SSH_PUBKEY_ALGORITHMS,
     _bech32_decode,
     _bech32_convertbits_decode,
+    load_session_store,
+    save_session_store,
 )
 
 logger = logging.getLogger(__name__)
@@ -144,8 +146,27 @@ HUB_SESSION_SECRET_FILE = "/var/lib/secrets/hub-session-secret"
 SESSION_COOKIE_NAME     = "hub_session"
 SESSION_MAX_AGE         = 86400  # 24 hours
 
-# In-memory session store: token → expiry timestamp (float)
+# Sessions are persisted here so logins survive a restart of the Hub service.
+# nixos-rebuild switch restarts sovran-hub-web.service during activation (its
+# unit definition changes with every feature toggle — e.g. the Bitcoin
+# Knots → Core switch changes services.bitcoind.package, which is on the Hub's
+# PATH).  Without persistence the browser session dies mid-rebuild, the
+# /api/rebuild/status polling starts receiving 401s and the rebuild modal
+# hangs forever showing "Applying changes…".  The file lives in
+# /var/lib/secrets so a security reset wipes it and forces a re-login, like
+# the session secret itself.
+SESSIONS_FILE = "/var/lib/secrets/hub-sessions.json"
+
+# Session store: token → expiry timestamp (float).  Loaded lazily from
+# SESSIONS_FILE on first use and written back on every meaningful change.
 _sessions: dict[str, float] = {}
+_sessions_loaded = False
+_sessions_lock = Lock()
+
+# Sliding the expiry on every authenticated request would rewrite the store on
+# every poll, so persist slide-only updates at most this often.
+_SESSION_PERSIST_MIN_INTERVAL = 30.0  # seconds
+_sessions_last_persist = 0.0
 
 # Failed login tracking: ip → list of failure timestamps
 _login_failures: dict[str, list[float]] = {}
@@ -606,38 +627,74 @@ def _get_or_create_session_secret() -> bytes:
     return token_hex
 
 
+def _load_sessions_once() -> None:
+    """Lazily load the persisted session store on first use (idempotent)."""
+    global _sessions_loaded
+    with _sessions_lock:
+        if _sessions_loaded:
+            return
+        _sessions.update(load_session_store(SESSIONS_FILE))
+        _sessions_loaded = True
+
+
+def _persist_sessions(force: bool = False) -> None:
+    """Write the session store to SESSIONS_FILE (best-effort).
+
+    Expiry slides happen on every authenticated request, so non-forced
+    persists are throttled; create/destroy/purge pass ``force=True``.
+    """
+    global _sessions_last_persist
+    now = time.time()
+    with _sessions_lock:
+        if not force and (now - _sessions_last_persist) < _SESSION_PERSIST_MIN_INTERVAL:
+            return
+        snapshot = dict(_sessions)
+        _sessions_last_persist = now
+    save_session_store(SESSIONS_FILE, snapshot)
+
+
 def _create_session() -> str:
     """Create a new opaque session token and register it in the store."""
+    _load_sessions_once()
     _purge_expired_sessions()
     token = secrets.token_hex(32)
     _sessions[token] = time.time() + SESSION_MAX_AGE
+    _persist_sessions(force=True)
     return token
 
 
 def _destroy_session(token: str) -> None:
     """Remove a session token from the store."""
-    _sessions.pop(token, None)
+    _load_sessions_once()
+    if _sessions.pop(token, None) is not None:
+        _persist_sessions(force=True)
 
 
 def _purge_expired_sessions() -> None:
-    """Remove all expired sessions from the in-memory store."""
+    """Remove all expired sessions from the store."""
+    _load_sessions_once()
     now = time.time()
     expired = [tok for tok, exp in _sessions.items() if exp <= now]
     for tok in expired:
         del _sessions[tok]
+    if expired:
+        _persist_sessions(force=True)
 
 
 def _is_authenticated(request: Request) -> bool:
     """Return True if the request carries a valid, unexpired session cookie."""
+    _load_sessions_once()
     token = request.cookies.get(SESSION_COOKIE_NAME)
     if not token:
         return False
     expiry = _sessions.get(token)
     if expiry is None or time.time() >= expiry:
-        _sessions.pop(token, None)
+        if _sessions.pop(token, None) is not None:
+            _persist_sessions(force=True)
         return False
     # Slide the expiry window on activity
     _sessions[token] = time.time() + SESSION_MAX_AGE
+    _persist_sessions()  # throttled — don't rewrite the store on every poll
     return True
 
 
@@ -6152,6 +6209,9 @@ async def _startup_session_secret():
     """Ensure the session secret exists on disk at startup."""
     loop = asyncio.get_event_loop()
     await loop.run_in_executor(None, _get_or_create_session_secret)
+    # Preload persisted sessions so browser logins survive this restart (the
+    # Hub service is restarted by nixos-rebuild switch during every rebuild).
+    await loop.run_in_executor(None, _load_sessions_once)
 
 
 # ── Startup: recover stale RUNNING status files ──────────────────

app/sovran_systemsos_web/static/js/constants.js

@@ -5,6 +5,10 @@
 const POLL_INTERVAL_SERVICES    = 5000;
 const POLL_INTERVAL_UPDATES     = 1800000;
 const UPDATE_POLL_INTERVAL      = 2000;
+// Max consecutive failed rebuild/update status polls before the page gives up
+// waiting and reloads to re-sync (2s interval → ~2 minutes of failures).
+// A brief Hub restart during activation only causes a handful of failures.
+const STATUS_POLL_MAX_FAILURES  = 60;
 const REBOOT_CHECK_INTERVAL     = 5000;
 const REBOOT_FETCH_TIMEOUT      = 12000;
 const REBOOT_REQUEST_TIMEOUT    = 4000;

app/sovran_systemsos_web/static/js/rebuild.js

@@ -8,6 +8,7 @@ function openRebuildModal() {
   _rebuildLogOffset = 0;
   _rebuildServerDown = false;
   _rebuildFinished = false;
+  _rebuildPollFailures = 0;
   if ($rebuildLog) { $rebuildLog.textContent = ""; $rebuildLog.style.display = "none"; }
   var action = _rebuildIsEnabling ? "Enabling" : "Disabling";
   var label = _rebuildFeatureName || "feature";
@@ -45,6 +46,7 @@ async function pollRebuildStatus() {
   if (_rebuildFinished) return;
   try {
     var data = await apiFetch("/api/rebuild/status?offset=" + _rebuildLogOffset);
+    _rebuildPollFailures = 0;
     if (_rebuildServerDown) { _rebuildServerDown = false; }
     if (data.log) appendRebuildLog(data.log);
     _rebuildLogOffset = data.offset;
@@ -57,6 +59,18 @@ async function pollRebuildStatus() {
       onRebuildDone(data.result === "success");
     }
   } catch (err) {
+    _rebuildPollFailures += 1;
+    // The Hub restarts itself during activation, which briefly drops this poll.
+    // If polling stays broken long past a normal restart, the page's session
+    // almost certainly no longer matches the server (e.g. the Hub restarted
+    // and sessions were not recovered).  Reload to re-authenticate and show
+    // the real feature state instead of spinning forever.
+    if (_rebuildPollFailures >= STATUS_POLL_MAX_FAILURES) {
+      _rebuildFinished = true;
+      stopRebuildPoll();
+      window.location.reload();
+      return;
+    }
     if (!_rebuildServerDown) { _rebuildServerDown = true; if ($rebuildStatus) $rebuildStatus.textContent = "Applying changes…"; }
   }
 }

app/sovran_systemsos_web/static/js/state.js

@@ -9,6 +9,7 @@ let _updatePollTimer  = null;
 let _updateLogOffset  = 0;
 let _serverWasDown    = false;
 let _updateFinished   = false;
+let _updatePollFailures = 0; // consecutive failed update-status polls
 let _supportTimerInt      = null;
 let _supportEnabledAt     = null;
 let _supportStatus        = null;   // last fetched /api/support/status payload
@@ -25,6 +26,7 @@ let _rebuildLogOffset     = 0;
 let _rebuildPollTimer     = null;
 let _rebuildFinished      = false;
 let _rebuildServerDown    = false;
+let _rebuildPollFailures  = 0; // consecutive failed rebuild-status polls
 let _pendingToggle        = null; // {feature, extra} waiting for domain/confirm
 let _rebuildFeatureName   = "";
 let _rebuildIsEnabling    = true;

app/sovran_systemsos_web/static/js/update.js

@@ -33,6 +33,7 @@ function _doOpenUpdateModal() {
   _updateLogOffset = 0;
   _serverWasDown = false;
   _updateFinished = false;
+  _updatePollFailures = 0;
   if ($modalLog) $modalLog.textContent = "";
   if ($modalStatus) $modalStatus.textContent = "Starting update…";
   if ($modalSpinner) $modalSpinner.classList.add("spinning");
@@ -94,6 +95,7 @@ async function pollUpdateStatus() {
   if (_updateFinished) return;
   try {
     var data = await apiFetch("/api/updates/status?offset=" + _updateLogOffset);
+    _updatePollFailures = 0;
     if (_serverWasDown) {
       _serverWasDown = false;
       if (!data.running) {
@@ -143,6 +145,16 @@ async function pollUpdateStatus() {
       onUpdateDone(false);
     }
   } catch (err) {
+    _updatePollFailures += 1;
+    // Same guard as the rebuild modal: if polling stays broken long past a
+    // normal Hub restart, reload to re-authenticate and show the real state
+    // instead of spinning forever.
+    if (_updatePollFailures >= STATUS_POLL_MAX_FAILURES) {
+      _updateFinished = true;
+      stopUpdatePoll();
+      window.location.reload();
+      return;
+    }
     if (!_serverWasDown) { _serverWasDown = true; appendLog("\n[Server restarting — waiting for it to come back…]\n"); if ($modalStatus) $modalStatus.textContent = "Server restarting…"; }
   }
 }