From 8f89a4350ab8450c111164da58b977b4f1239b94 Mon Sep 17 00:00:00 2001 From: naturallaw77 Date: Tue, 11 Aug 2026 18:47:40 -0500 Subject: [PATCH] fix: prevent Bitcoin Core switch from hanging the Hub UI --- CHANGELOG.md | 17 ++++ app/sovran_systemsos_web/security_helpers.py | 72 ++++++++++++++ app/sovran_systemsos_web/server.py | 68 +++++++++++++- .../static/js/constants.js | 4 + app/sovran_systemsos_web/static/js/rebuild.js | 14 +++ app/sovran_systemsos_web/static/js/state.js | 2 + app/sovran_systemsos_web/static/js/update.js | 12 +++ modules/core/sovran-hub.nix | 18 ++-- tests/test_security.py | 93 +++++++++++++++++++ 9 files changed, 289 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7c717..2e8ea8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [Unreleased] + +### Fixed +- Fix Bitcoin Knots → Bitcoin Core switch hanging in the Hub UI + - Sessions are now persisted to `/var/lib/secrets/hub-sessions.json` so the + browser login survives the Hub service restart that `nixos-rebuild switch` + performs during activation. Previously the in-memory session store was + wiped by that restart, the `/api/rebuild/status` poll started returning + 401, and the rebuild modal spun forever showing "Applying changes…" while + the switch result was never displayed. + - Rebuild and update modals now bail out and reload the page after sustained + polling failures instead of hanging indefinitely. + - Rebuild/update scripts stream `nixos-rebuild` output into the live log + (it was buffered until completion, making long rebuilds look frozen). + +--- + ## [1.1.0] - 2026-08-11 ### Added diff --git a/app/sovran_systemsos_web/security_helpers.py b/app/sovran_systemsos_web/security_helpers.py index b1b0e0a..31f0196 100644 --- a/app/sovran_systemsos_web/security_helpers.py +++ b/app/sovran_systemsos_web/security_helpers.py @@ -11,7 +11,11 @@ from __future__ import annotations import base64 import ipaddress +import json +import os import re +import tempfile +import time import urllib.parse # ── Nix string escaping ──────────────────────────────────────────────────────── @@ -239,3 +243,71 @@ def _validate_ssh_pubkey(key: str) -> str: if len(decoded) < 20: raise ValueError("SSH public key payload is too short") return key + + +# ── Persistent Hub session store ───────────────────────────────────────────── + +def load_session_store(path: str) -> dict[str, float]: + """Load persisted Hub sessions from *path*. + + Returns a mapping of session token → expiry timestamp (epoch seconds). + Expired entries are discarded. A missing, unreadable or malformed file + yields an empty mapping — losing sessions is a UX inconvenience (the user + must log in again), never a fatal error. + + Persistence exists so that authenticated sessions survive a restart of + the Hub service itself. ``nixos-rebuild switch`` restarts + ``sovran-hub-web.service`` during activation (its unit definition changes + with every feature toggle), and without persistence the in-progress + rebuild/update status polling loses authentication and the UI hangs. + """ + try: + with open(path, "r") as f: + data = json.load(f) + except (OSError, ValueError): + return {} + if not isinstance(data, dict): + return {} + now = time.time() + sessions: dict[str, float] = {} + for token, expiry in data.items(): + if not isinstance(token, str) or not token: + continue + if isinstance(expiry, bool) or not isinstance(expiry, (int, float)): + continue + if expiry > now: + sessions[token] = float(expiry) + return sessions + + +def save_session_store(path: str, sessions: dict[str, float]) -> bool: + """Atomically persist *sessions* (token → expiry) to *path* with mode 0600. + + Writes to a temp file in the same directory and renames it into place so + the store is never left partially written. Returns True on success, + False otherwise (persistence is best-effort). + """ + directory = os.path.dirname(path) or "." + fd = None + tmp_path = None + try: + os.makedirs(directory, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=directory, prefix=".hub_sessions_tmp") + with os.fdopen(fd, "w") as f: + fd = None # os.fdopen takes ownership of the descriptor + json.dump(sessions, f) + os.chmod(tmp_path, 0o600) + os.replace(tmp_path, path) + return True + except OSError: + if fd is not None: + try: + os.close(fd) + except OSError: + pass + if tmp_path is not None: + try: + os.unlink(tmp_path) + except OSError: + pass + return False diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index b1cfa5b..0c54164 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -52,6 +52,8 @@ from .security_helpers import ( _SSH_PUBKEY_ALGORITHMS, _bech32_decode, _bech32_convertbits_decode, + load_session_store, + save_session_store, ) logger = logging.getLogger(__name__) @@ -144,8 +146,27 @@ HUB_SESSION_SECRET_FILE = "/var/lib/secrets/hub-session-secret" SESSION_COOKIE_NAME = "hub_session" SESSION_MAX_AGE = 86400 # 24 hours -# In-memory session store: token → expiry timestamp (float) +# Sessions are persisted here so logins survive a restart of the Hub service. +# nixos-rebuild switch restarts sovran-hub-web.service during activation (its +# unit definition changes with every feature toggle — e.g. the Bitcoin +# Knots → Core switch changes services.bitcoind.package, which is on the Hub's +# PATH). Without persistence the browser session dies mid-rebuild, the +# /api/rebuild/status polling starts receiving 401s and the rebuild modal +# hangs forever showing "Applying changes…". The file lives in +# /var/lib/secrets so a security reset wipes it and forces a re-login, like +# the session secret itself. +SESSIONS_FILE = "/var/lib/secrets/hub-sessions.json" + +# Session store: token → expiry timestamp (float). Loaded lazily from +# SESSIONS_FILE on first use and written back on every meaningful change. _sessions: dict[str, float] = {} +_sessions_loaded = False +_sessions_lock = Lock() + +# Sliding the expiry on every authenticated request would rewrite the store on +# every poll, so persist slide-only updates at most this often. +_SESSION_PERSIST_MIN_INTERVAL = 30.0 # seconds +_sessions_last_persist = 0.0 # Failed login tracking: ip → list of failure timestamps _login_failures: dict[str, list[float]] = {} @@ -606,38 +627,74 @@ def _get_or_create_session_secret() -> bytes: return token_hex +def _load_sessions_once() -> None: + """Lazily load the persisted session store on first use (idempotent).""" + global _sessions_loaded + with _sessions_lock: + if _sessions_loaded: + return + _sessions.update(load_session_store(SESSIONS_FILE)) + _sessions_loaded = True + + +def _persist_sessions(force: bool = False) -> None: + """Write the session store to SESSIONS_FILE (best-effort). + + Expiry slides happen on every authenticated request, so non-forced + persists are throttled; create/destroy/purge pass ``force=True``. + """ + global _sessions_last_persist + now = time.time() + with _sessions_lock: + if not force and (now - _sessions_last_persist) < _SESSION_PERSIST_MIN_INTERVAL: + return + snapshot = dict(_sessions) + _sessions_last_persist = now + save_session_store(SESSIONS_FILE, snapshot) + + def _create_session() -> str: """Create a new opaque session token and register it in the store.""" + _load_sessions_once() _purge_expired_sessions() token = secrets.token_hex(32) _sessions[token] = time.time() + SESSION_MAX_AGE + _persist_sessions(force=True) return token def _destroy_session(token: str) -> None: """Remove a session token from the store.""" - _sessions.pop(token, None) + _load_sessions_once() + if _sessions.pop(token, None) is not None: + _persist_sessions(force=True) def _purge_expired_sessions() -> None: - """Remove all expired sessions from the in-memory store.""" + """Remove all expired sessions from the store.""" + _load_sessions_once() now = time.time() expired = [tok for tok, exp in _sessions.items() if exp <= now] for tok in expired: del _sessions[tok] + if expired: + _persist_sessions(force=True) def _is_authenticated(request: Request) -> bool: """Return True if the request carries a valid, unexpired session cookie.""" + _load_sessions_once() token = request.cookies.get(SESSION_COOKIE_NAME) if not token: return False expiry = _sessions.get(token) if expiry is None or time.time() >= expiry: - _sessions.pop(token, None) + if _sessions.pop(token, None) is not None: + _persist_sessions(force=True) return False # Slide the expiry window on activity _sessions[token] = time.time() + SESSION_MAX_AGE + _persist_sessions() # throttled — don't rewrite the store on every poll return True @@ -6152,6 +6209,9 @@ async def _startup_session_secret(): """Ensure the session secret exists on disk at startup.""" loop = asyncio.get_event_loop() await loop.run_in_executor(None, _get_or_create_session_secret) + # Preload persisted sessions so browser logins survive this restart (the + # Hub service is restarted by nixos-rebuild switch during every rebuild). + await loop.run_in_executor(None, _load_sessions_once) # ── Startup: recover stale RUNNING status files ────────────────── diff --git a/app/sovran_systemsos_web/static/js/constants.js b/app/sovran_systemsos_web/static/js/constants.js index c1a49c3..fe0b080 100644 --- a/app/sovran_systemsos_web/static/js/constants.js +++ b/app/sovran_systemsos_web/static/js/constants.js @@ -5,6 +5,10 @@ const POLL_INTERVAL_SERVICES = 5000; const POLL_INTERVAL_UPDATES = 1800000; const UPDATE_POLL_INTERVAL = 2000; +// Max consecutive failed rebuild/update status polls before the page gives up +// waiting and reloads to re-sync (2s interval → ~2 minutes of failures). +// A brief Hub restart during activation only causes a handful of failures. +const STATUS_POLL_MAX_FAILURES = 60; const REBOOT_CHECK_INTERVAL = 5000; const REBOOT_FETCH_TIMEOUT = 12000; const REBOOT_REQUEST_TIMEOUT = 4000; diff --git a/app/sovran_systemsos_web/static/js/rebuild.js b/app/sovran_systemsos_web/static/js/rebuild.js index 91189a4..bb1df61 100644 --- a/app/sovran_systemsos_web/static/js/rebuild.js +++ b/app/sovran_systemsos_web/static/js/rebuild.js @@ -8,6 +8,7 @@ function openRebuildModal() { _rebuildLogOffset = 0; _rebuildServerDown = false; _rebuildFinished = false; + _rebuildPollFailures = 0; if ($rebuildLog) { $rebuildLog.textContent = ""; $rebuildLog.style.display = "none"; } var action = _rebuildIsEnabling ? "Enabling" : "Disabling"; var label = _rebuildFeatureName || "feature"; @@ -45,6 +46,7 @@ async function pollRebuildStatus() { if (_rebuildFinished) return; try { var data = await apiFetch("/api/rebuild/status?offset=" + _rebuildLogOffset); + _rebuildPollFailures = 0; if (_rebuildServerDown) { _rebuildServerDown = false; } if (data.log) appendRebuildLog(data.log); _rebuildLogOffset = data.offset; @@ -57,6 +59,18 @@ async function pollRebuildStatus() { onRebuildDone(data.result === "success"); } } catch (err) { + _rebuildPollFailures += 1; + // The Hub restarts itself during activation, which briefly drops this poll. + // If polling stays broken long past a normal restart, the page's session + // almost certainly no longer matches the server (e.g. the Hub restarted + // and sessions were not recovered). Reload to re-authenticate and show + // the real feature state instead of spinning forever. + if (_rebuildPollFailures >= STATUS_POLL_MAX_FAILURES) { + _rebuildFinished = true; + stopRebuildPoll(); + window.location.reload(); + return; + } if (!_rebuildServerDown) { _rebuildServerDown = true; if ($rebuildStatus) $rebuildStatus.textContent = "Applying changes…"; } } } diff --git a/app/sovran_systemsos_web/static/js/state.js b/app/sovran_systemsos_web/static/js/state.js index 235a080..709cd7a 100644 --- a/app/sovran_systemsos_web/static/js/state.js +++ b/app/sovran_systemsos_web/static/js/state.js @@ -9,6 +9,7 @@ let _updatePollTimer = null; let _updateLogOffset = 0; let _serverWasDown = false; let _updateFinished = false; +let _updatePollFailures = 0; // consecutive failed update-status polls let _supportTimerInt = null; let _supportEnabledAt = null; let _supportStatus = null; // last fetched /api/support/status payload @@ -25,6 +26,7 @@ let _rebuildLogOffset = 0; let _rebuildPollTimer = null; let _rebuildFinished = false; let _rebuildServerDown = false; +let _rebuildPollFailures = 0; // consecutive failed rebuild-status polls let _pendingToggle = null; // {feature, extra} waiting for domain/confirm let _rebuildFeatureName = ""; let _rebuildIsEnabling = true; diff --git a/app/sovran_systemsos_web/static/js/update.js b/app/sovran_systemsos_web/static/js/update.js index 0537f49..4c0aff9 100644 --- a/app/sovran_systemsos_web/static/js/update.js +++ b/app/sovran_systemsos_web/static/js/update.js @@ -33,6 +33,7 @@ function _doOpenUpdateModal() { _updateLogOffset = 0; _serverWasDown = false; _updateFinished = false; + _updatePollFailures = 0; if ($modalLog) $modalLog.textContent = ""; if ($modalStatus) $modalStatus.textContent = "Starting update…"; if ($modalSpinner) $modalSpinner.classList.add("spinning"); @@ -94,6 +95,7 @@ async function pollUpdateStatus() { if (_updateFinished) return; try { var data = await apiFetch("/api/updates/status?offset=" + _updateLogOffset); + _updatePollFailures = 0; if (_serverWasDown) { _serverWasDown = false; if (!data.running) { @@ -143,6 +145,16 @@ async function pollUpdateStatus() { onUpdateDone(false); } } catch (err) { + _updatePollFailures += 1; + // Same guard as the rebuild modal: if polling stays broken long past a + // normal Hub restart, reload to re-authenticate and show the real state + // instead of spinning forever. + if (_updatePollFailures >= STATUS_POLL_MAX_FAILURES) { + _updateFinished = true; + stopUpdatePoll(); + window.location.reload(); + return; + } if (!_serverWasDown) { _serverWasDown = true; appendLog("\n[Server restarting — waiting for it to come back…]\n"); if ($modalStatus) $modalStatus.textContent = "Server restarting…"; } } } diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index 6d1590a..79a5d80 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -185,13 +185,14 @@ let if [ "$RC" -eq 0 ]; then echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──" - BOOT_OUT=$(nixos-rebuild boot --flake /etc/nixos --print-build-logs \ + # Stream output straight into $LOG (see rebuild-script) so the Hub UI + # shows live progress instead of an empty log during long builds. + nixos-rebuild boot --flake /etc/nixos --print-build-logs \ --option connect-timeout 10 \ --option stalled-download-timeout 90 \ --option download-attempts 7 \ - --option fallback true 2>&1) + --option fallback true BOOT_RC=$? - echo "$BOOT_OUT" if [ "$BOOT_RC" -ne 0 ]; then echo "[ERROR] nixos-rebuild boot failed" RC=1 @@ -240,20 +241,23 @@ let echo "══════════════════════════════════════════════════" echo "" echo "── Rebuilding system configuration ──────────────" - SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \ + # Stream output straight into $LOG (tee'd by the exec redirect above) so + # the Hub UI shows live progress. Capturing the output in a variable + # kept the log empty for the entire build+activation, which made long + # rebuilds (e.g. the Bitcoin Knots → Core switch) look like a hang. + nixos-rebuild switch --flake /etc/nixos --print-build-logs \ --option connect-timeout 10 \ --option stalled-download-timeout 90 \ --option download-attempts 7 \ - --option fallback true 2>&1) + --option fallback true SWITCH_RC=$? - echo "$SWITCH_OUT" if [ "$SWITCH_RC" -eq 0 ]; then echo "" echo "══════════════════════════════════════════════════" echo " ✓ Rebuild completed successfully" echo "══════════════════════════════════════════════════" echo "SUCCESS" > "$STATUS" - elif echo "$SWITCH_OUT" | grep -q "switchInhibitors\|Pre-switch checks failed"; then + elif grep -q "switchInhibitors\|Pre-switch checks failed" "$LOG"; then echo "" echo " ✓ Build succeeded — a reboot is required to apply this rebuild" echo " (Critical system components changed; running nixos-rebuild boot instead)" diff --git a/tests/test_security.py b/tests/test_security.py index dd775f7..daa262b 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -14,6 +14,7 @@ import json import os import sys import tempfile +import time import unittest # Add the app package to the path so we can import without the full FastAPI tree. @@ -30,6 +31,8 @@ from sovran_systemsos_web.security_helpers import ( # noqa: E402 _validate_ssh_pubkey, _DDNS_ALLOWED_HOSTNAMES, _bech32_decode, + load_session_store, + save_session_store, ) from sovran_systemsos_web import support_ops # noqa: E402 @@ -362,6 +365,96 @@ class TestAuthExemptPaths(unittest.TestCase): self.assertIn("/api/ping", self._get_exempt_paths()) +# --------------------------------------------------------------------------- +# Persistent session store +# --------------------------------------------------------------------------- + +class TestSessionStore(unittest.TestCase): + """Sessions must persist across Hub restarts so rebuild/update polling + keeps working after nixos-rebuild switch restarts the Hub service.""" + + def _store_path(self, tmpdir, name="hub-sessions.json"): + return os.path.join(tmpdir, name) + + def test_roundtrip(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + future = time.time() + 3600 + sessions = {"token-a": future, "token-b": future + 10} + self.assertTrue(save_session_store(path, sessions)) + self.assertEqual(load_session_store(path), sessions) + + def test_missing_file_returns_empty(self): + with tempfile.TemporaryDirectory() as tmpdir: + self.assertEqual(load_session_store(self._store_path(tmpdir)), {}) + + def test_malformed_json_returns_empty(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + with open(path, "w") as f: + f.write("{not json") + self.assertEqual(load_session_store(path), {}) + + def test_non_dict_json_returns_empty(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + with open(path, "w") as f: + json.dump(["token"], f) + self.assertEqual(load_session_store(path), {}) + + def test_expired_sessions_discarded(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + now = time.time() + save_session_store(path, {"alive": now + 3600, "dead": now - 1}) + loaded = load_session_store(path) + self.assertIn("alive", loaded) + self.assertNotIn("dead", loaded) + + def test_invalid_entries_skipped(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + now = time.time() + with open(path, "w") as f: + json.dump({ + "good": now + 3600, + "": now + 3600, # empty token + "bool-expiry": True, # bool is not a valid expiry + "str-expiry": "soon", # non-numeric expiry + "none-expiry": None, + }, f) + loaded = load_session_store(path) + self.assertEqual(list(loaded.keys()), ["good"]) + + def test_file_mode_is_0600(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + save_session_store(path, {"token": time.time() + 60}) + mode = os.stat(path).st_mode & 0o777 + self.assertEqual(mode, 0o600) + + def test_save_overwrites_existing_store(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + future = time.time() + 3600 + save_session_store(path, {"old": future}) + save_session_store(path, {"new": future}) + self.assertEqual(load_session_store(path), {"new": future}) + + def test_empty_store_roundtrip(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + self.assertTrue(save_session_store(path, {})) + self.assertEqual(load_session_store(path), {}) + + def test_no_leftover_temp_files(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = self._store_path(tmpdir) + save_session_store(path, {"token": time.time() + 60}) + leftovers = [n for n in os.listdir(tmpdir) if n.startswith(".hub_sessions_tmp")] + self.assertEqual(leftovers, []) + + # --------------------------------------------------------------------------- # tech-support.nix validation # ---------------------------------------------------------------------------