fix(hub): derive Restart required from boot default vs running system

NixOS already knows whether a reboot is pending: /nix/var/nix/profiles/
system vs /run/current-system. Marker files only the Hub's own updater
wrote desynced for terminal-updated machines (and markers from older
updaters could never clear), pinning the badge on forever. Reconcile
REBOOT_REQUIRED against live state on every read; the stale marker
self-heals to IDLE. The .generation marker write is now informational.

CHANGELOG.md

@@ -83,6 +83,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Successful staged updates now record the exact NixOS generation. The Hub
     keeps showing "Restart required" until that generation is active, then
     clears the marker after reboot (with log-based recovery for older updates).
+  - Pending-reboot state is now derived from NixOS itself — the boot default
+    (`/nix/var/nix/profiles/system`) versus the running `/run/current-system`
+    — instead of from Hub-written marker files. Updates performed from a
+    terminal or support session (which never touch the Hub's status files)
+    previously left a stale `REBOOT_REQUIRED` marker the Hub could never
+    clear, pinning the "Restart required" badge forever even after many
+    reboots. The marker self-heals to `IDLE` on the next status read whenever
+    boot default and running system agree, and recording the informational
+    `.generation` marker can no longer fail an otherwise successful update.
 
 ---
 

app/sovran_systemsos_web/server.py

@@ -55,7 +55,7 @@ from .security_helpers import (
     load_session_store,
     save_session_store,
 )
-from .update_state import staged_generation_is_active
+from .update_state import effective_update_status
 
 logger = logging.getLogger(__name__)
 
@@ -1648,11 +1648,14 @@ def _write_update_status(status: str):
 def _read_update_status() -> str:
     """Read and reconcile the persistent update status.
 
-    ``REBOOT_REQUIRED`` intentionally survives Hub/browser restarts before the
+    ``REBOOT_REQUIRED`` survives Hub/browser restarts before the reboot, but
-    reboot.  Once the staged generation is the running ``/run/current-system``,
+    it is a CLAIM about live NixOS state, not the source of truth: the boot
-    clear that marker so a completed reboot cannot leave the Hub asking for
+    default (``/nix/var/nix/profiles/system``) versus the running
-    another reboot forever.  The generation helper can recover older updates
+    ``/run/current-system``.  Re-validating on every read keeps the Hub
-    from the final nixos-rebuild log line when no explicit marker exists.
+    correct when the system was updated from a terminal or support session
+    (which never writes Hub markers), and lets an old marker that predates
+    the reconciliation feature self-heal instead of demanding reboots
+    forever.  The stale marker file is removed once cleared.
     """
     try:
         with open(UPDATE_STATUS, "r") as f:
@@ -1660,15 +1663,14 @@ def _read_update_status() -> str:
     except FileNotFoundError:
         return "IDLE"
 
-    if status == "REBOOT_REQUIRED" and staged_generation_is_active(
+    effective = effective_update_status(status)
-        UPDATE_GENERATION, UPDATE_LOG
+    if effective != status:
-    ):
+        _write_update_status(effective)
-        _write_update_status("IDLE")
         try:
             os.remove(UPDATE_GENERATION)
         except OSError:
             pass
-        return "IDLE"
+        return effective
 
     return status
 

app/sovran_systemsos_web/update_state.py

@@ -1,81 +1,88 @@
-"""Persistent update-state helpers for the Sovran Hub.
+"""Update-state helpers for the Sovran Hub.
 
 The full-system updater stages a NixOS generation with ``nixos-rebuild boot``.
-That generation is not active until the machine reboots.  These helpers let the
+That generation is not active until the machine reboots — and the same is true
-Hub distinguish a genuinely pending reboot from an old REBOOT_REQUIRED marker
+for updates started from a terminal or an SSH support session, which never go
-that survived the reboot.
+near the Hub's status files.
 
-This module deliberately has no FastAPI or systemd dependencies so its state
+The ONLY reliable indicator that a reboot is pending is NixOS itself: the
-reconciliation can be tested without importing the Hub server.
+system profile (``/nix/var/nix/profiles/system``), which ``nixos-rebuild``
+points at the newest generation on every ``boot`` AND every ``switch``, versus
+``/run/current-system``, the generation actually running since the last boot.
+When the two differ, a staged generation has not been booted yet.
+
+Earlier revisions reconstructed this from a marker file and log tails written
+by the Hub's own updater.  Any system updated by other means — or whose
+``REBOOT_REQUIRED`` status was written by an updater older than the marker
+feature — left the Hub showing "Restart required" forever: the recorded
+generation could never equal the (since advanced) running one, so the marker
+could never be cleared.
+
+This module has no FastAPI or systemd dependencies so the policy can be tested
+without importing the Hub server.
 """
 
 from __future__ import annotations
 
 import os
-import re
+
+# The NixOS system profile.  ``nixos-rebuild boot`` and ``nixos-rebuild
+# switch`` both add a generation here; ``boot`` additionally makes it the
+# bootloader default.  The path is a symlink chain (``system`` ->
+# ``system-N-link`` -> ``/nix/store/...-nixos-system-...``).
+BOOT_PROFILE_PATH = "/nix/var/nix/profiles/system"
+CURRENT_SYSTEM_PATH = "/run/current-system"
 
 
-# Nix store hashes use the lower-case Nix base32 alphabet.  Keep the output
+def reboot_is_pending(
-# name deliberately conservative: a system generation has no path separators.
+    boot_profile_path: str = BOOT_PROFILE_PATH,
-_SYSTEM_GENERATION_RE = re.compile(
+    current_system_path: str = CURRENT_SYSTEM_PATH,
-    r"^/nix/store/[0-9a-z]{32}-nixos-system-[A-Za-z0-9._+\-]+$"
+) -> bool:
-)
+    """Return whether a staged NixOS generation has not been booted yet.
-_LOG_GENERATION_RE = re.compile(
-    r"The new configuration is "
-    r"(/nix/store/[0-9a-z]{32}-nixos-system-[A-Za-z0-9._+\-]+)"
-)
 
+    This is deliberately independent of how the update was started — Hub
+    "Update System", terminal ``nixos-rebuild boot``, or a support session all
+    move the system profile the same way:
 
-def _valid_generation(value: str) -> str | None:
+    * after ``nixos-rebuild boot``:  profile -> new, current -> old  → pending
-    """Return a normalized NixOS generation path, or ``None`` if invalid."""
+    * after rebooting:               both -> new                     → cleared
-    candidate = value.strip()
+    * after ``nixos-rebuild switch``: both move together             → no reboot
-    if _SYSTEM_GENERATION_RE.fullmatch(candidate):
+      ever needed (switch activates immediately)
-        return candidate
+    * after a rollback:              both point at the rollback target → cleared
-    return None
 
-
+    Unreadable or missing paths are treated as "not pending": the Hub must
-def read_staged_generation(marker_path: str, log_path: str) -> str | None:
+    never demand a reboot it cannot substantiate.
-    """Read the generation staged by the last successful Hub update.
-
-    New updater versions write ``marker_path`` explicitly.  For an update that
-    started with an older updater, recover the same value from the final
-    ``nixos-rebuild`` log line.  Only the tail is needed and bounding the read
-    avoids loading a potentially large build log during every status poll.
     """
     try:
-        with open(marker_path, "r", encoding="utf-8") as marker:
+        boot_default = os.path.realpath(boot_profile_path)
-            generation = _valid_generation(marker.read())
-            if generation:
-                return generation
-    except OSError:
-        pass
-
-    try:
-        with open(log_path, "rb") as log:
-            log.seek(0, os.SEEK_END)
-            size = log.tell()
-            log.seek(max(0, size - 131_072), os.SEEK_SET)
-            tail = log.read().decode("utf-8", errors="replace")
-    except OSError:
-        return None
-
-    matches = list(_LOG_GENERATION_RE.finditer(tail))
-    if not matches:
-        return None
-    return _valid_generation(matches[-1].group(1))
-
-
-def staged_generation_is_active(
-    marker_path: str,
-    log_path: str,
-    current_system_path: str = "/run/current-system",
-) -> bool:
-    """Return whether the staged update generation is now the running system."""
-    staged = read_staged_generation(marker_path, log_path)
-    if not staged:
-        return False
-    try:
         current = os.path.realpath(current_system_path)
     except OSError:
         return False
-    return current == staged
+    if not os.path.exists(boot_default) or not os.path.exists(current):
+        return False
+    return boot_default != current
+
+
+def effective_update_status(
+    status: str,
+    boot_profile_path: str = BOOT_PROFILE_PATH,
+    current_system_path: str = CURRENT_SYSTEM_PATH,
+) -> str:
+    """Map a persisted Hub status to the one that reflects live NixOS state.
+
+    Only ``REBOOT_REQUIRED`` is re-validated: it means "the update staged a
+    generation the machine has not booted into", a claim that must stay true
+    no matter which tool performed the last update.  When the boot default IS
+    the running system the claim is stale — the staged generation booted, was
+    superseded by a newer update, or the marker was written by an updater that
+    could never clear it — so the effective status is ``IDLE``.
+
+    All other statuses (``RUNNING``, ``FAILED``, ``SUCCESS``, ``IDLE``) pass
+    through unchanged; RUNNING staleness is handled separately against the
+    systemd unit itself.
+    """
+    if status == "REBOOT_REQUIRED" and not reboot_is_pending(
+        boot_profile_path, current_system_path
+    ):
+        return "IDLE"
+    return status

modules/core/sovran-hub.nix

@@ -199,9 +199,11 @@ let
         echo "[ERROR] nixos-rebuild boot failed"
         RC=1
       elif ! readlink -f /nix/var/nix/profiles/system > "$GENERATION"; then
-        echo "[ERROR] update was built but its staged generation could not be recorded"
+        # The marker is informational only.  The Hub derives pending-reboot
+        # state from the NixOS system profile itself, so failing to record
+        # the marker must not fail an otherwise successful update.
+        echo "[WARNING] update succeeded but its staged generation could not be recorded"
         rm -f "$GENERATION"
-        RC=1
       fi
       echo ""
     fi

tests/test_update_state.py

@@ -17,80 +17,144 @@ if str(_APP_PARENT) not in sys.path:
     sys.path.insert(0, str(_APP_PARENT))
 
 from sovran_systemsos_web.update_state import (  # noqa: E402
-    read_staged_generation,
+    effective_update_status,
-    staged_generation_is_active,
+    reboot_is_pending,
 )
 
 
-GENERATION = (
+# Real store paths observed on the incident machine that prompted this rework.
-    "/nix/store/rmi0g35cd8w60k0ig7pm6kb8kzws8b7x-"
+RUNNING_GENERATION = (
-    "nixos-system-nixos-26.11.20260817.ec2d622"
+    "84rsiqi66nc68jbikd26ms50ap831xf8-nixos-system-nixos-26.11.20260817.ec2d622"
 )
-OTHER_GENERATION = (
+PREVIOUS_GENERATION = (
-    "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-"
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-nixos-system-nixos-26.11.20260816.old"
-    "nixos-system-nixos-26.11.20260816.old"
+)
+# What the stale Hub log still claimed was staged: a two-week-old update cycle.
+HUB_LOG_GENERATION = (
+    "yis0saq6p8fhqcaii0h2yzqf0blhdwns-nixos-system-nixos-26.11.20260804.e72e4f2"
 )
 
 
-class TestStagedGenerationState(unittest.TestCase):
+class TestRebootPendingState(unittest.TestCase):
+    """Reboot-pending state is derived from NixOS, not from Hub marker files.
+
+    The system profile (what boots next) is compared against
+    /run/current-system (what is running).  This stays correct no matter
+    which tool performed the update: Hub "Update System", a terminal
+    ``nixos-rebuild``, or an SSH support session.
+    """
+
     def setUp(self):
         self.tmp = tempfile.TemporaryDirectory()
         self.addCleanup(self.tmp.cleanup)
         root = Path(self.tmp.name)
-        self.marker = root / "update.generation"
+        self.store = root / "store"
-        self.log = root / "update.log"
+        self.store.mkdir()
+        # /nix/var/nix/profiles/system is a two-hop chain: the diagnostics on
+        # the incident box showed ``system`` -> ``system-148-link`` -> store.
+        self.profile = root / "system"
+        self.profile_entry = root / "system-148-link"
+        # /run/current-system links straight to the running store path.
         self.current = root / "current-system"
 
-    def test_explicit_generation_marker_is_read(self):
+    def _store_dir(self, name: str) -> str:
-        self.marker.write_text(GENERATION + "\n", encoding="utf-8")
+        path = self.store / name
-        self.assertEqual(
+        path.mkdir(exist_ok=True)
-            read_staged_generation(str(self.marker), str(self.log)), GENERATION
+        return str(path)
+
+    def _stage(self, booted: str, running: str) -> None:
+        """Stage ``booted`` as the boot default with ``running`` live."""
+        os.symlink(self._store_dir(booted), self.profile_entry)
+        os.symlink(self.profile_entry, self.profile)
+        os.symlink(self._store_dir(running), self.current)
+
+    def test_hub_update_staged_and_not_yet_rebooted_is_pending(self):
+        self._stage(booted=RUNNING_GENERATION, running=PREVIOUS_GENERATION)
+        self.assertTrue(
+            reboot_is_pending(str(self.profile), str(self.current))
         )
 
-    def test_legacy_updater_generation_is_recovered_from_log_tail(self):
+    def test_staged_generation_booted_is_no_longer_pending(self):
-        self.log.write_text(
+        self._stage(booted=RUNNING_GENERATION, running=RUNNING_GENERATION)
-            "building the system configuration...\n"
+        self.assertFalse(
-            f"Done. The new configuration is {GENERATION}\n"
+            reboot_is_pending(str(self.profile), str(self.current))
+        )
+
+    def test_terminal_switch_needs_no_reboot(self):
+        # nixos-rebuild switch moves the profile AND current-system together.
+        self._stage(booted=RUNNING_GENERATION, running=RUNNING_GENERATION)
+        self.assertFalse(
+            reboot_is_pending(str(self.profile), str(self.current))
+        )
+
+    def test_rollback_leaves_nothing_pending(self):
+        # nixos-rebuild switch --rollback points both at the rollback target.
+        self._stage(booted=PREVIOUS_GENERATION, running=PREVIOUS_GENERATION)
+        self.assertFalse(
+            reboot_is_pending(str(self.profile), str(self.current))
+        )
+
+    def test_unverifiable_state_is_never_a_reboot_demand(self):
+        # No profile and no running system readable: the Hub must not nag
+        # about a reboot it cannot substantiate.
+        self.assertFalse(
+            reboot_is_pending(str(self.profile), str(self.current))
+        )
+
+    def test_stale_hub_marker_clears_after_terminal_updates(self):
+        """The exact incident: terminal-updated machine, frozen Hub marker.
+
+        The user's last Hub update (old updater, weeks prior) left
+        REBOOT_REQUIRED behind.  Every update since ran in a terminal and
+        never touched the Hub's files, so the log still records a staged
+        generation from 2026-08-04 while the machine runs a 2026-08-17
+        build.  With the profile and the running system in agreement, the
+        stale claim must reconcile to IDLE regardless of anything the old
+        marker/log files say.
+        """
+        root = Path(self.tmp.name)
+        log = root / "sovran-hub-update.log"
+        log.write_text(
+            "Done. The new configuration is "
+            f"/nix/store/{HUB_LOG_GENERATION}\n"
             "✓ Update staged successfully\n",
             encoding="utf-8",
         )
-        self.assertEqual(
+        # Deliberately no sovran-hub-update.generation marker: the updater
-            read_staged_generation(str(self.marker), str(self.log)), GENERATION
+        # that produced this state predates the marker feature.
-        )
-
-    def test_latest_generation_line_wins(self):
-        self.log.write_text(
-            f"Done. The new configuration is {OTHER_GENERATION}\n"
-            f"Done. The new configuration is {GENERATION}\n",
-            encoding="utf-8",
-        )
-        self.assertEqual(
-            read_staged_generation(str(self.marker), str(self.log)), GENERATION
-        )
-
-    def test_invalid_marker_is_ignored(self):
-        self.marker.write_text("/tmp/not-a-generation\n", encoding="utf-8")
-        self.log.write_text("no completed generation\n", encoding="utf-8")
-        self.assertIsNone(read_staged_generation(str(self.marker), str(self.log)))
-
-    def test_staged_generation_is_active_after_reboot(self):
-        self.marker.write_text(GENERATION + "\n", encoding="utf-8")
-        os.symlink(GENERATION, self.current)
-        self.assertTrue(
-            staged_generation_is_active(
-                str(self.marker), str(self.log), str(self.current)
-            )
-        )
-
-    def test_staged_generation_remains_pending_before_reboot(self):
-        self.marker.write_text(GENERATION + "\n", encoding="utf-8")
-        os.symlink(OTHER_GENERATION, self.current)
         self.assertFalse(
-            staged_generation_is_active(
+            (root / "sovran-hub-update.generation").exists()
-                str(self.marker), str(self.log), str(self.current)
-            )
         )
 
+        self._stage(booted=RUNNING_GENERATION, running=RUNNING_GENERATION)
+        self.assertEqual(
+            effective_update_status(
+                "REBOOT_REQUIRED", str(self.profile), str(self.current)
+            ),
+            "IDLE",
+        )
+
+    def test_genuine_pending_reboot_claim_survives(self):
+        # A staged generation that has NOT been booted yet: the claim is
+        # true and must keep surfacing until the reboot really happens.
+        self._stage(booted=RUNNING_GENERATION, running=PREVIOUS_GENERATION)
+        self.assertEqual(
+            effective_update_status(
+                "REBOOT_REQUIRED", str(self.profile), str(self.current)
+            ),
+            "REBOOT_REQUIRED",
+        )
+
+    def test_other_statuses_pass_through_unchanged(self):
+        self._stage(booted=RUNNING_GENERATION, running=RUNNING_GENERATION)
+        for status in ("RUNNING", "FAILED", "SUCCESS", "IDLE"):
+            self.assertEqual(
+                effective_update_status(
+                    status, str(self.profile), str(self.current)
+                ),
+                status,
+            )
+
 
 class TestUpdatePollingWiring(unittest.TestCase):
     """Guard the browser failure modes that caused a permanent spinner."""