fix(hub): self-heal truncated/corrupt Nix downloads and keep failed updates retryable

Updater/rebuild self-heal:
- Add a shared run_step wrapper used by both the update and rebuild
  scripts. On the first failure matching a transient fetch/cache signature
  (truncated tarball, corrupt NAR, hash mismatch, network timeout,
  interrupted download), clear Nix's fetch caches and repair the store,
  then retry once. Real config errors do not match and still fail loudly.
- The kernel-change boot fallback in the rebuild path is also wrapped.
- Fixes the reported 'cannot read file from tarball: Truncated tar archive
  detected' failure, which a plain re-run cannot clear because Nix reuses
  the corrupt cached archive.

Failed-update recovery / reporting:
- check_for_updates() now compares the running Hub version against the
  branch VERSION, so a failed 'nix flake update' (lock advanced but no
  generation staged) can no longer masquerade as 'up to date' and block
  retries.
- /api/updates/check surfaces a persistent 'failed' state; /api/updates/run
  never blocks a retry after a failure.
- Dashboard shows a red 'Update failed - click to retry' tile; the modal
  offers a Retry Update button and stops offering a reboot on failure.

app/sovran_systemsos_web/server.py

@@ -908,11 +908,68 @@ def _get_remote_rev(branch=None):
     return None
 
 
+def _parse_version(text):
+    """Return a (major, minor, patch) tuple from a VERSION string, or None."""
+    try:
+        match = re.search(r"(\d+)\.(\d+)\.(\d+)", str(text))
+        if match:
+            return tuple(int(g) for g in match.groups())
+    except Exception:
+        pass
+    return None
+
+
+def _get_remote_version(branch=None):
+    """Read VERSION on the tracked branch, e.g. the stable release version."""
+    try:
+        ref = branch or "stable"
+        url = (
+            "https://git.sovransystems.com/api/v1/repos/"
+            "Sovran_Systems/Sovran_SystemsOS/raw/VERSION?ref="
+            + urllib.parse.quote(ref)
+        )
+        req = urllib.request.Request(url, method="GET")
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            return _parse_version(resp.read().decode())
+    except Exception:
+        pass
+    return None
+
+
 def check_for_updates() -> bool | None:
+    """Whether an update is available.
+
+    Primary signal: the flake lock's pinned Sovran_Systems rev differs from
+    the remote branch head.  BUT a failed update rewrites ``flake.lock`` (the
+    ``nix flake update`` step) without staging a generation (the
+    ``nixos-rebuild boot`` step failed), so after a failure the lock and the
+    remote agree while the *running* system is still on the old version.  In
+    that case the rev comparison alone reports a false "up to date" and hides
+    the failed update from the dashboard.
+
+    Backstop: compare the running Hub version against the branch VERSION.
+    A newer released version while running an older one means the update did
+    not apply (build failed, reboot skipped, generation rolled back) and must
+    be offered again.
+    """
     locked_rev, branch = _get_locked_info()
     remote_rev = _get_remote_rev(branch)
     if locked_rev and remote_rev:
-        return locked_rev != remote_rev
+        rev_differs = locked_rev != remote_rev
+        if rev_differs:
+            return True
+        # Revs match — make sure the pinned (failed) rev isn't masking an
+        # older *running* system.
+        running_ver = _parse_version(_get_sovran_version())
+        remote_ver = _get_remote_version(branch)
+        if running_ver and remote_ver and remote_ver > running_ver:
+            return True
+        return False
+    # Couldn't compare revs — fall back to the version backstop.
+    running_ver = _parse_version(_get_sovran_version())
+    remote_ver = _get_remote_version(branch)
+    if running_ver and remote_ver:
+        return remote_ver > running_ver
     return None  # inconclusive — couldn't read lock or reach remote
 
 
@@ -3885,6 +3942,11 @@ async def api_updates_check():
         # Avoid a slow remote update check when there is already an operation
         # the dashboard needs to surface.
         return {"available": True, "status": status.lower()}
+    if status == "FAILED":
+        # The last update did not complete (build failed).  Keep offering the
+        # update so the user can re-run it rather than silently landing on a
+        # false "up to date".
+        return {"available": True, "status": "failed"}
 
     available = await loop.run_in_executor(None, check_for_updates)
     # None means inconclusive (check failed) — report as available so the UI doesn't block
@@ -3962,8 +4024,15 @@ async def api_updates_run():
         except OSError:
             pass
 
+    # Re-read status: a prior failed update leaves flake.lock advanced even
+    # though no generation was staged, so the rev-based check below can say
+    # "no updates" even though the system is still old.  A failed update must
+    # always be re-runnable to recover.
+    persisted_status = await loop.run_in_executor(None, _read_update_status)
+    last_failed = persisted_status == "FAILED"
+
     available = await loop.run_in_executor(None, check_for_updates)
-    if available is False:  # only block when positively confirmed no updates
+    if available is False and not last_failed:  # only block when positively confirmed no updates
         # Clear stale status/log so they don't contaminate future modal opens.
         _write_update_status("IDLE")
         try:

app/sovran_systemsos_web/static/js/events.js

@@ -7,6 +7,7 @@ if ($btnCloseModal) $btnCloseModal.addEventListener("click", closeUpdateModal);
 if ($btnReboot) $btnReboot.addEventListener("click", doReboot);
 if ($btnSave) $btnSave.addEventListener("click", saveErrorReport);
 if ($btnRetryUpdate) $btnRetryUpdate.addEventListener("click", retryUpdateStatus);
+if ($btnRetryRun) $btnRetryRun.addEventListener("click", retryUpdateRun);
 
 // Browser timers and requests may be suspended while an RDP session/tab is in
 // the background. Reconcile immediately when the user returns instead of

app/sovran_systemsos_web/static/js/state.js

@@ -53,6 +53,7 @@ const $modalLog       = document.getElementById("modal-log");
 const $btnReboot      = document.getElementById("btn-reboot");
 const $btnSave        = document.getElementById("btn-save-report");
 const $btnRetryUpdate = document.getElementById("btn-retry-update-status");
+const $btnRetryRun    = document.getElementById("btn-retry-update");
 const $btnCloseModal  = document.getElementById("btn-close-modal");
 
 const $rebootOverlay  = document.getElementById("reboot-overlay");

app/sovran_systemsos_web/static/js/tiles.js

@@ -275,7 +275,13 @@ async function checkUpdates() {
     var sidebarUpdateBtn = document.getElementById("sidebar-btn-update");
     var sidebarUpdateHint = document.getElementById("sidebar-update-hint");
     if (sidebarUpdateBtn) {
-      if (updateStatus === "reboot_required") {
+      if (updateStatus === "failed") {
+        // Last update errored and did not apply — surface it as a persistent
+        // red banner that re-opens the failed run with a "Retry Update" action.
+        sidebarUpdateBtn.style.borderColor = "#e01b24";
+        sidebarUpdateBtn.style.backgroundColor = "rgba(224, 27, 36, 0.10)";
+        if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Update failed — click to retry";
+      } else if (updateStatus === "reboot_required") {
         sidebarUpdateBtn.style.borderColor = "#e5a50a";
         sidebarUpdateBtn.style.backgroundColor = "rgba(229, 165, 10, 0.10)";
         if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Restart required";

app/sovran_systemsos_web/static/js/update.js

@@ -14,7 +14,10 @@ async function openUpdateModal() {
       { cache: "no-store" },
       STATUS_POLL_FETCH_TIMEOUT
     );
-    if (current.running || current.result === "reboot_required") {
+    if (current.running || current.result === "reboot_required" || current.result === "failed") {
+      // An in-progress update, a staged update awaiting reboot, or a prior
+      // failed update — reattach to the persisted systemd/log state instead of
+      // starting over or wrongly reporting "up to date".
       showExistingUpdate(current);
       return;
     }
@@ -41,6 +44,7 @@ async function openUpdateModal() {
         if ($btnReboot) $btnReboot.style.display = "none";
         if ($btnSave) $btnSave.style.display = "none";
         if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
+        if ($btnRetryRun) $btnRetryRun.style.display = "none";
         if ($btnCloseModal) $btnCloseModal.disabled = false;
         $modal.classList.add("open");
         return;
@@ -69,6 +73,7 @@ function prepareUpdateModal() {
   if ($btnReboot) $btnReboot.style.display = "none";
   if ($btnSave) $btnSave.style.display = "none";
   if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
+  if ($btnRetryRun) $btnRetryRun.style.display = "none";
   if ($btnCloseModal) $btnCloseModal.disabled = true;
   $modal.classList.add("open");
 }
@@ -154,6 +159,7 @@ function startUpdate() {
         if ($btnReboot) $btnReboot.style.display = "none";
         if ($btnSave) $btnSave.style.display = "none";
         if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
+        if ($btnRetryRun) $btnRetryRun.style.display = "none";
         if ($btnCloseModal) $btnCloseModal.disabled = false;
         _updateFinished = true;
         return;
@@ -283,6 +289,16 @@ function retryUpdateStatus() {
   startUpdatePoll();
 }
 
+// Re-run a failed (or never-applied) update from scratch.  The backend always
+// allows this after a FAILED attempt even though flake.lock may already be
+// advanced (the previous build never staged a bootable generation).
+function retryUpdateRun() {
+  if ($btnRetryRun) $btnRetryRun.style.display = "none";
+  if ($btnSave) $btnSave.style.display = "none";
+  if ($btnReboot) $btnReboot.style.display = "none";
+  _doOpenUpdateModal();
+}
+
 function resumeUpdateStatusAfterInterruption() {
   if (!$modal || !$modal.classList.contains("open")) return;
   if (_updateStatusUnavailable) {
@@ -304,9 +320,10 @@ function onUpdateDone(result) {
     if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else {
-    if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
+    if ($modalStatus) $modalStatus.textContent = "✗ Update failed — your system was not changed. Run the update again or save the error report for support.";
+    if ($btnRetryRun) $btnRetryRun.style.display = "inline-flex";
     if ($btnSave) $btnSave.style.display = "inline-flex";
-    if ($btnReboot) $btnReboot.style.display = "inline-flex";
+    if ($btnReboot) $btnReboot.style.display = "none";
   }
 }
 

app/sovran_systemsos_web/templates/index.html

@@ -75,6 +75,7 @@
       <div class="modal-footer">
         <button class="btn btn-save" id="btn-save-report" style="display:none">Save Error Report</button>
         <button class="btn btn-save" id="btn-retry-update-status" style="display:none">Retry Status</button>
+        <button class="btn btn-reboot" id="btn-retry-update" style="display:none">Retry Update</button>
         <button class="btn btn-reboot" id="btn-reboot" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="btn-close-modal" disabled>Close</button>
       </div>

modules/core/sovran-hub.nix

@@ -150,6 +150,61 @@ let
     "haven-relay.service" = if pkgs ? haven-relay then pkgs.haven-relay.version else (if pkgs ? haven then pkgs.haven.version else "0.1.0");
   });
 
+  # Shared shell prelude used by both the update and rebuild wrapper scripts.
+  # A flake/package fetch that is interrupted (network blip, reboot
+  # mid-download, disk filled, hiccup on the remote) can leave a truncated
+  # tarball or partial git clone in Nix's download caches. Nix then reuses the
+  # corrupt archive on every retry and dies with "cannot read file from
+  # tarball: Truncated tar archive detected" — a failure that is NOT fixed by
+  # simply re-running, but IS fixed by clearing the fetch caches. run_step runs
+  # a command and, on the first failure that matches a download/cache
+  # signature, clears the caches and retries once. Real config errors never
+  # match, so they still fail loudly. Each sourcing script must define $LOG.
+  nix-self-heal-prelude = ''
+    transient_failure() {
+      grep -Eqi 'truncated tar|unexpected end of (file|archive)|unexpected eof|corrupt(ed)? (archive|nar|download|file)|could not (fetch|download)|download.*(failed|interrupted)|timed out|timeout|connection (reset|refused|timed out)|network is unreachable|temporary failure in name resolution|checksum mismatch|hash mismatch|nar hash|unable to download|store path.*is not valid|cannot read file from tarball|into the git cache' "$LOG"
+    }
+
+    clear_fetch_caches() {
+      echo "[SELF-HEAL] Clearing stale Nix download caches and verifying the Nix store…"
+      # Re-fetchable caches only; /nix/store generations and the running system
+      # are never touched here.
+      rm -rf /root/.cache/nix/tarballs /root/.cache/nix/vcs-cache /root/.cache/nix/git* /root/.cache/nix/flakes 2>/dev/null || true
+      # Fast closure-level repair only. A full --check-contents scan hashes
+      # every store path and can take tens of minutes on a big node; the cache
+      # clear above is the actual fix for truncated/corrupt downloads.
+      nix-store --verify --repair >/dev/null 2>&1 || true
+      echo "[SELF-HEAL] Caches cleared; retrying…"
+      echo ""
+    }
+
+    # run_step LABEL CMD [ARGS...] — run a build step; on a transient
+    # fetch/cache failure, heal once and retry. Returns the command exit code
+    # but leaves error messaging to the caller.
+    run_step() {
+      label="$1"; shift
+      rc=1
+      for try in 1 2; do
+        if [ "$try" -eq 2 ]; then
+          echo "── $label — retry after cache repair ──"
+        fi
+        "$@"
+        rc=$?
+        if [ "$rc" -eq 0 ]; then
+          return 0
+        fi
+        if [ "$try" -eq 1 ] && transient_failure; then
+          echo ""
+          echo "[SELF-HEAL] $label failed on a download/cache error (see above)."
+          clear_fetch_caches
+          continue
+        fi
+        return "$rc"
+      done
+      return "$rc"
+    }
+  '';
+
   # ── Update wrapper script ──────────────────────────────────────
   update-script = pkgs.writeShellScript "sovran-hub-update.sh" ''
     set -uo pipefail
@@ -171,12 +226,14 @@ let
 
     RC=0
 
+    ${nix-self-heal-prelude}
+
     echo "── Step 1/3: nix flake update ────────────────────"
-    if ! nix flake update --flake /etc/nixos --print-build-logs \
+    if ! run_step "nix flake update" nix flake update --flake /etc/nixos --print-build-logs \
           --option connect-timeout 10 \
           --option stalled-download-timeout 90 \
           --option download-attempts 7 \
-          --option fallback true 2>&1; then
+          --option fallback true; then
       echo "[ERROR] nix flake update failed"
       RC=1
     fi
@@ -186,22 +243,22 @@ let
       echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──"
       # Stream output straight into $LOG (see rebuild-script) so the Hub UI
       # shows live progress instead of an empty log during long builds.
-      nixos-rebuild boot --flake /etc/nixos --print-build-logs \
+      if run_step "nixos-rebuild boot" nixos-rebuild boot --flake /etc/nixos --print-build-logs \
           --option connect-timeout 10 \
           --option stalled-download-timeout 90 \
           --option download-attempts 7 \
-        --option fallback true
+          --option fallback true; then
-      BOOT_RC=$?
+        if ! readlink -f /nix/var/nix/profiles/system > "$GENERATION"; then
-      if [ "$BOOT_RC" -ne 0 ]; then
-        echo "[ERROR] nixos-rebuild boot failed"
-        RC=1
-      elif ! readlink -f /nix/var/nix/profiles/system > "$GENERATION"; then
           # The marker is informational only.  The Hub derives pending-reboot
           # state from the NixOS system profile itself, so failing to record
           # the marker must not fail an otherwise successful update.
           echo "[WARNING] update succeeded but its staged generation could not be recorded"
           rm -f "$GENERATION"
         fi
+      else
+        echo "[ERROR] nixos-rebuild boot failed"
+        RC=1
+      fi
       echo ""
     fi
 
@@ -245,12 +302,15 @@ let
     echo "  Sovran_SystemsOS Rebuild — $(date)"
     echo "══════════════════════════════════════════════════"
     echo ""
+
+    ${nix-self-heal-prelude}
+
     echo "── Rebuilding system configuration ──────────────"
     # Stream output straight into $LOG (tee'd by the exec redirect above) so
     # the Hub UI shows live progress.  Capturing the output in a variable
     # kept the log empty for the entire build+activation, which made long
     # rebuilds can otherwise look like a hang.
-    nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+    run_step "nixos-rebuild switch" nixos-rebuild switch --flake /etc/nixos --print-build-logs \
       --option connect-timeout 10 \
       --option stalled-download-timeout 90 \
       --option download-attempts 7 \
@@ -266,11 +326,11 @@ let
       echo ""
       echo "  ✓ Build succeeded — a reboot is required to apply this rebuild"
       echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-      if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
+      if run_step "nixos-rebuild boot" nixos-rebuild boot --flake /etc/nixos --print-build-logs \
            --option connect-timeout 10 \
            --option stalled-download-timeout 90 \
            --option download-attempts 7 \
-           --option fallback true 2>&1; then
+           --option fallback true; then
         echo "REBOOT_REQUIRED" > "$STATUS"
       else
         echo "[ERROR] nixos-rebuild boot also failed"
@@ -278,6 +338,7 @@ let
         exit 1
       fi
     else
+      echo "[ERROR] nixos-rebuild switch failed"
       echo ""
       echo "══════════════════════════════════════════════════"
       echo "  ✗ Rebuild failed — see errors above"