fix(hub): self-heal truncated/corrupt Nix downloads and keep failed updates retryable

Updater/rebuild self-heal:
- Add a shared run_step wrapper used by both the update and rebuild
  scripts. On the first failure matching a transient fetch/cache signature
  (truncated tarball, corrupt NAR, hash mismatch, network timeout,
  interrupted download), clear Nix's fetch caches and repair the store,
  then retry once. Real config errors do not match and still fail loudly.
- The kernel-change boot fallback in the rebuild path is also wrapped.
- Fixes the reported 'cannot read file from tarball: Truncated tar archive
  detected' failure, which a plain re-run cannot clear because Nix reuses
  the corrupt cached archive.

Failed-update recovery / reporting:
- check_for_updates() now compares the running Hub version against the
  branch VERSION, so a failed 'nix flake update' (lock advanced but no
  generation staged) can no longer masquerade as 'up to date' and block
  retries.
- /api/updates/check surfaces a persistent 'failed' state; /api/updates/run
  never blocks a retry after a failure.
- Dashboard shows a red 'Update failed - click to retry' tile; the modal
  offers a Retry Update button and stops offering a reboot on failure.

app/sovran_systemsos_web/static/js/events.js

@@ -7,6 +7,7 @@ if ($btnCloseModal) $btnCloseModal.addEventListener("click", closeUpdateModal);
 if ($btnReboot) $btnReboot.addEventListener("click", doReboot);
 if ($btnSave) $btnSave.addEventListener("click", saveErrorReport);
 if ($btnRetryUpdate) $btnRetryUpdate.addEventListener("click", retryUpdateStatus);
+if ($btnRetryRun) $btnRetryRun.addEventListener("click", retryUpdateRun);
 
 // Browser timers and requests may be suspended while an RDP session/tab is in
 // the background. Reconcile immediately when the user returns instead of

app/sovran_systemsos_web/static/js/state.js

@@ -53,6 +53,7 @@ const $modalLog       = document.getElementById("modal-log");
 const $btnReboot      = document.getElementById("btn-reboot");
 const $btnSave        = document.getElementById("btn-save-report");
 const $btnRetryUpdate = document.getElementById("btn-retry-update-status");
+const $btnRetryRun    = document.getElementById("btn-retry-update");
 const $btnCloseModal  = document.getElementById("btn-close-modal");
 
 const $rebootOverlay  = document.getElementById("reboot-overlay");

app/sovran_systemsos_web/static/js/tiles.js

@@ -275,7 +275,13 @@ async function checkUpdates() {
     var sidebarUpdateBtn = document.getElementById("sidebar-btn-update");
     var sidebarUpdateHint = document.getElementById("sidebar-update-hint");
     if (sidebarUpdateBtn) {
-      if (updateStatus === "reboot_required") {
+      if (updateStatus === "failed") {
+        // Last update errored and did not apply — surface it as a persistent
+        // red banner that re-opens the failed run with a "Retry Update" action.
+        sidebarUpdateBtn.style.borderColor = "#e01b24";
+        sidebarUpdateBtn.style.backgroundColor = "rgba(224, 27, 36, 0.10)";
+        if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Update failed — click to retry";
+      } else if (updateStatus === "reboot_required") {
         sidebarUpdateBtn.style.borderColor = "#e5a50a";
         sidebarUpdateBtn.style.backgroundColor = "rgba(229, 165, 10, 0.10)";
         if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Restart required";

app/sovran_systemsos_web/static/js/update.js

@@ -14,7 +14,10 @@ async function openUpdateModal() {
       { cache: "no-store" },
       STATUS_POLL_FETCH_TIMEOUT
     );
-    if (current.running || current.result === "reboot_required") {
+    if (current.running || current.result === "reboot_required" || current.result === "failed") {
+      // An in-progress update, a staged update awaiting reboot, or a prior
+      // failed update — reattach to the persisted systemd/log state instead of
+      // starting over or wrongly reporting "up to date".
       showExistingUpdate(current);
       return;
     }
@@ -41,6 +44,7 @@ async function openUpdateModal() {
         if ($btnReboot) $btnReboot.style.display = "none";
         if ($btnSave) $btnSave.style.display = "none";
         if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
+        if ($btnRetryRun) $btnRetryRun.style.display = "none";
         if ($btnCloseModal) $btnCloseModal.disabled = false;
         $modal.classList.add("open");
         return;
@@ -69,6 +73,7 @@ function prepareUpdateModal() {
   if ($btnReboot) $btnReboot.style.display = "none";
   if ($btnSave) $btnSave.style.display = "none";
   if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
+  if ($btnRetryRun) $btnRetryRun.style.display = "none";
   if ($btnCloseModal) $btnCloseModal.disabled = true;
   $modal.classList.add("open");
 }
@@ -154,6 +159,7 @@ function startUpdate() {
         if ($btnReboot) $btnReboot.style.display = "none";
         if ($btnSave) $btnSave.style.display = "none";
         if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
+        if ($btnRetryRun) $btnRetryRun.style.display = "none";
         if ($btnCloseModal) $btnCloseModal.disabled = false;
         _updateFinished = true;
         return;
@@ -283,6 +289,16 @@ function retryUpdateStatus() {
   startUpdatePoll();
 }
 
+// Re-run a failed (or never-applied) update from scratch.  The backend always
+// allows this after a FAILED attempt even though flake.lock may already be
+// advanced (the previous build never staged a bootable generation).
+function retryUpdateRun() {
+  if ($btnRetryRun) $btnRetryRun.style.display = "none";
+  if ($btnSave) $btnSave.style.display = "none";
+  if ($btnReboot) $btnReboot.style.display = "none";
+  _doOpenUpdateModal();
+}
+
 function resumeUpdateStatusAfterInterruption() {
   if (!$modal || !$modal.classList.contains("open")) return;
   if (_updateStatusUnavailable) {
@@ -304,9 +320,10 @@ function onUpdateDone(result) {
     if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else {
-    if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
+    if ($modalStatus) $modalStatus.textContent = "✗ Update failed — your system was not changed. Run the update again or save the error report for support.";
+    if ($btnRetryRun) $btnRetryRun.style.display = "inline-flex";
     if ($btnSave) $btnSave.style.display = "inline-flex";
-    if ($btnReboot) $btnReboot.style.display = "inline-flex";
+    if ($btnReboot) $btnReboot.style.display = "none";
   }
 }