diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index 9bdc7e8..af848ce 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -908,11 +908,68 @@ def _get_remote_rev(branch=None): return None +def _parse_version(text): + """Return a (major, minor, patch) tuple from a VERSION string, or None.""" + try: + match = re.search(r"(\d+)\.(\d+)\.(\d+)", str(text)) + if match: + return tuple(int(g) for g in match.groups()) + except Exception: + pass + return None + + +def _get_remote_version(branch=None): + """Read VERSION on the tracked branch, e.g. the stable release version.""" + try: + ref = branch or "stable" + url = ( + "https://git.sovransystems.com/api/v1/repos/" + "Sovran_Systems/Sovran_SystemsOS/raw/VERSION?ref=" + + urllib.parse.quote(ref) + ) + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=15) as resp: + return _parse_version(resp.read().decode()) + except Exception: + pass + return None + + def check_for_updates() -> bool | None: + """Whether an update is available. + + Primary signal: the flake lock's pinned Sovran_Systems rev differs from + the remote branch head. BUT a failed update rewrites ``flake.lock`` (the + ``nix flake update`` step) without staging a generation (the + ``nixos-rebuild boot`` step failed), so after a failure the lock and the + remote agree while the *running* system is still on the old version. In + that case the rev comparison alone reports a false "up to date" and hides + the failed update from the dashboard. + + Backstop: compare the running Hub version against the branch VERSION. + A newer released version while running an older one means the update did + not apply (build failed, reboot skipped, generation rolled back) and must + be offered again. + """ locked_rev, branch = _get_locked_info() remote_rev = _get_remote_rev(branch) if locked_rev and remote_rev: - return locked_rev != remote_rev + rev_differs = locked_rev != remote_rev + if rev_differs: + return True + # Revs match — make sure the pinned (failed) rev isn't masking an + # older *running* system. + running_ver = _parse_version(_get_sovran_version()) + remote_ver = _get_remote_version(branch) + if running_ver and remote_ver and remote_ver > running_ver: + return True + return False + # Couldn't compare revs — fall back to the version backstop. + running_ver = _parse_version(_get_sovran_version()) + remote_ver = _get_remote_version(branch) + if running_ver and remote_ver: + return remote_ver > running_ver return None # inconclusive — couldn't read lock or reach remote @@ -3885,6 +3942,11 @@ async def api_updates_check(): # Avoid a slow remote update check when there is already an operation # the dashboard needs to surface. return {"available": True, "status": status.lower()} + if status == "FAILED": + # The last update did not complete (build failed). Keep offering the + # update so the user can re-run it rather than silently landing on a + # false "up to date". + return {"available": True, "status": "failed"} available = await loop.run_in_executor(None, check_for_updates) # None means inconclusive (check failed) — report as available so the UI doesn't block @@ -3962,8 +4024,15 @@ async def api_updates_run(): except OSError: pass + # Re-read status: a prior failed update leaves flake.lock advanced even + # though no generation was staged, so the rev-based check below can say + # "no updates" even though the system is still old. A failed update must + # always be re-runnable to recover. + persisted_status = await loop.run_in_executor(None, _read_update_status) + last_failed = persisted_status == "FAILED" + available = await loop.run_in_executor(None, check_for_updates) - if available is False: # only block when positively confirmed no updates + if available is False and not last_failed: # only block when positively confirmed no updates # Clear stale status/log so they don't contaminate future modal opens. _write_update_status("IDLE") try: diff --git a/app/sovran_systemsos_web/static/js/events.js b/app/sovran_systemsos_web/static/js/events.js index 58bd35c..8f4acc5 100644 --- a/app/sovran_systemsos_web/static/js/events.js +++ b/app/sovran_systemsos_web/static/js/events.js @@ -7,6 +7,7 @@ if ($btnCloseModal) $btnCloseModal.addEventListener("click", closeUpdateModal); if ($btnReboot) $btnReboot.addEventListener("click", doReboot); if ($btnSave) $btnSave.addEventListener("click", saveErrorReport); if ($btnRetryUpdate) $btnRetryUpdate.addEventListener("click", retryUpdateStatus); +if ($btnRetryRun) $btnRetryRun.addEventListener("click", retryUpdateRun); // Browser timers and requests may be suspended while an RDP session/tab is in // the background. Reconcile immediately when the user returns instead of diff --git a/app/sovran_systemsos_web/static/js/state.js b/app/sovran_systemsos_web/static/js/state.js index 4be13fe..200722f 100644 --- a/app/sovran_systemsos_web/static/js/state.js +++ b/app/sovran_systemsos_web/static/js/state.js @@ -53,6 +53,7 @@ const $modalLog = document.getElementById("modal-log"); const $btnReboot = document.getElementById("btn-reboot"); const $btnSave = document.getElementById("btn-save-report"); const $btnRetryUpdate = document.getElementById("btn-retry-update-status"); +const $btnRetryRun = document.getElementById("btn-retry-update"); const $btnCloseModal = document.getElementById("btn-close-modal"); const $rebootOverlay = document.getElementById("reboot-overlay"); diff --git a/app/sovran_systemsos_web/static/js/tiles.js b/app/sovran_systemsos_web/static/js/tiles.js index d432d35..d3154a5 100644 --- a/app/sovran_systemsos_web/static/js/tiles.js +++ b/app/sovran_systemsos_web/static/js/tiles.js @@ -275,7 +275,13 @@ async function checkUpdates() { var sidebarUpdateBtn = document.getElementById("sidebar-btn-update"); var sidebarUpdateHint = document.getElementById("sidebar-update-hint"); if (sidebarUpdateBtn) { - if (updateStatus === "reboot_required") { + if (updateStatus === "failed") { + // Last update errored and did not apply — surface it as a persistent + // red banner that re-opens the failed run with a "Retry Update" action. + sidebarUpdateBtn.style.borderColor = "#e01b24"; + sidebarUpdateBtn.style.backgroundColor = "rgba(224, 27, 36, 0.10)"; + if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Update failed — click to retry"; + } else if (updateStatus === "reboot_required") { sidebarUpdateBtn.style.borderColor = "#e5a50a"; sidebarUpdateBtn.style.backgroundColor = "rgba(229, 165, 10, 0.10)"; if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Restart required"; diff --git a/app/sovran_systemsos_web/static/js/update.js b/app/sovran_systemsos_web/static/js/update.js index 7d7b9f3..7ea98bd 100644 --- a/app/sovran_systemsos_web/static/js/update.js +++ b/app/sovran_systemsos_web/static/js/update.js @@ -14,7 +14,10 @@ async function openUpdateModal() { { cache: "no-store" }, STATUS_POLL_FETCH_TIMEOUT ); - if (current.running || current.result === "reboot_required") { + if (current.running || current.result === "reboot_required" || current.result === "failed") { + // An in-progress update, a staged update awaiting reboot, or a prior + // failed update — reattach to the persisted systemd/log state instead of + // starting over or wrongly reporting "up to date". showExistingUpdate(current); return; } @@ -41,6 +44,7 @@ async function openUpdateModal() { if ($btnReboot) $btnReboot.style.display = "none"; if ($btnSave) $btnSave.style.display = "none"; if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; + if ($btnRetryRun) $btnRetryRun.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = false; $modal.classList.add("open"); return; @@ -69,6 +73,7 @@ function prepareUpdateModal() { if ($btnReboot) $btnReboot.style.display = "none"; if ($btnSave) $btnSave.style.display = "none"; if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; + if ($btnRetryRun) $btnRetryRun.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = true; $modal.classList.add("open"); } @@ -154,6 +159,7 @@ function startUpdate() { if ($btnReboot) $btnReboot.style.display = "none"; if ($btnSave) $btnSave.style.display = "none"; if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; + if ($btnRetryRun) $btnRetryRun.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = false; _updateFinished = true; return; @@ -283,6 +289,16 @@ function retryUpdateStatus() { startUpdatePoll(); } +// Re-run a failed (or never-applied) update from scratch. The backend always +// allows this after a FAILED attempt even though flake.lock may already be +// advanced (the previous build never staged a bootable generation). +function retryUpdateRun() { + if ($btnRetryRun) $btnRetryRun.style.display = "none"; + if ($btnSave) $btnSave.style.display = "none"; + if ($btnReboot) $btnReboot.style.display = "none"; + _doOpenUpdateModal(); +} + function resumeUpdateStatusAfterInterruption() { if (!$modal || !$modal.classList.contains("open")) return; if (_updateStatusUnavailable) { @@ -304,9 +320,10 @@ function onUpdateDone(result) { if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required"; if ($btnReboot) $btnReboot.style.display = "inline-flex"; } else { - if ($modalStatus) $modalStatus.textContent = "✗ Update failed"; + if ($modalStatus) $modalStatus.textContent = "✗ Update failed — your system was not changed. Run the update again or save the error report for support."; + if ($btnRetryRun) $btnRetryRun.style.display = "inline-flex"; if ($btnSave) $btnSave.style.display = "inline-flex"; - if ($btnReboot) $btnReboot.style.display = "inline-flex"; + if ($btnReboot) $btnReboot.style.display = "none"; } } diff --git a/app/sovran_systemsos_web/templates/index.html b/app/sovran_systemsos_web/templates/index.html index 13f1e6d..803e224 100644 --- a/app/sovran_systemsos_web/templates/index.html +++ b/app/sovran_systemsos_web/templates/index.html @@ -75,6 +75,7 @@
+
diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index 2dc58e7..3deb7ce 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -150,6 +150,61 @@ let "haven-relay.service" = if pkgs ? haven-relay then pkgs.haven-relay.version else (if pkgs ? haven then pkgs.haven.version else "0.1.0"); }); + # Shared shell prelude used by both the update and rebuild wrapper scripts. + # A flake/package fetch that is interrupted (network blip, reboot + # mid-download, disk filled, hiccup on the remote) can leave a truncated + # tarball or partial git clone in Nix's download caches. Nix then reuses the + # corrupt archive on every retry and dies with "cannot read file from + # tarball: Truncated tar archive detected" — a failure that is NOT fixed by + # simply re-running, but IS fixed by clearing the fetch caches. run_step runs + # a command and, on the first failure that matches a download/cache + # signature, clears the caches and retries once. Real config errors never + # match, so they still fail loudly. Each sourcing script must define $LOG. + nix-self-heal-prelude = '' + transient_failure() { + grep -Eqi 'truncated tar|unexpected end of (file|archive)|unexpected eof|corrupt(ed)? (archive|nar|download|file)|could not (fetch|download)|download.*(failed|interrupted)|timed out|timeout|connection (reset|refused|timed out)|network is unreachable|temporary failure in name resolution|checksum mismatch|hash mismatch|nar hash|unable to download|store path.*is not valid|cannot read file from tarball|into the git cache' "$LOG" + } + + clear_fetch_caches() { + echo "[SELF-HEAL] Clearing stale Nix download caches and verifying the Nix store…" + # Re-fetchable caches only; /nix/store generations and the running system + # are never touched here. + rm -rf /root/.cache/nix/tarballs /root/.cache/nix/vcs-cache /root/.cache/nix/git* /root/.cache/nix/flakes 2>/dev/null || true + # Fast closure-level repair only. A full --check-contents scan hashes + # every store path and can take tens of minutes on a big node; the cache + # clear above is the actual fix for truncated/corrupt downloads. + nix-store --verify --repair >/dev/null 2>&1 || true + echo "[SELF-HEAL] Caches cleared; retrying…" + echo "" + } + + # run_step LABEL CMD [ARGS...] — run a build step; on a transient + # fetch/cache failure, heal once and retry. Returns the command exit code + # but leaves error messaging to the caller. + run_step() { + label="$1"; shift + rc=1 + for try in 1 2; do + if [ "$try" -eq 2 ]; then + echo "── $label — retry after cache repair ──" + fi + "$@" + rc=$? + if [ "$rc" -eq 0 ]; then + return 0 + fi + if [ "$try" -eq 1 ] && transient_failure; then + echo "" + echo "[SELF-HEAL] $label failed on a download/cache error (see above)." + clear_fetch_caches + continue + fi + return "$rc" + done + return "$rc" + } + ''; + # ── Update wrapper script ────────────────────────────────────── update-script = pkgs.writeShellScript "sovran-hub-update.sh" '' set -uo pipefail @@ -171,12 +226,14 @@ let RC=0 + ${nix-self-heal-prelude} + echo "── Step 1/3: nix flake update ────────────────────" - if ! nix flake update --flake /etc/nixos --print-build-logs \ + if ! run_step "nix flake update" nix flake update --flake /etc/nixos --print-build-logs \ --option connect-timeout 10 \ --option stalled-download-timeout 90 \ --option download-attempts 7 \ - --option fallback true 2>&1; then + --option fallback true; then echo "[ERROR] nix flake update failed" RC=1 fi @@ -186,21 +243,21 @@ let echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──" # Stream output straight into $LOG (see rebuild-script) so the Hub UI # shows live progress instead of an empty log during long builds. - nixos-rebuild boot --flake /etc/nixos --print-build-logs \ - --option connect-timeout 10 \ - --option stalled-download-timeout 90 \ - --option download-attempts 7 \ - --option fallback true - BOOT_RC=$? - if [ "$BOOT_RC" -ne 0 ]; then + if run_step "nixos-rebuild boot" nixos-rebuild boot --flake /etc/nixos --print-build-logs \ + --option connect-timeout 10 \ + --option stalled-download-timeout 90 \ + --option download-attempts 7 \ + --option fallback true; then + if ! readlink -f /nix/var/nix/profiles/system > "$GENERATION"; then + # The marker is informational only. The Hub derives pending-reboot + # state from the NixOS system profile itself, so failing to record + # the marker must not fail an otherwise successful update. + echo "[WARNING] update succeeded but its staged generation could not be recorded" + rm -f "$GENERATION" + fi + else echo "[ERROR] nixos-rebuild boot failed" RC=1 - elif ! readlink -f /nix/var/nix/profiles/system > "$GENERATION"; then - # The marker is informational only. The Hub derives pending-reboot - # state from the NixOS system profile itself, so failing to record - # the marker must not fail an otherwise successful update. - echo "[WARNING] update succeeded but its staged generation could not be recorded" - rm -f "$GENERATION" fi echo "" fi @@ -245,12 +302,15 @@ let echo " Sovran_SystemsOS Rebuild — $(date)" echo "══════════════════════════════════════════════════" echo "" + + ${nix-self-heal-prelude} + echo "── Rebuilding system configuration ──────────────" # Stream output straight into $LOG (tee'd by the exec redirect above) so # the Hub UI shows live progress. Capturing the output in a variable # kept the log empty for the entire build+activation, which made long # rebuilds can otherwise look like a hang. - nixos-rebuild switch --flake /etc/nixos --print-build-logs \ + run_step "nixos-rebuild switch" nixos-rebuild switch --flake /etc/nixos --print-build-logs \ --option connect-timeout 10 \ --option stalled-download-timeout 90 \ --option download-attempts 7 \ @@ -266,11 +326,11 @@ let echo "" echo " ✓ Build succeeded — a reboot is required to apply this rebuild" echo " (Critical system components changed; running nixos-rebuild boot instead)" - if nixos-rebuild boot --flake /etc/nixos --print-build-logs \ + if run_step "nixos-rebuild boot" nixos-rebuild boot --flake /etc/nixos --print-build-logs \ --option connect-timeout 10 \ --option stalled-download-timeout 90 \ --option download-attempts 7 \ - --option fallback true 2>&1; then + --option fallback true; then echo "REBOOT_REQUIRED" > "$STATUS" else echo "[ERROR] nixos-rebuild boot also failed" @@ -278,6 +338,7 @@ let exit 1 fi else + echo "[ERROR] nixos-rebuild switch failed" echo "" echo "══════════════════════════════════════════════════" echo " ✗ Rebuild failed — see errors above"