fix(hub): self-heal truncated/corrupt Nix downloads and keep failed updates retryable
Updater/rebuild self-heal: - Add a shared run_step wrapper used by both the update and rebuild scripts. On the first failure matching a transient fetch/cache signature (truncated tarball, corrupt NAR, hash mismatch, network timeout, interrupted download), clear Nix's fetch caches and repair the store, then retry once. Real config errors do not match and still fail loudly. - The kernel-change boot fallback in the rebuild path is also wrapped. - Fixes the reported 'cannot read file from tarball: Truncated tar archive detected' failure, which a plain re-run cannot clear because Nix reuses the corrupt cached archive. Failed-update recovery / reporting: - check_for_updates() now compares the running Hub version against the branch VERSION, so a failed 'nix flake update' (lock advanced but no generation staged) can no longer masquerade as 'up to date' and block retries. - /api/updates/check surfaces a persistent 'failed' state; /api/updates/run never blocks a retry after a failure. - Dashboard shows a red 'Update failed - click to retry' tile; the modal offers a Retry Update button and stops offering a reboot on failure.
This commit is contained in:
Sovran Systems
@@ -908,11 +908,68 @@ def _get_remote_rev(branch=None):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_version(text):
|
||||
"""Return a (major, minor, patch) tuple from a VERSION string, or None."""
|
||||
try:
|
||||
match = re.search(r"(\d+)\.(\d+)\.(\d+)", str(text))
|
||||
if match:
|
||||
return tuple(int(g) for g in match.groups())
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _get_remote_version(branch=None):
|
||||
"""Read VERSION on the tracked branch, e.g. the stable release version."""
|
||||
try:
|
||||
ref = branch or "stable"
|
||||
url = (
|
||||
"https://git.sovransystems.com/api/v1/repos/"
|
||||
"Sovran_Systems/Sovran_SystemsOS/raw/VERSION?ref="
|
||||
+ urllib.parse.quote(ref)
|
||||
)
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return _parse_version(resp.read().decode())
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def check_for_updates() -> bool | None:
|
||||
"""Whether an update is available.
|
||||
|
||||
Primary signal: the flake lock's pinned Sovran_Systems rev differs from
|
||||
the remote branch head. BUT a failed update rewrites ``flake.lock`` (the
|
||||
``nix flake update`` step) without staging a generation (the
|
||||
``nixos-rebuild boot`` step failed), so after a failure the lock and the
|
||||
remote agree while the *running* system is still on the old version. In
|
||||
that case the rev comparison alone reports a false "up to date" and hides
|
||||
the failed update from the dashboard.
|
||||
|
||||
Backstop: compare the running Hub version against the branch VERSION.
|
||||
A newer released version while running an older one means the update did
|
||||
not apply (build failed, reboot skipped, generation rolled back) and must
|
||||
be offered again.
|
||||
"""
|
||||
locked_rev, branch = _get_locked_info()
|
||||
remote_rev = _get_remote_rev(branch)
|
||||
if locked_rev and remote_rev:
|
||||
return locked_rev != remote_rev
|
||||
rev_differs = locked_rev != remote_rev
|
||||
if rev_differs:
|
||||
return True
|
||||
# Revs match — make sure the pinned (failed) rev isn't masking an
|
||||
# older *running* system.
|
||||
running_ver = _parse_version(_get_sovran_version())
|
||||
remote_ver = _get_remote_version(branch)
|
||||
if running_ver and remote_ver and remote_ver > running_ver:
|
||||
return True
|
||||
return False
|
||||
# Couldn't compare revs — fall back to the version backstop.
|
||||
running_ver = _parse_version(_get_sovran_version())
|
||||
remote_ver = _get_remote_version(branch)
|
||||
if running_ver and remote_ver:
|
||||
return remote_ver > running_ver
|
||||
return None # inconclusive — couldn't read lock or reach remote
|
||||
|
||||
|
||||
@@ -3885,6 +3942,11 @@ async def api_updates_check():
|
||||
# Avoid a slow remote update check when there is already an operation
|
||||
# the dashboard needs to surface.
|
||||
return {"available": True, "status": status.lower()}
|
||||
if status == "FAILED":
|
||||
# The last update did not complete (build failed). Keep offering the
|
||||
# update so the user can re-run it rather than silently landing on a
|
||||
# false "up to date".
|
||||
return {"available": True, "status": "failed"}
|
||||
|
||||
available = await loop.run_in_executor(None, check_for_updates)
|
||||
# None means inconclusive (check failed) — report as available so the UI doesn't block
|
||||
@@ -3962,8 +4024,15 @@ async def api_updates_run():
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Re-read status: a prior failed update leaves flake.lock advanced even
|
||||
# though no generation was staged, so the rev-based check below can say
|
||||
# "no updates" even though the system is still old. A failed update must
|
||||
# always be re-runnable to recover.
|
||||
persisted_status = await loop.run_in_executor(None, _read_update_status)
|
||||
last_failed = persisted_status == "FAILED"
|
||||
|
||||
available = await loop.run_in_executor(None, check_for_updates)
|
||||
if available is False: # only block when positively confirmed no updates
|
||||
if available is False and not last_failed: # only block when positively confirmed no updates
|
||||
# Clear stale status/log so they don't contaminate future modal opens.
|
||||
_write_update_status("IDLE")
|
||||
try:
|
||||
|
||||
@@ -7,6 +7,7 @@ if ($btnCloseModal) $btnCloseModal.addEventListener("click", closeUpdateModal);
|
||||
if ($btnReboot) $btnReboot.addEventListener("click", doReboot);
|
||||
if ($btnSave) $btnSave.addEventListener("click", saveErrorReport);
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.addEventListener("click", retryUpdateStatus);
|
||||
if ($btnRetryRun) $btnRetryRun.addEventListener("click", retryUpdateRun);
|
||||
|
||||
// Browser timers and requests may be suspended while an RDP session/tab is in
|
||||
// the background. Reconcile immediately when the user returns instead of
|
||||
|
||||
@@ -53,6 +53,7 @@ const $modalLog = document.getElementById("modal-log");
|
||||
const $btnReboot = document.getElementById("btn-reboot");
|
||||
const $btnSave = document.getElementById("btn-save-report");
|
||||
const $btnRetryUpdate = document.getElementById("btn-retry-update-status");
|
||||
const $btnRetryRun = document.getElementById("btn-retry-update");
|
||||
const $btnCloseModal = document.getElementById("btn-close-modal");
|
||||
|
||||
const $rebootOverlay = document.getElementById("reboot-overlay");
|
||||
|
||||
@@ -275,7 +275,13 @@ async function checkUpdates() {
|
||||
var sidebarUpdateBtn = document.getElementById("sidebar-btn-update");
|
||||
var sidebarUpdateHint = document.getElementById("sidebar-update-hint");
|
||||
if (sidebarUpdateBtn) {
|
||||
if (updateStatus === "reboot_required") {
|
||||
if (updateStatus === "failed") {
|
||||
// Last update errored and did not apply — surface it as a persistent
|
||||
// red banner that re-opens the failed run with a "Retry Update" action.
|
||||
sidebarUpdateBtn.style.borderColor = "#e01b24";
|
||||
sidebarUpdateBtn.style.backgroundColor = "rgba(224, 27, 36, 0.10)";
|
||||
if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Update failed — click to retry";
|
||||
} else if (updateStatus === "reboot_required") {
|
||||
sidebarUpdateBtn.style.borderColor = "#e5a50a";
|
||||
sidebarUpdateBtn.style.backgroundColor = "rgba(229, 165, 10, 0.10)";
|
||||
if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Restart required";
|
||||
|
||||
@@ -14,7 +14,10 @@ async function openUpdateModal() {
|
||||
{ cache: "no-store" },
|
||||
STATUS_POLL_FETCH_TIMEOUT
|
||||
);
|
||||
if (current.running || current.result === "reboot_required") {
|
||||
if (current.running || current.result === "reboot_required" || current.result === "failed") {
|
||||
// An in-progress update, a staged update awaiting reboot, or a prior
|
||||
// failed update — reattach to the persisted systemd/log state instead of
|
||||
// starting over or wrongly reporting "up to date".
|
||||
showExistingUpdate(current);
|
||||
return;
|
||||
}
|
||||
@@ -41,6 +44,7 @@ async function openUpdateModal() {
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnRetryRun) $btnRetryRun.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = false;
|
||||
$modal.classList.add("open");
|
||||
return;
|
||||
@@ -69,6 +73,7 @@ function prepareUpdateModal() {
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnRetryRun) $btnRetryRun.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = true;
|
||||
$modal.classList.add("open");
|
||||
}
|
||||
@@ -154,6 +159,7 @@ function startUpdate() {
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnRetryRun) $btnRetryRun.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = false;
|
||||
_updateFinished = true;
|
||||
return;
|
||||
@@ -283,6 +289,16 @@ function retryUpdateStatus() {
|
||||
startUpdatePoll();
|
||||
}
|
||||
|
||||
// Re-run a failed (or never-applied) update from scratch. The backend always
|
||||
// allows this after a FAILED attempt even though flake.lock may already be
|
||||
// advanced (the previous build never staged a bootable generation).
|
||||
function retryUpdateRun() {
|
||||
if ($btnRetryRun) $btnRetryRun.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
_doOpenUpdateModal();
|
||||
}
|
||||
|
||||
function resumeUpdateStatusAfterInterruption() {
|
||||
if (!$modal || !$modal.classList.contains("open")) return;
|
||||
if (_updateStatusUnavailable) {
|
||||
@@ -304,9 +320,10 @@ function onUpdateDone(result) {
|
||||
if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required";
|
||||
if ($btnReboot) $btnReboot.style.display = "inline-flex";
|
||||
} else {
|
||||
if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
|
||||
if ($modalStatus) $modalStatus.textContent = "✗ Update failed — your system was not changed. Run the update again or save the error report for support.";
|
||||
if ($btnRetryRun) $btnRetryRun.style.display = "inline-flex";
|
||||
if ($btnSave) $btnSave.style.display = "inline-flex";
|
||||
if ($btnReboot) $btnReboot.style.display = "inline-flex";
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,7 @@
|
||||
<div class="modal-footer">
|
||||
<button class="btn btn-save" id="btn-save-report" style="display:none">Save Error Report</button>
|
||||
<button class="btn btn-save" id="btn-retry-update-status" style="display:none">Retry Status</button>
|
||||
<button class="btn btn-reboot" id="btn-retry-update" style="display:none">Retry Update</button>
|
||||
<button class="btn btn-reboot" id="btn-reboot" style="display:none">Restart Entire System</button>
|
||||
<button class="btn btn-close-modal" id="btn-close-modal" disabled>Close</button>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user