diff --git a/CHANGELOG.md b/CHANGELOG.md index e75b7d8..aca01f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 polling failures instead of hanging indefinitely. - Rebuild/update scripts stream `nixos-rebuild` output into the live log (it was buffered until completion, making long rebuilds look frozen). + - The update modal now reconciles persisted update state after a page/RDP + reconnect, bounds every status request with a timeout, prevents overlapping + async polls, and reports an explicit "status unavailable" state with a + Retry Status action instead of spinning forever. Verbose Nix logs are + rendered incrementally and bounded so they cannot stall the browser UI. + - Successful staged updates now record the exact NixOS generation. The Hub + keeps showing "Restart required" until that generation is active, then + clears the marker after reboot (with log-based recovery for older updates). --- diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index 16e29cd..2c96e86 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -55,6 +55,7 @@ from .security_helpers import ( load_session_store, save_session_store, ) +from .update_state import staged_generation_is_active logger = logging.getLogger(__name__) @@ -64,9 +65,10 @@ FLAKE_LOCK_PATH = "/etc/nixos/flake.lock" FLAKE_INPUT_NAME = "Sovran_Systems" GITEA_API_BASE = "https://git.sovransystems.com/api/v1/repos/Sovran_Systems/Sovran_SystemsOS/commits" -UPDATE_LOG = "/var/log/sovran-hub-update.log" -UPDATE_STATUS = "/var/log/sovran-hub-update.status" -UPDATE_UNIT = "sovran-hub-update.service" +UPDATE_LOG = "/var/log/sovran-hub-update.log" +UPDATE_STATUS = "/var/log/sovran-hub-update.status" +UPDATE_GENERATION = "/var/log/sovran-hub-update.generation" +UPDATE_UNIT = "sovran-hub-update.service" REBUILD_LOG = "/var/log/sovran-hub-rebuild.log" REBUILD_STATUS = "/var/log/sovran-hub-rebuild.status" @@ -1634,15 +1636,6 @@ def _nwc_lnurl_bech32(alias: str, domain: str) -> str: # ── Update helpers (file-based, no systemctl) ──────────────────── -def _read_update_status() -> str: - """Read the status file. Returns RUNNING, SUCCESS, REBOOT_REQUIRED, FAILED, or IDLE.""" - try: - with open(UPDATE_STATUS, "r") as f: - return f.read().strip() - except FileNotFoundError: - return "IDLE" - - def _write_update_status(status: str): """Write to the status file.""" try: @@ -1652,6 +1645,34 @@ def _write_update_status(status: str): pass +def _read_update_status() -> str: + """Read and reconcile the persistent update status. + + ``REBOOT_REQUIRED`` intentionally survives Hub/browser restarts before the + reboot. Once the staged generation is the running ``/run/current-system``, + clear that marker so a completed reboot cannot leave the Hub asking for + another reboot forever. The generation helper can recover older updates + from the final nixos-rebuild log line when no explicit marker exists. + """ + try: + with open(UPDATE_STATUS, "r") as f: + status = f.read().strip() + except FileNotFoundError: + return "IDLE" + + if status == "REBOOT_REQUIRED" and staged_generation_is_active( + UPDATE_GENERATION, UPDATE_LOG + ): + _write_update_status("IDLE") + try: + os.remove(UPDATE_GENERATION) + except OSError: + pass + return "IDLE" + + return status + + def _read_log(offset: int = 0) -> tuple[str, int]: """Read the update log file from the given byte offset. Returns (new_text, new_offset).""" @@ -3832,9 +3853,15 @@ async def api_ports_health(): @app.get("/api/updates/check") async def api_updates_check(): loop = asyncio.get_event_loop() + status = await loop.run_in_executor(None, _read_update_status) + if status in {"RUNNING", "REBOOT_REQUIRED"}: + # Avoid a slow remote update check when there is already an operation + # the dashboard needs to surface. + return {"available": True, "status": status.lower()} + available = await loop.run_in_executor(None, check_for_updates) # None means inconclusive (check failed) — report as available so the UI doesn't block - return {"available": available is not False} + return {"available": available is not False, "status": status.lower()} @app.get("/api/ping") diff --git a/app/sovran_systemsos_web/static/js/constants.js b/app/sovran_systemsos_web/static/js/constants.js index fe0b080..91c56bd 100644 --- a/app/sovran_systemsos_web/static/js/constants.js +++ b/app/sovran_systemsos_web/static/js/constants.js @@ -4,11 +4,17 @@ const POLL_INTERVAL_SERVICES = 5000; const POLL_INTERVAL_UPDATES = 1800000; -const UPDATE_POLL_INTERVAL = 2000; -// Max consecutive failed rebuild/update status polls before the page gives up -// waiting and reloads to re-sync (2s interval → ~2 minutes of failures). -// A brief Hub restart during activation only causes a handful of failures. -const STATUS_POLL_MAX_FAILURES = 60; +const UPDATE_POLL_INTERVAL = 2000; +// A pending fetch never rejects by itself. Bound every status request so a +// wedged browser connection cannot leave the modal spinning forever. +const STATUS_POLL_FETCH_TIMEOUT = 15000; +// Eight timed-out requests plus the poll interval is a little over two minutes. +// A brief Hub restart or a heavily loaded Nix build remains well inside this. +const STATUS_POLL_MAX_FAILURES = 8; +// Keep verbose Nix output from making textContent updates quadratic and +// freezing the Hub renderer (especially noticeable over RDP). +const UPDATE_VISIBLE_LOG_MAX_CHARS = 250000; +const UPDATE_VISIBLE_LOG_TRIM_CHARS = 200000; const REBOOT_CHECK_INTERVAL = 5000; const REBOOT_FETCH_TIMEOUT = 12000; const REBOOT_REQUEST_TIMEOUT = 4000; diff --git a/app/sovran_systemsos_web/static/js/events.js b/app/sovran_systemsos_web/static/js/events.js index 35ed367..58bd35c 100644 --- a/app/sovran_systemsos_web/static/js/events.js +++ b/app/sovran_systemsos_web/static/js/events.js @@ -6,6 +6,16 @@ if ($btnCloseModal) $btnCloseModal.addEventListener("click", closeUpdateModal); if ($btnReboot) $btnReboot.addEventListener("click", doReboot); if ($btnSave) $btnSave.addEventListener("click", saveErrorReport); +if ($btnRetryUpdate) $btnRetryUpdate.addEventListener("click", retryUpdateStatus); + +// Browser timers and requests may be suspended while an RDP session/tab is in +// the background. Reconcile immediately when the user returns instead of +// waiting for the next interval. +window.addEventListener("focus", resumeUpdateStatusAfterInterruption); +window.addEventListener("online", resumeUpdateStatusAfterInterruption); +document.addEventListener("visibilitychange", function() { + if (document.visibilityState === "visible") resumeUpdateStatusAfterInterruption(); +}); if ($credsCloseBtn) $credsCloseBtn.addEventListener("click", closeCredsModal); if ($supportCloseBtn) $supportCloseBtn.addEventListener("click", closeSupportModal); @@ -248,6 +258,11 @@ async function init() { setInterval(checkUpdates, POLL_INTERVAL_UPDATES); loadAutolaunchToggle(); } + + // If the page was reloaded or the RDP/browser session resumed during an + // update, reopen the modal from the persisted backend state. This also + // surfaces a completed update that is waiting for its activation reboot. + await restoreUpdateModalIfNeeded(); } document.addEventListener("DOMContentLoaded", init); \ No newline at end of file diff --git a/app/sovran_systemsos_web/static/js/helpers.js b/app/sovran_systemsos_web/static/js/helpers.js index 69ddb59..974711b 100644 --- a/app/sovran_systemsos_web/static/js/helpers.js +++ b/app/sovran_systemsos_web/static/js/helpers.js @@ -144,3 +144,22 @@ async function apiFetch(path, options) { } return res.json(); } + +async function apiFetchWithTimeout(path, options, timeoutMs) { + var controller = new AbortController(); + var fetchOptions = Object.assign({}, options || {}); + fetchOptions.signal = controller.signal; + var timer = setTimeout(function() { controller.abort(); }, timeoutMs); + try { + return await apiFetch(path, fetchOptions); + } catch (err) { + if (controller.signal.aborted) { + var timeoutError = new Error("Request timed out"); + timeoutError.name = "TimeoutError"; + throw timeoutError; + } + throw err; + } finally { + clearTimeout(timer); + } +} diff --git a/app/sovran_systemsos_web/static/js/rebuild.js b/app/sovran_systemsos_web/static/js/rebuild.js index bb1df61..df606f1 100644 --- a/app/sovran_systemsos_web/static/js/rebuild.js +++ b/app/sovran_systemsos_web/static/js/rebuild.js @@ -8,6 +8,7 @@ function openRebuildModal() { _rebuildLogOffset = 0; _rebuildServerDown = false; _rebuildFinished = false; + _rebuildPollInFlight = false; _rebuildPollFailures = 0; if ($rebuildLog) { $rebuildLog.textContent = ""; $rebuildLog.style.display = "none"; } var action = _rebuildIsEnabling ? "Enabling" : "Disabling"; @@ -34,6 +35,7 @@ function appendRebuildLog(text) { } function startRebuildPoll() { + if (_rebuildPollTimer) clearInterval(_rebuildPollTimer); pollRebuildStatus(); _rebuildPollTimer = setInterval(pollRebuildStatus, UPDATE_POLL_INTERVAL); } @@ -43,9 +45,14 @@ function stopRebuildPoll() { } async function pollRebuildStatus() { - if (_rebuildFinished) return; + if (_rebuildFinished || _rebuildPollInFlight) return; + _rebuildPollInFlight = true; try { - var data = await apiFetch("/api/rebuild/status?offset=" + _rebuildLogOffset); + var data = await apiFetchWithTimeout( + "/api/rebuild/status?offset=" + _rebuildLogOffset, + { cache: "no-store" }, + STATUS_POLL_FETCH_TIMEOUT + ); _rebuildPollFailures = 0; if (_rebuildServerDown) { _rebuildServerDown = false; } if (data.log) appendRebuildLog(data.log); @@ -61,10 +68,8 @@ async function pollRebuildStatus() { } catch (err) { _rebuildPollFailures += 1; // The Hub restarts itself during activation, which briefly drops this poll. - // If polling stays broken long past a normal restart, the page's session - // almost certainly no longer matches the server (e.g. the Hub restarted - // and sessions were not recovered). Reload to re-authenticate and show - // the real feature state instead of spinning forever. + // If polling stays broken long past a normal restart, reload to + // re-authenticate and show the resulting feature state. if (_rebuildPollFailures >= STATUS_POLL_MAX_FAILURES) { _rebuildFinished = true; stopRebuildPoll(); @@ -72,6 +77,8 @@ async function pollRebuildStatus() { return; } if (!_rebuildServerDown) { _rebuildServerDown = true; if ($rebuildStatus) $rebuildStatus.textContent = "Applying changes…"; } + } finally { + _rebuildPollInFlight = false; } } diff --git a/app/sovran_systemsos_web/static/js/state.js b/app/sovran_systemsos_web/static/js/state.js index 709cd7a..4be13fe 100644 --- a/app/sovran_systemsos_web/static/js/state.js +++ b/app/sovran_systemsos_web/static/js/state.js @@ -6,9 +6,12 @@ let _servicesCache = []; let _categoryLabels = {}; let _updateLog = ""; let _updatePollTimer = null; +let _updatePollInFlight = false; let _updateLogOffset = 0; +let _updateVisibleLogChars = 0; let _serverWasDown = false; let _updateFinished = false; +let _updateStatusUnavailable = false; let _updatePollFailures = 0; // consecutive failed update-status polls let _supportTimerInt = null; let _supportEnabledAt = null; @@ -24,6 +27,7 @@ let _featuresData = null; let _rebuildLog = ""; let _rebuildLogOffset = 0; let _rebuildPollTimer = null; +let _rebuildPollInFlight = false; let _rebuildFinished = false; let _rebuildServerDown = false; let _rebuildPollFailures = 0; // consecutive failed rebuild-status polls @@ -48,6 +52,7 @@ const $modalStatus = document.getElementById("modal-status"); const $modalLog = document.getElementById("modal-log"); const $btnReboot = document.getElementById("btn-reboot"); const $btnSave = document.getElementById("btn-save-report"); +const $btnRetryUpdate = document.getElementById("btn-retry-update-status"); const $btnCloseModal = document.getElementById("btn-close-modal"); const $rebootOverlay = document.getElementById("reboot-overlay"); diff --git a/app/sovran_systemsos_web/static/js/tiles.js b/app/sovran_systemsos_web/static/js/tiles.js index 5a24772..d432d35 100644 --- a/app/sovran_systemsos_web/static/js/tiles.js +++ b/app/sovran_systemsos_web/static/js/tiles.js @@ -271,10 +271,19 @@ async function checkUpdates() { try { var data = await apiFetch("/api/updates/check"); var hasUpdates = !!data.available; + var updateStatus = data.status || "idle"; var sidebarUpdateBtn = document.getElementById("sidebar-btn-update"); var sidebarUpdateHint = document.getElementById("sidebar-update-hint"); if (sidebarUpdateBtn) { - if (hasUpdates) { + if (updateStatus === "reboot_required") { + sidebarUpdateBtn.style.borderColor = "#e5a50a"; + sidebarUpdateBtn.style.backgroundColor = "rgba(229, 165, 10, 0.10)"; + if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Restart required"; + } else if (updateStatus === "running") { + sidebarUpdateBtn.style.borderColor = "#3584e4"; + sidebarUpdateBtn.style.backgroundColor = "rgba(53, 132, 228, 0.10)"; + if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Update in progress…"; + } else if (hasUpdates) { sidebarUpdateBtn.style.borderColor = "#2ec27e"; sidebarUpdateBtn.style.backgroundColor = "rgba(46, 194, 126, 0.08)"; if (sidebarUpdateHint) sidebarUpdateHint.textContent = "Updates available!"; diff --git a/app/sovran_systemsos_web/static/js/update.js b/app/sovran_systemsos_web/static/js/update.js index 4c0aff9..7d7b9f3 100644 --- a/app/sovran_systemsos_web/static/js/update.js +++ b/app/sovran_systemsos_web/static/js/update.js @@ -2,20 +2,45 @@ // ── Update modal ────────────────────────────────────────────────── -function openUpdateModal() { +async function openUpdateModal() { if (!$modal) return; - apiFetch("/api/updates/check") + + // Reattach before checking for new updates. This makes a browser reload, + // RDP reconnect, or suspended tab recover the authoritative systemd-backed + // state instead of starting over or claiming the system is merely up to date. + try { + var current = await apiFetchWithTimeout( + "/api/updates/status?offset=0", + { cache: "no-store" }, + STATUS_POLL_FETCH_TIMEOUT + ); + if (current.running || current.result === "reboot_required") { + showExistingUpdate(current); + return; + } + } catch (_) { + // The normal start path below has its own visible error handling. + } + + apiFetchWithTimeout( + "/api/updates/check", + { cache: "no-store" }, + STATUS_POLL_FETCH_TIMEOUT + ) .then(function(data) { if (!data.available) { stopUpdatePoll(); _updateLog = ""; _updateLogOffset = 0; + _updateVisibleLogChars = 0; _updateFinished = true; + _updateStatusUnavailable = false; if ($modalLog) $modalLog.textContent = ""; if ($modalStatus) $modalStatus.textContent = "✓ System is already up to date"; if ($modalSpinner) $modalSpinner.classList.remove("spinning"); if ($btnReboot) $btnReboot.style.display = "none"; if ($btnSave) $btnSave.style.display = "none"; + if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = false; $modal.classList.add("open"); return; @@ -27,23 +52,69 @@ function openUpdateModal() { }); } -function _doOpenUpdateModal() { +function prepareUpdateModal() { if (!$modal) return; + stopUpdatePoll(); _updateLog = ""; _updateLogOffset = 0; + _updateVisibleLogChars = 0; + _updatePollInFlight = false; _serverWasDown = false; _updateFinished = false; + _updateStatusUnavailable = false; _updatePollFailures = 0; if ($modalLog) $modalLog.textContent = ""; if ($modalStatus) $modalStatus.textContent = "Starting update…"; if ($modalSpinner) $modalSpinner.classList.add("spinning"); if ($btnReboot) $btnReboot.style.display = "none"; if ($btnSave) $btnSave.style.display = "none"; + if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = true; $modal.classList.add("open"); +} + +function _doOpenUpdateModal() { + prepareUpdateModal(); startUpdate(); } +function showExistingUpdate(data) { + prepareUpdateModal(); + if (data.log) appendLog(data.log); + _updateLogOffset = Number(data.offset) || 0; + + if (data.running) { + if ($modalStatus) $modalStatus.textContent = "Updating…"; + startUpdatePoll(); + return; + } + + _updateFinished = true; + if (data.result === "reboot_required") { + onUpdateDone("reboot_required"); + } else if (data.result === "success") { + onUpdateDone(true); + } else { + onUpdateDone(false); + } +} + +async function restoreUpdateModalIfNeeded() { + if (!$modal || $modal.classList.contains("open")) return; + try { + var data = await apiFetchWithTimeout( + "/api/updates/status?offset=0", + { cache: "no-store" }, + STATUS_POLL_FETCH_TIMEOUT + ); + if (data.running || data.result === "reboot_required") { + showExistingUpdate(data); + } + } catch (_) { + // Dashboard startup must remain usable when status cannot be reached. + } +} + function closeUpdateModal() { if (!$modal) return; $modal.classList.remove("open"); @@ -53,21 +124,36 @@ function closeUpdateModal() { function appendLog(text) { if (!text) return; _updateLog += text; - if ($modalLog) { $modalLog.textContent += text; $modalLog.scrollTop = $modalLog.scrollHeight; } + if ($modalLog) { + // Appending a text node avoids reparsing/replacing the complete log on + // every two-second poll. Trim only occasionally once the visible log is + // large; the complete _updateLog remains available for error reports. + if (_updateVisibleLogChars + text.length > UPDATE_VISIBLE_LOG_MAX_CHARS) { + var tail = _updateLog.slice(-UPDATE_VISIBLE_LOG_TRIM_CHARS); + var notice = "[Earlier update output hidden from this view; it remains in the saved report.]\n\n"; + $modalLog.textContent = notice + tail; + _updateVisibleLogChars = notice.length + tail.length; + } else { + $modalLog.appendChild(document.createTextNode(text)); + _updateVisibleLogChars += text.length; + } + $modalLog.scrollTop = $modalLog.scrollHeight; + } } function startUpdate() { - fetch("/api/updates/run", { method: "POST" }) - .then(function(response) { - if (!response.ok) return response.text().then(function(t) { throw new Error(t); }); - return response.json(); - }) + apiFetchWithTimeout( + "/api/updates/run", + { method: "POST" }, + STATUS_POLL_FETCH_TIMEOUT * 2 + ) .then(function(data) { if (data.status === "no_updates") { if ($modalStatus) $modalStatus.textContent = "✓ System is already up to date"; if ($modalSpinner) $modalSpinner.classList.remove("spinning"); if ($btnReboot) $btnReboot.style.display = "none"; if ($btnSave) $btnSave.style.display = "none"; + if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = false; _updateFinished = true; return; @@ -83,6 +169,7 @@ function startUpdate() { } function startUpdatePoll() { + if (_updatePollTimer) clearInterval(_updatePollTimer); pollUpdateStatus(); _updatePollTimer = setInterval(pollUpdateStatus, UPDATE_POLL_INTERVAL); } @@ -92,33 +179,45 @@ function stopUpdatePoll() { } async function pollUpdateStatus() { - if (_updateFinished) return; + // setInterval does not wait for an async callback. The guard prevents a slow + // request from creating overlapping, out-of-order status polls. + if (_updateFinished || _updatePollInFlight) return; + _updatePollInFlight = true; try { - var data = await apiFetch("/api/updates/status?offset=" + _updateLogOffset); + var data = await apiFetchWithTimeout( + "/api/updates/status?offset=" + _updateLogOffset, + { cache: "no-store" }, + STATUS_POLL_FETCH_TIMEOUT + ); _updatePollFailures = 0; if (_serverWasDown) { _serverWasDown = false; if (!data.running) { - // The update finished while the server was restarting. Reset to - // offset 0 and re-fetch so the complete log is shown from the top. + // The update finished while the server or browser connection was away. + // Re-fetch from offset 0 so the final result and complete tail agree. _updateLog = ""; _updateLogOffset = 0; + _updateVisibleLogChars = 0; if ($modalLog) $modalLog.textContent = ""; try { - var fullData = await apiFetch("/api/updates/status?offset=0"); + var fullData = await apiFetchWithTimeout( + "/api/updates/status?offset=0", + { cache: "no-store" }, + STATUS_POLL_FETCH_TIMEOUT + ); if (fullData.log) appendLog(fullData.log); _updateLogOffset = fullData.offset; - } catch (e) { - // If the re-fetch fails, fall through with whatever we have. + data = fullData; + } catch (_) { if (data.log) appendLog(data.log); _updateLogOffset = data.offset; } if (data.result === "reboot_required") { - appendLog("[Server restarted — update completed, reboot required.]\n"); + appendLog("[Reconnected — update completed, reboot required.]\n"); } else if (data.result === "success") { - appendLog("[Server restarted — update completed successfully.]\n"); + appendLog("[Reconnected — update completed successfully.]\n"); } else { - appendLog("[Server restarted — update encountered an error.]\n"); + appendLog("[Reconnected — update encountered an error.]\n"); } _updateFinished = true; stopUpdatePoll(); @@ -129,7 +228,7 @@ async function pollUpdateStatus() { } return; } - appendLog("[Server reconnected]\n"); + appendLog("[Update status reconnected]\n"); if ($modalStatus) $modalStatus.textContent = "Updating…"; } if (data.log) appendLog(data.log); @@ -146,21 +245,57 @@ async function pollUpdateStatus() { } } catch (err) { _updatePollFailures += 1; - // Same guard as the rebuild modal: if polling stays broken long past a - // normal Hub restart, reload to re-authenticate and show the real state - // instead of spinning forever. if (_updatePollFailures >= STATUS_POLL_MAX_FAILURES) { - _updateFinished = true; - stopUpdatePoll(); - window.location.reload(); + showUpdateStatusUnavailable(); return; } - if (!_serverWasDown) { _serverWasDown = true; appendLog("\n[Server restarting — waiting for it to come back…]\n"); if ($modalStatus) $modalStatus.textContent = "Server restarting…"; } + if (!_serverWasDown) { + _serverWasDown = true; + appendLog("\n[Update status connection interrupted — retrying…]\n"); + if ($modalStatus) $modalStatus.textContent = "Reconnecting to update…"; + } + } finally { + _updatePollInFlight = false; + } +} + +function showUpdateStatusUnavailable() { + _updateFinished = true; + _updateStatusUnavailable = true; + stopUpdatePoll(); + if ($modalSpinner) $modalSpinner.classList.remove("spinning"); + if ($modalStatus) $modalStatus.textContent = "Update status unavailable — update may still be running"; + appendLog("\n[The Hub could not confirm update status. The background update was not stopped. Select Retry Status after reconnecting.]\n"); + if ($btnRetryUpdate) $btnRetryUpdate.style.display = "inline-flex"; + if ($btnCloseModal) $btnCloseModal.disabled = false; +} + +function retryUpdateStatus() { + if (!$modal) return; + _updateFinished = false; + _updateStatusUnavailable = false; + _updatePollFailures = 0; + _serverWasDown = true; + if ($modalSpinner) $modalSpinner.classList.add("spinning"); + if ($modalStatus) $modalStatus.textContent = "Reconnecting to update…"; + if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; + if ($btnCloseModal) $btnCloseModal.disabled = true; + startUpdatePoll(); +} + +function resumeUpdateStatusAfterInterruption() { + if (!$modal || !$modal.classList.contains("open")) return; + if (_updateStatusUnavailable) { + retryUpdateStatus(); + } else if (!_updateFinished) { + pollUpdateStatus(); } } function onUpdateDone(result) { + _updateStatusUnavailable = false; if ($modalSpinner) $modalSpinner.classList.remove("spinning"); + if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none"; if ($btnCloseModal) $btnCloseModal.disabled = false; if (result === true) { if ($modalStatus) $modalStatus.textContent = "✓ Update complete"; @@ -187,6 +322,7 @@ function saveErrorReport() { URL.revokeObjectURL(url); } + // ── Reboot ──────────────────────────────────────────────────────── var _rebootStartTime = 0; diff --git a/app/sovran_systemsos_web/templates/index.html b/app/sovran_systemsos_web/templates/index.html index 66f9646..3e3c447 100644 --- a/app/sovran_systemsos_web/templates/index.html +++ b/app/sovran_systemsos_web/templates/index.html @@ -74,6 +74,7 @@
diff --git a/app/sovran_systemsos_web/update_state.py b/app/sovran_systemsos_web/update_state.py new file mode 100644 index 0000000..5c68c5e --- /dev/null +++ b/app/sovran_systemsos_web/update_state.py @@ -0,0 +1,81 @@ +"""Persistent update-state helpers for the Sovran Hub. + +The full-system updater stages a NixOS generation with ``nixos-rebuild boot``. +That generation is not active until the machine reboots. These helpers let the +Hub distinguish a genuinely pending reboot from an old REBOOT_REQUIRED marker +that survived the reboot. + +This module deliberately has no FastAPI or systemd dependencies so its state +reconciliation can be tested without importing the Hub server. +""" + +from __future__ import annotations + +import os +import re + + +# Nix store hashes use the lower-case Nix base32 alphabet. Keep the output +# name deliberately conservative: a system generation has no path separators. +_SYSTEM_GENERATION_RE = re.compile( + r"^/nix/store/[0-9a-z]{32}-nixos-system-[A-Za-z0-9._+\-]+$" +) +_LOG_GENERATION_RE = re.compile( + r"The new configuration is " + r"(/nix/store/[0-9a-z]{32}-nixos-system-[A-Za-z0-9._+\-]+)" +) + + +def _valid_generation(value: str) -> str | None: + """Return a normalized NixOS generation path, or ``None`` if invalid.""" + candidate = value.strip() + if _SYSTEM_GENERATION_RE.fullmatch(candidate): + return candidate + return None + + +def read_staged_generation(marker_path: str, log_path: str) -> str | None: + """Read the generation staged by the last successful Hub update. + + New updater versions write ``marker_path`` explicitly. For an update that + started with an older updater, recover the same value from the final + ``nixos-rebuild`` log line. Only the tail is needed and bounding the read + avoids loading a potentially large build log during every status poll. + """ + try: + with open(marker_path, "r", encoding="utf-8") as marker: + generation = _valid_generation(marker.read()) + if generation: + return generation + except OSError: + pass + + try: + with open(log_path, "rb") as log: + log.seek(0, os.SEEK_END) + size = log.tell() + log.seek(max(0, size - 131_072), os.SEEK_SET) + tail = log.read().decode("utf-8", errors="replace") + except OSError: + return None + + matches = list(_LOG_GENERATION_RE.finditer(tail)) + if not matches: + return None + return _valid_generation(matches[-1].group(1)) + + +def staged_generation_is_active( + marker_path: str, + log_path: str, + current_system_path: str = "/run/current-system", +) -> bool: + """Return whether the staged update generation is now the running system.""" + staged = read_staged_generation(marker_path, log_path) + if not staged: + return False + try: + current = os.path.realpath(current_system_path) + except OSError: + return False + return current == staged diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index fec5cb6..ef6133d 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -160,8 +160,10 @@ let LOG="/var/log/sovran-hub-update.log" STATUS="/var/log/sovran-hub-update.status" + GENERATION="/var/log/sovran-hub-update.generation" echo "RUNNING" > "$STATUS" + rm -f "$GENERATION" : > "$LOG" exec > >(tee -a "$LOG") 2>&1 @@ -196,6 +198,10 @@ let if [ "$BOOT_RC" -ne 0 ]; then echo "[ERROR] nixos-rebuild boot failed" RC=1 + elif ! readlink -f /nix/var/nix/profiles/system > "$GENERATION"; then + echo "[ERROR] update was built but its staged generation could not be recorded" + rm -f "$GENERATION" + RC=1 fi echo "" fi diff --git a/tests/test_update_state.py b/tests/test_update_state.py new file mode 100644 index 0000000..322896a --- /dev/null +++ b/tests/test_update_state.py @@ -0,0 +1,158 @@ +"""Regression tests for Hub update completion and polling recovery.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_APP_PARENT = _REPO_ROOT / "app" +if str(_APP_PARENT) not in sys.path: + sys.path.insert(0, str(_APP_PARENT)) + +from sovran_systemsos_web.update_state import ( # noqa: E402 + read_staged_generation, + staged_generation_is_active, +) + + +GENERATION = ( + "/nix/store/rmi0g35cd8w60k0ig7pm6kb8kzws8b7x-" + "nixos-system-nixos-26.11.20260817.ec2d622" +) +OTHER_GENERATION = ( + "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-" + "nixos-system-nixos-26.11.20260816.old" +) + + +class TestStagedGenerationState(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + root = Path(self.tmp.name) + self.marker = root / "update.generation" + self.log = root / "update.log" + self.current = root / "current-system" + + def test_explicit_generation_marker_is_read(self): + self.marker.write_text(GENERATION + "\n", encoding="utf-8") + self.assertEqual( + read_staged_generation(str(self.marker), str(self.log)), GENERATION + ) + + def test_legacy_updater_generation_is_recovered_from_log_tail(self): + self.log.write_text( + "building the system configuration...\n" + f"Done. The new configuration is {GENERATION}\n" + "✓ Update staged successfully\n", + encoding="utf-8", + ) + self.assertEqual( + read_staged_generation(str(self.marker), str(self.log)), GENERATION + ) + + def test_latest_generation_line_wins(self): + self.log.write_text( + f"Done. The new configuration is {OTHER_GENERATION}\n" + f"Done. The new configuration is {GENERATION}\n", + encoding="utf-8", + ) + self.assertEqual( + read_staged_generation(str(self.marker), str(self.log)), GENERATION + ) + + def test_invalid_marker_is_ignored(self): + self.marker.write_text("/tmp/not-a-generation\n", encoding="utf-8") + self.log.write_text("no completed generation\n", encoding="utf-8") + self.assertIsNone(read_staged_generation(str(self.marker), str(self.log))) + + def test_staged_generation_is_active_after_reboot(self): + self.marker.write_text(GENERATION + "\n", encoding="utf-8") + os.symlink(GENERATION, self.current) + self.assertTrue( + staged_generation_is_active( + str(self.marker), str(self.log), str(self.current) + ) + ) + + def test_staged_generation_remains_pending_before_reboot(self): + self.marker.write_text(GENERATION + "\n", encoding="utf-8") + os.symlink(OTHER_GENERATION, self.current) + self.assertFalse( + staged_generation_is_active( + str(self.marker), str(self.log), str(self.current) + ) + ) + + +class TestUpdatePollingWiring(unittest.TestCase): + """Guard the browser failure modes that caused a permanent spinner.""" + + @classmethod + def setUpClass(cls): + js_dir = _REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" + cls.update_js = (js_dir / "update.js").read_text(encoding="utf-8") + cls.rebuild_js = (js_dir / "rebuild.js").read_text(encoding="utf-8") + cls.helpers_js = (js_dir / "helpers.js").read_text(encoding="utf-8") + cls.events_js = (js_dir / "events.js").read_text(encoding="utf-8") + cls.template = ( + _REPO_ROOT / "app" / "sovran_systemsos_web" / "templates" / "index.html" + ).read_text(encoding="utf-8") + + def test_status_fetches_have_an_abort_timeout(self): + self.assertIn("function apiFetchWithTimeout", self.helpers_js) + self.assertIn("new AbortController()", self.helpers_js) + self.assertIn("controller.abort()", self.helpers_js) + self.assertIn("apiFetchWithTimeout(", self.update_js) + self.assertIn("STATUS_POLL_FETCH_TIMEOUT", self.update_js) + + def test_async_update_polls_cannot_overlap(self): + self.assertIn("_updatePollInFlight", self.update_js) + self.assertIn( + "if (_updateFinished || _updatePollInFlight) return;", self.update_js + ) + self.assertIn("finally", self.update_js) + + def test_connection_failure_has_explicit_non_running_ui(self): + self.assertIn("showUpdateStatusUnavailable", self.update_js) + self.assertIn("Update status unavailable", self.update_js) + self.assertIn("Retry Status", self.template) + self.assertIn("retryUpdateStatus", self.events_js) + + def test_rdp_or_tab_resume_forces_reconciliation(self): + self.assertIn("resumeUpdateStatusAfterInterruption", self.events_js) + self.assertIn('window.addEventListener("focus"', self.events_js) + self.assertIn('document.addEventListener("visibilitychange"', self.events_js) + + def test_verbose_log_rendering_is_bounded(self): + self.assertIn("UPDATE_VISIBLE_LOG_MAX_CHARS", self.update_js) + self.assertIn("document.createTextNode(text)", self.update_js) + self.assertNotIn("$modalLog.textContent += text", self.update_js) + + def test_page_reload_restores_running_or_completed_update(self): + self.assertIn("restoreUpdateModalIfNeeded", self.update_js) + self.assertIn("await restoreUpdateModalIfNeeded();", self.events_js) + self.assertIn('current.result === "reboot_required"', self.update_js) + + def test_all_javascript_remains_syntax_valid(self): + node = shutil.which("node") + if not node: + self.skipTest("node is not available in this test environment") + for script in ( + _REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" + ).glob("*.js"): + result = subprocess.run( + [node, "--check", str(script)], capture_output=True, text=True + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main()