fix(hub): reconcile completed updates after polling stalls
The full-system updater runs as a detached systemd service and can finish successfully even when the browser loses its status connection. In that case the update log and status file correctly report REBOOT_REQUIRED, but the Hub modal can remain on "Updating..." with its controls disabled. There were four independent ways for the frontend to get stuck: * update status fetches had no deadline, so a request that stayed pending never rejected and never advanced the existing failure counter; * setInterval started async polls without waiting for the previous poll, allowing slow requests to overlap and responses to arrive out of order; * each log chunk used textContent +=, replacing the complete and growing Nix build log every two seconds, which could stall browser rendering and was especially visible over RDP; and * page reload, tab resume, and RDP reconnect did not reattach the modal to the update status persisted by the backend. This produced a dangerous UX mismatch: the machine had a fully staged NixOS generation and was ready to reboot, while the Hub continued telling the user that the update was still running. Bound status requests with AbortController, prevent overlapping polls, and replace the endless spinner after sustained failures with an explicit "Update status unavailable" state and Retry Status action. Reconcile state immediately on focus, visibility, online, page startup, and before starting a new update. Use no-store requests and render verbose logs incrementally with a bounded visible tail while retaining the complete report in memory. Apply the same timeout and single-flight protection to rebuild polling. Record the exact generation produced by `nixos-rebuild boot`. The Hub now keeps REBOOT_REQUIRED visible until that generation matches /run/current-system, then clears the marker after reboot. For an update started by an older updater that did not write the marker, recover the staged generation from the final nixos-rebuild log line. The dashboard sidebar also distinguishes update-in-progress and restart-required states. Regression coverage verifies generation marker/log recovery, pre- versus post-reboot detection, request timeout wiring, single-flight polling, connection-loss UX, RDP/tab resume reconciliation, bounded log rendering, page-reload recovery, and JavaScript syntax. Validation: * python3 -m unittest discover -s tests -p 'test_*.py' -v (170 passed) * node --check app/sovran_systemsos_web/static/js/*.js * python3 -m py_compile for changed Python modules * git diff --check A Nix evaluation was not available in the development sandbox; the NixOS module should still be evaluated and built in CI or on a test machine before release.
This commit is contained in:
naturallaw777
@@ -2,20 +2,45 @@
|
||||
|
||||
// ── Update modal ──────────────────────────────────────────────────
|
||||
|
||||
function openUpdateModal() {
|
||||
async function openUpdateModal() {
|
||||
if (!$modal) return;
|
||||
apiFetch("/api/updates/check")
|
||||
|
||||
// Reattach before checking for new updates. This makes a browser reload,
|
||||
// RDP reconnect, or suspended tab recover the authoritative systemd-backed
|
||||
// state instead of starting over or claiming the system is merely up to date.
|
||||
try {
|
||||
var current = await apiFetchWithTimeout(
|
||||
"/api/updates/status?offset=0",
|
||||
{ cache: "no-store" },
|
||||
STATUS_POLL_FETCH_TIMEOUT
|
||||
);
|
||||
if (current.running || current.result === "reboot_required") {
|
||||
showExistingUpdate(current);
|
||||
return;
|
||||
}
|
||||
} catch (_) {
|
||||
// The normal start path below has its own visible error handling.
|
||||
}
|
||||
|
||||
apiFetchWithTimeout(
|
||||
"/api/updates/check",
|
||||
{ cache: "no-store" },
|
||||
STATUS_POLL_FETCH_TIMEOUT
|
||||
)
|
||||
.then(function(data) {
|
||||
if (!data.available) {
|
||||
stopUpdatePoll();
|
||||
_updateLog = "";
|
||||
_updateLogOffset = 0;
|
||||
_updateVisibleLogChars = 0;
|
||||
_updateFinished = true;
|
||||
_updateStatusUnavailable = false;
|
||||
if ($modalLog) $modalLog.textContent = "";
|
||||
if ($modalStatus) $modalStatus.textContent = "✓ System is already up to date";
|
||||
if ($modalSpinner) $modalSpinner.classList.remove("spinning");
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = false;
|
||||
$modal.classList.add("open");
|
||||
return;
|
||||
@@ -27,23 +52,69 @@ function openUpdateModal() {
|
||||
});
|
||||
}
|
||||
|
||||
function _doOpenUpdateModal() {
|
||||
function prepareUpdateModal() {
|
||||
if (!$modal) return;
|
||||
stopUpdatePoll();
|
||||
_updateLog = "";
|
||||
_updateLogOffset = 0;
|
||||
_updateVisibleLogChars = 0;
|
||||
_updatePollInFlight = false;
|
||||
_serverWasDown = false;
|
||||
_updateFinished = false;
|
||||
_updateStatusUnavailable = false;
|
||||
_updatePollFailures = 0;
|
||||
if ($modalLog) $modalLog.textContent = "";
|
||||
if ($modalStatus) $modalStatus.textContent = "Starting update…";
|
||||
if ($modalSpinner) $modalSpinner.classList.add("spinning");
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = true;
|
||||
$modal.classList.add("open");
|
||||
}
|
||||
|
||||
function _doOpenUpdateModal() {
|
||||
prepareUpdateModal();
|
||||
startUpdate();
|
||||
}
|
||||
|
||||
function showExistingUpdate(data) {
|
||||
prepareUpdateModal();
|
||||
if (data.log) appendLog(data.log);
|
||||
_updateLogOffset = Number(data.offset) || 0;
|
||||
|
||||
if (data.running) {
|
||||
if ($modalStatus) $modalStatus.textContent = "Updating…";
|
||||
startUpdatePoll();
|
||||
return;
|
||||
}
|
||||
|
||||
_updateFinished = true;
|
||||
if (data.result === "reboot_required") {
|
||||
onUpdateDone("reboot_required");
|
||||
} else if (data.result === "success") {
|
||||
onUpdateDone(true);
|
||||
} else {
|
||||
onUpdateDone(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreUpdateModalIfNeeded() {
|
||||
if (!$modal || $modal.classList.contains("open")) return;
|
||||
try {
|
||||
var data = await apiFetchWithTimeout(
|
||||
"/api/updates/status?offset=0",
|
||||
{ cache: "no-store" },
|
||||
STATUS_POLL_FETCH_TIMEOUT
|
||||
);
|
||||
if (data.running || data.result === "reboot_required") {
|
||||
showExistingUpdate(data);
|
||||
}
|
||||
} catch (_) {
|
||||
// Dashboard startup must remain usable when status cannot be reached.
|
||||
}
|
||||
}
|
||||
|
||||
function closeUpdateModal() {
|
||||
if (!$modal) return;
|
||||
$modal.classList.remove("open");
|
||||
@@ -53,21 +124,36 @@ function closeUpdateModal() {
|
||||
function appendLog(text) {
|
||||
if (!text) return;
|
||||
_updateLog += text;
|
||||
if ($modalLog) { $modalLog.textContent += text; $modalLog.scrollTop = $modalLog.scrollHeight; }
|
||||
if ($modalLog) {
|
||||
// Appending a text node avoids reparsing/replacing the complete log on
|
||||
// every two-second poll. Trim only occasionally once the visible log is
|
||||
// large; the complete _updateLog remains available for error reports.
|
||||
if (_updateVisibleLogChars + text.length > UPDATE_VISIBLE_LOG_MAX_CHARS) {
|
||||
var tail = _updateLog.slice(-UPDATE_VISIBLE_LOG_TRIM_CHARS);
|
||||
var notice = "[Earlier update output hidden from this view; it remains in the saved report.]\n\n";
|
||||
$modalLog.textContent = notice + tail;
|
||||
_updateVisibleLogChars = notice.length + tail.length;
|
||||
} else {
|
||||
$modalLog.appendChild(document.createTextNode(text));
|
||||
_updateVisibleLogChars += text.length;
|
||||
}
|
||||
$modalLog.scrollTop = $modalLog.scrollHeight;
|
||||
}
|
||||
}
|
||||
|
||||
function startUpdate() {
|
||||
fetch("/api/updates/run", { method: "POST" })
|
||||
.then(function(response) {
|
||||
if (!response.ok) return response.text().then(function(t) { throw new Error(t); });
|
||||
return response.json();
|
||||
})
|
||||
apiFetchWithTimeout(
|
||||
"/api/updates/run",
|
||||
{ method: "POST" },
|
||||
STATUS_POLL_FETCH_TIMEOUT * 2
|
||||
)
|
||||
.then(function(data) {
|
||||
if (data.status === "no_updates") {
|
||||
if ($modalStatus) $modalStatus.textContent = "✓ System is already up to date";
|
||||
if ($modalSpinner) $modalSpinner.classList.remove("spinning");
|
||||
if ($btnReboot) $btnReboot.style.display = "none";
|
||||
if ($btnSave) $btnSave.style.display = "none";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = false;
|
||||
_updateFinished = true;
|
||||
return;
|
||||
@@ -83,6 +169,7 @@ function startUpdate() {
|
||||
}
|
||||
|
||||
function startUpdatePoll() {
|
||||
if (_updatePollTimer) clearInterval(_updatePollTimer);
|
||||
pollUpdateStatus();
|
||||
_updatePollTimer = setInterval(pollUpdateStatus, UPDATE_POLL_INTERVAL);
|
||||
}
|
||||
@@ -92,33 +179,45 @@ function stopUpdatePoll() {
|
||||
}
|
||||
|
||||
async function pollUpdateStatus() {
|
||||
if (_updateFinished) return;
|
||||
// setInterval does not wait for an async callback. The guard prevents a slow
|
||||
// request from creating overlapping, out-of-order status polls.
|
||||
if (_updateFinished || _updatePollInFlight) return;
|
||||
_updatePollInFlight = true;
|
||||
try {
|
||||
var data = await apiFetch("/api/updates/status?offset=" + _updateLogOffset);
|
||||
var data = await apiFetchWithTimeout(
|
||||
"/api/updates/status?offset=" + _updateLogOffset,
|
||||
{ cache: "no-store" },
|
||||
STATUS_POLL_FETCH_TIMEOUT
|
||||
);
|
||||
_updatePollFailures = 0;
|
||||
if (_serverWasDown) {
|
||||
_serverWasDown = false;
|
||||
if (!data.running) {
|
||||
// The update finished while the server was restarting. Reset to
|
||||
// offset 0 and re-fetch so the complete log is shown from the top.
|
||||
// The update finished while the server or browser connection was away.
|
||||
// Re-fetch from offset 0 so the final result and complete tail agree.
|
||||
_updateLog = "";
|
||||
_updateLogOffset = 0;
|
||||
_updateVisibleLogChars = 0;
|
||||
if ($modalLog) $modalLog.textContent = "";
|
||||
try {
|
||||
var fullData = await apiFetch("/api/updates/status?offset=0");
|
||||
var fullData = await apiFetchWithTimeout(
|
||||
"/api/updates/status?offset=0",
|
||||
{ cache: "no-store" },
|
||||
STATUS_POLL_FETCH_TIMEOUT
|
||||
);
|
||||
if (fullData.log) appendLog(fullData.log);
|
||||
_updateLogOffset = fullData.offset;
|
||||
} catch (e) {
|
||||
// If the re-fetch fails, fall through with whatever we have.
|
||||
data = fullData;
|
||||
} catch (_) {
|
||||
if (data.log) appendLog(data.log);
|
||||
_updateLogOffset = data.offset;
|
||||
}
|
||||
if (data.result === "reboot_required") {
|
||||
appendLog("[Server restarted — update completed, reboot required.]\n");
|
||||
appendLog("[Reconnected — update completed, reboot required.]\n");
|
||||
} else if (data.result === "success") {
|
||||
appendLog("[Server restarted — update completed successfully.]\n");
|
||||
appendLog("[Reconnected — update completed successfully.]\n");
|
||||
} else {
|
||||
appendLog("[Server restarted — update encountered an error.]\n");
|
||||
appendLog("[Reconnected — update encountered an error.]\n");
|
||||
}
|
||||
_updateFinished = true;
|
||||
stopUpdatePoll();
|
||||
@@ -129,7 +228,7 @@ async function pollUpdateStatus() {
|
||||
}
|
||||
return;
|
||||
}
|
||||
appendLog("[Server reconnected]\n");
|
||||
appendLog("[Update status reconnected]\n");
|
||||
if ($modalStatus) $modalStatus.textContent = "Updating…";
|
||||
}
|
||||
if (data.log) appendLog(data.log);
|
||||
@@ -146,21 +245,57 @@ async function pollUpdateStatus() {
|
||||
}
|
||||
} catch (err) {
|
||||
_updatePollFailures += 1;
|
||||
// Same guard as the rebuild modal: if polling stays broken long past a
|
||||
// normal Hub restart, reload to re-authenticate and show the real state
|
||||
// instead of spinning forever.
|
||||
if (_updatePollFailures >= STATUS_POLL_MAX_FAILURES) {
|
||||
_updateFinished = true;
|
||||
stopUpdatePoll();
|
||||
window.location.reload();
|
||||
showUpdateStatusUnavailable();
|
||||
return;
|
||||
}
|
||||
if (!_serverWasDown) { _serverWasDown = true; appendLog("\n[Server restarting — waiting for it to come back…]\n"); if ($modalStatus) $modalStatus.textContent = "Server restarting…"; }
|
||||
if (!_serverWasDown) {
|
||||
_serverWasDown = true;
|
||||
appendLog("\n[Update status connection interrupted — retrying…]\n");
|
||||
if ($modalStatus) $modalStatus.textContent = "Reconnecting to update…";
|
||||
}
|
||||
} finally {
|
||||
_updatePollInFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
function showUpdateStatusUnavailable() {
|
||||
_updateFinished = true;
|
||||
_updateStatusUnavailable = true;
|
||||
stopUpdatePoll();
|
||||
if ($modalSpinner) $modalSpinner.classList.remove("spinning");
|
||||
if ($modalStatus) $modalStatus.textContent = "Update status unavailable — update may still be running";
|
||||
appendLog("\n[The Hub could not confirm update status. The background update was not stopped. Select Retry Status after reconnecting.]\n");
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "inline-flex";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = false;
|
||||
}
|
||||
|
||||
function retryUpdateStatus() {
|
||||
if (!$modal) return;
|
||||
_updateFinished = false;
|
||||
_updateStatusUnavailable = false;
|
||||
_updatePollFailures = 0;
|
||||
_serverWasDown = true;
|
||||
if ($modalSpinner) $modalSpinner.classList.add("spinning");
|
||||
if ($modalStatus) $modalStatus.textContent = "Reconnecting to update…";
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = true;
|
||||
startUpdatePoll();
|
||||
}
|
||||
|
||||
function resumeUpdateStatusAfterInterruption() {
|
||||
if (!$modal || !$modal.classList.contains("open")) return;
|
||||
if (_updateStatusUnavailable) {
|
||||
retryUpdateStatus();
|
||||
} else if (!_updateFinished) {
|
||||
pollUpdateStatus();
|
||||
}
|
||||
}
|
||||
|
||||
function onUpdateDone(result) {
|
||||
_updateStatusUnavailable = false;
|
||||
if ($modalSpinner) $modalSpinner.classList.remove("spinning");
|
||||
if ($btnRetryUpdate) $btnRetryUpdate.style.display = "none";
|
||||
if ($btnCloseModal) $btnCloseModal.disabled = false;
|
||||
if (result === true) {
|
||||
if ($modalStatus) $modalStatus.textContent = "✓ Update complete";
|
||||
@@ -187,6 +322,7 @@ function saveErrorReport() {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
|
||||
// ── Reboot ────────────────────────────────────────────────────────
|
||||
|
||||
var _rebootStartTime = 0;
|
||||
|
||||
Reference in New Issue
Block a user