From 38733daffc3367f802674a16e0d3d631efe23ea5 Mon Sep 17 00:00:00 2001 From: naturallaw77 Date: Thu, 2 Apr 2026 13:09:07 -0500 Subject: [PATCH] updated logging --- app/sovran_systemsos_web/server.py | 70 +++++++++++++++++--------- app/sovran_systemsos_web/static/app.js | 57 +++++++++++++++------ modules/core/sovran-hub.nix | 7 +-- 3 files changed, 93 insertions(+), 41 deletions(-) diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index b454b0a..c5f301c 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -26,7 +26,6 @@ FLAKE_INPUT_NAME = "Sovran_Systems" GITEA_API_BASE = "https://git.sovransystems.com/api/v1/repos/Sovran_Systems/Sovran_SystemsOS/commits" UPDATE_UNIT = "sovran-hub-update.service" -UPDATE_LOG = "/var/log/sovran-hub-update.log" REBOOT_COMMAND = [ "reboot", @@ -172,7 +171,7 @@ def _update_is_active() -> bool: def _update_result() -> str: - """Return 'success', 'failed', or 'inactive'.""" + """Return 'success', 'failed', or 'unknown'.""" r = subprocess.run( ["systemctl", "show", "-p", "Result", "--value", UPDATE_UNIT], capture_output=True, text=True, @@ -182,18 +181,47 @@ def _update_result() -> str: return "success" elif val: return "failed" - return "inactive" + return "unknown" -def _read_update_log(offset: int = 0) -> tuple[str, int]: - """Read update log from offset. Return (new_text, new_offset).""" - try: - with open(UPDATE_LOG, "r") as f: - f.seek(offset) - text = f.read() - return text, f.tell() - except FileNotFoundError: - return "", 0 +def _get_update_invocation_id() -> str: + """Get the current InvocationID of the update unit.""" + r = subprocess.run( + ["systemctl", "show", "-p", "InvocationID", "--value", UPDATE_UNIT], + capture_output=True, text=True, + ) + return r.stdout.strip() + + +def _read_journal_logs(since_cursor: str = "") -> tuple[list[str], str]: + """ + Read journal logs for the update unit. + Returns (lines, last_cursor). + Uses cursors so we never miss lines even if the server restarts. + """ + cmd = [ + "journalctl", "-u", UPDATE_UNIT, + "--no-pager", "-o", "cat", + "--show-cursor", + ] + if since_cursor: + cmd += ["--after-cursor", since_cursor] + else: + # Only get logs from the most recent invocation + cmd += ["-n", "10000"] + + r = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + output = r.stdout + + lines = [] + cursor = since_cursor + for raw_line in output.split("\n"): + if raw_line.startswith("-- cursor: "): + cursor = raw_line[len("-- cursor: "):] + elif raw_line: + lines.append(raw_line) + + return lines, cursor # ── Routes ─────────────────────────────────────────────────────── @@ -333,12 +361,6 @@ async def api_updates_run(): if running: return {"ok": True, "status": "already_running"} - # Clear the old log - try: - open(UPDATE_LOG, "w").close() - except OSError: - pass - # Reset the failed state (if any) and start the unit await asyncio.create_subprocess_exec( "systemctl", "reset-failed", UPDATE_UNIT, @@ -356,17 +378,19 @@ async def api_updates_run(): @app.get("/api/updates/status") -async def api_updates_status(offset: int = 0): - """Poll endpoint: returns running state, result, and new log lines.""" +async def api_updates_status(cursor: str = ""): + """Poll endpoint: returns running state, result, and new journal lines.""" loop = asyncio.get_event_loop() running = await loop.run_in_executor(None, _update_is_active) result = await loop.run_in_executor(None, _update_result) - new_log, new_offset = await loop.run_in_executor(None, _read_update_log, offset) + lines, new_cursor = await loop.run_in_executor( + None, _read_journal_logs, cursor, + ) return { "running": running, "result": result, - "log": new_log, - "offset": new_offset, + "lines": lines, + "cursor": new_cursor, } \ No newline at end of file diff --git a/app/sovran_systemsos_web/static/app.js b/app/sovran_systemsos_web/static/app.js index f1b2e3e..afb2842 100644 --- a/app/sovran_systemsos_web/static/app.js +++ b/app/sovran_systemsos_web/static/app.js @@ -26,7 +26,8 @@ let _categoryLabels = {}; let _updateSource = null; let _updateLog = ""; let _updatePollTimer = null; -let _updateLogOffset = 0; +let _updateCursor = ""; +let _serverDownSince = 0; // ── DOM refs ────────────────────────────────────────────────────── @@ -264,7 +265,8 @@ async function checkUpdates() { function openUpdateModal() { if (!$modal) return; _updateLog = ""; - _updateLogOffset = 0; + _updateCursor = ""; + _serverDownSince = 0; if ($modalLog) $modalLog.textContent = ""; if ($modalStatus) $modalStatus.textContent = "Updating…"; if ($modalSpinner) $modalSpinner.classList.add("spinning"); @@ -284,15 +286,16 @@ function closeUpdateModal() { function appendLog(text) { if (!text) return; - _updateLog += text; + _updateLog += text + "\n"; if ($modalLog) { - $modalLog.textContent += text; + $modalLog.textContent += text + "\n"; $modalLog.scrollTop = $modalLog.scrollHeight; } } function startUpdate() { - appendLog("$ cd /etc/nixos && nix flake update && nixos-rebuild switch && flatpak update -y\n\n"); + appendLog("$ cd /etc/nixos && nix flake update && nixos-rebuild switch && flatpak update -y"); + appendLog(""); // Trigger the systemd unit via POST fetch("/api/updates/run", { method: "POST" }) @@ -303,11 +306,14 @@ function startUpdate() { return response.json(); }) .then(data => { - // Start polling for status + log lines + if (data.status === "already_running") { + appendLog("[Update already in progress, attaching…]"); + } + // Start polling for journal output startUpdatePoll(); }) .catch(err => { - appendLog(`[Error: failed to start update — ${err}]\n`); + appendLog(`[Error: failed to start update — ${err}]`); onUpdateDone(false); }); } @@ -327,16 +333,31 @@ function stopUpdatePoll() { async function pollUpdateStatus() { try { - const data = await apiFetch(`/api/updates/status?offset=${_updateLogOffset}`); + const data = await apiFetch( + `/api/updates/status?cursor=${encodeURIComponent(_updateCursor)}` + ); - // Append new log text - if (data.log) { - appendLog(data.log); + // Server is back — reset the down counter + if (_serverDownSince > 0) { + appendLog("[Server reconnected, resuming…]"); + _serverDownSince = 0; } - _updateLogOffset = data.offset; - // Check if finished - if (!data.running) { + // Append new journal lines + if (data.lines && data.lines.length > 0) { + for (const line of data.lines) { + appendLog(line); + } + } + if (data.cursor) { + _updateCursor = data.cursor; + } + + // Update status text while running + if (data.running) { + if ($modalStatus) $modalStatus.textContent = "Updating…"; + } else { + // Finished stopUpdatePoll(); if (data.result === "success") { onUpdateDone(true); @@ -345,7 +366,12 @@ async function pollUpdateStatus() { } } } catch (err) { - // Server may be restarting during nixos-rebuild switch — keep polling + // Server is likely restarting during nixos-rebuild switch + if (_serverDownSince === 0) { + _serverDownSince = Date.now(); + appendLog("[Server restarting — waiting for it to come back…]"); + if ($modalStatus) $modalStatus.textContent = "Server restarting…"; + } console.warn("Update poll failed (server may be restarting):", err); } } @@ -360,6 +386,7 @@ function onUpdateDone(success) { } else { if ($modalStatus) $modalStatus.textContent = "✗ Update failed"; if ($btnSave) $btnSave.style.display = "inline-flex"; + if ($btnReboot) $btnReboot.style.display = "inline-flex"; } } diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index 8f403e1..96cb817 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -134,9 +134,10 @@ in description = "Sovran_SystemsOS System Update"; serviceConfig = { Type = "oneshot"; - ExecStart = "${pkgs.bash}/bin/bash -c 'cd /etc/nixos && nix flake update && nixos-rebuild switch && flatpak update -y'"; - StandardOutput = "file:/var/log/sovran-hub-update.log"; - StandardError = "file:/var/log/sovran-hub-update.log"; + ExecStart = "${pkgs.bash}/bin/bash -c 'cd /etc/nixos && nix flake update 2>&1 && nixos-rebuild switch 2>&1 && flatpak update -y 2>&1'"; + StandardOutput = "journal"; + StandardError = "journal"; + SyslogIdentifier = "sovran-hub-update"; }; path = [ pkgs.nix pkgs.nixos-rebuild pkgs.git pkgs.flatpak ]; };