updated logging

app/sovran_systemsos_web/server.py

@@ -26,7 +26,6 @@ FLAKE_INPUT_NAME = "Sovran_Systems"
 GITEA_API_BASE = "https://git.sovransystems.com/api/v1/repos/Sovran_Systems/Sovran_SystemsOS/commits"
 
 UPDATE_UNIT = "sovran-hub-update.service"
-UPDATE_LOG  = "/var/log/sovran-hub-update.log"
 
 REBOOT_COMMAND = [
     "reboot",
@@ -172,7 +171,7 @@ def _update_is_active() -> bool:
 
 
 def _update_result() -> str:
-    """Return 'success', 'failed', or 'inactive'."""
+    """Return 'success', 'failed', or 'unknown'."""
     r = subprocess.run(
         ["systemctl", "show", "-p", "Result", "--value", UPDATE_UNIT],
         capture_output=True, text=True,
@@ -182,18 +181,47 @@ def _update_result() -> str:
         return "success"
     elif val:
         return "failed"
-    return "inactive"
+    return "unknown"
 
 
-def _read_update_log(offset: int = 0) -> tuple[str, int]:
-    """Read update log from offset. Return (new_text, new_offset)."""
-    try:
-        with open(UPDATE_LOG, "r") as f:
-            f.seek(offset)
-            text = f.read()
-            return text, f.tell()
-    except FileNotFoundError:
-        return "", 0
+def _get_update_invocation_id() -> str:
+    """Get the current InvocationID of the update unit."""
+    r = subprocess.run(
+        ["systemctl", "show", "-p", "InvocationID", "--value", UPDATE_UNIT],
+        capture_output=True, text=True,
+    )
+    return r.stdout.strip()
+
+
+def _read_journal_logs(since_cursor: str = "") -> tuple[list[str], str]:
+    """
+    Read journal logs for the update unit.
+    Returns (lines, last_cursor).
+    Uses cursors so we never miss lines even if the server restarts.
+    """
+    cmd = [
+        "journalctl", "-u", UPDATE_UNIT,
+        "--no-pager", "-o", "cat",
+        "--show-cursor",
+    ]
+    if since_cursor:
+        cmd += ["--after-cursor", since_cursor]
+    else:
+        # Only get logs from the most recent invocation
+        cmd += ["-n", "10000"]
+
+    r = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+    output = r.stdout
+
+    lines = []
+    cursor = since_cursor
+    for raw_line in output.split("\n"):
+        if raw_line.startswith("-- cursor: "):
+            cursor = raw_line[len("-- cursor: "):]
+        elif raw_line:
+            lines.append(raw_line)
+
+    return lines, cursor
 
 
 # ── Routes ───────────────────────────────────────────────────────
@@ -333,12 +361,6 @@ async def api_updates_run():
     if running:
         return {"ok": True, "status": "already_running"}
 
-    # Clear the old log
-    try:
-        open(UPDATE_LOG, "w").close()
-    except OSError:
-        pass
-
     # Reset the failed state (if any) and start the unit
     await asyncio.create_subprocess_exec(
         "systemctl", "reset-failed", UPDATE_UNIT,
@@ -356,17 +378,19 @@ async def api_updates_run():
 
 
 @app.get("/api/updates/status")
-async def api_updates_status(offset: int = 0):
-    """Poll endpoint: returns running state, result, and new log lines."""
+async def api_updates_status(cursor: str = ""):
+    """Poll endpoint: returns running state, result, and new journal lines."""
     loop = asyncio.get_event_loop()
 
     running = await loop.run_in_executor(None, _update_is_active)
     result = await loop.run_in_executor(None, _update_result)
-    new_log, new_offset = await loop.run_in_executor(None, _read_update_log, offset)
+    lines, new_cursor = await loop.run_in_executor(
+        None, _read_journal_logs, cursor,
+    )
 
     return {
         "running": running,
         "result": result,
-        "log": new_log,
-        "offset": new_offset,
+        "lines": lines,
+        "cursor": new_cursor,
     }

app/sovran_systemsos_web/static/app.js

@@ -26,7 +26,8 @@ let _categoryLabels = {};
 let _updateSource   = null;
 let _updateLog      = "";
 let _updatePollTimer = null;
-let _updateLogOffset = 0;
+let _updateCursor    = "";
+let _serverDownSince = 0;
 
 // ── DOM refs ──────────────────────────────────────────────────────
 
@@ -264,7 +265,8 @@ async function checkUpdates() {
 function openUpdateModal() {
   if (!$modal) return;
   _updateLog = "";
-  _updateLogOffset = 0;
+  _updateCursor = "";
+  _serverDownSince = 0;
   if ($modalLog)    $modalLog.textContent = "";
   if ($modalStatus) $modalStatus.textContent = "Updating…";
   if ($modalSpinner) $modalSpinner.classList.add("spinning");
@@ -284,15 +286,16 @@ function closeUpdateModal() {
 
 function appendLog(text) {
   if (!text) return;
-  _updateLog += text;
+  _updateLog += text + "\n";
   if ($modalLog) {
-    $modalLog.textContent += text;
+    $modalLog.textContent += text + "\n";
     $modalLog.scrollTop = $modalLog.scrollHeight;
   }
 }
 
 function startUpdate() {
-  appendLog("$ cd /etc/nixos && nix flake update && nixos-rebuild switch && flatpak update -y\n\n");
+  appendLog("$ cd /etc/nixos && nix flake update && nixos-rebuild switch && flatpak update -y");
+  appendLog("");
 
   // Trigger the systemd unit via POST
   fetch("/api/updates/run", { method: "POST" })
@@ -303,11 +306,14 @@ function startUpdate() {
       return response.json();
     })
     .then(data => {
-      // Start polling for status + log lines
+      if (data.status === "already_running") {
+        appendLog("[Update already in progress, attaching…]");
+      }
+      // Start polling for journal output
       startUpdatePoll();
     })
     .catch(err => {
-      appendLog(`[Error: failed to start update — ${err}]\n`);
+      appendLog(`[Error: failed to start update — ${err}]`);
       onUpdateDone(false);
     });
 }
@@ -327,16 +333,31 @@ function stopUpdatePoll() {
 
 async function pollUpdateStatus() {
   try {
-    const data = await apiFetch(`/api/updates/status?offset=${_updateLogOffset}`);
+    const data = await apiFetch(
+      `/api/updates/status?cursor=${encodeURIComponent(_updateCursor)}`
+    );
 
-    // Append new log text
-    if (data.log) {
-      appendLog(data.log);
+    // Server is back — reset the down counter
+    if (_serverDownSince > 0) {
+      appendLog("[Server reconnected, resuming…]");
+      _serverDownSince = 0;
     }
-    _updateLogOffset = data.offset;
 
-    // Check if finished
-    if (!data.running) {
+    // Append new journal lines
+    if (data.lines && data.lines.length > 0) {
+      for (const line of data.lines) {
+        appendLog(line);
+      }
+    }
+    if (data.cursor) {
+      _updateCursor = data.cursor;
+    }
+
+    // Update status text while running
+    if (data.running) {
+      if ($modalStatus) $modalStatus.textContent = "Updating…";
+    } else {
+      // Finished
       stopUpdatePoll();
       if (data.result === "success") {
         onUpdateDone(true);
@@ -345,7 +366,12 @@ async function pollUpdateStatus() {
       }
     }
   } catch (err) {
-    // Server may be restarting during nixos-rebuild switch — keep polling
+    // Server is likely restarting during nixos-rebuild switch
+    if (_serverDownSince === 0) {
+      _serverDownSince = Date.now();
+      appendLog("[Server restarting — waiting for it to come back…]");
+      if ($modalStatus) $modalStatus.textContent = "Server restarting…";
+    }
     console.warn("Update poll failed (server may be restarting):", err);
   }
 }
@@ -360,6 +386,7 @@ function onUpdateDone(success) {
   } else {
     if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
     if ($btnSave)     $btnSave.style.display    = "inline-flex";
+    if ($btnReboot)   $btnReboot.style.display  = "inline-flex";
   }
 }