updated logging

This commit is contained in:
2026-04-02 13:27:25 -05:00
parent eb11231e34
commit 3e1f672c00
2 changed files with 59 additions and 9 deletions

View File

@@ -7,6 +7,7 @@ import json
import os import os
import socket import socket
import subprocess import subprocess
import time
import urllib.request import urllib.request
from fastapi import FastAPI, HTTPException from fastapi import FastAPI, HTTPException
@@ -69,6 +70,12 @@ if os.path.isdir(_ICONS_DIR):
templates = Jinja2Templates(directory=os.path.join(_BASE_DIR, "templates")) templates = Jinja2Templates(directory=os.path.join(_BASE_DIR, "templates"))
# ── Track when we started an update ──────────────────────────────
# This timestamp lets us know that an update was recently kicked off,
# so we don't prematurely declare it finished if the unit hasn't
# transitioned to "active" yet.
_update_started_at: float = 0.0
# ── Update check helpers ───────────────────────────────────────── # ── Update check helpers ─────────────────────────────────────────
def _get_locked_info(): def _get_locked_info():
@@ -166,6 +173,15 @@ def _update_is_active() -> bool:
return r.returncode == 0 return r.returncode == 0
def _update_state() -> str:
"""Return the ActiveState of the update unit."""
r = subprocess.run(
["systemctl", "show", "-p", "ActiveState", "--value", UPDATE_UNIT],
capture_output=True, text=True,
)
return r.stdout.strip()
def _update_result() -> str: def _update_result() -> str:
"""Return 'success', 'failed', or 'unknown'.""" """Return 'success', 'failed', or 'unknown'."""
r = subprocess.run( r = subprocess.run(
@@ -326,6 +342,7 @@ async def api_reboot():
@app.post("/api/updates/run") @app.post("/api/updates/run")
async def api_updates_run(): async def api_updates_run():
"""Kick off the detached update systemd unit.""" """Kick off the detached update systemd unit."""
global _update_started_at
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
running = await loop.run_in_executor(None, _update_is_active) running = await loop.run_in_executor(None, _update_is_active)
@@ -339,6 +356,9 @@ async def api_updates_run():
stderr=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
) )
# Record the start time so we can handle the race condition
_update_started_at = time.monotonic()
proc = await asyncio.create_subprocess_exec( proc = await asyncio.create_subprocess_exec(
"systemctl", "start", "--no-block", UPDATE_UNIT, "systemctl", "start", "--no-block", UPDATE_UNIT,
stdout=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.DEVNULL,
@@ -352,14 +372,33 @@ async def api_updates_run():
@app.get("/api/updates/status") @app.get("/api/updates/status")
async def api_updates_status(offset: int = 0): async def api_updates_status(offset: int = 0):
"""Poll endpoint: returns running state, result, and new log content.""" """Poll endpoint: returns running state, result, and new log content."""
global _update_started_at
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
running = await loop.run_in_executor(None, _update_is_active) active = await loop.run_in_executor(None, _update_is_active)
state = await loop.run_in_executor(None, _update_state)
result = await loop.run_in_executor(None, _update_result) result = await loop.run_in_executor(None, _update_result)
new_log, new_offset = await loop.run_in_executor(None, _read_log, offset) new_log, new_offset = await loop.run_in_executor(None, _read_log, offset)
# Race condition guard: if we just started the unit and it hasn't
# transitioned to "activating"/"active" yet, report it as still running.
# Give it up to 10 seconds to appear as active.
if not active and _update_started_at > 0:
elapsed = time.monotonic() - _update_started_at
if elapsed < 10 and state in ("inactive", ""):
# Unit hasn't started yet — tell the frontend it's still running
return {
"running": True,
"result": "pending",
"log": new_log,
"offset": new_offset,
}
else:
# Either it finished or the grace period expired
_update_started_at = 0.0
return { return {
"running": running, "running": active,
"result": result, "result": result,
"log": new_log, "log": new_log,
"offset": new_offset, "offset": new_offset,

View File

@@ -5,6 +5,7 @@ const POLL_INTERVAL_SERVICES = 5000; // 5 s
const POLL_INTERVAL_UPDATES = 1800000; // 30 min const POLL_INTERVAL_UPDATES = 1800000; // 30 min
const ACTION_REFRESH_DELAY = 1500; // 1.5 s after start/stop/restart const ACTION_REFRESH_DELAY = 1500; // 1.5 s after start/stop/restart
const UPDATE_POLL_INTERVAL = 1500; // 1.5 s while update is running const UPDATE_POLL_INTERVAL = 1500; // 1.5 s while update is running
const UPDATE_POLL_DELAY = 3000; // 3 s before first poll (let unit start)
const CATEGORY_ORDER = [ const CATEGORY_ORDER = [
"infrastructure", "infrastructure",
@@ -28,6 +29,7 @@ let _updatePollTimer = null;
let _updateLogOffset = 0; let _updateLogOffset = 0;
let _serverWasDown = false; let _serverWasDown = false;
let _updateFinished = false; let _updateFinished = false;
let _sawRunning = false;
// ── DOM refs ────────────────────────────────────────────────────── // ── DOM refs ──────────────────────────────────────────────────────
@@ -255,7 +257,6 @@ async function checkUpdates() {
if ($updateBadge) { if ($updateBadge) {
$updateBadge.classList.toggle("visible", hasUpdates); $updateBadge.classList.toggle("visible", hasUpdates);
} }
// Toggle button color: blue (default) → green (updates available)
if ($updateBtn) { if ($updateBtn) {
$updateBtn.classList.toggle("has-updates", hasUpdates); $updateBtn.classList.toggle("has-updates", hasUpdates);
} }
@@ -270,6 +271,7 @@ function openUpdateModal() {
_updateLogOffset = 0; _updateLogOffset = 0;
_serverWasDown = false; _serverWasDown = false;
_updateFinished = false; _updateFinished = false;
_sawRunning = false;
if ($modalLog) $modalLog.textContent = ""; if ($modalLog) $modalLog.textContent = "";
if ($modalStatus) $modalStatus.textContent = "Starting update…"; if ($modalStatus) $modalStatus.textContent = "Starting update…";
if ($modalSpinner) $modalSpinner.classList.add("spinning"); if ($modalSpinner) $modalSpinner.classList.add("spinning");
@@ -307,9 +309,11 @@ function startUpdate() {
.then(data => { .then(data => {
if (data.status === "already_running") { if (data.status === "already_running") {
appendLog("[Update already in progress, attaching…]\n\n"); appendLog("[Update already in progress, attaching…]\n\n");
_sawRunning = true;
} }
if ($modalStatus) $modalStatus.textContent = "Updating…"; if ($modalStatus) $modalStatus.textContent = "Updating…";
startUpdatePoll(); // Delay the first poll to give the systemd unit time to start
setTimeout(startUpdatePoll, UPDATE_POLL_DELAY);
}) })
.catch(err => { .catch(err => {
appendLog(`[Error: failed to start update — ${err}]\n`); appendLog(`[Error: failed to start update — ${err}]\n`);
@@ -330,7 +334,6 @@ function stopUpdatePoll() {
} }
async function pollUpdateStatus() { async function pollUpdateStatus() {
// Don't poll if we already know it's done
if (_updateFinished) return; if (_updateFinished) return;
try { try {
@@ -348,8 +351,14 @@ async function pollUpdateStatus() {
} }
_updateLogOffset = data.offset; _updateLogOffset = data.offset;
// Check if finished // Track if we ever saw the unit as running
if (!data.running) { if (data.running) {
_sawRunning = true;
}
// Only declare finished if we previously saw it running (or server says so)
// This prevents the race where the unit hasn't started yet
if (!data.running && _sawRunning) {
_updateFinished = true; _updateFinished = true;
stopUpdatePoll(); stopUpdatePoll();
if (data.result === "success") { if (data.result === "success") {
@@ -359,7 +368,9 @@ async function pollUpdateStatus() {
} }
} }
} catch (err) { } catch (err) {
// Server is likely restarting during nixos-rebuild switch — keep polling // Server is likely restarting during nixos-rebuild switch
// This counts as "saw running" since it was running before it died
_sawRunning = true;
if (!_serverWasDown) { if (!_serverWasDown) {
_serverWasDown = true; _serverWasDown = true;
appendLog("\n[Server restarting — waiting for it to come back…]\n\n"); appendLog("\n[Server restarting — waiting for it to come back…]\n\n");
@@ -413,7 +424,7 @@ if ($modal) {
}); });
} }
// ── Init ──────<EFBFBD><EFBFBD>─────────────────────────────────────────────────── // ── Init ─────────────────────────────────────────────────────────
async function init() { async function init() {
try { try {