Merge remote-tracking branch 'github/main' into staging-dev

Stage Hub full OS updates for next reboot (boot-first) instead of live switch

app/sovran_systemsos_web/server.py

@@ -1219,7 +1219,7 @@ def _generate_qr_base64(data: str) -> str | None:
 # ── Update helpers (file-based, no systemctl) ────────────────────
 
 def _read_update_status() -> str:
-    """Read the status file. Returns RUNNING, SUCCESS, FAILED, or IDLE."""
+    """Read the status file. Returns RUNNING, SUCCESS, REBOOT_REQUIRED, FAILED, or IDLE."""
     try:
         with open(UPDATE_STATUS, "r") as f:
             return f.read().strip()
@@ -1950,10 +1950,13 @@ def _verify_support_removed() -> bool:
 
 @app.get("/login", response_class=HTMLResponse)
 async def login_page(request: Request):
-    return templates.TemplateResponse("login.html", {
-        "request": request,
+    return templates.TemplateResponse(
+        request=request,
+        name="login.html",
+        context={
             "asset_version": ASSET_VERSION,
-    })
+        },
+    )
 
 
 @app.get("/auto-login")
@@ -2018,20 +2021,26 @@ async def api_logout(request: Request):
 
 @app.get("/", response_class=HTMLResponse)
 async def index(request: Request):
-    return templates.TemplateResponse("index.html", {
-        "request": request,
+    return templates.TemplateResponse(
+        request=request,
+        name="index.html",
+        context={
             "asset_version": ASSET_VERSION,
-    })
+        },
+    )
 
 
 @app.get("/onboarding", response_class=HTMLResponse)
 async def onboarding(request: Request):
     _ensure_onboarding_reopened_for_migration()
-    return templates.TemplateResponse("onboarding.html", {
-        "request": request,
+    return templates.TemplateResponse(
+        request=request,
+        name="onboarding.html",
+        context={
             "asset_version": ASSET_VERSION,
             "onboarding_js_hash": _ONBOARDING_JS_HASH,
-    })
+        },
+    )
 
 
 @app.get("/api/onboarding/status")
@@ -2986,18 +2995,22 @@ async def api_service_detail(unit: str, icon: str | None = None):
         if has_domain_issues:
             domain_check_steps.append({
                 "step": 4,
-                "label": "Additional Ports Required",
+                "label": "Router Setup Needed",
                 "status": "skipped",
-                "detail": "Skipped until Steps 1-3 are complete",
+                "detail": "Finish the domain steps first, then forward the Element Call ports in your router.",
             })
         else:
-            extra_open = all(p["status"] != "closed" for p in extra_ports)
+            # These checks are local-only (listening/firewall state on this computer),
+            # not an outside-in verification of router/NAT forwarding.
+            all_local_ready = all(p["status"] != "closed" for p in extra_ports)
             domain_check_steps.append({
                 "step": 4,
-                "label": "Additional Ports Required",
-                "status": "ok" if extra_open else "error",
+                "label": "Router Setup Needed" if all_local_ready else "Sovran_SystemsOS Port Setup Needed",
+                "status": "warning" if all_local_ready else "error",
                 "detail": (
-                    "Element-Call/LiveKit requires additional forwarded ports for WebRTC and TURN traffic."
+                    "Sovran_SystemsOS is ready to use these ports on this computer. Now forward them in your router so Element Call can work from outside your home network."
+                    if all_local_ready
+                    else "Sovran_SystemsOS is not ready to use all required Element Call ports on this computer yet. Fix the ports marked “Not ready yet” below, then forward them in your router."
                 ),
             })
 
@@ -4742,17 +4755,21 @@ def _recover_stale_status(status_file: str, log_file: str, unit_name: str) -> bo
     except Exception:
         pass
 
-    new_status = "SUCCESS" if unit_result == "success" else "FAILED"
+    if unit_result == "success":
+        new_status = "REBOOT_REQUIRED" if unit_name == UPDATE_UNIT else "SUCCESS"
+    else:
+        new_status = "FAILED"
     try:
         with open(status_file, "w") as f:
             f.write(new_status)
     except OSError:
         pass
-    msg = (
-        "\n[Update completed successfully while the server was restarting.]\n"
-        if new_status == "SUCCESS"
-        else "\n[Update encountered an error. See log above for details.]\n"
-    )
+    if new_status == "REBOOT_REQUIRED":
+        msg = "\n[Update staged successfully while the server was restarting. Reboot required.]\n"
+    elif new_status == "SUCCESS":
+        msg = "\n[Update completed successfully while the server was restarting.]\n"
+    else:
+        msg = "\n[Update encountered an error. See log above for details.]\n"
     try:
         with open(log_file, "a") as f:
             f.write(msg)

app/sovran_systemsos_web/static/js/service-detail.js

@@ -154,20 +154,36 @@ async function openServiceDetailModal(unit, name, icon) {
         '</div>';
 
       if (unit === "livekit.service" && data.extra_ports && data.extra_ports.length > 0) {
+        var trimmedInternalIp = data.internal_ip ? String(data.internal_ip).trim() : "";
+        var internalIp = trimmedInternalIp || "";
+        var internalIpHtml = internalIp ? escHtml(internalIp) : "Could not detect";
+        var routerIpHelp = internalIp
+          ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+          : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+        var routerNextStep = internalIp
+          ? 'Next step: Log in to your router and create forwarding rules for the ports above. Set the destination/internal IP to <strong>' + internalIpHtml + '</strong>.'
+          : 'Next step: Log in to your router and create forwarding rules for the ports above. Use this computer’s internal IP as the destination/internal IP.';
+        var domainConfigured = !!(data.domain && String(data.domain).trim());
         var extraRows = "";
         data.extra_ports.forEach(function(p) {
           var statusIcon, statusClass2;
-          if (p.status === "listening") {
-            statusIcon = "✅ Open";
+          if (!effectiveEnabled) {
+            statusIcon = "⚠ Configure Element Call first";
+            statusClass2 = "port-status-open";
+          } else if (!domainConfigured) {
+            statusIcon = "⚠ Configure domain first";
+            statusClass2 = "port-status-open";
+          } else if (p.status === "listening") {
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-listening";
           } else if (p.status === "firewall_open") {
-            statusIcon = "🟡 Firewall open";
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-open";
           } else if (p.status === "closed") {
-            statusIcon = "❌ Closed";
+            statusIcon = "❌ Not ready yet";
             statusClass2 = "port-status-closed";
           } else {
-            statusIcon = "— Unknown";
+            statusIcon = "— Could not check";
             statusClass2 = "port-status-unknown";
           }
           extraRows += '<tr>' +
@@ -178,11 +194,16 @@ async function openServiceDetailModal(unit, name, icon) {
             '</tr>';
         });
         html += '<div class="svc-detail-section">' +
-          '<div class="svc-detail-section-title">Step 4: Additional Ports</div>' +
+          '<div class="svc-detail-section-title">Ports to Forward in Your Router</div>' +
+          '<div class="svc-detail-port-note">Forward these ports in your router to this Sovran_SystemsOS computer.</div>' +
+          '<div class="svc-detail-port-note"><strong>Router Forward-To IP:</strong> ' + internalIpHtml + '</div>' +
+          '<div class="svc-detail-port-note">' + routerIpHelp + '</div>' +
           '<table class="svc-detail-port-table">' +
-            '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+            '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
             '<tbody>' + extraRows + '</tbody>' +
           '</table>' +
+          '<div class="svc-detail-port-note">The Hub can check whether Sovran_SystemsOS is ready on this computer, but full public port verification requires an outside internet check.</div>' +
+          '<div class="svc-detail-port-note">' + routerNextStep + '</div>' +
           '</div>';
       }
     } else if (data.port_statuses && data.port_statuses.length > 0) {
@@ -191,16 +212,16 @@ async function openServiceDetailModal(unit, name, icon) {
       data.port_statuses.forEach(function(p) {
         var statusIcon, statusClass2;
         if (p.status === "listening") {
-          statusIcon = "✅ Open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-listening";
         } else if (p.status === "firewall_open") {
-          statusIcon = "🟡 Firewall open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-open";
         } else if (p.status === "closed") {
-          statusIcon = "🔴 Closed";
+          statusIcon = "❌ Not ready";
           statusClass2 = "port-status-closed";
         } else {
-          statusIcon = "— Unknown";
+          statusIcon = "— Could not check";
           statusClass2 = "port-status-unknown";
         }
         portTableRows += '<tr>' +
@@ -211,9 +232,10 @@ async function openServiceDetailModal(unit, name, icon) {
           '</tr>';
       });
       html += '<div class="svc-detail-section">' +
-        '<div class="svc-detail-section-title">Port Status</div>' +
+        '<div class="svc-detail-section-title">Port Requirements</div>' +
+        '<div class="svc-detail-port-note">This shows whether Sovran_SystemsOS is ready to use this port on this computer. If you need access from outside your home network, forward this port in your router.</div>' +
         '<table class="svc-detail-port-table">' +
-          '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+          '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
           '<tbody>' + portTableRows + '</tbody>' +
         '</table>' +
         '</div>';

app/sovran_systemsos_web/static/onboarding.js

@@ -516,7 +516,7 @@ async function saveStep3() {
 async function loadStep4() {
   var body = document.getElementById("step-4-body");
   if (!body) return;
-  body.innerHTML = '<p class="onboarding-loading">Checking ports…</p>';
+  body.innerHTML = '<p class="onboarding-loading">Loading router setup…</p>';
 
   var networkData = null;
 
@@ -527,51 +527,59 @@ async function loadStep4() {
     return;
   }
 
-  var internalIp = (networkData && networkData.internal_ip) || "unknown";
-
-  var ip = escHtml(internalIp);
+  var trimmedInternalIp = (networkData && networkData.internal_ip) ? String(networkData.internal_ip).trim() : "";
+  var internalIp = trimmedInternalIp || "";
+  var hasInternalIp = !!internalIp;
+  var ip = escHtml(internalIp || "Could not detect");
+  var routerIpHelp = hasInternalIp
+    ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+    : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+  var destinationInstruction = hasInternalIp
+    ? 'Set the destination/internal IP to <strong>' + ip + '</strong>'
+    : 'Use this computer’s internal IP as the destination/internal IP';
 
   var html = '<p class="onboarding-port-note" style="margin-bottom:14px;">'
     + '⚠ <strong>Each port only needs to be forwarded once — all services share the same ports.</strong>'
     + '</p>';
 
   html += '<div class="onboarding-port-ip">';
-  html += '  <span class="onboarding-port-ip-label">Forward ports to this machine\'s internal IP:</span>';
+  html += '  <span class="onboarding-port-ip-label">Forward router traffic to this Sovran_SystemsOS computer:</span>';
   html += '  <span class="port-req-internal-ip">' + ip + '</span>';
   html += '</div>';
+  html += '<div class="onboarding-port-note" style="margin:8px 0 16px;">' + routerIpHelp + '</div>';
 
   // Required ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Ports — open these on your router:</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Router Rules</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
-  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP</td></tr>';
+  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP / SSL setup</td></tr>';
   html += '<tr><td class="port-req-port">443</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTPS</td></tr>';
-  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">SSH Remote Access</td></tr>';
+  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">Remote SSH access</td></tr>';
   html += '</tbody></table>';
   html += '</div>';
 
   // Optional ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Optional — Only needed if you enable Element Calling:</div>';
-  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">These 5 additional port openings are required on top of the 3 required ports above.</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Element Call Router Rules</div>';
+  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">Only add these if you enable Element Call. These ports help video and audio calls connect reliably.</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
   html += '<tr><td class="port-req-port">7881</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit WebRTC signalling</td></tr>';
   html += '<tr><td class="port-req-port">7882</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit media (UDP mux)</td></tr>';
   html += '<tr><td class="port-req-port">5349</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN over TLS</td></tr>';
   html += '<tr><td class="port-req-port">3478</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN (STUN/relay)</td></tr>';
-  html += '<tr><td class="port-req-port">30000–40000</td><td class="port-req-proto">TCP &amp; UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
+  html += '<tr><td class="port-req-port">30000-40000</td><td class="port-req-proto">TCP &amp; UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
   html += '</tbody></table>';
-  html += '<div style="font-size:0.85em;margin-top:6px;color:var(--color-text-muted,#888);">ℹ The <strong>30000–40000</strong> range is a single forwarding rule — just set its protocol to <strong>both TCP and UDP</strong> (often shown as "Both" or "TCP/UDP" on your router).</div>';
+  html += '<div style="font-size:0.85em;margin-top:6px;color:var(--color-text-muted,#888);">ℹ The <strong>30000-40000</strong> range is a single forwarding rule — just set its protocol to <strong>both TCP and UDP</strong> (often shown as "Both" or "TCP/UDP" on your router).</div>';
   html += '</div>';
 
   // Totals
   html += '<div class="onboarding-port-totals">';
-  html += '<strong>Total port openings: 3</strong> (without Element Calling)<br>';
-  html += '<strong>Total port openings: 8</strong> (with Element Calling — 3 required + 5 optional)';
+  html += '<strong>Total port openings: 3</strong> (without Element Call)<br>';
+  html += '<strong>Total port openings: 8</strong> (with Element Call — 3 required + 5 optional)';
   html += '</div>';
 
   html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
@@ -586,12 +594,16 @@ async function loadStep4() {
     + '<li>Open your router\'s admin panel — usually <code>http://192.168.1.1</code> or <code>http://192.168.0.1</code></li>'
     + '<li>Look for <strong>"Port Forwarding"</strong>, <strong>"NAT"</strong>, or <strong>"Virtual Server"</strong> in the settings</li>'
     + '<li>Create a new rule for each port listed above</li>'
-    + '<li>Set the destination/internal IP to <strong>' + ip + '</strong></li>'
+    + '<li>' + destinationInstruction + '</li>'
     + '<li>Set both internal and external port to the same number</li>'
     + '<li>Save and apply changes</li>'
     + '</ol>'
     + '</details>';
 
+  html += '<div class="onboarding-port-note" style="margin-top:12px;">'
+    + '<strong>Important:</strong> The Hub can show which ports Sovran_SystemsOS needs, but it cannot fully confirm router forwarding from inside your home network. Full public port verification requires an outside internet check.'
+    + '</div>';
+
   body.innerHTML = html;
 }
 

app/sovran_systemsos_web/templates/onboarding.html

@@ -148,14 +148,14 @@
       <div class="onboarding-panel" id="step-4" style="display:none">
         <div class="onboarding-step-header">
           <span class="onboarding-step-icon">🔌</span>
-          <h2 class="onboarding-step-title">Port Forwarding Check</h2>
+          <h2 class="onboarding-step-title">Router Setup</h2>
           <p class="onboarding-step-desc">
-            Forward these ports on your router to this machine. Each port only needs to be opened once — they are shared across all your services.
-            <strong>Ports 80 and 443 must be open for SSL certificates to work.</strong>
+            Forward these ports in your router to this Sovran_SystemsOS computer. These rules let people reach your services from outside your home network.
+            <strong>Ports 80 and 443 are required for HTTPS and SSL certificates.</strong>
           </p>
         </div>
         <div class="onboarding-card" id="step-4-body">
-          <p class="onboarding-loading">Checking ports…</p>
+          <p class="onboarding-loading">Loading router setup…</p>
         </div>
         <div class="onboarding-footer">
           <button class="btn btn-close-modal onboarding-btn-back" data-prev="3">← Back</button>

app/tests/test_hub_update_boot_staging.py

@@ -0,0 +1,44 @@
+import unittest
+from pathlib import Path
+
+
+HUB_NIX = Path(__file__).resolve().parents[2] / "modules" / "core" / "sovran-hub.nix"
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class HubUpdateBootStagingTests(unittest.TestCase):
+    def setUp(self):
+        self.source = HUB_NIX.read_text()
+        self.update_section = _section(
+            self.source,
+            'update-script = pkgs.writeShellScript "sovran-hub-update.sh" \'\'',
+            "# ── Rebuild wrapper script",
+        )
+        self.rebuild_section = _section(
+            self.source,
+            'rebuild-script = pkgs.writeShellScript "sovran-hub-rebuild.sh" \'\'',
+            "# ── Brave launcher wrapper",
+        )
+
+    def test_full_update_uses_boot_not_switch(self):
+        self.assertIn("nixos-rebuild boot --flake /etc/nixos", self.update_section)
+        self.assertNotIn("nixos-rebuild switch --flake /etc/nixos", self.update_section)
+
+    def test_full_update_marks_reboot_required(self):
+        self.assertIn('echo "REBOOT_REQUIRED" > "$STATUS"', self.update_section)
+
+    def test_rebuild_path_keeps_switch_semantics(self):
+        self.assertIn("nixos-rebuild switch --flake /etc/nixos", self.rebuild_section)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_service_detail_router_wording.py

@@ -0,0 +1,171 @@
+import unittest
+from pathlib import Path
+from unittest.mock import mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+
+                return _decorator
+
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server
+
+
+class ServiceDetailRouterWordingTests(unittest.IsolatedAsyncioTestCase):
+    async def test_livekit_service_detail_includes_internal_ip(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": {7881}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["internal_ip"], "192.168.1.44")
+        self.assertEqual(result["extra_ports"][0]["status"], "listening")
+        self.assertEqual(result["domain_check_steps"][-1]["label"], "Router Setup Needed")
+
+    async def test_livekit_router_step_uses_not_ready_yet_wording(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["extra_ports"][0]["status"], "closed")
+        self.assertIn("Not ready yet", result["domain_check_steps"][-1]["detail"])
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_template_response_signature.py

@@ -0,0 +1,99 @@
+"""Regression test for Starlette 1.1.0+ TemplateResponse keyword-argument style.
+
+Prior to this fix, the three HTML routes called:
+    templates.TemplateResponse("name.html", {"request": request, ...})
+which passes the context dict as the second positional argument.  With the
+updated Starlette/FastAPI versions shipped in NixOS unstable (Starlette 1.1.0,
+FastAPI 0.136.3) that positional argument is the template name, causing Jinja2
+to receive a dict as a cache key and raise:
+    TypeError: unhashable type: 'dict'
+
+The fix updates every call to use keyword arguments:
+    templates.TemplateResponse(request=request, name="name.html", context={...})
+"""
+
+import ast
+import unittest
+from pathlib import Path
+
+
+SERVER_PY = Path(__file__).resolve().parents[1] / "sovran_systemsos_web" / "server.py"
+
+
+def _template_response_calls(source: str):
+    """Return a list of ast.Call nodes that are TemplateResponse calls."""
+    tree = ast.parse(source)
+    calls = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func = node.func
+        if isinstance(func, ast.Attribute) and func.attr == "TemplateResponse":
+            calls.append(node)
+    return calls
+
+
+class TemplateResponseSignatureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = SERVER_PY.read_text()
+        self.calls = _template_response_calls(self.source)
+
+    def test_at_least_one_template_response_call_found(self):
+        self.assertGreater(len(self.calls), 0, "No TemplateResponse calls found in server.py")
+
+    def test_no_old_style_positional_dict_context(self):
+        """No TemplateResponse call should pass a dict literal as its second positional arg.
+
+        The old style was:
+            templates.TemplateResponse("name.html", {"request": request, ...})
+        where args[0] is a string and args[1] is a Dict node.  That pattern
+        triggers the Starlette 1.1.0 bug.
+        """
+        for call in self.calls:
+            positional = call.args
+            if len(positional) >= 2 and isinstance(positional[1], ast.Dict):
+                self.fail(
+                    f"Found old-style TemplateResponse call at line {call.lineno}: "
+                    "second positional argument is a dict literal.  "
+                    "Use keyword arguments (request=, name=, context=) instead."
+                )
+
+    def test_request_not_duplicated_in_context(self):
+        """The 'request' key must not appear inside the context= dict when
+        request= is already passed as a dedicated keyword argument."""
+        for call in self.calls:
+            kw_dict = {kw.arg: kw.value for kw in call.keywords if isinstance(kw, ast.keyword)}
+
+            if "request" not in kw_dict:
+                continue  # no request= kwarg, nothing to check
+
+            context_node = kw_dict.get("context")
+            if not isinstance(context_node, ast.Dict):
+                continue
+
+            for key_node in context_node.keys:
+                if isinstance(key_node, ast.Constant) and key_node.value == "request":
+                    self.fail(
+                        f"TemplateResponse at line {call.lineno} passes 'request' both as "
+                        "request= keyword argument and inside the context dict."
+                    )
+
+    def test_all_calls_use_keyword_arguments(self):
+        """Every TemplateResponse call should use keyword arguments for request, name,
+        and context rather than relying on positional ordering."""
+        for call in self.calls:
+            kw_args = {kw.arg for kw in call.keywords if isinstance(kw, ast.keyword)}
+            self.assertIn(
+                "request",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'request='.",
+            )
+            self.assertIn(
+                "name",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'name='.",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

flake.lock

@@ -5,11 +5,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1781789880,
-        "narHash": "sha256-HU/J4pFFkC2XXsYO8B3QFneTV2NWEXFuNi4QmV/4ZA8=",
+        "lastModified": 1783086783,
+        "narHash": "sha256-NxXpNF/9tq2nI+SxFHUxjro3u11SF3l4vs7bawdMKkQ=",
         "owner": "emmanuelrosa",
         "repo": "btc-clients-nix",
-        "rev": "10f0300231075e6c7417030fbcbf9f056d0a7c21",
+        "rev": "4f6d07cae877ef58f0fbc9e731c99800ddb80859",
         "type": "github"
       },
       "original": {
@@ -52,11 +52,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1778716662,
-        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "lastModified": 1782949081,
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1780218263,
-        "narHash": "sha256-T/f0pPDrH3Qc1VXyQXbK7yfHWRn90l3xwplc/nsxin4=",
+        "lastModified": 1782911660,
+        "narHash": "sha256-PbR+tJ5E/Ux+01UtdFKqblccVA4/FgWbkym4ev3VHHQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7fc393d1b46fa000d48ff14e8b6a3c9985f03af0",
+        "rev": "cf720c15e108d432d29041cc5a185630809acefb",
         "type": "github"
       },
       "original": {
@@ -139,11 +139,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1781216227,
-        "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=",
+        "lastModified": 1782999065,
+        "narHash": "sha256-5Dgj5+pIQYZKrXUGaLCk7CKfN3MmpwIhO94++WVxvng=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0",
+        "rev": "80d591ed473cfc46329932c2aadac9b435342c7c",
         "type": "github"
       },
       "original": {
@@ -187,11 +187,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1781577229,
-        "narHash": "sha256-lrp67w8AulE9Ks53n27I45ADSzbOCn4H+CNW1Ck8B+8=",
+        "lastModified": 1782959384,
+        "narHash": "sha256-xnJJk+ct+D2+wdRxj1wk36w5zV9RVESwRqcklPdt3fM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "567a49d1913ce81ac6e9582e3553dd90a955875f",
+        "rev": "65179426c83bb3f6bc14898b42ea1c6f01d374b0",
         "type": "github"
       },
       "original": {
@@ -203,11 +203,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1781607440,
-        "narHash": "sha256-rxO+uc/KFbSJp+pgyXRuAX6QlG9hJdnt0BXpEQRXY+U=",
+        "lastModified": 1782948114,
+        "narHash": "sha256-AXmz9ho4Lud5CsbrZsuSVwpQZ4o5FgZ1chxBn5cJ8+0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3e41b24abd260e8f71dbe2f5737d24122f972158",
+        "rev": "9e92285f211dad236540fd617d7e30e0b99bc0e1",
         "type": "github"
       },
       "original": {
@@ -224,11 +224,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1781713417,
-        "narHash": "sha256-Kaj44jTNmnaFhKrcADx8nXmUYPa7l2HYfb7m6lEPy7Q=",
+        "lastModified": 1783173302,
+        "narHash": "sha256-nlnOw/zsD2H2NHSZ5oNWwcjuM17vipyAapfXsO78GjY=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "caee4e5d4161778815f522d9ea1c9e3dc42462b7",
+        "rev": "a402fdf2a1ef297d8ea7c95b90d6af0dbe90ab11",
         "type": "github"
       },
       "original": {

modules/core/caddy.nix

@@ -16,7 +16,10 @@ let
 in
 {
   services.caddy = {
-    enable = true;
+    # Only enable Caddy when at least one domain-based service needs it or
+    # the operator has defined custom vhosts.  This prevents Caddy from
+    # running on Desktop Only installs that have no web services configured.
+    enable = needsHttpsPorts || extraVhosts != "";
     user = "caddy";
     group = "root";
   };

modules/core/role-logic.nix

@@ -3,6 +3,13 @@
 {
   config = lib.mkMerge [
 
+    # nix-bitcoin is globally imported by the flake (nixosModules.Sovran_SystemsOS).
+    # This default satisfies nix-bitcoin's generateSecrets assertion so that Desktop
+    # Only systems can evaluate without enabling any Bitcoin services.
+    {
+      nix-bitcoin.generateSecrets = lib.mkDefault true;
+    }
+
     # ── Server+Desktop Role (default) ─────────────────────────
     (lib.mkIf config.sovran_systemsOS.roles.server_plus_desktop {
       sovran_systemsOS.web.btcpayserver = lib.mkDefault true;
@@ -12,15 +19,24 @@
     (lib.mkIf config.sovran_systemsOS.roles.desktop {
       services.desktopManager.gnome.enable = true;
 
+      # Force all server/node services and features off so they cannot be
+      # accidentally enabled via custom.nix or option defaults on Desktop Only.
       sovran_systemsOS.services = {
-        synapse = lib.mkDefault false;
-        bitcoin = lib.mkDefault false;
-        vaultwarden = lib.mkDefault false;
-        wordpress = lib.mkDefault false;
-        nextcloud = lib.mkDefault false;
+        synapse = lib.mkForce false;
+        bitcoin = lib.mkForce false;
+        vaultwarden = lib.mkForce false;
+        wordpress = lib.mkForce false;
+        nextcloud = lib.mkForce false;
       };
 
-      sovran_systemsOS.web.btcpayserver = lib.mkDefault false;
+      sovran_systemsOS.features = {
+        haven = lib.mkForce false;
+        mempool = lib.mkForce false;
+        element-calling = lib.mkForce false;
+        bitcoin-core = lib.mkForce false;
+      };
+
+      sovran_systemsOS.web.btcpayserver = lib.mkForce false;
     })
 
     # ── Bitcoin Node Only Role ────────────────────────────────

modules/core/sovran-hub.nix

@@ -146,33 +146,16 @@ let
     echo ""
 
     if [ "$RC" -eq 0 ]; then
-      echo "── Step 2/3: nixos-rebuild ──────────────────────────"
-      SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+      echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──"
+      BOOT_OUT=$(nixos-rebuild boot --flake /etc/nixos --print-build-logs \
         --option connect-timeout 10 \
         --option stalled-download-timeout 90 \
         --option download-attempts 7 \
         --option fallback true 2>&1)
-      SWITCH_RC=$?
-      echo "$SWITCH_OUT"
-      if [ "$SWITCH_RC" -eq 0 ]; then
-        echo "[OK] switch succeeded"
-      elif echo "$SWITCH_OUT" | grep -q "switchInhibitors\|Pre-switch checks failed"; then
-        echo ""
-        echo "  ✓ Build succeeded — a reboot is required to apply this update"
-        echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-        if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
-             --option connect-timeout 10 \
-             --option stalled-download-timeout 90 \
-             --option download-attempts 7 \
-             --option fallback true 2>&1; then
-          echo "REBOOT_REQUIRED" > "$STATUS"
-          exit 0
-        else
-          echo "[ERROR] nixos-rebuild boot also failed"
-          RC=1
-        fi
-      else
-        echo "[ERROR] nixos-rebuild switch failed"
+      BOOT_RC=$?
+      echo "$BOOT_OUT"
+      if [ "$BOOT_RC" -ne 0 ]; then
+        echo "[ERROR] nixos-rebuild boot failed"
         RC=1
       fi
       echo ""
@@ -188,9 +171,10 @@ let
 
     if [ "$RC" -eq 0 ]; then
       echo "══════════════════════════════════════════════════"
-      echo "  ✓ Update completed successfully"
+      echo "  ✓ Update staged successfully"
+      echo "  Reboot required to activate the new system"
       echo "══════════════════════════════════════════════════"
-      echo "SUCCESS" > "$STATUS"
+      echo "REBOOT_REQUIRED" > "$STATUS"
     else
       echo "══════════════════════════════════════════════════"
       echo "  ✗ Update failed — see errors above"

modules/core/ssh-bootstrap.nix

@@ -31,7 +31,7 @@ lib.mkIf userExists {
   };
 
   systemd.services.factory-ssh-keygen = {
-    description = "Generate factory SSH key for ${userName} if missing";
+    description = "Generate or repair factory SSH key for ${userName}";
     wantedBy = [ "multi-user.target" ];
     after = [ "ssh-passphrase-setup.service" ];
     requires = [ "ssh-passphrase-setup.service" ];
@@ -39,14 +39,47 @@ lib.mkIf userExists {
       Type = "oneshot";
       RemainAfterExit = true;
     };
-    path = [ pkgs.openssh pkgs.coreutils ];
+    path = [ pkgs.openssh pkgs.coreutils pkgs.util-linux ];
     script = ''
-      if [ ! -f "${keyPath}" ]; then
+      set -eu
+
       PASSPHRASE=$(cat /var/lib/secrets/ssh-passphrase)
+      lock_file="${keyPath}.lock"
+
+      exec 9>"$lock_file"
+
+      if ! flock -n 9; then
+        echo "Factory SSH key setup is already running." >&2
+        exit 1
+      fi
+
+      generate_factory_key() {
         ssh-keygen -q -N "$PASSPHRASE" -t ed25519 -f "${keyPath}"
         chown ${userName}:users "${keyPath}" "${keyPath}.pub"
         chmod 600 "${keyPath}"
         chmod 644 "${keyPath}.pub"
+      }
+
+      if [ ! -f "${keyPath}" ]; then
+        generate_factory_key
+      elif ! ssh-keygen -y -P "$PASSPHRASE" -f "${keyPath}" >/dev/null 2>&1; then
+        backup_suffix="$(date -u +%Y%m%d_%H%M%S)-$$"
+        backup_path="${keyPath}.bak-$backup_suffix"
+        backup_index=0
+
+        while [ -e "$backup_path" ] || [ -e "$backup_path.pub" ]; do
+          backup_index=$((backup_index + 1))
+          backup_path="${keyPath}.bak-$backup_suffix-$backup_index"
+        done
+
+        echo "Existing factory SSH key does not match current passphrase; backing it up to $backup_path and generating a replacement."
+        mv "${keyPath}" "$backup_path"
+
+        if [ -f "${keyPath}.pub" ]; then
+          mv "${keyPath}.pub" "$backup_path.pub"
+        fi
+
+        generate_factory_key
       fi
     '';
   };