Merge pull request #428 from naturallaw777/fix/hub-load-performance

perf: speed up Hub service status loading

app/sovran_systemsos_web/server.py

@@ -84,6 +84,17 @@ _DOMAIN_REACHABILITY_STARTUP_DELAY = 5
 _domain_reachability_task: asyncio.Task | None = None
 _domain_reachability_task_lock = asyncio.Lock()
 
+# Short-lived local diagnostics caches.  These values are only used for the
+# dashboard's health hints, so a few seconds of staleness is preferable to
+# launching several subprocesses and DNS lookups on every five-second poll.
+_PORT_CACHE_TTL = 5
+_port_cache_lock = Lock()
+_listening_ports_cache: tuple[float, dict[str, set[int]]] = (0.0, {"tcp": set(), "udp": set()})
+_firewall_ports_cache: tuple[float, dict[str, set[int]]] = (0.0, {"tcp": set(), "udp": set()})
+_DOMAIN_DNS_CACHE_TTL = 60
+_domain_dns_cache_lock = Lock()
+_domain_dns_cache: dict[str, tuple[float, list[str]]] = {}
+
 # Units to start after the next successful rebuild (feature enable flow)
 _pending_service_starts: set[str] = set()
 _pending_service_starts_lock = Lock()
@@ -918,7 +929,7 @@ def _get_listening_ports() -> dict[str, set[int]]:
         try:
             proc = subprocess.run(
                 ["ss", flag],
-                capture_output=True, text=True, timeout=10,
+                capture_output=True, text=True, timeout=5,
             )
             logger.debug("ss %s rc=%s stderr=%r", flag, proc.returncode, proc.stderr.strip())
             logger.debug("ss %s output sample: %r", flag, "\n".join(proc.stdout.splitlines()[:8]))
@@ -969,7 +980,7 @@ def _get_firewall_allowed_ports() -> dict[str, set[int]]:
     try:
         proc = subprocess.run(
             ["nft", "list", "ruleset"],
-            capture_output=True, text=True, timeout=10,
+            capture_output=True, text=True, timeout=5,
         )
         logger.debug("nft list ruleset rc=%s stderr=%r", proc.returncode, proc.stderr.strip())
         logger.debug("nft output sample: %r", "\n".join(proc.stdout.splitlines()[:12]))
@@ -1006,7 +1017,7 @@ def _get_firewall_allowed_ports() -> dict[str, set[int]]:
     try:
         proc = subprocess.run(
             ["iptables", "-L", "-n"],
-            capture_output=True, text=True, timeout=10,
+            capture_output=True, text=True, timeout=5,
         )
         if proc.returncode == 0:
             for line in proc.stdout.splitlines():
@@ -1023,6 +1034,50 @@ def _get_firewall_allowed_ports() -> dict[str, set[int]]:
     return result
 
 
+def _get_listening_ports_cached() -> dict[str, set[int]]:
+    """Return listening ports from a short-lived cache."""
+    global _listening_ports_cache
+    now = time.monotonic()
+    with _port_cache_lock:
+        cached_at, cached_value = _listening_ports_cache
+        if now - cached_at < _PORT_CACHE_TTL:
+            return cached_value
+
+    value = _get_listening_ports()
+    with _port_cache_lock:
+        _listening_ports_cache = (now, value)
+    return value
+
+
+def _get_firewall_allowed_ports_cached() -> dict[str, set[int]]:
+    """Return firewall ports from a short-lived cache."""
+    global _firewall_ports_cache
+    now = time.monotonic()
+    with _port_cache_lock:
+        cached_at, cached_value = _firewall_ports_cache
+        if now - cached_at < _PORT_CACHE_TTL:
+            return cached_value
+
+    value = _get_firewall_allowed_ports()
+    with _port_cache_lock:
+        _firewall_ports_cache = (now, value)
+    return value
+
+
+def _resolve_all_addresses_cached(domain: str) -> list[str]:
+    """Resolve a domain once per cache window for dashboard health checks."""
+    now = time.monotonic()
+    with _domain_dns_cache_lock:
+        cached = _domain_dns_cache.get(domain)
+        if cached is not None and now - cached[0] < _DOMAIN_DNS_CACHE_TTL:
+            return cached[1]
+
+    addresses = _resolve_all_addresses(domain)
+    with _domain_dns_cache_lock:
+        _domain_dns_cache[domain] = (now, addresses)
+    return addresses
+
+
 def _port_range_to_ints(port_str: str) -> list[int]:
     """Convert a port string like ``"443"``, ``"30000-40000"`` to a list of ints."""
     port_str = port_str.strip()
@@ -2886,7 +2941,9 @@ def _get_bitcoin_version_info() -> dict | None:
             ["bitcoin-cli", f"-datadir={BITCOIN_DATADIR}", "getnetworkinfo"],
             capture_output=True,
             text=True,
-            timeout=10,
+            # This is a dashboard hint; do not make an unavailable RPC delay
+            # the entire service tile response.
+            timeout=3,
         )
         if result.returncode != 0:
             _btc_version_cache = (now, None)
@@ -2915,7 +2972,9 @@ def _get_bitcoin_deployment_info() -> dict | None:
             ["bitcoin-cli", f"-datadir={BITCOIN_DATADIR}", "getdeploymentinfo"],
             capture_output=True,
             text=True,
-            timeout=10,
+            # BIP-110 is supplementary tile metadata; an RPC timeout must not
+            # block the dashboard from rendering.
+            timeout=3,
         )
         if result.returncode != 0:
             _btc_deployment_cache = (now, None)
@@ -3114,7 +3173,9 @@ def _get_bitcoin_sync_info() -> dict | None:
             ["bitcoin-cli", f"-datadir={BITCOIN_DATADIR}", "getblockchaininfo"],
             capture_output=True,
             text=True,
-            timeout=10,
+            # Sync progress is optional tile metadata.  Keep a cold/unavailable
+            # node from delaying the initial dashboard response.
+            timeout=3,
         )
         if result.returncode != 0:
             _btc_sync_cache = (now, None)
@@ -3176,6 +3237,7 @@ async def api_bitcoin_bip110():
 
 @app.get("/api/services")
 async def api_services():
+    started_at = time.monotonic()
     cfg = load_config()
     services = cfg.get("services", [])
 
@@ -3188,32 +3250,115 @@ async def api_services():
 
     loop = asyncio.get_event_loop()
 
-    # Read runtime feature overrides from custom.nix Hub Managed section
+    # Read runtime feature overrides from custom.nix Hub Managed section and
+    # calculate each service's effective enabled state once.  The old code did
+    # this inside every status coroutine and then launched one systemctl process
+    # per tile; batching the state lookup keeps a slow D-Bus from multiplying
+    # the dashboard's first-render latency.
     overrides, *_ = await loop.run_in_executor(None, _read_hub_overrides)
-
+    effective_entries: list[tuple[dict, bool]] = []
-    # Cache port/firewall data once for the entire /api/services request
+    active_units_by_scope: dict[str, list[str]] = {}
-    listening_ports, firewall_ports = await asyncio.gather(
+    domain_names: set[str] = set()
-        loop.run_in_executor(None, _get_listening_ports),
+    for entry in services:
-        loop.run_in_executor(None, _get_firewall_allowed_ports),
-    )
-
-    async def get_status(entry):
         unit = entry.get("unit", "")
         scope = entry.get("type", "system")
         icon = entry.get("icon", "")
         enabled = entry.get("enabled", True)
 
-        # Overlay runtime feature state from custom.nix Hub Managed section
         feat_id = unit_to_feature.get(unit)
         if feat_id is None:
             feat_id = FEATURE_ICON_MAP.get(icon)
         if feat_id is not None and feat_id in overrides:
             enabled = overrides[feat_id]
 
+        effective_entries.append((entry, enabled))
+        if enabled and unit:
+            active_units_by_scope.setdefault(scope, []).append(unit)
+
         if enabled:
-            status = await loop.run_in_executor(
+            domain_key = SERVICE_DOMAIN_MAP.get(unit)
-                None, lambda: sysctl.is_active(unit, scope)
+            if domain_key:
+                domain_path = os.path.join(DOMAINS_DIR, domain_key)
+                try:
+                    with open(domain_path, "r") as f:
+                        domain = f.read(512).strip()
+                    if domain:
+                        domain_names.add(domain)
+                except OSError:
+                    pass
+
+    async def resolve_domain(domain: str) -> tuple[str, list[str] | None]:
+        """Resolve DNS off the event loop with a hard upper bound.
+
+        A broken DNS server must not hold the entire dashboard response hostage.
+        ``None`` means the lookup was inconclusive and the background checker can
+        fill in the health result later; an empty list means DNS definitively
+        returned no address.
+        """
+        try:
+            addresses = await asyncio.wait_for(
+                loop.run_in_executor(None, _resolve_all_addresses_cached, domain),
+                timeout=2,
             )
+            return domain, addresses
+        except (asyncio.TimeoutError, OSError):
+            return domain, None
+
+    active_states_future = loop.run_in_executor(
+        None, sysctl.active_states, active_units_by_scope
+    )
+    port_states_future = asyncio.gather(
+        loop.run_in_executor(None, _get_listening_ports_cached),
+        loop.run_in_executor(None, _get_firewall_allowed_ports_cached),
+    )
+    dns_states_future = asyncio.gather(
+        *(resolve_domain(domain) for domain in sorted(domain_names))
+    )
+    # Bitcoin sync and BIP-110 are supplementary tile metadata.  Fetch each
+    # once, concurrently with the other diagnostics, instead of doing duplicate
+    # RPC calls inside both bitcoind tile coroutines.
+    has_enabled_bitcoin = any(
+        entry.get("unit") == "bitcoind.service" and enabled
+        for entry, enabled in effective_entries
+    )
+    has_enabled_bip110 = any(
+        entry.get("icon") == "bip110" and enabled
+        for entry, enabled in effective_entries
+    )
+    bitcoin_sync_future = (
+        loop.run_in_executor(None, _get_bitcoin_sync_info)
+        if has_enabled_bitcoin
+        else asyncio.sleep(0, result=None)
+    )
+    bip110_future = (
+        loop.run_in_executor(None, _get_bip110_status)
+        if has_enabled_bip110
+        else asyncio.sleep(0, result=None)
+    )
+    (
+        active_states,
+        port_states,
+        dns_states,
+        bitcoin_sync_info,
+        bip110_status,
+    ) = await asyncio.gather(
+        active_states_future,
+        port_states_future,
+        dns_states_future,
+        bitcoin_sync_future,
+        bip110_future,
+    )
+    listening_ports, firewall_ports = port_states
+    resolved_domains = dict(dns_states)
+
+    async def get_status(item: tuple[dict, bool]):
+        entry, enabled = item
+        unit = entry.get("unit", "")
+        scope = entry.get("type", "system")
+        icon = entry.get("icon", "")
+
+        if enabled:
+            status = active_states.get((scope, unit), "unknown")
         else:
             status = "disabled"
 
@@ -3269,18 +3414,23 @@ async def api_services():
                     break
         has_domain_issues = False
         if needs_domain and domain and enabled:
-            addrs = _resolve_all_addresses(domain)
+            # DNS was resolved concurrently above.  A timed-out lookup is
+            # treated as unknown rather than as a hard failure; the background
+            # reachability checker will refresh the health state without
+            # delaying the first tile render.
+            addrs = resolved_domains.get(domain)
             dns_ok = True
-            if not addrs:
+            if addrs is not None:
-                dns_ok = False
+                if not addrs:
-            elif all(_is_loopback_address(a) for a in addrs):
+                    dns_ok = False
-                # Intentional server-local /etc/hosts override — not a mismatch.
+                elif all(_is_loopback_address(a) for a in addrs):
-                dns_ok = True
+                    # Intentional server-local /etc/hosts override — not a mismatch.
-            elif (
+                    dns_ok = True
-                _cached_external_ip != "unavailable"
+                elif (
-                and not any(a == _cached_external_ip for a in addrs)
+                    _cached_external_ip != "unavailable"
-            ):
+                    and not any(a == _cached_external_ip for a in addrs)
-                dns_ok = False
+                ):
+                    dns_ok = False
 
             if not dns_ok:
                 has_domain_issues = True
@@ -3304,14 +3454,13 @@ async def api_services():
                     health = "checking_reachability" if cached_reachable is None else "healthy"
                 else:
                     health = "healthy"
-            # Check Bitcoin IBD state
+            # Check Bitcoin IBD state from the single shared lookup above.
             if unit == "bitcoind.service" and enabled:
-                sync = await loop.run_in_executor(None, _get_bitcoin_sync_info)
+                if bitcoin_sync_info and bitcoin_sync_info.get("initialblockdownload"):
-                if sync and sync.get("initialblockdownload"):
                     health = "syncing"
-                    sync_progress = sync.get("verificationprogress", 0)
+                    sync_progress = bitcoin_sync_info.get("verificationprogress", 0)
-                    sync_blocks = sync.get("blocks", 0)
+                    sync_blocks = bitcoin_sync_info.get("blocks", 0)
-                    sync_headers = sync.get("headers", 0)
+                    sync_headers = bitcoin_sync_info.get("headers", 0)
                     sync_ibd = True
         elif status == "inactive":
             # For enabled services that are inactive (e.g. socket-activated PHP-FPM),
@@ -3356,8 +3505,8 @@ async def api_services():
                 btc_ver = _format_bitcoin_version(raw_ver, icon=icon)
                 service_data["bitcoin_version"] = btc_ver  # backwards compat
                 service_data["version"] = btc_ver
-            if icon == "bip110":
+            if icon == "bip110" and bip110_status is not None:
-                service_data["bip110"] = await loop.run_in_executor(None, _get_bip110_status)
+                service_data["bip110"] = bip110_status
         # ── Generic version for all services (Nix store path) ──────────
         if enabled and unit and "version" not in service_data:
             ver = await loop.run_in_executor(None, _get_service_version, unit)
@@ -3365,7 +3514,9 @@ async def api_services():
                 service_data["version"] = ver
         return service_data
 
-    results = await asyncio.gather(*[get_status(s) for s in services])
+    results = await asyncio.gather(*[get_status(item) for item in effective_entries])
+    elapsed = time.monotonic() - started_at
+    logger.info("/api/services: %d services in %.3fs", len(results), elapsed)
     return list(results)
 
 

app/sovran_systemsos_web/static/css/tiles.css

@@ -1,5 +1,28 @@
 /* ── Service tile card (status-only) ─────────────────────────────── */
 
+.dashboard-loading {
+  min-height: 180px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 10px;
+  color: var(--text-secondary);
+  font-size: 0.9rem;
+}
+
+.dashboard-loading-spinner {
+  width: 16px;
+  height: 16px;
+  border: 2px solid var(--border-color);
+  border-top-color: var(--accent-color);
+  border-radius: 50%;
+  animation: dashboard-loading-spin 0.8s linear infinite;
+}
+
+@keyframes dashboard-loading-spin {
+  to { transform: rotate(360deg); }
+}
+
 .service-tile {
   width: 160px;
   min-height: 130px;

app/sovran_systemsos_web/static/js/events.js

@@ -187,30 +187,39 @@ function showSecurityBanner() {
 // ── Init ──────────────────────────────────────────────────────────
 
 async function init() {
-  // Check onboarding status first — redirect to wizard if not complete
+  // These lightweight requests are independent.  Running them together avoids
+  // making the dashboard wait through three serial network round-trips before
+  // it can even start loading the service tiles.
+  var onboardingStatus = null;
+  var bannerData = null;
+  var cfg;
   try {
-    var onboardingStatus = await apiFetch("/api/onboarding/status");
+    var startupResults = await Promise.all([
-    if (!onboardingStatus.complete) {
+      apiFetch("/api/onboarding/status").catch(function() { return null; }),
-      window.location.href = "/onboarding";
+      apiFetch("/api/security/banner-status").catch(function() { return null; }),
-      return;
+      apiFetch("/api/config"),
-    }
+    ]);
+    onboardingStatus = startupResults[0];
+    bannerData = startupResults[1];
+    cfg = startupResults[2];
   } catch (_) {
-    // If we can't reach the endpoint, continue to normal dashboard
+    // If config cannot be loaded, continue with the normal service fallback.
+    cfg = null;
+  }
+
+  if (onboardingStatus && !onboardingStatus.complete) {
+    window.location.href = "/onboarding";
+    return;
   }
 
   // Show first-login security banner only for machines that went through onboarding
   // (legacy machines without the onboarding flag will never see this)
-  try {
+  if (bannerData && bannerData.show) {
-    var bannerData = await apiFetch("/api/security/banner-status");
+    showSecurityBanner();
-    if (bannerData && bannerData.show) {
-      showSecurityBanner();
-    }
-  } catch (_) {
-    // Non-fatal — silently ignore
   }
 
   try {
-    var cfg = await apiFetch("/api/config");
+    if (!cfg) throw new Error("Hub config unavailable");
     _currentRole = cfg.role || "server_plus_desktop";
     if (cfg.category_order) {
       for (var i = 0; i < cfg.category_order.length; i++) {

app/sovran_systemsos_web/systemctl.py

@@ -18,6 +18,45 @@ def is_active(unit: str, scope: Literal["system", "user"] = "system") -> str:
     return _run(["systemctl", f"--{scope}", "is-active", unit]) or "unknown"
 
 
+def active_states(units_by_scope: dict[str, list[str]], timeout: float = 5) -> dict[tuple[str, str], str]:
+    """Return active states for many units with one systemctl call per scope.
+
+    The dashboard polls service state frequently. Spawning one ``systemctl``
+    process per tile makes a slow or unavailable system bus multiply the delay
+    (and can make the first dashboard render wait through many timeouts).  The
+    ``is-active`` command accepts multiple units, so batch them and cap the
+    whole batch at a single timeout.
+    """
+    states: dict[tuple[str, str], str] = {}
+    valid_scopes = {"system", "user"}
+
+    for scope, units in units_by_scope.items():
+        unique_units = list(dict.fromkeys(unit for unit in units if unit))
+        if not unique_units:
+            continue
+        if scope not in valid_scopes:
+            for unit in unique_units:
+                states[(scope, unit)] = "unknown"
+            continue
+
+        try:
+            result = subprocess.run(
+                ["systemctl", f"--{scope}", "is-active", "--", *unique_units],
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+            lines = result.stdout.splitlines()
+            for index, unit in enumerate(unique_units):
+                state = lines[index].strip() if index < len(lines) else ""
+                states[(scope, unit)] = state or "unknown"
+        except Exception:
+            for unit in unique_units:
+                states[(scope, unit)] = "unknown"
+
+    return states
+
+
 def is_enabled(unit: str, scope: Literal["system", "user"] = "system") -> str:
     return _run(["systemctl", f"--{scope}", "is-enabled", unit]) or "unknown"
 

app/sovran_systemsos_web/templates/index.html

@@ -55,7 +55,12 @@
       <div id="sidebar-support"></div>
       <div id="sidebar-features"></div>
     </aside>
-    <div id="tiles-area"></div>
+    <div id="tiles-area">
+      <div class="dashboard-loading" role="status" aria-live="polite">
+        <span class="dashboard-loading-spinner" aria-hidden="true"></span>
+        <span>Loading service status…</span>
+      </div>
+    </div>
   </main>
 
   <!-- Update modal -->