Merge pull request #304 from naturallaw777/copilot/fix-sovran-hub-updater-retry-issue

Harden Hub updater against cache.nixos.org narinfo stalls

app/sovran_systemsos_web/server.py

@@ -652,6 +652,37 @@ templates = Jinja2Templates(directory=os.path.join(_BASE_DIR, "templates"))
 
 # ── Static asset cache-busting ────────────────────────────────────
 
+def _compute_asset_version() -> str:
+    """Return a 16-char asset version from Nix store hash or static/template metadata."""
+    nix_match = re.search(r"/nix/store/([a-z0-9]{32})-", os.path.realpath(_BASE_DIR))
+    if nix_match:
+        return nix_match.group(1)[:16]
+
+    hasher = hashlib.sha256()
+    for root in (
+        os.path.join(_BASE_DIR, "static"),
+        os.path.join(_BASE_DIR, "templates"),
+    ):
+        if not os.path.isdir(root):
+            continue
+        for dirpath, dirnames, filenames in os.walk(root):
+            dirnames.sort()
+            for filename in sorted(filenames):
+                path = os.path.join(dirpath, filename)
+                try:
+                    stat = os.stat(path)
+                except OSError:
+                    continue
+                hasher.update(path.encode())
+                hasher.update(b"\0")
+                hasher.update(f"{stat.st_mtime_ns}:{stat.st_size}".encode())
+                hasher.update(b"\0")
+    return hasher.hexdigest()[:16]
+
+
+ASSET_VERSION = _compute_asset_version()
+
+
 def _file_hash(filename: str) -> str:
     """Return first 8 chars of the MD5 hex digest for a static file."""
     path = os.path.join(_BASE_DIR, "static", filename)
@@ -1894,7 +1925,10 @@ def _verify_support_removed() -> bool:
 
 @app.get("/login", response_class=HTMLResponse)
 async def login_page(request: Request):
-    return templates.TemplateResponse("login.html", {"request": request})
+    return templates.TemplateResponse("login.html", {
+        "request": request,
+        "asset_version": ASSET_VERSION,
+    })
 
 
 @app.get("/auto-login")
@@ -1961,6 +1995,7 @@ async def api_logout(request: Request):
 async def index(request: Request):
     return templates.TemplateResponse("index.html", {
         "request": request,
+        "asset_version": ASSET_VERSION,
     })
 
 
@@ -1969,6 +2004,7 @@ async def onboarding(request: Request):
     _ensure_onboarding_reopened_for_migration()
     return templates.TemplateResponse("onboarding.html", {
         "request": request,
+        "asset_version": ASSET_VERSION,
         "onboarding_js_hash": _ONBOARDING_JS_HASH,
     })
 

app/sovran_systemsos_web/templates/index.html

@@ -4,17 +4,17 @@
   <meta charset="UTF-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>Sovran_SystemsOS Hub</title>
-  <link rel="stylesheet" href="/static/css/base.css" />
+  <link rel="stylesheet" href="/static/css/base.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/buttons.css" />
+  <link rel="stylesheet" href="/static/css/buttons.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/header.css" />
+  <link rel="stylesheet" href="/static/css/header.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/layout.css" />
+  <link rel="stylesheet" href="/static/css/layout.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/tiles.css" />
+  <link rel="stylesheet" href="/static/css/tiles.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/modals.css" />
+  <link rel="stylesheet" href="/static/css/modals.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/features.css" />
+  <link rel="stylesheet" href="/static/css/features.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/onboarding.css" />
+  <link rel="stylesheet" href="/static/css/onboarding.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/support.css" />
+  <link rel="stylesheet" href="/static/css/support.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/domain-setup.css" />
+  <link rel="stylesheet" href="/static/css/domain-setup.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/security.css" />
+  <link rel="stylesheet" href="/static/css/security.css?v={{ asset_version }}" />
 </head>
 <body>
 
@@ -263,16 +263,16 @@
     </div>
   </div>
 
-  <script src="/static/js/constants.js"></script>
+  <script src="/static/js/constants.js?v={{ asset_version }}"></script>
-  <script src="/static/js/state.js"></script>
+  <script src="/static/js/state.js?v={{ asset_version }}"></script>
-  <script src="/static/js/helpers.js"></script>
+  <script src="/static/js/helpers.js?v={{ asset_version }}"></script>
-  <script src="/static/js/tiles.js"></script>
+  <script src="/static/js/tiles.js?v={{ asset_version }}"></script>
-  <script src="/static/js/service-detail.js"></script>
+  <script src="/static/js/service-detail.js?v={{ asset_version }}"></script>
-  <script src="/static/js/support.js"></script>
+  <script src="/static/js/support.js?v={{ asset_version }}"></script>
-  <script src="/static/js/update.js"></script>
+  <script src="/static/js/update.js?v={{ asset_version }}"></script>
-  <script src="/static/js/rebuild.js"></script>
+  <script src="/static/js/rebuild.js?v={{ asset_version }}"></script>
-  <script src="/static/js/features.js"></script>
+  <script src="/static/js/features.js?v={{ asset_version }}"></script>
-  <script src="/static/js/security.js"></script>
+  <script src="/static/js/security.js?v={{ asset_version }}"></script>
-  <script src="/static/js/events.js"></script>
+  <script src="/static/js/events.js?v={{ asset_version }}"></script>
 </body>
 </html>

app/sovran_systemsos_web/templates/login.html

@@ -4,8 +4,8 @@
   <meta charset="UTF-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>Sovran Hub — Login</title>
-  <link rel="stylesheet" href="/static/css/base.css" />
+  <link rel="stylesheet" href="/static/css/base.css?v={{ asset_version }}" />
-  <link rel="stylesheet" href="/static/css/buttons.css" />
+  <link rel="stylesheet" href="/static/css/buttons.css?v={{ asset_version }}" />
 </head>
 <body>
   <div class="login-wrapper">

configuration.nix

@@ -26,6 +26,13 @@
   nix.settings = {
     experimental-features = [ "nix-command" "flakes" ];
     download-buffer-size = 524288000;
+
+    # Network resilience for cache.nixos.org (Fastly) flakiness.
+    connect-timeout = 10;          # fail-fast on dead TCP connects (default: 0 = unlimited)
+    stalled-download-timeout = 90; # default 300s; retry sooner on stalled transfers
+    download-attempts = 7;         # default 5
+    http-connections = 25;         # cap concurrency (helps MTU/middlebox paths)
+    fallback = true;               # build locally if a substitute can't be fetched
   };
 
   # ── Networking ──────────────────────────────────────────────

modules/core/sovran-hub.nix

@@ -138,7 +138,11 @@ let
     RC=0
 
     echo "── Step 1/3: nix flake update ────────────────────"
-    if ! nix flake update --flake /etc/nixos --print-build-logs 2>&1; then
+    if ! nix flake update --flake /etc/nixos --print-build-logs \
+          --option connect-timeout 10 \
+          --option stalled-download-timeout 90 \
+          --option download-attempts 7 \
+          --option fallback true 2>&1; then
       echo "[ERROR] nix flake update failed"
       RC=1
     fi
@@ -146,7 +150,11 @@ let
 
     if [ "$RC" -eq 0 ]; then
       echo "── Step 2/3: nixos-rebuild ──────────────────────────"
-      SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs 2>&1)
+      SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+        --option connect-timeout 10 \
+        --option stalled-download-timeout 90 \
+        --option download-attempts 7 \
+        --option fallback true 2>&1)
       SWITCH_RC=$?
       echo "$SWITCH_OUT"
       if [ "$SWITCH_RC" -eq 0 ]; then
@@ -155,7 +163,11 @@ let
         echo ""
         echo "  ✓ Build succeeded — a reboot is required to apply this update"
         echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-        if nixos-rebuild boot --flake /etc/nixos --print-build-logs 2>&1; then
+        if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
+             --option connect-timeout 10 \
+             --option stalled-download-timeout 90 \
+             --option download-attempts 7 \
+             --option fallback true 2>&1; then
           echo "REBOOT_REQUIRED" > "$STATUS"
           exit 0
         else
@@ -209,7 +221,11 @@ let
     echo "══════════════════════════════════════════════════"
     echo ""
     echo "── Rebuilding system configuration ──────────────"
-    SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs 2>&1)
+    SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+      --option connect-timeout 10 \
+      --option stalled-download-timeout 90 \
+      --option download-attempts 7 \
+      --option fallback true 2>&1)
     SWITCH_RC=$?
     echo "$SWITCH_OUT"
     if [ "$SWITCH_RC" -eq 0 ]; then
@@ -222,7 +238,11 @@ let
       echo ""
       echo "  ✓ Build succeeded — a reboot is required to apply this rebuild"
       echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-      if nixos-rebuild boot --flake /etc/nixos --print-build-logs 2>&1; then
+      if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
+           --option connect-timeout 10 \
+           --option stalled-download-timeout 90 \
+           --option download-attempts 7 \
+           --option fallback true 2>&1; then
         echo "REBOOT_REQUIRED" > "$STATUS"
       else
         echo "[ERROR] nixos-rebuild boot also failed"