fix(hub): make Hub login verification loadable and diagnosable

Correction to the patch-0002 report: after a reboot the Hub icon asks for a password AND the correct free (desktop) password is rejected there, while desktop sign-in accepts it. Patch 0002/0003 explain the icon prompt (legacy hub_manual_logout cookie; fixed in 0003), but the rejection itself means verify_system_password() never gets a conclusive crypt answer in the Hub unit and its PAM fallback cannot load either: on NixOS a bare libcrypt.so.1/libpam.so.0 resolves through LD_LIBRARY_PATH or not at all (or to glibc's failure-token stub), and until now every stage failed silently into a generic 'Incorrect password'.

password-model.nix now puts pkgs.libxcrypt and pkgs.pam on sovran-hub-web's LD_LIBRARY_PATH so both the crypt fallbacks and PAM can actually load. verify_system_password() now logs, journal-visible and never containing a password or hash: the shadow hash scheme, every libcrypt source tried and whether it loaded, and PAM's service + return code when it has to decide.

app/sovran_systemsos_web/system_password.py

@@ -18,12 +18,16 @@ from __future__ import annotations
 import ctypes
 import ctypes.util
 import hmac
+import logging
 import os
 import secrets
+import shutil
 import subprocess
 import threading
 from typing import Callable
 
+logger = logging.getLogger(__name__)
+
 USER = "free"
 SECRETS_DIR = "/var/lib/secrets"
 DISPLAY_FILE = f"{SECRETS_DIR}/free-password"
@@ -33,6 +37,7 @@ LIVE_ISO_MARKER = "/iso"   # NixOS live media mounts the image here; never start
 
 _lock = threading.Lock()   # crypt() returns a static buffer; serialise calls
 _libcrypt_libs = None      # list of every loadable libcrypt, in preference order
+_libcrypt_sources: list[str] = []   # where those libraries were loaded from
 
 
 # ── low level ──────────────────────────────────────────────────────────────
@@ -65,12 +70,14 @@ def _crypt_libs() -> list:
                 lib = ctypes.CDLL(path)
                 lib.crypt.restype = ctypes.c_char_p
                 lib.crypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
-            except (OSError, AttributeError):
+            except (OSError, AttributeError) as exc:
+                _libcrypt_sources.append(f"{path} (unloadable: {exc})")
                 continue
             if lib._handle in seen:   # same library reached by two names
                 continue
             seen.add(lib._handle)
             libs.append(lib)
+            _libcrypt_sources.append(path)
         _libcrypt_libs = libs
     return _libcrypt_libs
 
@@ -161,11 +168,21 @@ def _verify_via_pam(password: str, user: str) -> bool | None:
                                 ctypes.byref(conv), ctypes.byref(handle)) != 0:
                 continue
             try:
-                return libpam.pam_authenticate(handle, 0) == 0
+                rc = libpam.pam_authenticate(handle, 0)
+                if rc != 0:
+                    # e.g. 7 = auth failure, 9/10 = bad item / user unknown.
+                    # Logged (code only, never the password) so a Hub-only
+                    # rejection can be told apart from a wrong password.
+                    logger.warning(
+                        "verify via PAM: service=%s user=%s pam_authenticate rc=%d",
+                        service, user, rc)
+                return rc == 0
             finally:
                 libpam.pam_end(handle, 0)
+        logger.warning("verify via PAM: no PAM service could be started for user=%s", user)
         return None
-    except Exception:
+    except Exception as exc:
+        logger.warning("verify via PAM unavailable for user=%s: %s", user, exc)
         return None
 
 
@@ -211,7 +228,18 @@ def verify_system_password(password: str, user: str = USER) -> bool:
                 return hmac.compare_digest(out, h.encode())
     except Exception:
         pass
+    # No libcrypt could decide. Log *why* (hash scheme only -- never the
+    # password or hash) so a false "wrong password" at Hub login, while the
+    # same password signs in at the desktop, is diagnosable from
+    # `journalctl -u sovran-hub-web` instead of looking like user error.
+    scheme = h[1:].split("$", 1)[0] if h.startswith("$") else "legacy"
+    logger.warning(
+        "verify_system_password: libcrypt inconclusive for user=%s (scheme=%s, "
+        "sources=%r); falling back to PAM",
+        user, scheme, _libcrypt_sources,
+    )
     pam_result = _verify_via_pam(password, user)
+    logger.warning("verify_system_password: PAM verdict for user=%s: %s", user, pam_result)
     return bool(pam_result)
 
 

modules/password-model.nix

@@ -149,4 +149,12 @@ in
 
   # ── 3. The Hub verifies passwords against /etc/shadow via libxcrypt ───────
   systemd.services.sovran-hub-web.environment.LIBCRYPT = "${pkgs.libxcrypt}/lib/libcrypt.so.1";
+  # ctypes fallback lookups (bare "libcrypt.so.1" / "libpam.so.0") resolve
+  # through LD_LIBRARY_PATH inside the unit. Without this they find nothing
+  # on NixOS (no ld.so.conf paths) or glibc's failure-token libcrypt stub,
+  # whose "*0"/NULL answers read as a wrong password at Hub login even
+  # though the same password signs in at the desktop. Put the real
+  # libxcrypt and PAM libraries on the Hub's library path.
+  systemd.services.sovran-hub-web.environment.LD_LIBRARY_PATH =
+    lib.makeLibraryPath [ pkgs.libxcrypt pkgs.pam ];
 }

tests/test_system_password.py

@@ -140,6 +140,25 @@ class WrongLibraryMustNotLockTheUserOut(unittest.TestCase):
             self.assertNotIn("sign out and back in to show it here", text)
 
 
+class HubUnitMustBeAbleToLoadItsVerifiers(unittest.TestCase):
+    """Patch 0002's PAM fallback could only work if libpam.so.0 is findable
+    inside the Hub unit. On NixOS a bare soname resolves via LD_LIBRARY_PATH
+    or not at all, so the unit must put the real libxcrypt + PAM libraries
+    on it -- otherwise a correct desktop password reads as 'wrong password'
+    at Hub login while the desktop accepts it."""
+
+    def test_hub_unit_gets_real_libcrypt_and_pam_on_library_path(self):
+        nix = _read("modules", "password-model.nix")
+        self.assertIn("LD_LIBRARY_PATH", nix)
+        self.assertIn("pkgs.libxcrypt pkgs.pam", nix)
+
+    def test_no_unix_chkpwd_shortcut(self):
+        # unix_chkpwd refuses (and stalls) when invoked outside pam_unix;
+        # PAM is the only supported route to it.
+        src = _read("app", "sovran_systemsos_web", "system_password.py")
+        self.assertNotIn("_verify_via_unix_chkpwd", src)
+
+
 class RootHintMustNotClaimItIsTheFreePassword(unittest.TestCase):
     """The root row in the System Passwords tile showed the free/hub hint
     verbatim. Root has no desktop or Hub sign-in, so that text was false