fix(hub): Hub login must accept the password the desktop accepts

verify_system_password() trusted the first loadable libcrypt. A library that cannot speak the shadow hash's algorithm (yescrypt, NixOS's default) answers with the '*0' sentinel, which was compared to the hash and reported as a wrong password -- even though the same password still signed in at the desktop. Following the System Passwords tile's 'sign out and back in' hint then locked the user out of the Hub, asking for a 'new' Hub password that does not exist: Hub login is the free account password, verified against /etc/shadow.

Try every loadable libcrypt, treat None/'*' replies as inconclusive rather than wrong, and when no library can decide, verify via PAM -- the same check GDM performs. Also replaces the sign-out hint (server + JS fallback) with wording that says the Hub has no separate password and not to sign out unless you know the current desktop password.

Recovery for an already locked-out user whose desktop login still works: on the machine itself, clear the localhost:8937 cookies and open http://localhost:8937/auto-login (localhost-only), or run 'sudo change-free-password' in a terminal to set a known free/Hub password.

app/sovran_systemsos_web/server.py

@@ -728,7 +728,10 @@ def _check_password(submitted: str) -> bool:
     proved (e.g. changed in GNOME Settings or the pre-upgrade Hub)."""
     if not sp.verify_system_password(submitted):
         return False
-    sp.record_known_password(submitted, only_if_stale=True)
+    try:
+        sp.record_known_password(submitted, only_if_stale=True)
+    except Exception as exc:  # healing the display copy is best-effort
+        logger.warning("Could not heal stale password display copy: %s", exc)
     return True
 
 
@@ -1808,12 +1811,20 @@ def _resolve_credential(cred: dict) -> dict | None:
     # free/root login passwords: shown only if the display copy really is the
     # current one (system_password verifies it against /etc/shadow).
     if filepath in (sp.DISPLAY_FILE, sp.ROOT_DISPLAY_FILE):
-        pw = (sp.read_display_password() if filepath == sp.DISPLAY_FILE
-              else sp.read_root_display_password())
+        try:
+            pw = (sp.read_display_password() if filepath == sp.DISPLAY_FILE
+                  else sp.read_root_display_password())
+            migration = sp.migration_pending() if pw is None else None
+        except Exception as exc:
+            # Never let one credential 500 the whole service-detail modal:
+            # the System Passwords tile must still open and say the password
+            # is unavailable instead of "Could not load service details."
+            logger.warning("Could not resolve %s for display: %s", filepath, exc)
+            pw, migration = None, None
         if pw is None:
             hint = ("Changed outside the Hub — finish the password step in onboarding to see it here."
-                    if sp.migration_pending() else
-                    "Changed outside the Hub — sign out and back in to show it here.")
+                    if migration else
+                    "Changed outside the Hub — the current password is your desktop (free) sign-in password; it will show here after you next sign in to the Hub with it. Do not sign out unless you know that password — the Hub has no separate password.")
             return {"label": label, "value": "", "unknown": True, "hint": hint}
         return {"label": label, "value": prefix + pw + suffix, "multiline": multiline}
 

app/sovran_systemsos_web/static/js/service-detail.js

@@ -35,7 +35,7 @@ function _renderCredsHtml(credentials, unit) {
       continue;
     }
     if (cred.unknown) {
-      html += '<div class="creds-row"><div class="creds-label">' + escHtml(cred.label) + '</div><div class="creds-value-wrap"><div class="creds-value creds-unknown">' + escHtml(cred.hint || "Changed outside the Hub — sign out and back in to show it here.") + '</div></div></div>';
+      html += '<div class="creds-row"><div class="creds-label">' + escHtml(cred.label) + '</div><div class="creds-value-wrap"><div class="creds-value creds-unknown">' + escHtml(cred.hint || "Changed outside the Hub — the current password is your desktop (free) sign-in password; it will show here after you next sign in to the Hub with it. Do not sign out unless you know that password — the Hub has no separate password.") + '</div></div></div>';
       continue;
     }
     var displayValue = linkify(cred.value);

app/sovran_systemsos_web/system_password.py

@@ -32,20 +32,141 @@ MIGRATION_FILE = f"{SECRETS_DIR}/free-password-migration-newpass"   # same path
 LIVE_ISO_MARKER = "/iso"   # NixOS live media mounts the image here; never start a migration there
 
 _lock = threading.Lock()   # crypt() returns a static buffer; serialise calls
-_libcrypt = None
+_libcrypt_libs = None      # list of every loadable libcrypt, in preference order
 
 
 # ── low level ──────────────────────────────────────────────────────────────
 
+def _crypt_libs() -> list:
+    """Every libcrypt we can load, best first ($LIBCRYPT, then system ones).
+
+    More than one library can be *loadable* while only one can actually
+    verify the system's hashes: e.g. a bare-soname libcrypt that loads fine
+    but returns the "*0" failure sentinel for yescrypt ($y$) hashes, which
+    NixOS uses by default. Verification therefore tries each library until
+    one gives a conclusive answer instead of trusting the first that loads.
+    A stale $LIBCRYPT (Nix store path from before an update / GC) is skipped,
+    never fatal.
+    """
+    global _libcrypt_libs
+    if _libcrypt_libs is None:
+        candidates: list[str] = []
+        env_path = os.environ.get("LIBCRYPT")
+        if env_path:
+            candidates.append(env_path)
+        found = ctypes.util.find_library("crypt")
+        if found:
+            candidates.append(found)
+        candidates += ["libcrypt.so.1", "libcrypt.so.2", "libcrypt.so"]
+        libs = []
+        seen = set()
+        for path in candidates:
+            try:
+                lib = ctypes.CDLL(path)
+                lib.crypt.restype = ctypes.c_char_p
+                lib.crypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+            except (OSError, AttributeError):
+                continue
+            if lib._handle in seen:   # same library reached by two names
+                continue
+            seen.add(lib._handle)
+            libs.append(lib)
+        _libcrypt_libs = libs
+    return _libcrypt_libs
+
+
 def _crypt():
-    global _libcrypt
-    if _libcrypt is None:
-        path = os.environ.get("LIBCRYPT") or ctypes.util.find_library("crypt") or "libcrypt.so.1"
-        lib = ctypes.CDLL(path)
-        lib.crypt.restype = ctypes.c_char_p
-        lib.crypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
-        _libcrypt = lib
-    return _libcrypt
+    libs = _crypt_libs()
+    if not libs:
+        raise OSError("could not load any libcrypt")
+    return libs[0]
+
+
+# ── PAM fallback ───────────────────────────────────────────────────────────
+# crypt() only answers conclusively when a loaded library understands the
+# shadow hash's algorithm. When none does (sentinel replies, no library),
+# ask PAM instead -- it is literally the check GDM/sudo perform, so a
+# password the desktop accepts can never be refused by the Hub just
+# because libcrypt could not speak its hash format.
+
+_PAM_SERVICE_CANDIDATES = ("login", "system-auth", "gdm-password", "sudo")
+
+
+class _PamMessage(ctypes.Structure):
+    _fields_ = [("msg_style", ctypes.c_int), ("msg", ctypes.c_char_p)]
+
+
+class _PamResponse(ctypes.Structure):
+    _fields_ = [("resp", ctypes.c_char_p), ("resp_retcode", ctypes.c_int)]
+
+
+_PAM_CONV_FUNC = ctypes.CFUNCTYPE(
+    ctypes.c_int, ctypes.c_int,
+    ctypes.POINTER(ctypes.POINTER(_PamMessage)),
+    ctypes.POINTER(ctypes.POINTER(_PamResponse)), ctypes.c_void_p,
+)
+
+
+class _PamConv(ctypes.Structure):
+    _fields_ = [("conv", _PAM_CONV_FUNC), ("appdata_ptr", ctypes.c_void_p)]
+
+
+def _verify_via_pam(password: str, user: str) -> bool | None:
+    """True/False if PAM gave a verdict, None if PAM is unavailable here."""
+    try:
+        libpam = None
+        for candidate in dict.fromkeys(
+                filter(None, [ctypes.util.find_library("pam"), "libpam.so.0"])):
+            try:
+                libpam = ctypes.CDLL(candidate)
+                break
+            except OSError:
+                continue
+        if libpam is None:
+            return None
+        libc = ctypes.CDLL(None)
+        libc.calloc.restype = ctypes.c_void_p
+        libc.calloc.argtypes = [ctypes.c_size_t, ctypes.c_size_t]
+        libc.strdup.restype = ctypes.c_void_p
+        libc.strdup.argtypes = [ctypes.c_char_p]
+        libpam.pam_start.restype = ctypes.c_int
+        libpam.pam_start.argtypes = [ctypes.c_char_p, ctypes.c_char_p,
+                                     ctypes.POINTER(_PamConv), ctypes.POINTER(ctypes.c_void_p)]
+        libpam.pam_authenticate.restype = ctypes.c_int
+        libpam.pam_authenticate.argtypes = [ctypes.c_void_p, ctypes.c_int]
+        libpam.pam_end.restype = ctypes.c_int
+        libpam.pam_end.argtypes = [ctypes.c_void_p, ctypes.c_int]
+
+        pw_bytes = password.encode()
+
+        def _conv(num_msg, msg, resp, _appdata):
+            if num_msg <= 0:
+                return 1  # PAM_CONV_ERR
+            responses = libc.calloc(num_msg, ctypes.sizeof(_PamResponse))
+            if not responses:
+                return 1
+            resp_array = ctypes.cast(responses, ctypes.POINTER(_PamResponse))
+            for i in range(num_msg):
+                style = msg[i].contents.msg_style
+                if style in (1, 2):  # PROMPT_ECHO_OFF / PROMPT_ECHO_ON
+                    resp_array[i].resp = ctypes.cast(libc.strdup(pw_bytes), ctypes.c_char_p)
+            resp[0] = ctypes.cast(responses, ctypes.POINTER(_PamResponse))
+            return 0  # PAM_SUCCESS
+
+        conv_fn = _PAM_CONV_FUNC(_conv)   # keep alive for the pam_* calls
+        conv = _PamConv(conv=conv_fn, appdata_ptr=None)
+        for service in _PAM_SERVICE_CANDIDATES:
+            handle = ctypes.c_void_p()
+            if libpam.pam_start(service.encode(), user.encode(),
+                                ctypes.byref(conv), ctypes.byref(handle)) != 0:
+                continue
+            try:
+                return libpam.pam_authenticate(handle, 0) == 0
+            finally:
+                libpam.pam_end(handle, 0)
+        return None
+    except Exception:
+        return None
 
 
 def shadow_hash(user: str = USER) -> str | None:
@@ -68,9 +189,30 @@ def verify_system_password(password: str, user: str = USER) -> bool:
     h = shadow_hash(user)
     if not h or not password:
         return False
-    with _lock:
-        out = _crypt().crypt(password.encode(), h.encode())
-    return out is not None and hmac.compare_digest(out, h.encode())
+    # Try every loadable libcrypt. A reply of None or "*0"/"*1" is a library
+    # saying "I don't understand this hash" (e.g. no yescrypt support) --
+    # inconclusive, NOT a wrong password -- so keep trying, and finally ask
+    # PAM. A real hash that simply doesn't match is conclusive: wrong
+    # password. Nothing here may raise: raising crashed the System
+    # Passwords tile ("Could not load service details."), and returning a
+    # false "wrong password" at login would lock a user whose desktop
+    # login still works out of the Hub.
+    try:
+        with _lock:
+            for lib in _crypt_libs():
+                try:
+                    out = lib.crypt(password.encode(), h.encode())
+                except Exception:
+                    continue
+                if isinstance(out, str):
+                    out = out.encode()
+                if out is None or out.startswith(b"*"):
+                    continue
+                return hmac.compare_digest(out, h.encode())
+    except Exception:
+        pass
+    pam_result = _verify_via_pam(password, user)
+    return bool(pam_result)
 
 
 def _write_secret(path: str, value: str) -> None:

flake.lock

@@ -152,11 +152,11 @@
         "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1792361662,
-        "narHash": "sha256-EySy/HbufH9trjH0kEuZ2nVVk0+Sn5FoxJI7YtHrspo=",
+        "lastModified": 1792372617,
+        "narHash": "sha256-MLFWEcj6ZyFgFWubNPYK7IEI8NUtjbrhxhXQiI6JbPM=",
         "owner": "naturallaw777",
         "repo": "Sovran_Bitcoin",
-        "rev": "5c13bf5e01beccc4a225a36cde833f00b2595718",
+        "rev": "78e4430525069172a8d1a9c08a951f8b87a03bf3",
         "type": "github"
       },
       "original": {

tests/test_system_password.py

@@ -0,0 +1,165 @@
+"""Regression tests for the System Passwords Hub tile.
+
+Bug: after the password-model update (4a25ef2), clicking "System Passwords"
+in The Hub showed only "Could not load service details."
+
+That message is the frontend catch-all for a failed
+GET /api/service-detail/root-password-setup.service. The new credential path
+was the only one that called system_password.verify_system_password(), which
+loaded libcrypt with no error handling: if LIBCRYPT pointed at a stale /
+unloadable Nix store path (or libcrypt was otherwise unavailable), CDLL raised
+OSError, _resolve_credential() raised, FastAPI returned 500, and the whole
+modal failed -- instead of just marking that one password "unknown".
+
+Verification must be fail-closed (return False / show the hint), never fatal.
+"""
+
+import importlib.util
+import os
+import sys
+import unittest
+import unittest.mock
+
+_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
+
+
+def _load_system_password():
+    path = os.path.join(_ROOT, "app", "sovran_systemsos_web", "system_password.py")
+    spec = importlib.util.spec_from_file_location("system_password_under_test", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _read(*parts):
+    with open(os.path.join(_ROOT, *parts), encoding="utf-8") as f:
+        return f.read()
+
+
+# A real sha512crypt hash (of "hello", salt "saltsalt") so crypt() can succeed
+# against the system libcrypt when the preferred path is stale.
+_KNOWN_HASH = "$6$saltsalt$YTeBOLnmm3CoeJTzBuijUEtOEWCqrw/nQ8/AeMmON4LGp3k0ZiMjC1OdqzWFUAfDQBvYkOch9BWtQTBcZ1E/p0"
+
+
+class VerifyNeverRaises(unittest.TestCase):
+    def setUp(self):
+        self.sp = _load_system_password()
+        self._old_env = os.environ.get("LIBCRYPT")
+
+    def tearDown(self):
+        if self._old_env is None:
+            os.environ.pop("LIBCRYPT", None)
+        else:
+            os.environ["LIBCRYPT"] = self._old_env
+
+    def test_unloadable_libcrypt_is_false_not_an_exception(self):
+        os.environ["LIBCRYPT"] = "/nix/store/stale-before-update/lib/libcrypt.so.1"
+        self.sp.shadow_hash = lambda user="free": _KNOWN_HASH
+        # No working libcrypt anywhere: every load attempt fails.
+        with unittest.mock.patch.object(
+            self.sp.ctypes, "CDLL", side_effect=OSError("cannot open shared object")
+        ):
+            self.assertFalse(self.sp.verify_system_password("hello"))
+
+    def test_stale_libcrypt_falls_back_and_still_verifies(self):
+        if not os.path.exists("/usr/lib/x86_64-linux-gnu/libcrypt.so.1") and \
+           not os.path.exists("/lib/x86_64-linux-gnu/libcrypt.so.1"):
+            # Wherever a system libcrypt exists, the fallback should find it
+            # via find_library / the bare soname; only assert the no-raise
+            # contract if this host has none to fall back to.
+            pass
+        os.environ["LIBCRYPT"] = "/nix/store/stale-before-update/lib/libcrypt.so.1"
+        self.sp.shadow_hash = lambda user="free": _KNOWN_HASH
+        try:
+            result = self.sp.verify_system_password("hello")
+        except Exception as exc:  # pragma: no cover - the regression itself
+            self.fail(f"verify_system_password raised {exc!r} instead of failing closed")
+        self.assertIn(result, (True, False))
+
+    def test_display_password_is_none_not_an_exception_when_unverifiable(self):
+        os.environ["LIBCRYPT"] = "/nix/store/stale-before-update/lib/libcrypt.so.1"
+        self.sp.shadow_hash = lambda user="free": _KNOWN_HASH
+        self.sp._read_secret = lambda path: "hello"
+        with unittest.mock.patch.object(
+            self.sp.ctypes, "CDLL", side_effect=OSError("cannot open shared object")
+        ):
+            self.assertIsNone(self.sp.read_display_password())
+            self.assertIsNone(self.sp.read_root_display_password())
+
+
+class WrongLibraryMustNotLockTheUserOut(unittest.TestCase):
+    """A loadable libcrypt that cannot speak the shadow hash's algorithm
+    (yescrypt $y$ on NixOS) answers with the "*0" sentinel. That used to be
+    compared against the hash, found unequal, and reported as a *wrong
+    password* at Hub login -- locking out a user whose desktop login with
+    the very same password still worked, after the tile's hint told them
+    to sign out. Inconclusive answers must fall through to the next
+    library, and finally to PAM (the check GDM itself performs)."""
+
+    def setUp(self):
+        self.sp = _load_system_password()
+        self.sp.shadow_hash = lambda user="free": _KNOWN_HASH
+        self._old_libs = self.sp._libcrypt_libs
+
+    def tearDown(self):
+        self.sp._libcrypt_libs = self._old_libs
+
+    def test_sentinel_from_first_library_falls_through_to_the_next(self):
+        class _SentinelLib:
+            @staticmethod
+            def crypt(_pw, _salt):
+                return b"*0"
+
+        class _RealLib:
+            @staticmethod
+            def crypt(pw, salt):
+                return _KNOWN_HASH.encode() if pw == b"hello" else b"$6$saltsalt$nomatch"
+
+        self.sp._libcrypt_libs = [_SentinelLib, _RealLib]
+        self.assertTrue(self.sp.verify_system_password("hello"))
+        self.assertFalse(self.sp.verify_system_password("not-the-password"))
+
+    def test_when_no_library_understands_the_hash_pam_decides(self):
+        class _SentinelLib:
+            @staticmethod
+            def crypt(_pw, _salt):
+                return b"*0"
+
+        self.sp._libcrypt_libs = [_SentinelLib]
+        with unittest.mock.patch.object(
+            self.sp, "_verify_via_pam", return_value=True
+        ) as pam:
+            self.assertTrue(self.sp.verify_system_password("hello"))
+            pam.assert_called_once()
+
+    def test_hint_no_longer_tells_the_user_to_sign_out_blind(self):
+        for text in (
+            _read("app", "sovran_systemsos_web", "server.py"),
+            _read("app", "sovran_systemsos_web", "static", "js", "service-detail.js"),
+        ):
+            self.assertNotIn("sign out and back in to show it here", text)
+
+
+class ServiceDetailCannotBeKilledByOneCredential(unittest.TestCase):
+    """Source guards (server.py is not importable without the full Hub deps,
+    same reason the other suites read it as text)."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.server = _read("app", "sovran_systemsos_web", "server.py")
+
+    def test_resolve_credential_guards_system_password_calls(self):
+        start = self.server.index("if filepath in (sp.DISPLAY_FILE, sp.ROOT_DISPLAY_FILE):")
+        block = self.server[start:start + 1600]
+        self.assertIn("try:", block)
+        self.assertIn("except Exception", block)
+        self.assertIn('"unknown": True', block)
+
+    def test_login_heal_is_best_effort(self):
+        start = self.server.index("sp.record_known_password(submitted, only_if_stale=True)")
+        block = self.server[max(0, start - 200):start + 200]
+        self.assertIn("except Exception", block)
+
+
+if __name__ == "__main__":
+    unittest.main()