fix(hub): Hub login must accept the password the desktop accepts

verify_system_password() trusted the first loadable libcrypt. A library that cannot speak the shadow hash's algorithm (yescrypt, NixOS's default) answers with the '*0' sentinel, which was compared to the hash and reported as a wrong password -- even though the same password still signed in at the desktop. Following the System Passwords tile's 'sign out and back in' hint then locked the user out of the Hub, asking for a 'new' Hub password that does not exist: Hub login is the free account password, verified against /etc/shadow.

Try every loadable libcrypt, treat None/'*' replies as inconclusive rather than wrong, and when no library can decide, verify via PAM -- the same check GDM performs. Also replaces the sign-out hint (server + JS fallback) with wording that says the Hub has no separate password and not to sign out unless you know the current desktop password.

Recovery for an already locked-out user whose desktop login still works: on the machine itself, clear the localhost:8937 cookies and open http://localhost:8937/auto-login (localhost-only), or run 'sudo change-free-password' in a terminal to set a known free/Hub password.

app/sovran_systemsos_web/server.py

@@ -1824,7 +1824,7 @@ def _resolve_credential(cred: dict) -> dict | None:
         if pw is None:
             hint = ("Changed outside the Hub — finish the password step in onboarding to see it here."
                     if migration else
-                    "Changed outside the Hub — sign out and back in to show it here.")
+                    "Changed outside the Hub — the current password is your desktop (free) sign-in password; it will show here after you next sign in to the Hub with it. Do not sign out unless you know that password — the Hub has no separate password.")
             return {"label": label, "value": "", "unknown": True, "hint": hint}
         return {"label": label, "value": prefix + pw + suffix, "multiline": multiline}
 

app/sovran_systemsos_web/static/js/service-detail.js

@@ -35,7 +35,7 @@ function _renderCredsHtml(credentials, unit) {
       continue;
     }
     if (cred.unknown) {
-      html += '<div class="creds-row"><div class="creds-label">' + escHtml(cred.label) + '</div><div class="creds-value-wrap"><div class="creds-value creds-unknown">' + escHtml(cred.hint || "Changed outside the Hub — sign out and back in to show it here.") + '</div></div></div>';
+      html += '<div class="creds-row"><div class="creds-label">' + escHtml(cred.label) + '</div><div class="creds-value-wrap"><div class="creds-value creds-unknown">' + escHtml(cred.hint || "Changed outside the Hub — the current password is your desktop (free) sign-in password; it will show here after you next sign in to the Hub with it. Do not sign out unless you know that password — the Hub has no separate password.") + '</div></div></div>';
       continue;
     }
     var displayValue = linkify(cred.value);

app/sovran_systemsos_web/system_password.py

@@ -32,14 +32,24 @@ MIGRATION_FILE = f"{SECRETS_DIR}/free-password-migration-newpass"   # same path
 LIVE_ISO_MARKER = "/iso"   # NixOS live media mounts the image here; never start a migration there
 
 _lock = threading.Lock()   # crypt() returns a static buffer; serialise calls
-_libcrypt = None
+_libcrypt_libs = None      # list of every loadable libcrypt, in preference order
 
 
 # ── low level ──────────────────────────────────────────────────────────────
 
-def _crypt():
-    global _libcrypt
-    if _libcrypt is None:
+def _crypt_libs() -> list:
+    """Every libcrypt we can load, best first ($LIBCRYPT, then system ones).
+
+    More than one library can be *loadable* while only one can actually
+    verify the system's hashes: e.g. a bare-soname libcrypt that loads fine
+    but returns the "*0" failure sentinel for yescrypt ($y$) hashes, which
+    NixOS uses by default. Verification therefore tries each library until
+    one gives a conclusive answer instead of trusting the first that loads.
+    A stale $LIBCRYPT (Nix store path from before an update / GC) is skipped,
+    never fatal.
+    """
+    global _libcrypt_libs
+    if _libcrypt_libs is None:
         candidates: list[str] = []
         env_path = os.environ.get("LIBCRYPT")
         if env_path:
@@ -48,22 +58,115 @@ def _crypt():
         if found:
             candidates.append(found)
         candidates += ["libcrypt.so.1", "libcrypt.so.2", "libcrypt.so"]
-        last_exc: Exception | None = None
-        for path in dict.fromkeys(candidates):
+        libs = []
+        seen = set()
+        for path in candidates:
             try:
                 lib = ctypes.CDLL(path)
                 lib.crypt.restype = ctypes.c_char_p
                 lib.crypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
-            except (OSError, AttributeError) as exc:
-                # A stale LIBCRYPT path (e.g. a Nix store path from before an
-                # update / GC) must not be fatal: try the next candidate.
-                last_exc = exc
+            except (OSError, AttributeError):
                 continue
-            _libcrypt = lib
+            if lib._handle in seen:   # same library reached by two names
+                continue
+            seen.add(lib._handle)
+            libs.append(lib)
+        _libcrypt_libs = libs
+    return _libcrypt_libs
+
+
+def _crypt():
+    libs = _crypt_libs()
+    if not libs:
+        raise OSError("could not load any libcrypt")
+    return libs[0]
+
+
+# ── PAM fallback ───────────────────────────────────────────────────────────
+# crypt() only answers conclusively when a loaded library understands the
+# shadow hash's algorithm. When none does (sentinel replies, no library),
+# ask PAM instead -- it is literally the check GDM/sudo perform, so a
+# password the desktop accepts can never be refused by the Hub just
+# because libcrypt could not speak its hash format.
+
+_PAM_SERVICE_CANDIDATES = ("login", "system-auth", "gdm-password", "sudo")
+
+
+class _PamMessage(ctypes.Structure):
+    _fields_ = [("msg_style", ctypes.c_int), ("msg", ctypes.c_char_p)]
+
+
+class _PamResponse(ctypes.Structure):
+    _fields_ = [("resp", ctypes.c_char_p), ("resp_retcode", ctypes.c_int)]
+
+
+_PAM_CONV_FUNC = ctypes.CFUNCTYPE(
+    ctypes.c_int, ctypes.c_int,
+    ctypes.POINTER(ctypes.POINTER(_PamMessage)),
+    ctypes.POINTER(ctypes.POINTER(_PamResponse)), ctypes.c_void_p,
+)
+
+
+class _PamConv(ctypes.Structure):
+    _fields_ = [("conv", _PAM_CONV_FUNC), ("appdata_ptr", ctypes.c_void_p)]
+
+
+def _verify_via_pam(password: str, user: str) -> bool | None:
+    """True/False if PAM gave a verdict, None if PAM is unavailable here."""
+    try:
+        libpam = None
+        for candidate in dict.fromkeys(
+                filter(None, [ctypes.util.find_library("pam"), "libpam.so.0"])):
+            try:
+                libpam = ctypes.CDLL(candidate)
                 break
-        if _libcrypt is None:
-            raise OSError(f"could not load libcrypt (tried {candidates!r})") from last_exc
-    return _libcrypt
+            except OSError:
+                continue
+        if libpam is None:
+            return None
+        libc = ctypes.CDLL(None)
+        libc.calloc.restype = ctypes.c_void_p
+        libc.calloc.argtypes = [ctypes.c_size_t, ctypes.c_size_t]
+        libc.strdup.restype = ctypes.c_void_p
+        libc.strdup.argtypes = [ctypes.c_char_p]
+        libpam.pam_start.restype = ctypes.c_int
+        libpam.pam_start.argtypes = [ctypes.c_char_p, ctypes.c_char_p,
+                                     ctypes.POINTER(_PamConv), ctypes.POINTER(ctypes.c_void_p)]
+        libpam.pam_authenticate.restype = ctypes.c_int
+        libpam.pam_authenticate.argtypes = [ctypes.c_void_p, ctypes.c_int]
+        libpam.pam_end.restype = ctypes.c_int
+        libpam.pam_end.argtypes = [ctypes.c_void_p, ctypes.c_int]
+
+        pw_bytes = password.encode()
+
+        def _conv(num_msg, msg, resp, _appdata):
+            if num_msg <= 0:
+                return 1  # PAM_CONV_ERR
+            responses = libc.calloc(num_msg, ctypes.sizeof(_PamResponse))
+            if not responses:
+                return 1
+            resp_array = ctypes.cast(responses, ctypes.POINTER(_PamResponse))
+            for i in range(num_msg):
+                style = msg[i].contents.msg_style
+                if style in (1, 2):  # PROMPT_ECHO_OFF / PROMPT_ECHO_ON
+                    resp_array[i].resp = ctypes.cast(libc.strdup(pw_bytes), ctypes.c_char_p)
+            resp[0] = ctypes.cast(responses, ctypes.POINTER(_PamResponse))
+            return 0  # PAM_SUCCESS
+
+        conv_fn = _PAM_CONV_FUNC(_conv)   # keep alive for the pam_* calls
+        conv = _PamConv(conv=conv_fn, appdata_ptr=None)
+        for service in _PAM_SERVICE_CANDIDATES:
+            handle = ctypes.c_void_p()
+            if libpam.pam_start(service.encode(), user.encode(),
+                                ctypes.byref(conv), ctypes.byref(handle)) != 0:
+                continue
+            try:
+                return libpam.pam_authenticate(handle, 0) == 0
+            finally:
+                libpam.pam_end(handle, 0)
+        return None
+    except Exception:
+        return None
 
 
 def shadow_hash(user: str = USER) -> str | None:
@@ -86,21 +189,30 @@ def verify_system_password(password: str, user: str = USER) -> bool:
     h = shadow_hash(user)
     if not h or not password:
         return False
+    # Try every loadable libcrypt. A reply of None or "*0"/"*1" is a library
+    # saying "I don't understand this hash" (e.g. no yescrypt support) --
+    # inconclusive, NOT a wrong password -- so keep trying, and finally ask
+    # PAM. A real hash that simply doesn't match is conclusive: wrong
+    # password. Nothing here may raise: raising crashed the System
+    # Passwords tile ("Could not load service details."), and returning a
+    # false "wrong password" at login would lock a user whose desktop
+    # login still works out of the Hub.
     try:
         with _lock:
-            out = _crypt().crypt(password.encode(), h.encode())
+            for lib in _crypt_libs():
+                try:
+                    out = lib.crypt(password.encode(), h.encode())
                 except Exception:
-        # Verification is fail-closed, never fatal. libcrypt can be missing /
-        # unloadable (stale LIBCRYPT Nix store path, library not yet in the
-        # Hub's environment after an update, unsupported hash): that must
-        # degrade to "password not verified / not showable", not raise into
-        # callers. Raising here crashed GET /api/service-detail/<unit> for
-        # the System Passwords tile, so the Hub showed only
-        # "Could not load service details." for that one tile.
-        return False
+                    continue
                 if isinstance(out, str):
                     out = out.encode()
-    return out is not None and hmac.compare_digest(out, h.encode())
+                if out is None or out.startswith(b"*"):
+                    continue
+                return hmac.compare_digest(out, h.encode())
+    except Exception:
+        pass
+    pam_result = _verify_via_pam(password, user)
+    return bool(pam_result)
 
 
 def _write_secret(path: str, value: str) -> None:

tests/test_system_password.py

@@ -87,6 +87,59 @@ class VerifyNeverRaises(unittest.TestCase):
             self.assertIsNone(self.sp.read_root_display_password())
 
 
+class WrongLibraryMustNotLockTheUserOut(unittest.TestCase):
+    """A loadable libcrypt that cannot speak the shadow hash's algorithm
+    (yescrypt $y$ on NixOS) answers with the "*0" sentinel. That used to be
+    compared against the hash, found unequal, and reported as a *wrong
+    password* at Hub login -- locking out a user whose desktop login with
+    the very same password still worked, after the tile's hint told them
+    to sign out. Inconclusive answers must fall through to the next
+    library, and finally to PAM (the check GDM itself performs)."""
+
+    def setUp(self):
+        self.sp = _load_system_password()
+        self.sp.shadow_hash = lambda user="free": _KNOWN_HASH
+        self._old_libs = self.sp._libcrypt_libs
+
+    def tearDown(self):
+        self.sp._libcrypt_libs = self._old_libs
+
+    def test_sentinel_from_first_library_falls_through_to_the_next(self):
+        class _SentinelLib:
+            @staticmethod
+            def crypt(_pw, _salt):
+                return b"*0"
+
+        class _RealLib:
+            @staticmethod
+            def crypt(pw, salt):
+                return _KNOWN_HASH.encode() if pw == b"hello" else b"$6$saltsalt$nomatch"
+
+        self.sp._libcrypt_libs = [_SentinelLib, _RealLib]
+        self.assertTrue(self.sp.verify_system_password("hello"))
+        self.assertFalse(self.sp.verify_system_password("not-the-password"))
+
+    def test_when_no_library_understands_the_hash_pam_decides(self):
+        class _SentinelLib:
+            @staticmethod
+            def crypt(_pw, _salt):
+                return b"*0"
+
+        self.sp._libcrypt_libs = [_SentinelLib]
+        with unittest.mock.patch.object(
+            self.sp, "_verify_via_pam", return_value=True
+        ) as pam:
+            self.assertTrue(self.sp.verify_system_password("hello"))
+            pam.assert_called_once()
+
+    def test_hint_no_longer_tells_the_user_to_sign_out_blind(self):
+        for text in (
+            _read("app", "sovran_systemsos_web", "server.py"),
+            _read("app", "sovran_systemsos_web", "static", "js", "service-detail.js"),
+        ):
+            self.assertNotIn("sign out and back in to show it here", text)
+
+
 class ServiceDetailCannotBeKilledByOneCredential(unittest.TestCase):
     """Source guards (server.py is not importable without the full Hub deps,
     same reason the other suites read it as text)."""