feat(public-ip): unify public-IP detection into one privacy-first script

The public IP was previously detected independently in three places,
each contacting a different third party: the Hub (HTTPS echo via
api.ipify.org / ifconfig.me / icanhazip.com on every API call and
background tick), DDNS (myip.opendns.com via OpenDNS), and LiveKit
(embedded STUN). Consolidate into a single detector with one shared
cache so every consumer reads the same value with minimal exposure.

- add modules/core/public-ip.nix: installs /var/lib/sovran/public-ip.py
  (pure Python stdlib, no new deps) writing /var/lib/secrets/external-ip
- detection chain (first success wins): explicit pin, fresh cache
  (default TTL 300s), STUN binding request over UDP (one packet, no
  metadata), DNS myip.opendns.com query, then OPT-IN HTTPS echo
  (publicIP.httpsEcho, empty by default — never contacted unless listed)
- privacy: while the cache is fresh zero third parties are contacted;
  at most one party learns the IP per refresh interval, via the least
  exposing mechanism available
- hub (server.py): _get_external_ip() now reads the shared detector /
  cache instead of calling ipify/ifconfig/icanhazip directly
- ddns (njalla.nix): use the shared detector instead of a separate
  OpenDNS dig; allow the hardened service to write /var/lib/secrets
- element-calling: livekit-turn-setup falls back to the shared
  detector on cold boot; add LiveKit webhooks to lk-jwt-service
  (sfu_webhook) so abrupt disconnects are cleaned up immediately;
  set LIVEKIT_SANITY_CHECK_INTERVAL_SECONDS=60 as a missed-webhook
  guard; drop the dead services.livekit.settings block and set
  openFirewall=false (Caddy fronts the SFU; no public 7880/tcp)
- new options: sovran_systemsOS.publicIP.{stunServer,stunPort,
  dnsResolver,httpsEcho,cacheTTL}

app/sovran_systemsos_web/server.py

@@ -21,6 +21,7 @@ import subprocess
 import tempfile
 import threading
 import time
+import sys
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -966,20 +967,30 @@ def _save_external_ip(ip: str):
 
 
 def _get_external_ip() -> str:
-    MAX_IP_LENGTH = 46
+    """Public IP via the shared detector (/var/lib/sovran/public-ip.py).
-    for url in [
+
-        "https://api.ipify.org",
+    The detector owns discovery (STUN -> DNS -> opt-in HTTPS echo), caches the
-        "https://ifconfig.me/ip",
+    result in /var/lib/secrets/external-ip, and contacts at most one third
-        "https://icanhazip.com",
+    party per refresh interval. This function only reads the cache and asks
-    ]:
+    the detector to refresh when it is missing or stale — it performs no
+    per-call external queries of its own.
+    """
     try:
-            req = urllib.request.Request(url, method="GET")
+        r = subprocess.run(
-            with urllib.request.urlopen(req, timeout=8) as resp:
+            [sys.executable, "/var/lib/sovran/public-ip.py", "check"],
-                ip = resp.read().decode().strip()
+            capture_output=True, text=True, timeout=20,
-                if ip and len(ip) < MAX_IP_LENGTH:
+        )
-                    return ip
+        if r.returncode == 0 and r.stdout.strip():
+            return r.stdout.strip().splitlines()[0]
     except Exception:
-            continue
+        pass
+    try:
+        with open(EXTERNAL_IP_FILE) as f:
+            ip = f.read().strip()
+        if ip:
+            return ip
+    except OSError:
+        pass
     return "unavailable"
 
 

modules/core/njalla.nix

@@ -33,7 +33,7 @@
       # /var/lib/njalla/ddns_urls.json.
       NoNewPrivileges    = true;
       ProtectSystem      = "strict";
-      ReadWritePaths     = [ "/var/lib/njalla" ];
+      ReadWritePaths     = [ "/var/lib/njalla" "/var/lib/secrets" ];
       ReadOnlyPaths      = [ "/etc/sovran" ];
       ProtectHome        = true;
       PrivateTmp         = true;
@@ -88,12 +88,15 @@ try:
 except Exception:
     sys.exit(0)  # no URLs configured — nothing to do
 
-# Resolve current public IP once
+# Resolve current public IP via the shared detector — one script, one cache
+# (STUN -> DNS -> opt-in HTTPS echo; see /var/lib/sovran/public-ip.py).
+# The detector refreshes /var/lib/secrets/external-ip, which the Hub and
+# LiveKit read as well, so the whole system shares a single detected value.
 public_ip = ""
 try:
     r = subprocess.run(
-        ["dig", "@resolver4.opendns.com", "myip.opendns.com", "+short", "-4"],
+        [sys.executable, "/var/lib/sovran/public-ip.py", "check"],
-        capture_output=True, text=True, timeout=10,
+        capture_output=True, text=True, timeout=20,
     )
     raw = r.stdout.strip().splitlines()[0] if r.stdout.strip() else ""
     ipaddress.ip_address(raw)  # validates — raises if not a real IP
@@ -101,6 +104,16 @@ try:
 except Exception:
     pass
 
+if not public_ip:
+    # Last resort: the shared cache file, if the detector is unavailable.
+    try:
+        with open("/var/lib/secrets/external-ip") as f:
+            raw = f.read().strip()
+        ipaddress.ip_address(raw)
+        public_ip = raw
+    except Exception:
+        pass
+
 if not public_ip:
     sys.exit(0)  # no IP resolved — skip to avoid sending bare ''${IP}
 

modules/core/public-ip.nix

@@ -0,0 +1,319 @@
+# ── Unified public-IP detection (privacy-first) ─────────────────────────────
+#
+# One script, one cache file, every consumer on the system reads the same
+# value. Previously the public IP was detected independently in three places,
+# each phoning home to a different third party:
+#   * the Hub (server.py _get_external_ip) → api.ipify.org / ifconfig.me /
+#     icanhazip.com over HTTPS on every /api/network call and every
+#     background-loop tick
+#   * DDNS (ddns-update.py) → myip.opendns.com via OpenDNS
+#   * LiveKit → STUN (its own embedded detection)
+#
+# This module replaces all of that with a single script
+# (/var/lib/sovran/public-ip.py) that detects the IP once per TTL using the
+# least-exposing mechanism available, and caches it in
+# /var/lib/secrets/external-ip. Consumers (Hub, DDNS, LiveKit) read the cache
+# and only invoke the script when it is missing or stale.
+#
+# Detection chain (first success wins, stops immediately):
+#   1. pin         — sovran_systemsOS.elementCalling.externalIP (baked in)
+#   2. cache       — /var/lib/secrets/external-ip if newer than cacheTTL
+#   3. STUN        — UDP binding request (one packet, no application data,
+#                    no HTTP metadata; the same protocol every WebRTC client
+#                    uses). Server configurable via publicIP.stunServer.
+#   4. DNS         — "myip.opendns.com" A query via publicIP.dnsResolver
+#                    (single DNS query, no HTTP headers)
+#   5. HTTPS echo  — ONLY endpoints listed in publicIP.httpsEcho (empty by
+#                    default → never contacted)
+#
+# Privacy property: while the cache is fresh, zero third parties are
+# contacted. When detection runs, at most ONE party learns the IP per
+# refresh interval (default 5 minutes), and the STUN/DNS mechanisms expose
+# nothing beyond the bare address.
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+let
+  stunServer  = config.sovran_systemsOS.publicIP.stunServer;
+  stunPort    = config.sovran_systemsOS.publicIP.stunPort;
+  dnsResolver = config.sovran_systemsOS.publicIP.dnsResolver;
+  httpsEcho   = config.sovran_systemsOS.publicIP.httpsEcho;
+  cacheTTL    = config.sovran_systemsOS.publicIP.cacheTTL;
+
+  # Optional pin shared with element-calling (baked in at build time).
+  pin = if config.sovran_systemsOS.elementCalling.externalIP != null then config.sovran_systemsOS.elementCalling.externalIP else "";
+
+  echoList = lib.concatStringsSep "," (map (u: "'${u}'") httpsEcho);
+in
+{
+  options.sovran_systemsOS.publicIP = {
+    stunServer = lib.mkOption {
+      type = lib.types.str;
+      default = "stun.l.google.com";
+      description = ''
+        STUN server used to discover the public IP over UDP. STUN is the most
+        privacy-preserving detection mechanism: a single stateless packet,
+        no HTTP metadata. Only used when the cache is stale.
+      '';
+    };
+    stunPort = lib.mkOption {
+      type = lib.types.port;
+      default = 19302;
+    };
+    dnsResolver = lib.mkOption {
+      type = lib.types.str;
+      default = "resolver4.opendns.com";
+      description = ''
+        DNS resolver used as fallback (myip.opendns.com trick) when STUN is
+        unavailable (e.g. ISP blocks UDP egress). A single DNS query, no
+        HTTP headers.
+      '';
+    };
+    httpsEcho = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [ "https://api.ipify.org" ];
+      description = ''
+        OPT-IN HTTPS endpoints that return the caller's public IP as a bare
+        IPv4 literal. Each listed endpoint observes this server's public IP
+        and HTTP metadata every time detection runs. Empty by default — no
+        HTTPS echo service is ever contacted unless you add one here. This is
+        the last-resort fallback after STUN and DNS.
+      '';
+    };
+    cacheTTL = lib.mkOption {
+      type = lib.types.int;
+      default = 300;
+      description = "Seconds the detected public IP is cached before re-detection.";
+    };
+  };
+
+  # ── Install the unified detector ──────────────────────────────────────────
+  system.activationScripts.sovranPublicIpInstall = lib.stringAfter [ "users" ] ''
+    install -d -m 0755 /var/lib/sovran
+    cat > /var/lib/sovran/public-ip.py <<'PYEOF'
+#!/usr/bin/env python3
+"""sovran-public-ip — one detector, one cache, every consumer reads the same IP.
+
+Privacy-first detection chain (first success wins):
+  1. pin     — baked in from sovran_systemsOS.elementCalling.externalIP
+  2. cache   — /var/lib/secrets/external-ip if newer than CACHE_TTL seconds
+  3. STUN    — UDP binding request (one packet, no application data)
+  4. DNS     — myip.opendns.com A query via the configured resolver
+  5. HTTPS   — ONLY endpoints baked in from publicIP.httpsEcho (opt-in)
+
+Usage:
+  public-ip.py check     print current public IP (cache first; refresh if stale)
+  public-ip.py refresh   force re-detection, update the cache file, print IP
+
+Exit status: 0 with the IP on stdout on success; 1 if no IP is available
+(cached value, if any, is still printed to stdout with a warning on stderr).
+"""
+import ipaddress
+import os
+import random
+import socket
+import struct
+import sys
+import time
+import urllib.request
+
+CACHE_FILE = "/var/lib/secrets/external-ip"
+PIN        = "${pin}"
+STUN_SERVER = "${stunServer}"
+STUN_PORT   = ${toString stunPort}
+DNS_RESOLVER = "${dnsResolver}"
+DNS_HOST     = "myip.opendns.com"
+ECHO_URLS    = [ ${echoList} ]
+CACHE_TTL    = ${toString cacheTTL}
+TIMEOUT      = 3.0
+
+# ---------------------------------------------------------------------------
+# Detection primitives
+# ---------------------------------------------------------------------------
+
+def is_usable_ip(text: str) -> bool:
+    """True if text is a globally routable IPv4 that LiveKit may advertise."""
+    try:
+        ip = ipaddress.ip_address(text)
+    except ValueError:
+        return False
+    if ip.version != 4:
+        return False
+    if (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast
+            or ip.is_reserved or ip.is_unspecified or not ip.is_global):
+        return False
+    # RFC 6598 shared (CGNAT) space — not reachable from the internet.
+    if ip in ipaddress.ip_network("100.64.0.0/10"):
+        return False
+    return True
+
+
+def stun_public_ip() -> str | None:
+    """RFC 5389 Binding request over UDP; returns the mapped (public) IPv4."""
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(TIMEOUT)
+    try:
+        txid = random.randbytes(12)
+        req = struct.pack("!HHI", 0x0001, 0, 0) + txid  # Binding request
+        sock.sendto(req, (STUN_SERVER, STUN_PORT))
+        data, _ = sock.recvfrom(2048)
+    except OSError:
+        return None
+    finally:
+        sock.close()
+
+    if len(data) < 20:
+        return None
+    mtype, _mlen = struct.unpack("!HH", data[:4])
+    if mtype != 0x0101:  # Binding success response
+        return None
+
+    cookie = data[4:8]
+    i = 20
+    while i + 4 <= len(data):
+        atype, alen = struct.unpack("!HH", data[i : i + 4])
+        aval = data[i + 4 : i + 4 + alen]
+        if atype in (0x0001, 0x0020) and len(aval) >= 8:  # MAPPED / XOR-MAPPED
+            family = aval[1]
+            if family == 0x01:  # IPv4
+                raw = aval[4:8]
+                if atype == 0x0020:  # XOR with magic cookie + txid prefix
+                    raw = bytes(b ^ c for b, c in zip(raw, cookie + txid[:4]))
+                return socket.inet_ntop(socket.AF_INET, raw)
+        i += 4 + ((alen + 3) // 4) * 4
+    return None
+
+
+def dns_public_ip() -> str | None:
+    """Minimal DNS A query for myip.opendns.com against the given resolver."""
+    qid = random.randint(0, 0xFFFF)
+    qname = b"".join(bytes([len(p)]) + p.encode() for p in DNS_HOST.split(".")) + b"\x00"
+    query = struct.pack("!HHHHHH", qid, 0x0100, 1, 0, 0, 0) + qname + struct.pack("!HH", 1, 1)
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.settimeout(TIMEOUT)
+    try:
+        sock.sendto(query, (DNS_RESOLVER, 53))
+        data, _ = sock.recvfrom(4096)
+    except OSError:
+        return None
+    finally:
+        sock.close()
+
+    try:
+        if len(data) < 12:
+            return None
+        rid, _flags, _qd, an, _ns, _ar = struct.unpack("!HHHHHH", data[:12])
+        if rid != qid or an == 0:
+            return None
+        i = 12
+        for _ in range(_qd):  # skip question
+            while data[i] != 0:
+                i += 1 + data[i]
+            i += 5
+        for _ in range(an):
+            if data[i] & 0xC0 == 0xC0:
+                i += 2
+            else:
+                while data[i] != 0:
+                    i += 1 + data[i]
+                i += 1
+            rtype, _rclass, _ttl, rdlen = struct.unpack("!HHIH", data[i : i + 10])
+            i += 10
+            if rtype == 1 and rdlen == 4:
+                return socket.inet_ntop(socket.AF_INET, data[i : i + 4])
+            i += rdlen
+    except (IndexError, struct.error):
+        return None
+    return None
+
+
+def echo_public_ip() -> str | None:
+    """Opt-in HTTPS echo endpoints (baked in at build time; empty by default)."""
+    for url in ECHO_URLS:
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "sovran-public-ip"})
+            with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
+                text = resp.read().decode().strip()
+            if is_usable_ip(text):
+                return text
+        except Exception:
+            continue
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Cache handling
+# ---------------------------------------------------------------------------
+
+def read_cache() -> str:
+    try:
+        with open(CACHE_FILE) as f:
+            return f.read().strip()
+    except OSError:
+        return ""
+
+
+def write_cache(ip: str) -> None:
+    try:
+        os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True)
+        tmp = f"{CACHE_FILE}.tmp"
+        with open(tmp, "w") as f:
+            f.write(ip + "\n")
+        os.replace(tmp, CACHE_FILE)
+    except OSError:
+        pass
+
+
+def cache_fresh() -> bool:
+    try:
+        return time.time() - os.path.getmtime(CACHE_FILE) < CACHE_TTL
+    except OSError:
+        return False
+
+
+def detect() -> str:
+    """Run the chain; returns usable IP or an empty string."""
+    if PIN and is_usable_ip(PIN):
+        return PIN
+    for fn in (stun_public_ip, dns_public_ip, echo_public_ip):
+        try:
+            cand = fn()
+        except Exception:
+            continue
+        if cand and is_usable_ip(cand):
+            return cand
+    return ""
+
+
+def main() -> int:
+    force = len(sys.argv) > 1 and sys.argv[1] == "refresh"
+    ip = ""
+    if not force and cache_fresh():
+        ip = read_cache()
+    if not ip:
+        ip = detect()
+        if ip:
+            write_cache(ip)
+        else:
+            stale = read_cache()
+            if stale:
+                print(stale)
+                print("WARNING: detection failed; using last known public IP", file=sys.stderr)
+                return 0
+            print("ERROR: could not determine a public IP (STUN/DNS unreachable)", file=sys.stderr)
+            return 1
+    print(ip)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+PYEOF
+    chmod 0555 /var/lib/sovran/public-ip.py
+  '';
+}

modules/element-calling.nix

@@ -136,9 +136,10 @@ EOF
     unitConfig = {
       ConditionPathExists = "/var/lib/domains/element-calling";
     };
-    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ];
+    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk pkgs.python3 ];
     script = ''
       MATRIX=$(cat /var/lib/domains/matrix)
+      ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
 
       mkdir -p /run/livekit
 
@@ -195,12 +196,13 @@ EOF
       # NAT with port-forwarding. It does not need to be assigned to this box,
       # and it may be dynamic.
       #
-      # Reuse the Hub's detection instead of running our own: the Hub already
+      # Reuse the shared detector (/var/lib/sovran/public-ip.py — see
-      # resolves the external IP (server.py _get_external_ip) and persists it
+      # modules/core/public-ip.nix) instead of running our own: one script,
-      # to /var/lib/secrets/external-ip. Priority:
+      # one cache, privacy-first (STUN -> DNS -> opt-in HTTPS echo). Priority:
       #   1. sovran_systemsOS.elementCalling.externalIP (explicit pin, if set)
-      #   2. /var/lib/secrets/external-ip (written by the Sovran Hub)
+      #   2. /var/lib/secrets/external-ip (the shared cache)
-      #   3. STUN auto-detection (use_external_ip) as the fallback, with a
+      #   3. run the detector now (it refreshes the cache)
+      #   4. STUN auto-detection (use_external_ip) as the fallback, with a
       #      warning — this is where broken installs used to silently end up
       #      advertising a private IP, causing "call connects but no video".
       EXTERNAL_IP='${if config.sovran_systemsOS.elementCalling.externalIP != null then config.sovran_systemsOS.elementCalling.externalIP else ""}'
@@ -209,6 +211,9 @@ EOF
       if [ -z "$PUBLIC_IP" ] && [ -f /var/lib/secrets/external-ip ]; then
         PUBLIC_IP=$(tr -d '[:space:]' < /var/lib/secrets/external-ip 2>/dev/null)
       fi
+      if [ -z "$PUBLIC_IP" ] && [ -x /var/lib/sovran/public-ip.py ]; then
+        PUBLIC_IP=$(python3 /var/lib/sovran/public-ip.py check 2>/dev/null | head -n1)
+      fi
 
       # Reject non-routable addresses (loopback, private, link-local, CGNAT).
       # A detected/pinned address like this must never be advertised.
@@ -250,6 +255,15 @@ EOF
         echo "WARNING: could not determine a public IP for LiveKit; using STUN auto-detection. If calls connect without media, check STUN egress or set sovran_systemsOS.elementCalling.externalIP." >&2
       fi
 
+      # Webhooks → lk-jwt-service. The JWT service validates the HMAC
+      # signature against the same key file it issues tokens with, and uses
+      # the events (participant_left / room_finished) to detect abruptly
+      # disconnected participants instead of waiting for the delayed-event
+      # timeout. The URL hits local Caddy via the /etc/hosts loopback
+      # override and is routed to the JWT service by the element-calling
+      # vhost (/livekit/jwt/sfu_webhook → 8073).
+      LK_KEY=$(cut -d: -f1 < ${livekitKeyFile} | tr -d '[:space:]')
+
       cat >> /run/livekit/livekit.yaml <<EOF
 room:
   auto_create: false
@@ -260,6 +274,10 @@ turn:
   udp_port: 3478
   cert_file: /run/credentials/livekit.service/turn-cert
   key_file: /run/credentials/livekit.service/turn-key
+webhook:
+  api_key: $LK_KEY
+  urls:
+    - https://$ELEMENT_CALLING/livekit/jwt/sfu_webhook
 EOF
 
       chmod 644 /run/livekit/livekit.yaml
@@ -267,24 +285,17 @@ EOF
   };
 
   ####### LIVEKIT SERVICE #######
+  # NOTE: the runtime config (rtc ports, TURN, webhook, node_ip) is generated
+  # by livekit-turn-setup and delivered via LoadCredential; the upstream
+  # module's `settings` block is therefore intentionally NOT used (it would
+  # be dead config that silently diverges from what LiveKit actually loads).
+  # The firewall ports are opened explicitly below; openFirewall is left off
+  # so the upstream module does not also open 7880/tcp publicly (Caddy fronts
+  # the SFU on this host).
   services.livekit = {
     enable = true;
-    openFirewall = true;
+    openFirewall = false;
     keyFile = livekitKeyFile;
-    settings = {
-      rtc.use_external_ip = true;
-      rtc.skip_external_ip_validation = true;
-      rtc.tcp_port = 7881;
-      rtc.udp_port = 7882;
-      rtc.port_range_start = 30000;
-      rtc.port_range_end = 40000;
-      room.auto_create = false;
-      turn = {
-        enabled = true;
-        tls_port = 5349;
-        udp_port = 3478;
-      };
-    };
   };
 
   # Override ExecStart to load the runtime-generated config (which carries the
@@ -344,6 +355,10 @@ EOF
       cat > /run/lk-jwt-service/env <<EOF
 LIVEKIT_URL=wss://$ELEMENT_CALLING
 LIVEKIT_FULL_ACCESS_HOMESERVERS=$FULL_ACCESS_HOMESERVERS
+# Re-check, every 60s, that connected participants are still on the SFU;
+# guards against missed SFU webhooks (e.g. an SFU restart) leaving stale
+# call members in Matrix rooms.
+LIVEKIT_SANITY_CHECK_INTERVAL_SECONDS=60
 EOF
 
       chmod 640 /run/lk-jwt-service/env
@@ -355,6 +370,9 @@ EOF
     enable = true;
     port = 8073;
     keyFile = livekitKeyFile;
+    # Required by the upstream module's option type, but overridden at runtime
+    # by EnvironmentFile (/run/lk-jwt-service/env, generated above from the
+    # element-calling domain). Kept as a harmless placeholder.
     livekitUrl = "wss://placeholder.local";
   };
 

modules/modules.nix

@@ -17,6 +17,7 @@
     ./core/no-sleep.nix
     ./core/cpu-performance.nix
     ./core/local-domain-loopback.nix
+    ./core/public-ip.nix
 
     # ── Always on (no flag) ───────────────────────────────────
     ./php.nix