From 224ea99ce42f8570d6b52ccc2586dfdcc9d9b9c7 Mon Sep 17 00:00:00 2001 From: naturallaw77 Date: Thu, 20 Aug 2026 16:12:00 -0500 Subject: [PATCH] fix(element-calling): reuse Hub external IP for LiveKit, drop egress service calls --- app/sovran_systemsos_web/server.py | 26 ++++++++++- modules/element-calling.nix | 72 +++++++++++++++++++----------- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index c4bec2a..ef2c6a8 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -134,6 +134,7 @@ _SERVICE_DOMAIN_KEYS = frozenset([ ]) INTERNAL_IP_FILE = "/var/lib/secrets/internal-ip" +EXTERNAL_IP_FILE = "/var/lib/secrets/external-ip" ZEUS_CONNECT_FILE = "/var/lib/secrets/zeus-connect-url" ONBOARDING_FLAG = "/var/lib/sovran/onboarding-complete" @@ -952,6 +953,18 @@ def _save_internal_ip(ip: str): pass +def _save_external_ip(ip: str): + """Write the external IP to a file so other services (e.g. LiveKit) can + reference it without running their own detection.""" + if ip and ip != "unavailable": + try: + os.makedirs(os.path.dirname(EXTERNAL_IP_FILE), exist_ok=True) + with open(EXTERNAL_IP_FILE, "w") as f: + f.write(ip) + except OSError: + pass + + def _get_external_ip() -> str: MAX_IP_LENGTH = 46 for url in [ @@ -3718,6 +3731,9 @@ async def api_network(): # Keep the internal-ip file in sync for credential lookups _save_internal_ip(internal) _cached_external_ip = external + # Persist the external IP so other services (e.g. LiveKit) can reuse the + # Hub's detection instead of running their own. + _save_external_ip(external) return {"internal_ip": internal, "external_ip": external} @@ -6210,10 +6226,19 @@ async def _startup_migrate_deprecated_features(): async def _background_domain_reachability_checker(): """Periodically curl configured domains and cache reachability results.""" + global _cached_external_ip await asyncio.sleep(_DOMAIN_REACHABILITY_STARTUP_DELAY) consecutive_failures = 0 while True: try: + # Keep the persisted external IP fresh (dynamic WAN IPs), so + # services like LiveKit can read /var/lib/secrets/external-ip. + loop = asyncio.get_event_loop() + external = await loop.run_in_executor(None, _get_external_ip) + if external != "unavailable": + _cached_external_ip = external + _save_external_ip(external) + cfg = load_config() services = cfg.get("services", []) @@ -6223,7 +6248,6 @@ async def _background_domain_reachability_checker(): if unit is not None } - loop = asyncio.get_event_loop() overrides, *_ = await loop.run_in_executor(None, _read_hub_overrides) domains_to_check: list[str] = [] diff --git a/modules/element-calling.nix b/modules/element-calling.nix index fff4a8e..c458c91 100755 --- a/modules/element-calling.nix +++ b/modules/element-calling.nix @@ -136,7 +136,7 @@ EOF unitConfig = { ConditionPathExists = "/var/lib/domains/element-calling"; }; - path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk pkgs.curl ]; + path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ]; script = '' MATRIX=$(cat /var/lib/domains/matrix) @@ -190,37 +190,31 @@ EOF # copies under /run/credentials. # # Determine the public IPv4 to advertise in LiveKit ICE candidates. - # Priority: + # Remote peers must be able to reach this address, so it must be the + # server's public IP — or the router's WAN IP when the server is behind + # NAT with port-forwarding. It does not need to be assigned to this box, + # and it may be dynamic. + # + # Reuse the Hub's detection instead of running our own: the Hub already + # resolves the external IP (server.py _get_external_ip) and persists it + # to /var/lib/secrets/external-ip. Priority: # 1. sovran_systemsOS.elementCalling.externalIP (explicit pin, if set) - # 2. runtime HTTPS egress detection — the server's own egress IP behind - # NAT. More reliable than STUN for this OS, because Caddy's ACME - # certificate issuance already proves outbound 443/TCP works, while - # STUN's UDP egress is often blocked by ISPs. Returns the same WAN - # IP that STUN would, so existing working setups are unaffected. + # 2. /var/lib/secrets/external-ip (written by the Sovran Hub) # 3. STUN auto-detection (use_external_ip) as the fallback, with a # warning — this is where broken installs used to silently end up # advertising a private IP, causing "call connects but no video". EXTERNAL_IP='${if config.sovran_systemsOS.elementCalling.externalIP != null then config.sovran_systemsOS.elementCalling.externalIP else ""}' PUBLIC_IP="$EXTERNAL_IP" - if [ -z "$PUBLIC_IP" ]; then - for SVC in "https://api.ipify.org" "https://checkip.amazonaws.com" "https://ifconfig.me/ip"; do - CANDIDATE=$(curl -fsS --max-time 5 "$SVC" 2>/dev/null | tr -d '[:space:]') - [ -z "$CANDIDATE" ] && continue - # Keep only plausible IPv4 literals (rejects hostnames, IPv6, junk). - case "$CANDIDATE" in - *[!0-9.]*) continue ;; - *) PUBLIC_IP="$CANDIDATE" ;; - esac - break - done + if [ -z "$PUBLIC_IP" ] && [ -f /var/lib/secrets/external-ip ]; then + PUBLIC_IP=$(tr -d '[:space:]' < /var/lib/secrets/external-ip 2>/dev/null) fi # Reject non-routable addresses (loopback, private, link-local, CGNAT). # A detected/pinned address like this must never be advertised. if [ -n "$PUBLIC_IP" ] && printf '%s' "$PUBLIC_IP" | grep -qE \ '^(0\.|127\.|10\.|100\.64\.|169\.254\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.)'; then - echo "WARNING: public IP candidate '$PUBLIC_IP' is not routable; falling back to STUN auto-detection." >&2 + echo "WARNING: external IP '$PUBLIC_IP' is not routable; falling back to STUN auto-detection." >&2 PUBLIC_IP="" fi @@ -371,6 +365,12 @@ EOF # Restart LiveKit / lk-jwt-service when a rebuild regenerates their runtime # configs (new domains, externalIP, full-access list), mirroring the domain # change flow. + # Re-run the config generator and restart LiveKit when a rebuild regenerates + # the runtime config, or when the Hub persists a new external IP (dynamic + # WAN IPs), so the advertised ICE candidate stays current without a manual + # restart. The trigger chain: external-ip change → livekit-turn-setup + # re-runs → rewrites livekit.yaml → livekit restarts with the new config. + systemd.services.livekit-turn-setup.restartTriggers = [ "/var/lib/secrets/external-ip" ]; systemd.services.livekit.restartTriggers = [ "/run/livekit/livekit.yaml" ]; systemd.services.lk-jwt-service.restartTriggers = [ "/run/lk-jwt-service/env" ]; @@ -383,10 +383,12 @@ EOF # * the lk-jwt-service being unreachable through Caddy, # * the MatrixRTC transports endpoint being absent (Element X cannot # discover calling and shows MISSING_MATRIX_RTC_TRANSPORT). - # The check deliberately queries a public resolver (1.1.1.1) rather than the - # system resolver, because this OS installs server-local loopback overrides - # for its own domains in /etc/hosts (see modules/core/local-domain-loopback.nix) - # — those are expected and fine for server-originated traffic. + # The check queries only the operator's own DNS provider (the domain's + # authoritative nameservers, resolved via the local resolver) plus the + # server's own Caddy and public IP — no third-party resolver or service is + # contacted. dig queries resolvers directly, so the /etc/hosts loopback + # overrides (modules/core/local-domain-loopback.nix) do not influence the + # result. systemd.services.element-calling-public-check = { description = "Verify Element Calling domain, JWT service and MatrixRTC transports endpoint are publicly reachable"; after = [ "network-online.target" "caddy.service" "livekit.service" "lk-jwt-service.service" ]; @@ -407,17 +409,33 @@ EOF echo "── Element Calling public reachability self-check ──" - # 1) Public DNS (bypassing the /etc/hosts loopback overrides). - IPS=$( { dig +short A "$ELEMENT_CALLING" @1.1.1.1 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" @1.1.1.1 2>/dev/null; } | tr '\n' ' ' ) + # 1) Authoritative DNS view (bypassing the /etc/hosts loopback + # overrides). Resolve the domain's own nameservers via the local + # resolver, then query those nameservers directly — the only party + # that sees the query is the DNS provider the operator already uses + # for the domain. + NS_LIST=$(dig +short NS "$ELEMENT_CALLING" 2>/dev/null | tr '\n' ' ') + if [ -n "$NS_LIST" ]; then + IPS="" + for NSRV in $NS_LIST; do + IPS=$( { dig +short A "$ELEMENT_CALLING" "@$NSRV" 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" "@$NSRV" 2>/dev/null; } | tr '\n' ' ' ) + [ -n "$IPS" ] && break + done + echo "Authoritative nameservers for $ELEMENT_CALLING: $NS_LIST" + else + echo "WARNING: could not resolve nameservers for $ELEMENT_CALLING via the local resolver; using the local resolver's answer instead." >&2 + IPS=$( { dig +short A "$ELEMENT_CALLING" 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" 2>/dev/null; } | tr '\n' ' ' ) + fi + if [ -z "$IPS" ]; then - echo "ERROR: $ELEMENT_CALLING has no public A/AAAA records (via 1.1.1.1). Remote peers cannot reach this LiveKit; calls will connect without media." >&2 + echo "ERROR: no A/AAAA records for $ELEMENT_CALLING at its authoritative nameservers. Remote peers cannot reach this LiveKit; calls will connect without media." >&2 FAIL=1 else echo "Public DNS for $ELEMENT_CALLING: $IPS" for IP in $IPS; do case "$IP" in 0.*|127.*|169.254.*|100.64.*|::1|fe80:*|fc*:*|fd*:*) - echo "ERROR: $ELEMENT_CALLING publicly resolves to $IP (loopback/link-local/CGNAT). Remote peers cannot reach it." >&2 + echo "ERROR: $ELEMENT_CALLING resolves to $IP (loopback/link-local/CGNAT). Remote peers cannot reach it." >&2 FAIL=1 ;; esac done