feat(element-calling): fix Element X discovery and harden federated calling

The element-calling feature only advertised the LiveKit focus via the
well-known org.matrix.msc4143.rtc_foci file, and relied on STUN
auto-detection for the public IP. Element X queries the MatrixRTC
transports registry endpoint and fails with MISSING_MATRIX_RTC_TRANSPORT
when it is absent, and blocked STUN egress silently left LiveKit
advertising a private IP (call connects but no video across servers).

- synapse: enable msc4143_enabled and advertise matrix_rtc.transports
  (MSC4519) with the site's element-calling URL, so Element X can
  discover the LiveKit focus instead of erroring out
- livekit: determine the public IP to advertise at runtime —
  explicit pin, then HTTPS egress detection (api.ipify.org /
  checkip.amazonaws.com / ifconfig.me), then STUN fallback with a
  warning; reject non-routable results (private/loopback/CGNAT)
- lk-jwt-service: append optional extra homeservers to
  LIVEKIT_FULL_ACCESS_HOMESERVERS via the new
  sovran_systemsOS.elementCalling.fullAccessHomeservers option
- add sovran_systemsOS.elementCalling.externalIP option to pin the
  advertised public IP for multi-WAN/VPN setups
- add element-calling-public-check.service: boot-time diagnostics for
  public DNS (via 1.1.1.1, bypassing local loopback overrides), JWT
  healthz through Caddy and via the public IP, and the transports
  endpoint — turns the silent -no media- failure into a visible error
- add restartTriggers so livekit/lk-jwt-service pick up regenerated
  runtime configs on rebuild

modules/core/roles.nix

@@ -79,6 +79,41 @@
       };
     };
 
+    # ── Element Calling (video/audio) tuning ──────────────────
+    elementCalling = {
+      fullAccessHomeservers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [ "matrix.peer.example.com" ];
+        description = ''
+          Additional Matrix server_names (beyond this server itself) that may
+          trigger LiveKit room creation on this server's SFU via lk-jwt-service.
+
+          Not needed for the common federated setup: each participant's client
+          always obtains its token from its own homeserver's JWT service and
+          publishes to its own SFU, and the participant who starts a call
+          creates the room on their own SFU — the remote user merely joins
+          (joining does not require full access).
+
+          Only set this for asymmetric cases: e.g. a peer homeserver that has
+          no focus of its own, or calls whose first participant lands on this
+          server's SFU but belongs to the peer.
+        '';
+      };
+      externalIP = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = "203.0.113.10";
+        description = ''
+          Optional pin: force LiveKit to advertise this public IPv4 in its
+          host/TURN ICE candidates. Not required in normal operation — the
+          module auto-detects the public IP at runtime (HTTPS egress
+          detection, falling back to STUN). Set it only to override a
+          mis-detected address (e.g. multi-WAN/VPN setups).
+        '';
+      };
+    };
+
     # ── Domain setup registry ─────────────────────────────────
     domainRequirements = lib.mkOption {
       type = lib.types.listOf (lib.types.submodule {

modules/element-calling.nix

@@ -136,7 +136,7 @@ EOF
     unitConfig = {
       ConditionPathExists = "/var/lib/domains/element-calling";
     };
-    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ];
+    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk pkgs.curl ];
     script = ''
       MATRIX=$(cat /var/lib/domains/matrix)
 
@@ -188,7 +188,59 @@ EOF
       # rtc.interfaces.includes are only known at runtime, so they are
       # substituted here. The cert/key paths point at the LoadCredential-staged
       # copies under /run/credentials.
-      cat > /run/livekit/livekit.yaml <<EOF
+      #
+      # Determine the public IPv4 to advertise in LiveKit ICE candidates.
+      # Priority:
+      #   1. sovran_systemsOS.elementCalling.externalIP (explicit pin, if set)
+      #   2. runtime HTTPS egress detection — the server's own egress IP behind
+      #      NAT. More reliable than STUN for this OS, because Caddy's ACME
+      #      certificate issuance already proves outbound 443/TCP works, while
+      #      STUN's UDP egress is often blocked by ISPs. Returns the same WAN
+      #      IP that STUN would, so existing working setups are unaffected.
+      #   3. STUN auto-detection (use_external_ip) as the fallback, with a
+      #      warning — this is where broken installs used to silently end up
+      #      advertising a private IP, causing "call connects but no video".
+      EXTERNAL_IP='${if config.sovran_systemsOS.elementCalling.externalIP != null then config.sovran_systemsOS.elementCalling.externalIP else ""}'
+
+      PUBLIC_IP="$EXTERNAL_IP"
+      if [ -z "$PUBLIC_IP" ]; then
+        for SVC in "https://api.ipify.org" "https://checkip.amazonaws.com" "https://ifconfig.me/ip"; do
+          CANDIDATE=$(curl -fsS --max-time 5 "$SVC" 2>/dev/null | tr -d '[:space:]')
+          [ -z "$CANDIDATE" ] && continue
+          # Keep only plausible IPv4 literals (rejects hostnames, IPv6, junk).
+          case "$CANDIDATE" in
+            *[!0-9.]*) continue ;;
+            *) PUBLIC_IP="$CANDIDATE" ;;
+          esac
+          break
+        done
+      fi
+
+      # Reject non-routable addresses (loopback, private, link-local, CGNAT).
+      # A detected/pinned address like this must never be advertised.
+      if [ -n "$PUBLIC_IP" ] && printf '%s' "$PUBLIC_IP" | grep -qE \
+        '^(0\.|127\.|10\.|100\.64\.|169\.254\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.)'; then
+        echo "WARNING: public IP candidate '$PUBLIC_IP' is not routable; falling back to STUN auto-detection." >&2
+        PUBLIC_IP=""
+      fi
+
+      if [ -n "$PUBLIC_IP" ]; then
+        cat > /run/livekit/livekit.yaml <<EOF
+port: 7880
+rtc:
+  use_external_ip: false
+  node_ip: $PUBLIC_IP
+  tcp_port: 7881
+  udp_port: 7882
+  port_range_start: 30000
+  port_range_end: 40000
+  interfaces:
+    includes:
+      - $IFACE
+EOF
+        echo "LiveKit will advertise public IP: $PUBLIC_IP"
+      else
+        cat > /run/livekit/livekit.yaml <<EOF
 port: 7880
 rtc:
   use_external_ip: true
@@ -200,6 +252,11 @@ rtc:
   interfaces:
     includes:
       - $IFACE
+EOF
+        echo "WARNING: could not determine a public IP for LiveKit; using STUN auto-detection. If calls connect without media, check STUN egress or set sovran_systemsOS.elementCalling.externalIP." >&2
+      fi
+
+      cat >> /run/livekit/livekit.yaml <<EOF
 room:
   auto_create: false
 turn:
@@ -277,12 +334,22 @@ EOF
     script = ''
       ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
       MATRIX=$(cat /var/lib/domains/matrix)
+      FULL_ACCESS_HOMESERVERS="$MATRIX"
+
+      # Federated peers may also be granted LiveKit room-creation (full access)
+      # on this SFU via sovran_systemsOS.elementCalling.fullAccessHomeservers.
+      # Without this, remote users can join existing calls but cannot be the
+      # first to start one on your SFU.
+      EXTRA_HS='${lib.concatStringsSep "," config.sovran_systemsOS.elementCalling.fullAccessHomeservers}'
+      if [ -n "$EXTRA_HS" ]; then
+        FULL_ACCESS_HOMESERVERS="$FULL_ACCESS_HOMESERVERS,$EXTRA_HS"
+      fi
 
       mkdir -p /run/lk-jwt-service
 
       cat > /run/lk-jwt-service/env <<EOF
 LIVEKIT_URL=wss://$ELEMENT_CALLING
-LIVEKIT_FULL_ACCESS_HOMESERVERS=$MATRIX
+LIVEKIT_FULL_ACCESS_HOMESERVERS=$FULL_ACCESS_HOMESERVERS
 EOF
 
       chmod 640 /run/lk-jwt-service/env
@@ -301,6 +368,102 @@ EOF
     "/run/lk-jwt-service/env"
   ];
 
+  # Restart LiveKit / lk-jwt-service when a rebuild regenerates their runtime
+  # configs (new domains, externalIP, full-access list), mirroring the domain
+  # change flow.
+  systemd.services.livekit.restartTriggers = [ "/run/livekit/livekit.yaml" ];
+  systemd.services.lk-jwt-service.restartTriggers = [ "/run/lk-jwt-service/env" ];
+
+  ####### PUBLIC REACHABILITY SELF-CHECK #######
+  # Diagnostic only — never a hard dependency of livekit/caddy. Catches the
+  # classic "call connects but no media" setup errors at boot instead of at
+  # call time:
+  #   * the element-calling domain having no public records (or resolving to
+  #     loopback/link-local/CGNAT for remote peers),
+  #   * the lk-jwt-service being unreachable through Caddy,
+  #   * the MatrixRTC transports endpoint being absent (Element X cannot
+  #     discover calling and shows MISSING_MATRIX_RTC_TRANSPORT).
+  # The check deliberately queries a public resolver (1.1.1.1) rather than the
+  # system resolver, because this OS installs server-local loopback overrides
+  # for its own domains in /etc/hosts (see modules/core/local-domain-loopback.nix)
+  # — those are expected and fine for server-originated traffic.
+  systemd.services.element-calling-public-check = {
+    description = "Verify Element Calling domain, JWT service and MatrixRTC transports endpoint are publicly reachable";
+    after = [ "network-online.target" "caddy.service" "livekit.service" "lk-jwt-service.service" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    unitConfig = {
+      ConditionPathExists = "/var/lib/domains/element-calling";
+    };
+    path = [ pkgs.coreutils pkgs.gawk pkgs.dnsutils pkgs.curl ];
+    script = ''
+      ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
+      MATRIX=$(cat /var/lib/domains/matrix)
+      FAIL=0
+
+      echo "── Element Calling public reachability self-check ──"
+
+      # 1) Public DNS (bypassing the /etc/hosts loopback overrides).
+      IPS=$( { dig +short A "$ELEMENT_CALLING" @1.1.1.1 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" @1.1.1.1 2>/dev/null; } | tr '\n' ' ' )
+      if [ -z "$IPS" ]; then
+        echo "ERROR: $ELEMENT_CALLING has no public A/AAAA records (via 1.1.1.1). Remote peers cannot reach this LiveKit; calls will connect without media." >&2
+        FAIL=1
+      else
+        echo "Public DNS for $ELEMENT_CALLING: $IPS"
+        for IP in $IPS; do
+          case "$IP" in
+            0.*|127.*|169.254.*|100.64.*|::1|fe80:*|fc*:*|fd*:*)
+              echo "ERROR: $ELEMENT_CALLING publicly resolves to $IP (loopback/link-local/CGNAT). Remote peers cannot reach it." >&2
+              FAIL=1 ;;
+          esac
+        done
+      fi
+
+      # 2) lk-jwt-service healthz through Caddy (validates the proxy chain).
+      if curl -fsS --max-time 10 "https://$ELEMENT_CALLING/livekit/jwt/healthz" >/dev/null 2>&1; then
+        echo "OK: https://$ELEMENT_CALLING/livekit/jwt/healthz responds"
+      else
+        echo "ERROR: https://$ELEMENT_CALLING/livekit/jwt/healthz not reachable through Caddy." >&2
+        FAIL=1
+      fi
+
+      # 3) Same healthz via the first public IP (tests the full NAT path).
+      #    NOTE: if this box is behind the same NAT you are testing through,
+      #    routers without hairpin NAT will fail this step — the warning is
+      #    then expected and harmless; verify from an external device instead.
+      if [ -n "$IPS" ]; then
+        PUBIP=$(echo "$IPS" | awk '{print $1}')
+        if curl -fsS --max-time 15 --resolve "$ELEMENT_CALLING:443:$PUBIP" "https://$ELEMENT_CALLING/livekit/jwt/healthz" >/dev/null 2>&1; then
+          echo "OK: healthz reachable via public IP $PUBIP (NAT path works)"
+        else
+          echo "WARNING: healthz NOT reachable via public IP $PUBIP — check router port-forwarding (443/TCP) and NAT hairpin. Expected if the router lacks hairpin NAT; verify from an external device." >&2
+        fi
+      fi
+
+      # 4) MatrixRTC transports registry (MSC4519) — required by Element X.
+      CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "https://$MATRIX/_matrix/client/unstable/org.matrix.msc4143/rtc/transports")
+      case "$CODE" in
+        401|200)
+          echo "OK: MatrixRTC transports endpoint present (HTTP $CODE; auth required is expected)" ;;
+        404)
+          echo "ERROR: /_matrix/client/unstable/org.matrix.msc4143/rtc/transports missing (HTTP 404) — Element X cannot discover calling. Enable msc4143_enabled and matrix_rtc.transports in Synapse." >&2
+          FAIL=1 ;;
+        *)
+          echo "WARNING: transports endpoint returned HTTP $CODE" >&2 ;;
+      esac
+
+      if [ "$FAIL" -eq 1 ]; then
+        echo "── Element Calling self-check FAILED — see errors above ──" >&2
+        exit 1
+      fi
+      echo "── Element Calling self-check passed ──"
+    '';
+  };
+
   ####### SYNAPSE RUNTIME CONFIG (element-calling additions) #######
   systemd.services.element-calling-synapse-config = {
     description = "Generate Synapse runtime config for Element Calling";
@@ -317,6 +480,7 @@ EOF
     path = [ pkgs.coreutils ];
     script = ''
       MATRIX=$(cat /var/lib/domains/matrix)
+      ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
 
       mkdir -p /run/matrix-synapse
 
@@ -326,7 +490,17 @@ public_baseurl: "https://$MATRIX"
 serve_server_wellknown: true
 experimental_features:
   msc3266_enabled: true
+  # MSC4143: enables the MatrixRTC transports registry endpoint
+  # (/_matrix/client/unstable/org.matrix.msc4143/rtc/transports, MSC4519).
+  # Element X requires this endpoint to discover the LiveKit focus; without it
+  # mobile clients fail with MISSING_MATRIX_RTC_TRANSPORT / cannot start calls.
+  msc4143_enabled: true
   msc4222_enabled: true
+# MSC4519: advertise this site's LiveKit focus via the transports registry.
+matrix_rtc:
+  transports:
+    - type: livekit
+      livekit_service_url: "https://$ELEMENT_CALLING/livekit/jwt"
 max_event_delay_duration: "24h"
 rc_message:
   per_second: 0.5