Sovran_Systems/Sovran_SystemsOS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

fix(element-calling): reuse Hub external IP for LiveKit, drop egress service calls

Browse Source
This commit is contained in:
naturallaw777
2026-08-20 16:12:00 -05:00
parent c54dbfe2a5
commit 224ea99ce4
2 changed files with 70 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/sovran_systemsos_web/server.py
+25 -1
View File
@@ -134,6 +134,7 @@ _SERVICE_DOMAIN_KEYS = frozenset([
])
INTERNAL_IP_FILE = "/var/lib/secrets/internal-ip"
EXTERNAL_IP_FILE = "/var/lib/secrets/external-ip"
ZEUS_CONNECT_FILE = "/var/lib/secrets/zeus-connect-url"
ONBOARDING_FLAG = "/var/lib/sovran/onboarding-complete"
@@ -952,6 +953,18 @@ def _save_internal_ip(ip: str):
pass
def _save_external_ip(ip: str):
"""Write the external IP to a file so other services (e.g. LiveKit) can
reference it without running their own detection."""
if ip and ip != "unavailable":
try:
os.makedirs(os.path.dirname(EXTERNAL_IP_FILE), exist_ok=True)
with open(EXTERNAL_IP_FILE, "w") as f:
f.write(ip)
except OSError:
pass
def _get_external_ip() -> str:
MAX_IP_LENGTH = 46
for url in [
@@ -3718,6 +3731,9 @@ async def api_network():
# Keep the internal-ip file in sync for credential lookups
_save_internal_ip(internal)
_cached_external_ip = external
# Persist the external IP so other services (e.g. LiveKit) can reuse the
# Hub's detection instead of running their own.
_save_external_ip(external)
return {"internal_ip": internal, "external_ip": external}
@@ -6210,10 +6226,19 @@ async def _startup_migrate_deprecated_features():
async def _background_domain_reachability_checker():
"""Periodically curl configured domains and cache reachability results."""
global _cached_external_ip
await asyncio.sleep(_DOMAIN_REACHABILITY_STARTUP_DELAY)
consecutive_failures = 0
while True:
try:
# Keep the persisted external IP fresh (dynamic WAN IPs), so
# services like LiveKit can read /var/lib/secrets/external-ip.
loop = asyncio.get_event_loop()
external = await loop.run_in_executor(None, _get_external_ip)
if external != "unavailable":
_cached_external_ip = external
_save_external_ip(external)
cfg = load_config()
services = cfg.get("services", [])
@@ -6223,7 +6248,6 @@ async def _background_domain_reachability_checker():
if unit is not None
}
loop = asyncio.get_event_loop()
overrides, *_ = await loop.run_in_executor(None, _read_hub_overrides)
domains_to_check: list[str] = []
modules/element-calling.nix
+45 -27
View File
@@ -136,7 +136,7 @@ EOF
unitConfig = {
ConditionPathExists = "/var/lib/domains/element-calling";
};
path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk pkgs.curl ];
path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ];
script = ''
MATRIX=$(cat /var/lib/domains/matrix)
@@ -190,37 +190,31 @@ EOF
# copies under /run/credentials.
#
# Determine the public IPv4 to advertise in LiveKit ICE candidates.
# Priority:
# Remote peers must be able to reach this address, so it must be the
# server's public IP — or the router's WAN IP when the server is behind
# NAT with port-forwarding. It does not need to be assigned to this box,
# and it may be dynamic.
#
# Reuse the Hub's detection instead of running our own: the Hub already
# resolves the external IP (server.py _get_external_ip) and persists it
# to /var/lib/secrets/external-ip. Priority:
# 1. sovran_systemsOS.elementCalling.externalIP (explicit pin, if set)
# 2. runtime HTTPS egress detection — the server's own egress IP behind
# NAT. More reliable than STUN for this OS, because Caddy's ACME
# certificate issuance already proves outbound 443/TCP works, while
# STUN's UDP egress is often blocked by ISPs. Returns the same WAN
# IP that STUN would, so existing working setups are unaffected.
# 2. /var/lib/secrets/external-ip (written by the Sovran Hub)
# 3. STUN auto-detection (use_external_ip) as the fallback, with a
# warning — this is where broken installs used to silently end up
# advertising a private IP, causing "call connects but no video".
EXTERNAL_IP='${if config.sovran_systemsOS.elementCalling.externalIP != null then config.sovran_systemsOS.elementCalling.externalIP else ""}'
PUBLIC_IP="$EXTERNAL_IP"
if [ -z "$PUBLIC_IP" ]; then
for SVC in "https://api.ipify.org" "https://checkip.amazonaws.com" "https://ifconfig.me/ip"; do
CANDIDATE=$(curl -fsS --max-time 5 "$SVC" 2>/dev/null | tr -d '[:space:]')
[ -z "$CANDIDATE" ] && continue
# Keep only plausible IPv4 literals (rejects hostnames, IPv6, junk).
case "$CANDIDATE" in
*[!0-9.]*) continue ;;
*) PUBLIC_IP="$CANDIDATE" ;;
esac
break
done
if [ -z "$PUBLIC_IP" ] && [ -f /var/lib/secrets/external-ip ]; then
PUBLIC_IP=$(tr -d '[:space:]' < /var/lib/secrets/external-ip 2>/dev/null)
fi
# Reject non-routable addresses (loopback, private, link-local, CGNAT).
# A detected/pinned address like this must never be advertised.
if [ -n "$PUBLIC_IP" ] && printf '%s' "$PUBLIC_IP" | grep -qE \
'^(0\.|127\.|10\.|100\.64\.|169\.254\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.)'; then
echo "WARNING: public IP candidate '$PUBLIC_IP' is not routable; falling back to STUN auto-detection." >&2
echo "WARNING: external IP '$PUBLIC_IP' is not routable; falling back to STUN auto-detection." >&2
PUBLIC_IP=""
fi
@@ -371,6 +365,12 @@ EOF
# Restart LiveKit / lk-jwt-service when a rebuild regenerates their runtime
# configs (new domains, externalIP, full-access list), mirroring the domain
# change flow.
# Re-run the config generator and restart LiveKit when a rebuild regenerates
# the runtime config, or when the Hub persists a new external IP (dynamic
# WAN IPs), so the advertised ICE candidate stays current without a manual
# restart. The trigger chain: external-ip change → livekit-turn-setup
# re-runs → rewrites livekit.yaml → livekit restarts with the new config.
systemd.services.livekit-turn-setup.restartTriggers = [ "/var/lib/secrets/external-ip" ];
systemd.services.livekit.restartTriggers = [ "/run/livekit/livekit.yaml" ];
systemd.services.lk-jwt-service.restartTriggers = [ "/run/lk-jwt-service/env" ];
@@ -383,10 +383,12 @@ EOF
# * the lk-jwt-service being unreachable through Caddy,
# * the MatrixRTC transports endpoint being absent (Element X cannot
# discover calling and shows MISSING_MATRIX_RTC_TRANSPORT).
# The check deliberately queries a public resolver (1.1.1.1) rather than the
# system resolver, because this OS installs server-local loopback overrides
# for its own domains in /etc/hosts (see modules/core/local-domain-loopback.nix)
# — those are expected and fine for server-originated traffic.
# The check queries only the operator's own DNS provider (the domain's
# authoritative nameservers, resolved via the local resolver) plus the
# server's own Caddy and public IP — no third-party resolver or service is
# contacted. dig queries resolvers directly, so the /etc/hosts loopback
# overrides (modules/core/local-domain-loopback.nix) do not influence the
# result.
systemd.services.element-calling-public-check = {
description = "Verify Element Calling domain, JWT service and MatrixRTC transports endpoint are publicly reachable";
after = [ "network-online.target" "caddy.service" "livekit.service" "lk-jwt-service.service" ];
@@ -407,17 +409,33 @@ EOF
echo "── Element Calling public reachability self-check ──"
# 1) Public DNS (bypassing the /etc/hosts loopback overrides).
IPS=$( { dig +short A "$ELEMENT_CALLING" @1.1.1.1 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" @1.1.1.1 2>/dev/null; } | tr '\n' ' ' )
# 1) Authoritative DNS view (bypassing the /etc/hosts loopback
# overrides). Resolve the domain's own nameservers via the local
# resolver, then query those nameservers directly — the only party
# that sees the query is the DNS provider the operator already uses
# for the domain.
NS_LIST=$(dig +short NS "$ELEMENT_CALLING" 2>/dev/null | tr '\n' ' ')
if [ -n "$NS_LIST" ]; then
IPS=""
for NSRV in $NS_LIST; do
IPS=$( { dig +short A "$ELEMENT_CALLING" "@$NSRV" 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" "@$NSRV" 2>/dev/null; } | tr '\n' ' ' )
[ -n "$IPS" ] && break
done
echo "Authoritative nameservers for $ELEMENT_CALLING: $NS_LIST"
else
echo "WARNING: could not resolve nameservers for $ELEMENT_CALLING via the local resolver; using the local resolver's answer instead." >&2
IPS=$( { dig +short A "$ELEMENT_CALLING" 2>/dev/null; dig +short AAAA "$ELEMENT_CALLING" 2>/dev/null; } | tr '\n' ' ' )
fi
if [ -z "$IPS" ]; then
echo "ERROR: $ELEMENT_CALLING has no public A/AAAA records (via 1.1.1.1). Remote peers cannot reach this LiveKit; calls will connect without media." >&2
echo "ERROR: no A/AAAA records for $ELEMENT_CALLING at its authoritative nameservers. Remote peers cannot reach this LiveKit; calls will connect without media." >&2
FAIL=1
else
echo "Public DNS for $ELEMENT_CALLING: $IPS"
for IP in $IPS; do
case "$IP" in
0.*|127.*|169.254.*|100.64.*|::1|fe80:*|fc*:*|fd*:*)
echo "ERROR: $ELEMENT_CALLING publicly resolves to $IP (loopback/link-local/CGNAT). Remote peers cannot reach it." >&2
echo "ERROR: $ELEMENT_CALLING resolves to $IP (loopback/link-local/CGNAT). Remote peers cannot reach it." >&2
FAIL=1 ;;
esac
done