From 13f15cb8451e38daa4b29c812805a4fceca41e0b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 12:34:39 +0000 Subject: [PATCH 1/2] Initial plan From 86942ebc332c45dfe7d749d82ca2015bbd7087e0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 12:45:02 +0000 Subject: [PATCH 2/2] feat: replace domain port table with sequential domain diagnostics Agent-Logs-Url: https://github.com/naturallaw777/staging_alpha/sessions/93de7af8-10f9-438e-b9bc-8c6e9d39d787 Co-authored-by: naturallaw777 <99053422+naturallaw777@users.noreply.github.com> --- app/sovran_systemsos_web/server.py | 326 +++++++++++++----- .../static/js/service-detail.js | 190 ++++------ modules/core/sovran-hub.nix | 1 + 3 files changed, 313 insertions(+), 204 deletions(-) diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index 0a62270..dcf554e 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -277,15 +277,10 @@ FEATURE_SERVICE_MAP = { } # Port requirements for service tiles (keyed by unit name or icon) -# Services using only 80/443 for domain access share the same base list. -_PORTS_WEB = [ - {"port": "80", "protocol": "TCP", "description": "HTTP (redirect to HTTPS)"}, - {"port": "443", "protocol": "TCP", "description": "HTTPS"}, -] -_PORTS_MATRIX_FEDERATION = _PORTS_WEB + [ +_PORTS_MATRIX_FEDERATION = [ {"port": "8448", "protocol": "TCP", "description": "Matrix server-to-server federation"}, ] -_PORTS_ELEMENT_CALLING = _PORTS_WEB + [ +_PORTS_ELEMENT_CALLING = [ {"port": "7881", "protocol": "TCP", "description": "LiveKit WebRTC signalling"}, {"port": "7882-7894", "protocol": "UDP", "description": "LiveKit media streams"}, {"port": "5349", "protocol": "TCP", "description": "TURN over TLS"}, @@ -299,12 +294,12 @@ SERVICE_PORT_REQUIREMENTS: dict[str, list[dict]] = { # Communication "matrix-synapse.service": _PORTS_MATRIX_FEDERATION, "livekit.service": _PORTS_ELEMENT_CALLING, - # Domain-based apps (80/443) - "btcpayserver.service": _PORTS_WEB, - "vaultwarden.service": _PORTS_WEB, - "phpfpm-nextcloud.service": _PORTS_WEB, - "phpfpm-wordpress.service": _PORTS_WEB, - "haven-relay.service": _PORTS_WEB, + # Domain-based apps (80/443 handled by end-to-end domain reachability checks) + "btcpayserver.service": [], + "vaultwarden.service": [], + "phpfpm-nextcloud.service": [], + "phpfpm-wordpress.service": [], + "haven-relay.service": [], # SSH (only open when feature is enabled) "sshd.service": [{"port": "22", "protocol": "TCP", "description": "SSH"}], } @@ -934,6 +929,185 @@ def _check_port_status( return "closed" +def _check_domain_reachable(domain: str) -> dict: + """Curl the domain to verify end-to-end HTTPS reachability.""" + try: + result = subprocess.run( + ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "10", f"https://{domain}"], + capture_output=True, + text=True, + timeout=15, + ) + status_code = result.stdout.strip() + if status_code and status_code.isdigit() and int(status_code) > 0: + return {"reachable": True, "status_code": int(status_code)} + return {"reachable": False, "error": result.stderr.strip() or "No response"} + except subprocess.TimeoutExpired: + return {"reachable": False, "error": "timeout"} + except Exception as e: + return {"reachable": False, "error": str(e)} + + +def _evaluate_domain_checklist(domain: str | None, external_ip: str, internal_ip: str | None = None) -> dict: + """Evaluate sequential domain diagnostics and return UI-ready checklist data.""" + steps: list[dict] = [] + domain_status: dict = { + "status": "not_set", + "resolved_ip": None, + "expected_ip": external_ip, + } + domain_reachable: dict | None = None + + if not domain: + steps.append({ + "step": 1, + "label": "Domain Configured", + "status": "error", + "detail": "No domain configured", + }) + steps.append({ + "step": 2, + "label": "DNS Points to Your Server", + "status": "skipped", + "detail": "Skipped until a domain is configured", + }) + steps.append({ + "step": 3, + "label": "Ports 80 & 443 Open", + "status": "skipped", + "detail": "Skipped until domain and DNS are configured", + }) + return { + "domain_status": domain_status, + "domain_reachable": domain_reachable, + "domain_check_steps": steps, + "has_issues": True, + } + + steps.append({ + "step": 1, + "label": "Domain Configured", + "status": "ok", + "detail": domain, + }) + + resolved_ip: str | None = None + try: + results = socket.getaddrinfo(domain, None) + if results: + resolved_ip = results[0][4][0] + except socket.gaierror: + resolved_ip = None + except Exception: + resolved_ip = None + + if not resolved_ip: + domain_status = { + "status": "unresolvable", + "resolved_ip": None, + "expected_ip": external_ip, + } + steps.append({ + "step": 2, + "label": "DNS Points to Your Server", + "status": "error", + "detail": "Domain does not resolve — check your DNS provider", + }) + steps.append({ + "step": 3, + "label": "Ports 80 & 443 Open", + "status": "skipped", + "detail": "Skipped until DNS resolves to your server", + }) + return { + "domain_status": domain_status, + "domain_reachable": domain_reachable, + "domain_check_steps": steps, + "has_issues": True, + } + + if external_ip == "unavailable": + domain_status = { + "status": "error", + "resolved_ip": resolved_ip, + "expected_ip": external_ip, + } + steps.append({ + "step": 2, + "label": "DNS Points to Your Server", + "status": "warning", + "detail": f"Resolves to {resolved_ip} (external IP unavailable for comparison)", + }) + elif resolved_ip != external_ip: + domain_status = { + "status": "dns_mismatch", + "resolved_ip": resolved_ip, + "expected_ip": external_ip, + } + steps.append({ + "step": 2, + "label": "DNS Points to Your Server", + "status": "error", + "detail": f"Resolves to {resolved_ip} but your external IP is {external_ip}", + }) + steps.append({ + "step": 3, + "label": "Ports 80 & 443 Open", + "status": "skipped", + "detail": "Skipped until DNS points to your external IP", + }) + return { + "domain_status": domain_status, + "domain_reachable": domain_reachable, + "domain_check_steps": steps, + "has_issues": True, + } + else: + domain_status = { + "status": "connected", + "resolved_ip": resolved_ip, + "expected_ip": external_ip, + } + steps.append({ + "step": 2, + "label": "DNS Points to Your Server", + "status": "ok", + "detail": f"Resolves to {resolved_ip} (matches your external IP)", + }) + + domain_reachable = _check_domain_reachable(domain) + if domain_reachable.get("reachable"): + status_code = domain_reachable.get("status_code") + steps.append({ + "step": 3, + "label": "Ports 80 & 443 Open", + "status": "ok", + "detail": f"HTTPS reachable (HTTP {status_code})", + }) + else: + internal = internal_ip or "your server" + error_text = domain_reachable.get("error") or "No response" + steps.append({ + "step": 3, + "label": "Ports 80 & 443 Open", + "status": "error", + "detail": ( + f"Could not reach https://{domain} ({error_text}).\n" + f"→ Forward ports 80 & 443 on your router to {internal}\n" + "→ This only needs to be done once — all domain services share these ports\n" + "→ Test from your phone on mobile data (your home network may not support hairpin NAT / loopback)" + ), + }) + + has_issues = any(step.get("status") == "error" for step in steps[:3]) + return { + "domain_status": domain_status, + "domain_reachable": domain_reachable, + "domain_check_steps": steps, + "has_issues": has_issues, + } + + # ── QR code helper ──────────────────────────────────────────────── def _generate_qr_base64(data: str) -> str | None: @@ -2184,19 +2358,14 @@ async def api_services(): break has_domain_issues = False if needs_domain: - if not domain: - has_domain_issues = True - else: - try: - results = socket.getaddrinfo(domain, None) - if not results: - has_domain_issues = True - else: - resolved_ip = results[0][4][0] - if _cached_external_ip != "unavailable" and resolved_ip != _cached_external_ip: - has_domain_issues = True - except (socket.gaierror, Exception): - has_domain_issues = True + domain_eval = await loop.run_in_executor( + None, + _evaluate_domain_checklist, + domain, + _cached_external_ip, + None, + ) + has_domain_issues = bool(domain_eval.get("has_issues")) health = "needs_attention" if (has_port_issues or has_domain_issues) else "healthy" # Check Bitcoin IBD state if unit == "bitcoind.service" and enabled: @@ -2347,57 +2516,23 @@ async def api_service_detail(unit: str, icon: str | None = None): external_ip = _cached_external_ip _save_internal_ip(internal_ip) - # Domain status check + # Domain diagnostics (sequential checklist) domain_status: dict | None = None + domain_reachable: dict | None = None + domain_check_steps: list[dict] = [] + has_domain_issues = False if needs_domain: - if domain: - def _check_one_domain(d: str) -> dict: - try: - results = socket.getaddrinfo(d, None) - if not results: - return { - "status": "unresolvable", - "resolved_ip": None, - "expected_ip": external_ip, - } - resolved_ip = results[0][4][0] - if external_ip == "unavailable": - return { - "status": "error", - "resolved_ip": resolved_ip, - "expected_ip": external_ip, - } - if resolved_ip == external_ip: - return { - "status": "connected", - "resolved_ip": resolved_ip, - "expected_ip": external_ip, - } - return { - "status": "dns_mismatch", - "resolved_ip": resolved_ip, - "expected_ip": external_ip, - } - except socket.gaierror: - return { - "status": "unresolvable", - "resolved_ip": None, - "expected_ip": external_ip, - } - except Exception: - return { - "status": "error", - "resolved_ip": None, - "expected_ip": external_ip, - } - - domain_status = await loop.run_in_executor(None, _check_one_domain, domain) - else: - domain_status = { - "status": "not_set", - "resolved_ip": None, - "expected_ip": external_ip, - } + domain_eval = await loop.run_in_executor( + None, + _evaluate_domain_checklist, + domain, + external_ip, + internal_ip, + ) + domain_status = domain_eval.get("domain_status") + domain_reachable = domain_eval.get("domain_reachable") + domain_check_steps = domain_eval.get("domain_check_steps", []) + has_domain_issues = bool(domain_eval.get("has_issues")) # Port requirements and statuses port_requirements = SERVICE_PORT_REQUIREMENTS.get(unit, []) @@ -2417,6 +2552,38 @@ async def api_service_detail(unit: str, icon: str | None = None): "status": ps, "description": p.get("description", ""), }) + extra_ports = port_statuses if unit in ("matrix-synapse.service", "livekit.service") else [] + + if needs_domain and unit in ("matrix-synapse.service", "livekit.service"): + if has_domain_issues: + domain_check_steps.append({ + "step": 4, + "label": "Federation Port" if unit == "matrix-synapse.service" else "Additional Ports Required", + "status": "skipped", + "detail": "Skipped until Steps 1-3 are complete", + }) + elif unit == "matrix-synapse.service": + if extra_ports: + matrix_open = extra_ports[0]["status"] != "closed" + domain_check_steps.append({ + "step": 4, + "label": "Federation Port", + "status": "ok" if matrix_open else "error", + "detail": ( + f"Matrix federation port 8448 (TCP) is {'open' if matrix_open else 'closed'}.\n" + f"Matrix federation requires port 8448 (TCP) forwarded to {internal_ip}" + ), + }) + else: + extra_open = all(p["status"] != "closed" for p in extra_ports) + domain_check_steps.append({ + "step": 4, + "label": "Additional Ports Required", + "status": "ok" if extra_open else "error", + "detail": ( + "Element-Call/LiveKit requires additional forwarded ports for WebRTC and TURN traffic." + ), + }) # Compute composite health sync_progress: float | None = None @@ -2427,12 +2594,6 @@ async def api_service_detail(unit: str, icon: str | None = None): health = "disabled" elif status == "active": has_port_issues = any(p["status"] == "closed" for p in port_statuses) - has_domain_issues = False - if needs_domain: - if not domain: - has_domain_issues = True - elif domain_status and domain_status.get("status") not in ("connected", None): - has_domain_issues = True health = "needs_attention" if (has_port_issues or has_domain_issues) else "healthy" # Check Bitcoin IBD state if unit == "bitcoind.service" and enabled: @@ -2499,8 +2660,11 @@ async def api_service_detail(unit: str, icon: str | None = None): "domain": domain, "domain_name": domain_key, "domain_status": domain_status, + "domain_reachable": domain_reachable, + "domain_check_steps": domain_check_steps, "port_requirements": port_requirements, "port_statuses": port_statuses, + "extra_ports": extra_ports, "external_ip": external_ip, "internal_ip": internal_ip, "feature": feature_entry, diff --git a/app/sovran_systemsos_web/static/js/service-detail.js b/app/sovran_systemsos_web/static/js/service-detail.js index 01f7155..e48f7f5 100644 --- a/app/sovran_systemsos_web/static/js/service-detail.js +++ b/app/sovran_systemsos_web/static/js/service-detail.js @@ -102,9 +102,71 @@ async function openServiceDetailModal(unit, name, icon) { '' + ''; - // Section C: Ports (only if service has port_requirements) - if (data.port_statuses && data.port_statuses.length > 0) { - var anyPortClosed = data.port_statuses.some(function(p) { return p.status === "closed"; }); + // Section C: Domain diagnostics (domain services) + if (data.needs_domain) { + var steps = data.domain_check_steps || []; + var stepsHtml = ""; + steps.forEach(function(step) { + var iconLabel = "—"; + if (step.status === "ok") iconLabel = "✅"; + else if (step.status === "error") iconLabel = "❌"; + else if (step.status === "warning") iconLabel = "⚠️"; + else if (step.status === "skipped") iconLabel = "⏭️"; + var detail = escHtml(step.detail || "").replace(/\n/g, "
"); + stepsHtml += '
' + + '' + iconLabel + ' Step ' + escHtml(String(step.step)) + ': ' + escHtml(step.label || "") + '' + + (detail ? '
' + detail + '
' : '') + + '
'; + }); + + var domainActionHtml = ""; + var ds = data.domain_status || {}; + if (!data.domain && data.domain_name) { + domainActionHtml = ''; + } else if (data.domain && (ds.status === "dns_mismatch" || ds.status === "unresolvable")) { + domainActionHtml = ''; + } + + html += '
' + + '
Domain Diagnostic Checklist
' + + stepsHtml + + domainActionHtml + + '
'; + + if (unit === "livekit.service" && data.extra_ports && data.extra_ports.length > 0) { + var extraRows = ""; + data.extra_ports.forEach(function(p) { + var statusIcon, statusClass2; + if (p.status === "listening") { + statusIcon = "✅ Open"; + statusClass2 = "port-status-listening"; + } else if (p.status === "firewall_open") { + statusIcon = "🟡 Firewall open"; + statusClass2 = "port-status-open"; + } else if (p.status === "closed") { + statusIcon = "❌ Closed"; + statusClass2 = "port-status-closed"; + } else { + statusIcon = "— Unknown"; + statusClass2 = "port-status-unknown"; + } + extraRows += '' + + '' + escHtml(p.port) + '' + + '' + escHtml(p.protocol) + '' + + '' + escHtml(p.description || "") + '' + + '' + statusIcon + '' + + ''; + }); + html += '
' + + '
Step 4: Additional Ports
' + + '' + + '' + + '' + extraRows + '' + + '
PortProtocolDescriptionStatus
' + + '
'; + } + } else if (data.port_statuses && data.port_statuses.length > 0) { + // Non-domain services (SSH) keep local single-port checks. var portTableRows = ""; data.port_statuses.forEach(function(p) { var statusIcon, statusClass2; @@ -121,137 +183,19 @@ async function openServiceDetailModal(unit, name, icon) { statusIcon = "— Unknown"; statusClass2 = "port-status-unknown"; } - var desc = p.description; - var portNum = parseInt(p.port, 10); - if (portNum === 80 || portNum === 443) { - desc += " (shared — all services)"; - } portTableRows += '' + '' + escHtml(p.port) + '' + '' + escHtml(p.protocol) + '' + - '' + escHtml(desc) + '' + + '' + escHtml(p.description || "") + '' + '' + statusIcon + '' + ''; }); - - var troubleshootHtml = ""; - if (anyPortClosed) { - var sharedPorts = []; - var specificPorts = []; - data.port_statuses.forEach(function(p) { - if (p.status === "closed") { - var portNum = parseInt(p.port, 10); - if (portNum === 80 || portNum === 443) { - sharedPorts.push(p); - } else { - specificPorts.push(p); - } - } - }); - - var troubleParts = []; - - if (sharedPorts.length > 0) { - troubleParts.push( - '⚠️ Ports 80 and 443 need to be forwarded on your router.' + - '

These are shared system ports — you only need to set them up once and they cover all your domain-based services ' + - '(BTCPayServer, Nextcloud, Matrix, WordPress, etc.).

' + - '

If you already forwarded these ports during onboarding, you don\'t need to do it again. Otherwise:

' + - '
    ' + - '
  1. Log into your router\'s admin panel (usually http://192.168.1.1)
    2. ' + - '
  2. Find the Port Forwarding section
    3. ' + - '
  3. Forward port 80 (TCP) and port 443 (TCP) to your machine\'s internal IP: ' + escHtml(data.internal_ip || "—") + '
    4. ' + - '
  4. Save your router settings
    5. ' + - '
' + - '

💡 Once these two ports are forwarded, you won\'t see this warning on any service again.

' - ); - } - - if (specificPorts.length > 0) { - var portList = specificPorts.map(function(p) { - return '' + escHtml(p.port) + ' (' + escHtml(p.protocol) + ') — ' + escHtml(p.description); - }).join('
'); - - troubleParts.push( - '⚠️ This service requires additional ports to be forwarded:' + - '

' + portList + '

' + - '
    ' + - '
  1. Log into your router\'s admin panel
    2. ' + - '
  2. Forward each port listed above to your machine\'s internal IP: ' + escHtml(data.internal_ip || "—") + '
    3. ' + - '
  3. Save your router settings
    4. ' + - '
' - ); - } - - troubleshootHtml = '
' + troubleParts.join('
') + '
'; - } - html += '
' + '
Port Status
' + '' + - '' + - '' + - '' + + '' + '' + portTableRows + '' + '
PortProtocolDescriptionStatus
PortProtocolDescriptionStatus
' + - troubleshootHtml + - '
'; - } - - // Section D: Domain (only if service needs_domain) - if (data.needs_domain) { - var domainStatusHtml = ""; - var ds = data.domain_status || {}; - var domainBadge = ""; - - if (data.domain) { - if (ds.status === "connected") { - domainBadge = '✓ ' + escHtml(data.domain) + ''; - } else if (ds.status === "dns_mismatch") { - domainBadge = '⚠ ' + escHtml(data.domain) + ' (IP mismatch)'; - domainStatusHtml = '
' + - '⚠️ Your domain resolves to ' + escHtml(ds.resolved_ip || "unknown") + ' but your external IP is ' + escHtml(ds.expected_ip || "unknown") + '.' + - '

This usually means the DNS record needs to be updated:

' + - '
    ' + - '
  1. Go to njal.la and log into your account
    2. ' + - '
  2. Find your domain and check the Dynamic DNS record
    3. ' + - '
  3. Make sure it points to your current external IP: ' + escHtml(ds.expected_ip || "—") + '
    4. ' + - '
  4. If you set up a DDNS curl command during onboarding, verify it\'s running correctly
    5. ' + - '
' + - '
'; - } else if (ds.status === "unresolvable") { - domainBadge = '✗ ' + escHtml(data.domain) + ' (DNS error)'; - domainStatusHtml = '
' + - '⚠️ This domain cannot be resolved. DNS is not configured yet.' + - '

Let\'s get it set up:

' + - '
    ' + - '
  1. Go to njal.la and log into your account
    2. ' + - '
  2. Find the domain you purchased for this service
    3. ' + - '
  3. Create a Dynamic DNS record pointing to your external IP: ' + escHtml(ds.expected_ip || "—") + '
    4. ' + - '
  4. Copy the DDNS curl command from Njal.la\'s dashboard
    5. ' + - '
' + - '' + - '
'; - } else { - domainBadge = '' + escHtml(data.domain) + ''; - } - } else { - domainBadge = 'Not configured'; - domainStatusHtml = '
' + - '⚠️ No domain has been configured for this service yet.' + - '

To get this service working:

' + - '
    ' + - '
  1. Purchase a subdomain at njal.la (if you haven\'t already)
    2. ' + - '
  2. Use the button below to configure your domain through the setup wizard
    3. ' + - '
' + - '' + - '
'; - } - - html += '
' + - '
Domain
' + - domainBadge + - domainStatusHtml + '
'; } diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index 1784ab1..c7e70ba 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -352,6 +352,7 @@ in path = [ pkgs.qrencode + pkgs.curl pkgs.iproute2 pkgs.nftables pkgs.iptables