updated nixpkgs and btc clients

Fix rsync destination-directory failure: sync_tree with mkdir -p, mount re-verification, and path-safety guards

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ── Sovran Hub External Backup Script ────────────────────────────
-# Backs up Sovran_SystemsOS data to an external USB hard drive.
+# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
 # Designed for the Hub web UI (no GUI dependencies).
 #
 # Your Sovran Pro already backs up your data automatically to its
@@ -8,6 +8,15 @@
 # This script creates an additional copy on an external USB drive —
 # storing your data in a third location for maximum protection.
 #
+# The external drive must be formatted as ext4. Files are stored as
+# directly browsable files under Sovran_SystemsOS_Backup/current/.
+# Later runs update the same mirror and only transfer changed or new
+# files, making repeat backups fast.
+#
+# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
+# blockchain and Electrs index data are NOT included (they live on
+# the internal second drive).
+#
 # Usage:
 #   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
 #   (or run with no env var to auto-detect the first external USB drive)
@@ -17,18 +26,28 @@ set -euo pipefail
 BACKUP_LOG="/var/log/sovran-hub-backup.log"
 BACKUP_STATUS="/var/log/sovran-hub-backup.status"
 MEDIA_ROOT="/run/media"
-MIN_FREE_GB=10
 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
 ROLE_STATE_NIX="/etc/nixos/role-state.nix"
+SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
+SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))
 
 # ── Internal drive labels/paths to NEVER use as backup targets ───
 INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
-INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/")
+INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
+
+FAILED_ALREADY=0
+BACKUP_COMPLETE=0
+RSYNC_WARNINGS=()
+
+# Stable rsync mirror sub-path under the target drive. Not timestamped
+# so later runs update the same destination and only transfer new or changed files.
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
 
 # ── Logging helpers ──────────────────────────────────────────────
 
 log() {
-  local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+  local msg
+  msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
   echo "$msg" | tee -a "$BACKUP_LOG"
 }
 
@@ -37,16 +56,47 @@ set_status() {
 }
 
 fail() {
+  FAILED_ALREADY=1
   log "ERROR: $*"
   set_status "FAILED"
   exit 1
 }
 
+cleanup() {
+  local rc=$?
+
+  # Release the concurrency lock file descriptor if it was opened
+  if [[ -n "${LOCK_FD:-}" ]]; then
+    exec {LOCK_FD}>&- 2>/dev/null || true
+  fi
+
+  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
+    return
+  fi
+
+  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
+    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
+    set_status "FAILED"
+  fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
+}
+
+trap cleanup EXIT
+trap 'exit 1' INT TERM
+
+require_cmd() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
+}
+
 # ── Check whether a mount point is an internal drive ────────────
 
 is_internal() {
   local mnt="$1"
-  # Reject known internal mount points and their subdirectories
   for internal in "${INTERNAL_MOUNTS[@]}"; do
     if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
       return 0
@@ -59,39 +109,37 @@ is_internal() {
 
 find_external_drive() {
   local target=""
-  # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
-  if command -v lsblk &>/dev/null; then
-    while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
-      # Must be a partition or disk, and be removable/hotplug
-      [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
-      [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
-      [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal labels
-      local skip=0
-      for lbl in "${INTERNAL_LABELS[@]}"; do
-        [[ "$label" == "$lbl" ]] && skip=1 && break
-      done
-      [[ "$skip" -eq 1 ]] && continue
+  while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
+    [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
+    [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
+    [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal mount points
-      is_internal "$mountpoint" && continue
+    local skip=0
+    for lbl in "${INTERNAL_LABELS[@]}"; do
+      [[ "$label" == "$lbl" ]] && skip=1 && break
+    done
+    [[ "$skip" -eq 1 ]] && continue
 
-      if mountpoint -q "$mountpoint" 2>/dev/null; then
-        target="$mountpoint"
-        break
-      fi
-    done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
-      python3 -c "
+    is_internal "$mountpoint" && continue
+
+    if mountpoint -q "$mountpoint" 2>/dev/null; then
+      target="$mountpoint"
+      break
+    fi
+  done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
+    python3 -c "
 import sys, json
-data = json.load(sys.stdin)
+
 def flatten(devs):
     for d in devs:
         yield d
         for c in d.get('children', []):
             yield from flatten([c])
+
+data = json.load(sys.stdin)
 for d in flatten(data.get('blockdevices', [])):
-    print('\t'.join([
+    print('\\t'.join([
         d.get('type') or '',
         str(d.get('hotplug') or '0'),
         str(d.get('rm') or '0'),
@@ -99,24 +147,10 @@ for d in flatten(data.get('blockdevices', [])):
         d.get('mountpoint') or '',
     ]))
 " 2>/dev/null || true)
-  fi
 
-  # Fallback: walk /run/media/ if lsblk produced nothing
   if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
     while IFS= read -r -d '' mnt; do
       is_internal "$mnt" && continue
-      # Check label via lsblk on the device backing this mount
-      local dev
-      dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
-      if [[ -n "$dev" ]]; then
-        local lbl
-        lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
-        local skip=0
-        for internal_lbl in "${INTERNAL_LABELS[@]}"; do
-          [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
-        done
-        [[ "$skip" -eq 1 ]] && continue
-      fi
       if mountpoint -q "$mnt" 2>/dev/null; then
         target="$mnt"
         break
@@ -128,16 +162,10 @@ for d in flatten(data.get('blockdevices', [])):
 }
 
 # ── Detect the configured system role ───────────────────────────
-#
-# Priority:
-#   1. Hub config JSON  (/var/lib/sovran-hub/config.json)  — "role" key
-#   2. role-state.nix   (/etc/nixos/role-state.nix)        — grep for true flag
-#   3. Default: server_plus_desktop
 
 detect_role() {
   local role="server_plus_desktop"
 
-  # 1. Try the Hub config JSON
   if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
     local r
     r=$(python3 -c \
@@ -149,7 +177,6 @@ detect_role() {
     fi
   fi
 
-  # 2. Fall back to parsing role-state.nix
   if [[ -f "$ROLE_STATE_NIX" ]]; then
     if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
       role="desktop"
@@ -161,6 +188,119 @@ detect_role() {
   echo "$role"
 }
 
+validate_target_mount() {
+  local target="$1"
+  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
+  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
+  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
+
+  local fstype=""
+  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
+  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
+
+  if [[ "$fstype" != "ext4" ]]; then
+    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
+  fi
+
+  local write_test
+  write_test="$target/.sovran-write-test-$$"
+  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
+    fail "Target '$target' is not writable."
+  fi
+
+  log "Verified backup target filesystem: $fstype"
+}
+
+estimate_path_bytes() {
+  local path="$1"
+  shift || true
+  [[ -e "$path" ]] || {
+    echo 0
+    return
+  }
+
+  local size
+  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
+  [[ -n "$size" ]] || size=0
+  echo "$size"
+}
+
+# ── Sync one source tree to its backup destination ───────────────
+# Usage: sync_tree <label> <allow_vanished> <source> <destination> [rsync options...]
+#
+# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
+# Used for /home only — files may disappear while the desktop is active.
+# All other nonzero exit codes are always fatal.
+#
+# Before every rsync call this helper:
+#   1. Re-verifies $TARGET is still a mount point (fails if drive disconnected).
+#   2. Verifies the destination path remains beneath $BACKUP_DIR and $BACKUP_DIR
+#      remains beneath $TARGET (safe-path check).
+#   3. Creates the full destination directory hierarchy with mkdir -p so that
+#      rsync never fails trying to create a directory whose parent is absent.
+sync_tree() {
+  local label="$1"
+  local allow_vanished="$2"
+  local source="$3"
+  local destination="$4"
+  shift 4
+  # Remaining "$@" are rsync options (--exclude, etc.)
+
+  # ── Re-verify the external drive is still mounted ────────────────
+  mountpoint -q "$TARGET" 2>/dev/null || \
+    fail "Stage $label: external drive '$TARGET' is no longer mounted. Refusing to write."
+
+  # ── Verify path safety ────────────────────────────────────────────
+  # BACKUP_DIR must remain beneath TARGET.
+  case "$BACKUP_DIR" in
+    "$TARGET"/*) ;;
+    *) fail "Stage $label: BACKUP_DIR '$BACKUP_DIR' is outside TARGET '$TARGET'." ;;
+  esac
+  # Destination must remain beneath BACKUP_DIR.
+  case "$destination" in
+    "$BACKUP_DIR"/*|"$BACKUP_DIR") ;;
+    *) fail "Stage $label: destination '$destination' is outside BACKUP_DIR '$BACKUP_DIR'. Refusing to write." ;;
+  esac
+
+  # ── Create complete destination directory hierarchy ───────────────
+  # This is the fix for the production failure:
+  #   rsync: [Receiver] mkdir ".../current/etc/nixos" failed: No such file or directory
+  # mkdir -p creates all intermediate parents (e.g. current/etc/) before rsync runs.
+  mkdir -p -- "$destination" || \
+    fail "Stage $label: failed to create destination directory '$destination' (source: '$source')."
+
+  local rsync_err_tmp
+  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
+
+  local rc=0
+  rsync \
+    --archive \
+    --acls \
+    --xattrs \
+    --hard-links \
+    --numeric-ids \
+    --one-file-system \
+    --partial \
+    "$@" "$source" "$destination" 2>"$rsync_err_tmp" || rc=$?
+
+  if [[ -s "$rsync_err_tmp" ]]; then
+    while IFS= read -r rline; do
+      log "rsync: $rline"
+    done < "$rsync_err_tmp"
+  fi
+  rm -f "$rsync_err_tmp"
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
+  fi
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -169,14 +309,38 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"
 
+# ── Acquire exclusive run lock ────────────────────────────────────
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd rsync
+require_cmd findmnt
+require_cmd lsblk
+require_cmd mountpoint
+require_cmd df
+require_cmd du
+require_cmd awk
+require_cmd find
+require_cmd hostname
+require_cmd date
+require_cmd python3
+require_cmd flock
+
 # ── Detect system role ───────────────────────────────────────────
 
 ROLE="$(detect_role)"
 case "$ROLE" in
-  desktop)            ROLE_LABEL="Desktop Only" ;;
-  node)               ROLE_LABEL="Node (Bitcoin-only)" ;;
-  server_plus_desktop) ROLE_LABEL="Server + Desktop" ;;
-  *)                  ROLE_LABEL="$ROLE" ;;
+  desktop)              ROLE_LABEL="Desktop Only" ;;
+  node)                 ROLE_LABEL="Node (Bitcoin-only)" ;;
+  server_plus_desktop)  ROLE_LABEL="Server + Desktop" ;;
+  *)                    ROLE_LABEL="$ROLE" ;;
 esac
 log "Detected role: $ROLE_LABEL"
 
@@ -184,7 +348,6 @@ log "Detected role: $ROLE_LABEL"
 
 if [[ -n "${BACKUP_TARGET:-}" ]]; then
   TARGET="$BACKUP_TARGET"
-  # Safety: never allow internal drives even if explicitly passed
   if is_internal "$TARGET"; then
     fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
   fi
@@ -193,39 +356,74 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. " \
-         "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
+    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
-# ── Verify mount point ───────────────────────────────────────────
+validate_target_mount "$TARGET"
 
-[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist."
-mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
+# ── Set up stable backup destination ────────────────────────────
+# Subsequent runs update the same mirror, transferring only new or changed files.
 
-# ── Check free disk space (require ≥ 10 GB) ──────────────────────
+BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
+mkdir -p -- "$BACKUP_DIR"
 
-FREE_KB=$(df -k --output=avail "$TARGET" | tail -1)
-FREE_GB=$(( FREE_KB / 1024 / 1024 ))
-log "Free space on drive: ${FREE_GB} GB"
-(( FREE_GB >= MIN_FREE_GB )) || \
-  fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)."
+# Remove any stale BACKUP_COMPLETE left by a previous successful run.
+# The new run will re-earn it only after all stages succeed.
+rm -f "$BACKUP_DIR/BACKUP_COMPLETE"
 
-# ── Create timestamped backup directory ─────────────────────────
-
-TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
-BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
-mkdir -p "$BACKUP_DIR"
+# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
+# after all rsync stages and manifest write succeed. Failed or interrupted
+# runs keep this marker so they are clearly identifiable.
+touch "$BACKUP_DIR/INCOMPLETE"
 log "Backup destination: $BACKUP_DIR"
 
+# ── Estimate required free space ─────────────────────────────────
+# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
+# from the estimate to avoid inflating the required size.
+
+ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
+HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
+SECRETS_BYTES=0
+if [[ "$ROLE" != "desktop" ]]; then
+  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
+fi
+
+VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='postgresql' \
+  --exclude='mysql' \
+  --exclude='mariadb' \
+  --exclude='bitcoind' \
+  --exclude='electrs' \
+  --exclude='*/log' \
+  --exclude='*/logs' \
+  --exclude='*/cache' \
+  --exclude='*/tmp')
+
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
+# Require 20% growth headroom plus a fixed 1 GiB safety margin.
+# Later incremental runs need far less space, but a conservative first-run
+# check protects against running out of space mid-backup.
+REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
+
+FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
+FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
+REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
+
+log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
+log "Required free space (with safety margin): ${REQUIRED_GB} GB"
+log "Free space on drive: ${FREE_GB} GB"
+
+(( FREE_BYTES >= REQUIRED_BYTES )) || \
+  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
+
 # ── Stage 1/4: NixOS configuration ──────────────────────────────
 
 log ""
 log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 1 failed while copying /etc/nixos"
+  sync_tree "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
@@ -234,64 +432,64 @@ fi
 # ── Stage 2/4: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/4: Secrets ───────────────────────────────────────"
-mkdir -p "$BACKUP_DIR/secrets"
-
+log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
+elif [[ -e /etc/nix-bitcoin-secrets ]]; then
+  sync_tree "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
 else
-  if [[ -e /etc/nix-bitcoin-secrets ]]; then
-    rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
-  else
-    log "  (not found: /etc/nix-bitcoin-secrets — skipping)"
-  fi
+  log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
-
 log "Stage 2 complete."
 
-# ── Stage 3/4: Home directory ────────────────────────────────────
+# ── Stage 3/4: Home directory ───────────────────────────────────
+# Rsync exit code 24 (vanished source files) is treated as nonfatal here
+# because the desktop may be active and files can disappear between the
+# directory scan and the copy. All other nonzero exit codes remain fatal.
 
 log ""
 log "── Stage 3/4: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  rsync -a --info=progress2 \
+  sync_tree "/home" yes /home/ "$BACKUP_DIR/home/" \
     --exclude='.cache/' \
     --exclude='.local/share/Trash/' \
-    --exclude='*/Trash/' \
-    /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 3 failed while copying /home"
+    --exclude='Trash/' \
+    --exclude='.mozilla/firefox/*/cache2/' \
+    --exclude='.mozilla/firefox/*/startupCache/' \
+    --exclude='.mozilla/firefox/*/thumbnails/' \
+    --exclude='.config/google-chrome/*/Cache/' \
+    --exclude='.config/google-chrome/*/Code Cache/' \
+    --exclude='.config/chromium/*/Cache/' \
+    --exclude='.config/chromium/*/Code Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
+    --exclude='.local/share/baloo/' \
+    --exclude='.thumbnails/' \
+    --exclude='.xsession-errors' \
+    --exclude='.xsession-errors.old'
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/4: System data ───────────────────────────────────────
+# ── Stage 4/4: System data ──────────────────────────────────────
+# PostgreSQL/MariaDB raw database directories are excluded. Application
+# databases must be backed up separately with native database tools.
+# Bitcoin/Electrs data are excluded; they live on the internal second drive.
 
 log ""
-log "── Stage 4/4: System data (/var/lib) ────────────────────────"
-if [[ "$ROLE" == "desktop" ]]; then
-  if [[ -d /var/lib ]]; then
-    rsync -a --info=progress2 \
-      --filter='- /lnd/***' \
-      --exclude='logs/' \
-      --exclude='log/' \
-      --exclude='*/logs/' \
-      --exclude='*/log/' \
-      /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      fail "Stage 4 failed while copying /var/lib for Desktop Only role"
-    log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)."
-  else
-    log "WARNING: /var/lib not found — skipping."
-  fi
-elif [[ -d /var/lib ]]; then
-  rsync -a --info=progress2 \
-    --exclude='logs/' \
-    --exclude='log/' \
-    --exclude='*/logs/' \
+log "── Stage 4/4: System data (/var/lib) ───────────────────────"
+if [[ -d /var/lib ]]; then
+  sync_tree "/var/lib" no /var/lib/ "$BACKUP_DIR/var/lib/" \
+    --exclude='postgresql/' \
+    --exclude='mysql/' \
+    --exclude='mariadb/' \
+    --exclude='bitcoind/' \
+    --exclude='electrs/' \
     --exclude='*/log/' \
-    /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 4 failed while copying /var/lib"
+    --exclude='*/logs/' \
+    --exclude='*/cache/' \
+    --exclude='*/tmp/'
   log "Stage 4 complete."
 else
   log "WARNING: /var/lib not found — skipping."
@@ -301,21 +499,91 @@ fi
 
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
+MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
+
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date)"
+  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Contents:"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort
-} > "$BACKUP_DIR/BACKUP_MANIFEST.txt"
-log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo "Backup type: Live rsync mirror (directly browsable files)"
+  echo "Location:    ${BACKUP_DIR}"
+  echo ""
+  echo "Source paths mirrored:"
+  echo "- /etc/nixos  →  current/etc/nixos/"
+  if [[ "$ROLE" != "desktop" ]]; then
+    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
+  fi
+  echo "- /home  →  current/home/"
+  echo "- /var/lib  →  current/var/lib/"
+  echo ""
+  echo "Exclusions:"
+  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
+  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
+  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
+  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
+  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
+  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
+  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
+  echo ""
+  echo "Important limitations:"
+  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
+  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
+  echo "  applications, their data must be backed up separately using native tools."
+  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
+  echo "  reconstructable or stored on the internal second drive."
+  echo "- This is a live file-level mirror, not a transactional database backup."
+  echo "  Files being written during the backup may be in an inconsistent state."
+  echo ""
+  echo "Restore guidance:"
+  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
+  echo "- To restore a directory:"
+  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
+  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
+  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
+  echo "- To copy individual files:"
+  echo "    sudo cp -a current/home/username/ /home/username/"
+  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
+  echo "  hardware-configuration.nix for the new hardware before rebuilding."
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for warning in "${RSYNC_WARNINGS[@]}"; do
+      echo "- $warning"
+    done
+  fi
+  echo ""
+  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "from manual external backup because they already live on the internal second drive"
+  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
+} > "$MANIFEST_FILE"
+
+log "Manifest written to $MANIFEST_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
+if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${RSYNC_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that"
+  log "vanished during backup, which is normal on an active desktop."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
+log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
 log "Please eject the drive safely before removing it from your Sovran Pro."
+
+# Remove incomplete marker and write completion marker only after all work succeeds.
+# A later successful run will update the same mirror and replace any INCOMPLETE state.
+rm -f "$BACKUP_DIR/INCOMPLETE"
+date -u '+%Y-%m-%dT%H:%M:%SZ' > "$BACKUP_DIR/BACKUP_COMPLETE"
+
+BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/static/css/header.css

@@ -91,3 +91,30 @@
   border-color: var(--accent-color);
   color: var(--accent-color);
 }
+
+/* ── Header reboot button ───────────────────────────────────────── */
+
+.btn-header-reboot {
+  background: transparent;
+  border: 1px solid rgba(184, 125, 0, 0.35);
+  color: #c98d08;
+  font-size: 0.78rem;
+  font-weight: 600;
+  padding: 4px 12px;
+  border-radius: var(--radius-btn);
+  cursor: pointer;
+  transition: border-color 0.15s, color 0.15s, background-color 0.15s;
+}
+
+.btn-header-reboot:hover {
+  border-color: #b87d00;
+  color: #e0a010;
+  background-color: rgba(184, 125, 0, 0.1);
+}
+
+@media (max-width: 480px) {
+  .btn-header-reboot {
+    padding: 4px 8px;
+    font-size: 0.72rem;
+  }
+}

app/sovran_systemsos_web/static/css/modals.css

@@ -102,6 +102,48 @@ button.btn-reboot:hover:not(:disabled) {
   background-color: #529E7E;
 }
 
+/* Restart = AMBER (manual restart action) */
+.btn-restart-amber {
+  background-color: #b87d00;
+  color: #fff;
+}
+
+.btn-restart-amber:hover:not(:disabled) {
+  background-color: #9a6800;
+}
+
+/* Restart conflict warning box */
+.restart-conflict-box {
+  background-color: rgba(180, 100, 0, 0.12);
+  border-left: 3px solid #c97a00;
+  border-radius: 6px;
+  padding: 12px 14px;
+  margin-bottom: 14px;
+}
+
+.restart-conflict-title {
+  font-size: 0.88rem;
+  font-weight: 700;
+  color: #e69000;
+  margin: 0 0 6px 0;
+}
+
+.restart-conflict-desc {
+  font-size: 0.83rem;
+  color: var(--text-secondary);
+  line-height: 1.5;
+  margin: 0;
+}
+
+/* Reboot error card actions row */
+.reboot-error-actions {
+  display: flex;
+  gap: 12px;
+  justify-content: center;
+  flex-wrap: wrap;
+  margin-top: 20px;
+}
+
 .btn-save {
   background-color: var(--yellow);
   color: #0A1A10;

app/sovran_systemsos_web/static/css/tiles.css

@@ -155,6 +155,69 @@
   white-space: nowrap;
 }
 
+/* ── BIP-110 status badge (tile + detail modal) ───────────────────── */
+
+.tile-bip110-badge {
+  display: inline-flex;
+  align-items: center;
+  gap: 3px;
+  font-size: 0.64rem;
+  font-weight: 600;
+  border-radius: 4px;
+  padding: 2px 6px;
+  margin-top: 4px;
+  white-space: nowrap;
+  letter-spacing: 0.02em;
+}
+
+.tile-bip110-badge--active {
+  background: rgba(109, 191, 139, 0.18);
+  color: var(--green);
+  border: 1px solid rgba(109, 191, 139, 0.3);
+}
+
+.tile-bip110-badge--locked_in {
+  background: rgba(94, 173, 138, 0.15);
+  color: var(--accent-color);
+  border: 1px solid rgba(94, 173, 138, 0.3);
+}
+
+.tile-bip110-badge--signaling {
+  background: rgba(94, 173, 138, 0.12);
+  color: var(--accent-color);
+  border: 1px solid rgba(94, 173, 138, 0.2);
+}
+
+.tile-bip110-badge--not_signaling {
+  background: rgba(229, 165, 10, 0.12);
+  color: var(--yellow);
+  border: 1px solid rgba(229, 165, 10, 0.25);
+}
+
+.tile-bip110-badge--unsupported {
+  background: rgba(94, 122, 106, 0.12);
+  color: var(--grey);
+  border: 1px solid rgba(94, 122, 106, 0.2);
+}
+
+.tile-bip110-badge--unknown {
+  background: transparent;
+  color: var(--text-dim);
+  border: 1px solid var(--border-color);
+}
+
+.bip110-status-row {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  flex-wrap: wrap;
+}
+
+.bip110-source-label {
+  color: var(--text-dim);
+  font-size: 0.75rem;
+}
+
 /* ── Service detail modal sections ───────────────────────────────── */
 
 .svc-detail-section {

app/sovran_systemsos_web/static/js/events.js

@@ -44,6 +44,28 @@ if ($upgradeCloseBtn) $upgradeCloseBtn.addEventListener("click", closeUpgradeMod
 if ($upgradeCancelBtn) $upgradeCancelBtn.addEventListener("click", closeUpgradeModal);
 if ($upgradeModal) $upgradeModal.addEventListener("click", function(e) { if (e.target === $upgradeModal) closeUpgradeModal(); });
 
+// Restart confirm dialog
+if ($restartConfirmCancel) $restartConfirmCancel.addEventListener("click", closeRestartConfirmDialog);
+if ($restartConfirmModal) $restartConfirmModal.addEventListener("click", function(e) { if (e.target === $restartConfirmModal) closeRestartConfirmDialog(); });
+if ($restartConfirmModal) $restartConfirmModal.addEventListener("keydown", function(e) { if (e.key === "Escape") closeRestartConfirmDialog(); });
+
+// Header Reboot button
+if ($headerRebootBtn) $headerRebootBtn.addEventListener("click", function() { openRestartConfirmDialog(); });
+if ($restartConfirmOk) $restartConfirmOk.addEventListener("click", function() {
+  if ($restartConfirmOk.disabled) return;
+  $restartConfirmOk.disabled = true;
+  closeRestartConfirmDialog();
+  doReboot();
+});
+
+// Reboot error card buttons
+var $rebootErrorCloseBtn = document.getElementById("reboot-error-close-btn");
+var $rebootErrorRetryBtn = document.getElementById("reboot-error-retry-btn");
+if ($rebootErrorCloseBtn) $rebootErrorCloseBtn.addEventListener("click", function() {
+  if ($rebootOverlay) $rebootOverlay.classList.remove("visible");
+});
+if ($rebootErrorRetryBtn) $rebootErrorRetryBtn.addEventListener("click", doReboot);
+
 // ── Upgrade modal functions ───────────────────────────────────────
 
 function openUpgradeModal() {
@@ -54,6 +76,37 @@ function closeUpgradeModal() {
   if ($upgradeModal) $upgradeModal.classList.remove("open");
 }
 
+// ── Restart confirm dialog functions ─────────────────────────────
+
+var _restartDialogOpener = null;
+
+function openRestartConfirmDialog() {
+  if (!$restartConfirmModal) return;
+  _restartDialogOpener = document.activeElement;
+
+  // Detect conflicting operations
+  var isOperationInProgress = !!_updatePollTimer || !!_rebuildPollTimer;
+  if ($restartConflictBox) $restartConflictBox.style.display = isOperationInProgress ? "" : "none";
+  if ($restartConfirmOk) $restartConfirmOk.disabled = isOperationInProgress;
+
+  $restartConfirmModal.classList.add("open");
+
+  // Focus Cancel initially for safety
+  var cancelBtn = document.getElementById("restart-confirm-cancel-btn");
+  if (cancelBtn) setTimeout(function() { cancelBtn.focus(); }, 50);
+}
+
+function closeRestartConfirmDialog() {
+  if ($restartConfirmModal) $restartConfirmModal.classList.remove("open");
+  // Re-enable confirm button for next open
+  if ($restartConfirmOk) $restartConfirmOk.disabled = false;
+  // Return focus to the element that opened the dialog
+  if (_restartDialogOpener && _restartDialogOpener.focus) {
+    try { _restartDialogOpener.focus(); } catch (_) {}
+    _restartDialogOpener = null;
+  }
+}
+
 async function doUpgradeToServer() {
   var confirmBtn = $upgradeConfirmBtn;
   if (confirmBtn) { confirmBtn.disabled = true; confirmBtn.textContent = "Upgrading…"; }

app/sovran_systemsos_web/static/js/features.js

@@ -73,22 +73,47 @@ function openDomainSetupModal(feat, onSaved) {
     npubField = '<div class="domain-field-group"><label class="domain-field-label" for="domain-npub-input">Nostr Public Key (npub1...):</label><input class="domain-field-input" type="text" id="domain-npub-input" placeholder="npub1..." value="' + escHtml(currentNpub) + '" /></div>';
   }
 
-  var externalIp = _cachedExternalIp || "your external IP";
+  var introHtml;
+  if (_currentRole === "node") {
+    introHtml =
+      '<p>To enable <strong>' + escHtml(feat.name) + '</strong>, it needs its own domain from Njal.la.</p>' +
+      '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+      '<li>Create an account at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">njal.la</a>.</li>' +
+      '<li>Set up a domain for it — either a free subdomain or a separate domain. Pick one option:</li>' +
+      '</ol>';
+  } else {
+    introHtml =
+      '<p>To enable <strong>' + escHtml(feat.name) + '</strong>, it needs its own domain from Njal.la. ' +
+      'In your Njal.la account, set up a domain for it — either a free subdomain or a separate domain. Pick one option:</p>';
+  }
 
   $domainSetupBody.innerHTML =
     '<div class="domain-setup-intro">' +
-    '<p><strong>Before continuing:</strong></p>' +
-    '<ol>' +
-    '<li>Create an account at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">https://njal.la</a></li>' +
-    '<li>Purchase a new domain on Njal.la, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.</li>' +
-    '<li>In the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine\'s external IP address:<br>' +
-    '<span style="display:inline-block;margin-top:4px;padding:4px 10px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>' +
-    '<li>Njal.la will give you a curl command like:<br>' +
-    '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=sub.domain.com&amp;k=abc123&amp;auto&quot;</code></li>' +
-    '<li>Enter the subdomain and paste that curl command below</li>' +
+    introHtml +
+    '<details style="margin-top:10px;">' +
+    '<summary style="cursor:pointer;font-weight:600;">Option A — Free subdomain (recommended)</summary>' +
+    '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+    '<li>In Njal.la, open a domain you own and click &quot;Add record&quot;.</li>' +
+    '<li>Set record type to <strong>Dynamic</strong>.</li>' +
+    '<li>In the <strong>Name</strong> field, type ONLY the host part — the word before your domain.<br>' +
+    '(Example only, your choice — for &quot;call.yourdomain.com&quot; you&apos;d type just: &nbsp;<code>call</code>)<br>' +
+    '&#9888; Do NOT type the full domain here — Njal.la adds it automatically.</li>' +
+    '<li>A Dynamic record has NO IP field — the IP auto-fills after the rebuild/reboot.</li>' +
+    '<li>Copy the curl command Njal.la gives you, e.g.:<br>' +
+    '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=call.yourdomain.com&amp;k=abc123&amp;auto&quot;</code></li>' +
     '</ol>' +
+    '</details>' +
+    '<details style="margin-top:6px;">' +
+    '<summary style="cursor:pointer;font-weight:600;">Option B — Separate / new domain</summary>' +
+    '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+    '<li>In Njal.la, buy the domain you want.</li>' +
+    '<li>Add a Dynamic record as in Option A. If this domain is dedicated to the service, leave the Name field blank or use <code>@</code>.</li>' +
+    '<li>Copy the curl command Njal.la gives you.</li>' +
+    '</ol>' +
+    '</details>' +
+    '<p style="margin-top:10px;">Below, enter the full domain for this service — a subdomain (e.g. call.yourdomain.com) or a separate domain (e.g. call.com) — and paste its curl command.</p>' +
     '</div>' +
-    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Subdomain (e.g. myservice.example.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" /></div>' +
+    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Service domain (e.g. call.yourdomain.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" /></div>' +
     '<div class="domain-field-group"><label class="domain-field-label" for="domain-ddns-input">Njal.la Dynamic DNS Update Command:</label><input class="domain-field-input" type="text" id="domain-ddns-input" placeholder="curl &quot;https://njal.la/update/?h=myservice.example.com&amp;k=abc123&amp;auto&quot;" /><p class="domain-field-hint">ℹ Paste the full curl command from your Njal.la dashboard\'s Dynamic record</p></div>' +
     npubField +
     '<div class="domain-field-actions"><button class="btn btn-close-modal" id="domain-setup-cancel-btn">Cancel</button><button class="btn btn-primary" id="domain-setup-save-btn">Save &amp; Enable</button></div>';
@@ -103,7 +128,7 @@ function openDomainSetupModal(feat, onSaved) {
     ddnsUrl   = ddnsUrl.trim();
     npub      = npub.trim();
 
-    if (!subdomain) { alert("Please enter a subdomain."); return; }
+    if (!subdomain) { alert("Please enter a domain."); return; }
     if (feat.id === "haven" && !npub) { alert("Please enter your Nostr public key."); return; }
 
     var saveBtn = document.getElementById("domain-setup-save-btn");
@@ -159,14 +184,14 @@ function openDomainReconfigureModal(feat, existingDomain, onSaved) {
     '<p><strong>Troubleshooting steps:</strong></p>' +
     '<ol>' +
     '<li>Log into your Njal.la dashboard at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">https://njal.la</a></li>' +
-    '<li>Find the DNS record for <strong>' + escHtml(currentDomain || "your domain") + '</strong></li>' +
+    '<li>Find the DNS record for <strong>' + escHtml(currentDomain || "your domain") + '</strong>. In Njal.la\'s Name field, note that only the host part is stored (the word before the domain) — not the full domain.</li>' +
     '<li>Verify it has a <strong>Dynamic</strong> record pointing to your current external IP:<br>' +
     '<span style="display:inline-block;margin-top:4px;padding:4px 10px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>' +
     '<li>If the IP is wrong or the record is missing, update it</li>' +
     '<li>If you changed the DDNS curl command, paste the updated one below</li>' +
     '</ol>' +
     '</div>' +
-    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Subdomain (e.g. myservice.example.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" value="' + escHtml(currentDomain) + '" /></div>' +
+    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Service domain (e.g. call.yourdomain.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" value="' + escHtml(currentDomain) + '" /></div>' +
     '<div class="domain-field-group"><label class="domain-field-label" for="domain-ddns-input">Njal.la Dynamic DNS Update Command:</label><input class="domain-field-input" type="text" id="domain-ddns-input" placeholder="curl &quot;https://njal.la/update/?h=myservice.example.com&amp;k=abc123&amp;auto&quot;" /><p class="domain-field-hint">ℹ Paste the full curl command from your Njal.la dashboard\'s Dynamic record</p></div>' +
     npubField +
     '<div class="domain-field-actions"><button class="btn btn-close-modal" id="domain-setup-cancel-btn">Cancel</button><button class="btn btn-primary" id="domain-setup-save-btn">Save &amp; Update</button></div>';
@@ -413,16 +438,11 @@ function handleFeatureToggle(feat, newEnabled) {
       });
   }
 
-  if (conflictNames.length > 0) {
-    var confirmMsg;
-    if (feat.id === "bip110") {
-      confirmMsg = "Only one Bitcoin node implementation can be active. Enabling Bitcoin Knots + BIP110 will disable Bitcoin Core (if active). Your timechain data will be preserved — you will not need to re-download the timechain. Continue?";
-    } else if (feat.id === "bitcoin-core") {
-      confirmMsg = "Only one Bitcoin node implementation can be active. Enabling Bitcoin Core will disable Bitcoin Knots + BIP110 (if active). Your timechain data will be preserved — you will not need to re-download the timechain. Continue?";
-    } else {
-      confirmMsg = "This will disable " + conflictNames.join(", ") + ". Continue?";
-    }
+  if (feat.id === "bitcoin-core") {
+    var confirmMsg = "Only one Bitcoin node implementation can be active. Enabling Bitcoin Core will replace Bitcoin Knots + BIP110 as the active node. Your timechain data will be preserved — you will not need to re-download the timechain. Continue?";
     openFeatureConfirm(confirmMsg, proceedAfterConflictCheck);
+  } else if (conflictNames.length > 0) {
+    openFeatureConfirm("This will disable " + conflictNames.join(", ") + ". Continue?", proceedAfterConflictCheck);
   } else {
     proceedAfterConflictCheck();
   }

app/sovran_systemsos_web/static/js/helpers.js

@@ -60,3 +60,17 @@ async function apiFetch(path, options) {
   }
   return res.json();
 }
+
+
+// ── BIP-110 badge state config ────────────────────────────────────
+// Shared lookup used by tiles.js and service-detail.js.
+// Keys match the "state" values returned by /api/bitcoin/bip110.
+
+var BIP110_BADGE_CONFIG = {
+  active:        { cls: 'tile-bip110-badge--active',       label: 'Active',         title: 'BIP-110 is active on this node' },
+  locked_in:     { cls: 'tile-bip110-badge--locked_in',    label: 'Locked In',      title: 'BIP-110 is locked in and will activate shortly' },
+  signaling:     { cls: 'tile-bip110-badge--signaling',    label: 'Signaling',      title: 'Node is signaling readiness for BIP-110' },
+  not_signaling: { cls: 'tile-bip110-badge--not_signaling',label: 'Not Signaling',  title: 'Node supports BIP-110 but is not signaling this period' },
+  unsupported:   { cls: 'tile-bip110-badge--unsupported',  label: 'Not Supported',  title: 'This node build does not include BIP-110' },
+  unknown:       { cls: 'tile-bip110-badge--unknown',      label: '\u2014',          title: 'Status unavailable (node syncing or RPC not ready)' }
+};

app/sovran_systemsos_web/static/js/rebuild.js

@@ -69,7 +69,7 @@ function onRebuildDone(result) {
     // Auto-reload the page after a short delay so tiles and toggles reflect the new state
     setTimeout(function() { window.location.reload(); }, 1200);
   } else if (result === "reboot_required") {
-    if ($rebuildStatus) $rebuildStatus.textContent = "✓ Done — reboot required";
+    if ($rebuildStatus) $rebuildStatus.textContent = "✓ Done — restart required";
     if ($rebuildReboot) $rebuildReboot.style.display = "inline-flex";
   } else {
     if ($rebuildStatus) $rebuildStatus.textContent = "✗ Something went wrong";

app/sovran_systemsos_web/static/js/security.js

@@ -145,28 +145,25 @@ function openSecurityModal() {
           if (rebootBtn) {
             // Keep button disabled for 5 seconds to prevent accidental clicks
             var countdown = 5;
-            rebootBtn.textContent = "I have written down my new password \u2014 Reboot now (" + countdown + ")";
+            rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System (" + countdown + ")";
             var timer = setInterval(function() {
               countdown--;
               if (countdown <= 0) {
                 clearInterval(timer);
                 rebootBtn.disabled = false;
-                rebootBtn.textContent = "I have written down my new password \u2014 Reboot now";
+                rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System";
               } else {
-                rebootBtn.textContent = "I have written down my new password \u2014 Reboot now (" + countdown + ")";
+                rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System (" + countdown + ")";
               }
             }, 1000);
 
             rebootBtn.addEventListener("click", function() {
               rebootBtn.disabled = true;
-              rebootBtn.textContent = "Rebooting\u2026";
-              if ($rebootOverlay) $rebootOverlay.classList.add("visible");
-              _rebootStartTime = Date.now();
-              _serverWentDown = false;
-              setTimeout(waitForServerReboot, REBOOT_INITIAL_DELAY);
-              var rebootCtrl = new AbortController();
-              setTimeout(function() { rebootCtrl.abort(); }, REBOOT_REQUEST_TIMEOUT);
-              fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal }).catch(function() {});
+              rebootBtn.textContent = "Restarting\u2026";
+              // Hide the security reset overlay so the shared reboot overlay is visible
+              var $secResetOverlay2 = document.getElementById("security-reset-overlay");
+              if ($secResetOverlay2) $secResetOverlay2.classList.remove("visible");
+              doReboot();
             }, { once: true });
           }
         } catch (err) {

app/sovran_systemsos_web/static/js/service-detail.js

@@ -107,6 +107,21 @@ async function openServiceDetailModal(unit, name, icon) {
       '</div>' +
       '</div>';
 
+    // Section B2: BIP-110 live status (bip110 tile only)
+    if (icon === 'bip110' && data.bip110) {
+      var bip110 = data.bip110;
+      var bip110State = bip110.state || 'unknown';
+      var bip110Cfg = BIP110_BADGE_CONFIG[bip110State] || BIP110_BADGE_CONFIG.unknown;
+      var bip110Source = bip110.source ? ' <span class="bip110-source-label">(source: ' + escHtml(bip110.source) + ')</span>' : '';
+      html += '<div class="svc-detail-section">' +
+        '<div class="svc-detail-section-title">BIP-110 Deployment Status</div>' +
+        '<div class="bip110-status-row">' +
+          '<span class="tile-bip110-badge ' + bip110Cfg.cls + '" title="' + escHtml(bip110Cfg.title) + '">' + escHtml(bip110Cfg.label) + '</span>' +
+          bip110Source +
+        '</div>' +
+        '</div>';
+    }
+
     // Section C: Domain diagnostics (domain services)
     if (data.needs_domain) {
       var steps = data.domain_check_steps || [];
@@ -139,20 +154,36 @@ async function openServiceDetailModal(unit, name, icon) {
         '</div>';
 
       if (unit === "livekit.service" && data.extra_ports && data.extra_ports.length > 0) {
+        var trimmedInternalIp = data.internal_ip ? String(data.internal_ip).trim() : "";
+        var internalIp = trimmedInternalIp || "";
+        var internalIpHtml = internalIp ? escHtml(internalIp) : "Could not detect";
+        var routerIpHelp = internalIp
+          ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+          : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+        var routerNextStep = internalIp
+          ? 'Next step: Log in to your router and create forwarding rules for the ports above. Set the destination/internal IP to <strong>' + internalIpHtml + '</strong>.'
+          : 'Next step: Log in to your router and create forwarding rules for the ports above. Use this computer’s internal IP as the destination/internal IP.';
+        var domainConfigured = !!(data.domain && String(data.domain).trim());
         var extraRows = "";
         data.extra_ports.forEach(function(p) {
           var statusIcon, statusClass2;
-          if (p.status === "listening") {
-            statusIcon = "✅ Open";
+          if (!effectiveEnabled) {
+            statusIcon = "⚠ Configure Element Call first";
+            statusClass2 = "port-status-open";
+          } else if (!domainConfigured) {
+            statusIcon = "⚠ Configure domain first";
+            statusClass2 = "port-status-open";
+          } else if (p.status === "listening") {
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-listening";
           } else if (p.status === "firewall_open") {
-            statusIcon = "🟡 Firewall open";
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-open";
           } else if (p.status === "closed") {
-            statusIcon = "❌ Closed";
+            statusIcon = "❌ Not ready yet";
             statusClass2 = "port-status-closed";
           } else {
-            statusIcon = "— Unknown";
+            statusIcon = "— Could not check";
             statusClass2 = "port-status-unknown";
           }
           extraRows += '<tr>' +
@@ -163,11 +194,16 @@ async function openServiceDetailModal(unit, name, icon) {
             '</tr>';
         });
         html += '<div class="svc-detail-section">' +
-          '<div class="svc-detail-section-title">Step 4: Additional Ports</div>' +
+          '<div class="svc-detail-section-title">Ports to Forward in Your Router</div>' +
+          '<div class="svc-detail-port-note">Forward these ports in your router to this Sovran_SystemsOS computer.</div>' +
+          '<div class="svc-detail-port-note"><strong>Router Forward-To IP:</strong> ' + internalIpHtml + '</div>' +
+          '<div class="svc-detail-port-note">' + routerIpHelp + '</div>' +
           '<table class="svc-detail-port-table">' +
-            '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+            '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
             '<tbody>' + extraRows + '</tbody>' +
           '</table>' +
+          '<div class="svc-detail-port-note">The Hub can check whether Sovran_SystemsOS is ready on this computer, but full public port verification requires an outside internet check.</div>' +
+          '<div class="svc-detail-port-note">' + routerNextStep + '</div>' +
           '</div>';
       }
     } else if (data.port_statuses && data.port_statuses.length > 0) {
@@ -176,16 +212,16 @@ async function openServiceDetailModal(unit, name, icon) {
       data.port_statuses.forEach(function(p) {
         var statusIcon, statusClass2;
         if (p.status === "listening") {
-          statusIcon = "✅ Open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-listening";
         } else if (p.status === "firewall_open") {
-          statusIcon = "🟡 Firewall open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-open";
         } else if (p.status === "closed") {
-          statusIcon = "🔴 Closed";
+          statusIcon = "❌ Not ready";
           statusClass2 = "port-status-closed";
         } else {
-          statusIcon = "— Unknown";
+          statusIcon = "— Could not check";
           statusClass2 = "port-status-unknown";
         }
         portTableRows += '<tr>' +
@@ -196,9 +232,10 @@ async function openServiceDetailModal(unit, name, icon) {
           '</tr>';
       });
       html += '<div class="svc-detail-section">' +
-        '<div class="svc-detail-section-title">Port Status</div>' +
+        '<div class="svc-detail-section-title">Port Requirements</div>' +
+        '<div class="svc-detail-port-note">This shows whether Sovran_SystemsOS is ready to use this port on this computer. If you need access from outside your home network, forward this port in your router.</div>' +
         '<table class="svc-detail-port-table">' +
-          '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+          '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
           '<tbody>' + portTableRows + '</tbody>' +
         '</table>' +
         '</div>';
@@ -242,7 +279,7 @@ async function openServiceDetailModal(unit, name, icon) {
       var addonBtnCls      = feat.enabled ? "btn btn-close-modal" : "btn btn-primary";
 
       // Section title: use a more specific label for mutually-exclusive Bitcoin node features
-      var addonSectionTitle = (feat.id === "bip110" || feat.id === "bitcoin-core")
+      var addonSectionTitle = (feat.id === "bitcoin-core")
         ? "\u20BF Bitcoin Node Selection"
         : "\uD83D\uDD27 Addon Feature";
 

app/sovran_systemsos_web/static/js/state.js

@@ -49,6 +49,9 @@ const $btnSave        = document.getElementById("btn-save-report");
 const $btnCloseModal  = document.getElementById("btn-close-modal");
 
 const $rebootOverlay  = document.getElementById("reboot-overlay");
+const $rebootMainCard = document.getElementById("reboot-main-card");
+const $rebootErrorCard = document.getElementById("reboot-error-card");
+const $rebootSubmessage = document.getElementById("reboot-submessage");
 
 const $credsModal     = document.getElementById("creds-modal");
 const $credsTitle     = document.getElementById("creds-modal-title");
@@ -101,5 +104,14 @@ const $upgradeConfirmBtn  = document.getElementById("upgrade-confirm-btn");
 const $upgradeCancelBtn   = document.getElementById("upgrade-cancel-btn");
 const $upgradeCloseBtn    = document.getElementById("upgrade-close-btn");
 
+// Restart confirm dialog
+const $restartConfirmModal  = document.getElementById("restart-confirm-modal");
+const $restartConfirmOk     = document.getElementById("restart-confirm-ok-btn");
+const $restartConfirmCancel = document.getElementById("restart-confirm-cancel-btn");
+const $restartConflictBox   = document.getElementById("restart-conflict-box");
+
+// Header reboot button
+const $headerRebootBtn = document.getElementById("btn-header-reboot");
+
 // System status banner
 // (removed — health is now shown per-tile via the composite health field)

app/sovran_systemsos_web/static/js/support.js

@@ -491,8 +491,9 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>At least 500 GB of free space on the drive</li>',
-        '<li>Drive must be formatted as <strong>exFAT</strong></li>',
+        '<li>Enough free space for your data (the backup checks this before starting)</li>',
+        '<li>Drive must be formatted as <strong>ext4</strong> (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or <code>mkfs.ext4</code> — note that formatting erases all data on the drive.</li>',
+        '<li>The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.</li>',
       '</ol>',
     '</div>',
 
@@ -501,17 +502,25 @@ function renderBackupReady(drives) {
       '<ol class="support-backup-steps">',
         '<li>NixOS configuration (<code>/etc/nixos</code>)</li>',
         '<li>nix-bitcoin secrets (<code>/etc/nix-bitcoin-secrets</code>)</li>',
-        '<li>System service data (<code>/var/lib</code>) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets</li>',
+        '<li>System service data (<code>/var/lib</code>) — excluding databases and blockchain data (see note below)</li>',
         '<li>Home directory (<code>/home</code>)</li>',
       '</ol>',
     '</div>',
 
     '<div class="support-wallet-box support-wallet-warning">',
+      '<div class="support-wallet-header">',
+        '<span class="support-wallet-icon">\u2139\ufe0f</span>',
+        '<span class="support-wallet-title">Database and Blockchain Data</span>',
+      '</div>',
+      '<p class="support-wallet-desc">Application databases stored in PostgreSQL or MariaDB/MySQL are <strong>not included</strong> in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.</p>',
+    '</div>',
+
+    '<div class="support-wallet-box support-wallet-protected">',
       '<div class="support-wallet-header">',
         '<span class="support-wallet-icon">\u23f1\ufe0f</span>',
         '<span class="support-wallet-title">Time Estimate</span>',
       '</div>',
-      '<p class="support-wallet-desc">This backup can take <strong>up to 4 hours</strong> depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026</p>',
+      '<p class="support-wallet-desc">The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.</p>',
     '</div>',
 
     driveSelector,
@@ -584,9 +593,10 @@ async function pollBackupStatus() {
       logDiv.scrollTop = logDiv.scrollHeight;
     }
     _backupLogOffset = data.offset;
-    if (!data.running) {
+    const result = (data.result || "").toLowerCase();
+    if (result === "success" || result === "failed") {
       stopBackupPoll();
-      renderBackupDone(data.result === "success");
+      renderBackupDone(result === "success");
     }
   } catch (_) {}
 }
@@ -618,7 +628,7 @@ function renderBackupDone(success) {
       '<div class="support-section">',
       '<div class="support-icon-big">\u26a0\ufe0f</div>',
       '<h3 class="support-heading">Backup Failed</h3>',
-      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.</p>',
+      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.</p>',
       '<div class="modal-log" id="backup-log-fail" style="text-align:left;"></div>',
       '<button class="btn support-btn-done" id="btn-backup-close">Close</button>',
       '</div>',

app/sovran_systemsos_web/static/js/tiles.js

@@ -4,6 +4,21 @@
 // Keyed by tileId: { progress: float, timestamp: ms }
 var _btcSyncPrev = {};
 
+// ── BIP-110 badge helper ──────────────────────────────────────────
+
+function _renderBip110Badge(bip110) {
+  if (!bip110) return '';
+  var state = bip110.state || 'unknown';
+  var cfg = BIP110_BADGE_CONFIG[state] || BIP110_BADGE_CONFIG.unknown;
+  return '<div class="tile-bip110-badge ' + cfg.cls + '" title="' + escHtml(cfg.title) + '">' + escHtml(cfg.label) + '</div>';
+}
+
+function _firstElementFromHtml(html) {
+  var tmp = document.createElement("div");
+  tmp.innerHTML = html;
+  return tmp.firstElementChild || null;
+}
+
 // ── Render: initial build ─────────────────────────────────────────
 
 function buildTiles(services, categoryLabels) {
@@ -165,7 +180,8 @@ function buildTile(svc) {
 
   var ver = svc.version || svc.bitcoin_version || '';
   var versionLabel = ver ? '<div class="tile-version">' + escHtml(ver) + '</div>' : '';
-  tile.innerHTML = '<img class="tile-icon" src="/static/icons/' + escHtml(svc.icon) + '.svg" alt="' + escHtml(svc.name) + '" onerror="this.style.display=\'none\';this.nextElementSibling.style.display=\'flex\'"><div class="tile-icon-fallback" style="display:none">?</div><div class="tile-name">' + escHtml(svc.name) + '</div>' + versionLabel + '<div class="tile-status"><span class="status-dot ' + sc + '"></span><span class="status-text">' + st + '</span></div>';
+  var bip110Badge = (svc.icon === 'bip110') ? _renderBip110Badge(svc.bip110) : '';
+  tile.innerHTML = '<img class="tile-icon" src="/static/icons/' + escHtml(svc.icon) + '.svg" alt="' + escHtml(svc.name) + '" onerror="this.style.display=\'none\';this.nextElementSibling.style.display=\'flex\'"><div class="tile-icon-fallback" style="display:none">?</div><div class="tile-name">' + escHtml(svc.name) + '</div>' + versionLabel + bip110Badge + '<div class="tile-status"><span class="status-dot ' + sc + '"></span><span class="status-text">' + st + '</span></div>';
 
   tile.style.cursor = "pointer";
   tile.addEventListener("click", function() {
@@ -265,6 +281,23 @@ function updateTiles(services) {
           }
         }
       }
+      // Update BIP-110 badge for bip110 tiles
+      if (svc.icon === 'bip110') {
+        var badgeHtml = _renderBip110Badge(svc.bip110);
+        var badgeEl = tile.querySelector(".tile-bip110-badge");
+        if (badgeEl) {
+          // Replace existing badge in-place
+          var newBadge = _firstElementFromHtml(badgeHtml);
+          if (newBadge) { badgeEl.replaceWith(newBadge); } else { badgeEl.remove(); }
+        } else if (badgeHtml) {
+          // Insert badge after version label (or after tile-name if no version)
+          var anchorEl = tile.querySelector(".tile-version") || tile.querySelector(".tile-name");
+          if (anchorEl) {
+            var newBadgeEl = _firstElementFromHtml(badgeHtml);
+            if (newBadgeEl) anchorEl.insertAdjacentElement("afterend", newBadgeEl);
+          }
+        }
+      }
     }
   }
 }

app/sovran_systemsos_web/static/js/update.js

@@ -154,7 +154,7 @@ function onUpdateDone(result) {
     if ($modalStatus) $modalStatus.textContent = "✓ Update complete";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else if (result === "reboot_required") {
-    if ($modalStatus) $modalStatus.textContent = "✓ Update complete — reboot required";
+    if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else {
     if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
@@ -179,23 +179,50 @@ function saveErrorReport() {
 
 var _rebootStartTime = 0;
 var _serverWentDown = false;
+var _rebootFailed = false;
+
+function _setRebootStatus(msg) {
+  if ($rebootSubmessage) $rebootSubmessage.textContent = msg;
+}
 
 function doReboot() {
   if ($modal) $modal.classList.remove("open");
   if ($rebuildModal) $rebuildModal.classList.remove("open");
   stopUpdatePoll();
   stopRebuildPoll();
+  // Reset overlay to main card
+  if ($rebootMainCard) $rebootMainCard.style.display = "";
+  if ($rebootErrorCard) $rebootErrorCard.style.display = "none";
+  _setRebootStatus("Sending restart request\u2026");
   if ($rebootOverlay) $rebootOverlay.classList.add("visible");
   _rebootStartTime = Date.now();
   _serverWentDown = false;
+  _rebootFailed = false;
   var rebootCtrl = new AbortController();
   setTimeout(function() { rebootCtrl.abort(); }, REBOOT_REQUEST_TIMEOUT);
-  fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal }).catch(function() {});
+  fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal })
+    .then(function(res) {
+      if (!res.ok) {
+        // Definitive HTTP error — server rejected the request before going down
+        _rebootFailed = true;
+        if ($rebootMainCard) $rebootMainCard.style.display = "none";
+        if ($rebootErrorCard) $rebootErrorCard.style.display = "";
+        // Leave overlay visible so the error card is shown
+      }
+      // HTTP 2xx: request accepted, proceed with polling
+    })
+    .catch(function() {
+      // Connection dropped or request aborted — the server is likely already going
+      // down as part of the restart. Treat as success and continue polling.
+    });
   // Wait before the first check — NixOS shutdown after an update can take 20-40s
   setTimeout(waitForServerReboot, REBOOT_INITIAL_DELAY);
 }
 
 function waitForServerReboot() {
+  if (_rebootFailed) return;
+  // Update status on first check (server hasn't gone down yet)
+  if (!_serverWentDown) _setRebootStatus("Waiting for the computer to shut down\u2026");
   var controller = new AbortController();
   var timeoutId = setTimeout(function() { controller.abort(); }, REBOOT_FETCH_TIMEOUT);
 
@@ -205,18 +232,23 @@ function waitForServerReboot() {
       if (_serverWentDown) {
         // Server is responding after having been down — reboot is complete.
         // Any response (even 401/500) means the server process is back.
+        _setRebootStatus("System is back online. Reconnecting\u2026");
         window.location.reload();
       } else if ((Date.now() - _rebootStartTime) < 90000) {
         // Server still responding but hasn't gone down yet — keep waiting
         setTimeout(waitForServerReboot, REBOOT_CHECK_INTERVAL);
       } else {
         // Been over 90 seconds and server is responding — just reload
+        _setRebootStatus("System is back online. Reconnecting\u2026");
         window.location.reload();
       }
     })
     .catch(function() {
       clearTimeout(timeoutId);
-      _serverWentDown = true;
+      if (!_serverWentDown) {
+        _serverWentDown = true;
+        _setRebootStatus("The computer is restarting\u2026");
+      }
       setTimeout(waitForServerReboot, REBOOT_CHECK_INTERVAL);
     });
 }

app/sovran_systemsos_web/static/onboarding.js

@@ -333,8 +333,6 @@ async function loadStep3() {
     return;
   }
 
-  var externalIp = (networkData && networkData.external_ip) || "Unknown (could not retrieve)";
-
   // Build set of enabled service units
   var enabledUnits = new Set();
   (_servicesData || []).forEach(function(svc) {
@@ -352,25 +350,31 @@ async function loadStep3() {
     html += '<p class="onboarding-body-text">No domain-based services are enabled for your role. You can skip this step.</p>';
   } else {
     html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
-      + '<strong>Before you continue:</strong>'
+      + '<p style="margin:0 0 8px;"><strong>Sovran_SystemsOS uses Njal.la for domains and Dynamic DNS.</strong></p>'
       + '<ol style="margin:8px 0 0 16px; padding:0; line-height:1.7;">'
-      + '<li>Create an account at <a href="https://njal.la" target="_blank" style="color:var(--accent-color);">https://njal.la</a></li>'
-      + '<li>Purchase a new domain on Njal.la, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.</li>'
-      + '<li>In the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine\'s external IP address:<br>'
-      + '<span style="display:inline-block;margin-top:4px;padding:4px 12px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1.1em;font-weight:700;letter-spacing:0.03em;">' + escHtml(externalIp) + '</span></li>'
-      + '<li>Njal.la will give you a curl command like:<br>'
-      + '<code style="font-size:0.8em;">curl "https://njal.la/update/?h=sub.domain.com&amp;k=abc123&amp;auto"</code></li>'
-      + '<li>Enter the subdomain and paste that curl command below for each service</li>'
+      + '<li>Create an account at <a href="https://njal.la" target="_blank" style="color:var(--accent-color);">https://njal.la</a>.</li>'
+      + '<li>Buy at least one domain. Each service below needs its own domain — you can either give each service its own subdomain of a single domain you buy (subdomains are free, and one domain can have many), OR use a separate domain for each. Your choice.</li>'
+      + '<li>For each service, add a <strong>Dynamic</strong> record in Njal.la:'
+      + '<ul style="margin:4px 0 0 16px;padding:0;line-height:1.7;">'
+      + '<li>In the Njal.la <strong>Name</strong> field, type ONLY the host part — the word before your domain.<br>'
+      + '(Example only, your choice — for &quot;call.yourdomain.com&quot; you&apos;d type just: <code>call</code>.)<br>'
+      + 'If you bought a whole separate domain just for this service, leave Name blank or use <code>@</code>.<br>'
+      + '&#9888; Do NOT type the full domain in the Name field — Njal.la adds it automatically.</li>'
+      + '<li>A Dynamic record has NO IP field. You don&apos;t enter an IP anywhere — it auto-fills once Sovran_SystemsOS updates it (on save, and again after reboot).</li>'
+      + '</ul>'
+      + '</li>'
+      + '<li>Njal.la gives you a curl command like:<br>'
+      + '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=call.yourdomain.com&amp;k=abc123&amp;auto&quot;</code></li>'
       + '</ol>'
       + '</div>';
-    html += '<p class="onboarding-hint">Enter each fully-qualified subdomain (e.g. <code>matrix.yourdomain.com</code>) and its Njal.la DDNS curl command.</p>';
+    html += '<p class="onboarding-hint">Enter each service\'s full domain — a subdomain (e.g. <code>call.yourdomain.com</code>) or a separate domain (e.g. <code>call.com</code>) — and its Njal.la DDNS curl command.</p>';
     relevantDomains.forEach(function(d) {
       var currentVal = (_domainsData && _domainsData[d.name]) || "";
       html += '<div class="onboarding-domain-group">';
       html += '<label class="onboarding-domain-label">' + escHtml(d.label) + '</label>';
       html += '<input class="onboarding-domain-input domain-field-input" type="text" id="domain-input-' + escHtml(d.name) + '" data-domain="' + escHtml(d.name) + '" placeholder="e.g. ' + escHtml(d.name) + '.yourdomain.com" value="' + escHtml(currentVal) + '" />';
       html += '<label class="onboarding-domain-label onboarding-domain-label--sub">Njal.la DDNS Curl Command</label>';
-      html += '<input class="onboarding-domain-input domain-field-input" type="text" id="ddns-input-' + escHtml(d.name) + '" data-ddns="' + escHtml(d.name) + '" placeholder="curl &quot;https://njal.la/update/?h=' + escHtml(d.name) + '.yourdomain.com&amp;k=abc123&amp;auto&quot;" />';
+      html += '<input class="onboarding-domain-input domain-field-input" type="text" id="ddns-input-' + escHtml(d.name) + '" data-ddns="' + escHtml(d.name) + '" placeholder="curl &quot;https://njal.la/update/?h=...&amp;k=...&amp;auto&quot;" />';
       html += '<p class="onboarding-hint" style="margin-top:4px;">ℹ Paste the curl URL from your Njal.la dashboard\'s Dynamic record</p>';
       html += '<button type="button" class="btn btn-primary onboarding-domain-save-btn" data-save-domain="' + escHtml(d.name) + '" style="align-self:flex-start;margin-top:8px;font-size:0.82rem;padding:6px 16px;">Save</button>';
       html += '<span class="onboarding-domain-save-status" id="domain-save-status-' + escHtml(d.name) + '" style="font-size:0.82rem;min-height:1.2em;"></span>';
@@ -512,7 +516,7 @@ async function saveStep3() {
 async function loadStep4() {
   var body = document.getElementById("step-4-body");
   if (!body) return;
-  body.innerHTML = '<p class="onboarding-loading">Checking ports…</p>';
+  body.innerHTML = '<p class="onboarding-loading">Loading router setup…</p>';
 
   var networkData = null;
 
@@ -523,51 +527,59 @@ async function loadStep4() {
     return;
   }
 
-  var internalIp = (networkData && networkData.internal_ip) || "unknown";
-
-  var ip = escHtml(internalIp);
+  var trimmedInternalIp = (networkData && networkData.internal_ip) ? String(networkData.internal_ip).trim() : "";
+  var internalIp = trimmedInternalIp || "";
+  var hasInternalIp = !!internalIp;
+  var ip = escHtml(internalIp || "Could not detect");
+  var routerIpHelp = hasInternalIp
+    ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+    : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+  var destinationInstruction = hasInternalIp
+    ? 'Set the destination/internal IP to <strong>' + ip + '</strong>'
+    : 'Use this computer’s internal IP as the destination/internal IP';
 
   var html = '<p class="onboarding-port-note" style="margin-bottom:14px;">'
     + '⚠ <strong>Each port only needs to be forwarded once — all services share the same ports.</strong>'
     + '</p>';
 
   html += '<div class="onboarding-port-ip">';
-  html += '  <span class="onboarding-port-ip-label">Forward ports to this machine\'s internal IP:</span>';
+  html += '  <span class="onboarding-port-ip-label">Forward router traffic to this Sovran_SystemsOS computer:</span>';
   html += '  <span class="port-req-internal-ip">' + ip + '</span>';
   html += '</div>';
+  html += '<div class="onboarding-port-note" style="margin:8px 0 16px;">' + routerIpHelp + '</div>';
 
   // Required ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Ports — open these on your router:</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Router Rules</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
-  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP</td></tr>';
+  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP / SSL setup</td></tr>';
   html += '<tr><td class="port-req-port">443</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTPS</td></tr>';
-  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">SSH Remote Access</td></tr>';
-  html += '<tr><td class="port-req-port">8448</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">Matrix Federation</td></tr>';
+  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">Remote SSH access</td></tr>';
   html += '</tbody></table>';
   html += '</div>';
 
   // Optional ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Optional — Only needed if you enable Element Calling:</div>';
-  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">These 5 additional port openings are required on top of the 4 required ports above.</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Element Call Router Rules</div>';
+  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">Only add these if you enable Element Call. These ports help video and audio calls connect reliably.</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
   html += '<tr><td class="port-req-port">7881</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit WebRTC signalling</td></tr>';
   html += '<tr><td class="port-req-port">7882</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit media (UDP mux)</td></tr>';
   html += '<tr><td class="port-req-port">5349</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN over TLS</td></tr>';
   html += '<tr><td class="port-req-port">3478</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN (STUN/relay)</td></tr>';
-  html += '<tr><td class="port-req-port">30000–40000</td><td class="port-req-proto">TCP/UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
+  html += '<tr><td class="port-req-port">30000-40000</td><td class="port-req-proto">TCP &amp; UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
   html += '</tbody></table>';
+  html += '<div style="font-size:0.85em;margin-top:6px;color:var(--color-text-muted,#888);">ℹ The <strong>30000-40000</strong> range is a single forwarding rule — just set its protocol to <strong>both TCP and UDP</strong> (often shown as "Both" or "TCP/UDP" on your router).</div>';
   html += '</div>';
 
   // Totals
   html += '<div class="onboarding-port-totals">';
-  html += '<strong>Total port openings: 4</strong> (without Element Calling)<br>';
-  html += '<strong>Total port openings: 9</strong> (with Element Calling — 4 required + 5 optional)';
+  html += '<strong>Total port openings: 3</strong> (without Element Call)<br>';
+  html += '<strong>Total port openings: 8</strong> (with Element Call — 3 required + 5 optional)';
   html += '</div>';
 
   html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
@@ -582,12 +594,16 @@ async function loadStep4() {
     + '<li>Open your router\'s admin panel — usually <code>http://192.168.1.1</code> or <code>http://192.168.0.1</code></li>'
     + '<li>Look for <strong>"Port Forwarding"</strong>, <strong>"NAT"</strong>, or <strong>"Virtual Server"</strong> in the settings</li>'
     + '<li>Create a new rule for each port listed above</li>'
-    + '<li>Set the destination/internal IP to <strong>' + ip + '</strong></li>'
+    + '<li>' + destinationInstruction + '</li>'
     + '<li>Set both internal and external port to the same number</li>'
     + '<li>Save and apply changes</li>'
     + '</ol>'
     + '</details>';
 
+  html += '<div class="onboarding-port-note" style="margin-top:12px;">'
+    + '<strong>Important:</strong> The Hub can show which ports Sovran_SystemsOS needs, but it cannot fully confirm router forwarding from inside your home network. Full public port verification requires an outside internet check.'
+    + '</div>';
+
   body.innerHTML = html;
 }
 

app/sovran_systemsos_web/templates/index.html

@@ -24,6 +24,7 @@
     <span class="title">Sovran_SystemsOS Hub</span>
     <div class="header-buttons">
       <span class="role-badge" id="role-badge">Loading…</span>
+      <button class="btn btn-header-reboot" id="btn-header-reboot" title="Restart the entire computer">Reboot</button>
       <button class="btn btn-logout" id="btn-logout" title="Sign out">Sign Out</button>
     </div>
   </header>
@@ -61,7 +62,7 @@
       <div class="modal-log" id="modal-log" aria-live="polite"></div>
       <div class="modal-footer">
         <button class="btn btn-save" id="btn-save-report" style="display:none">Save Error Report</button>
-        <button class="btn btn-reboot" id="btn-reboot" style="display:none">Reboot</button>
+        <button class="btn btn-reboot" id="btn-reboot" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="btn-close-modal" disabled>Close</button>
       </div>
     </div>
@@ -164,7 +165,7 @@
       <div class="modal-log" id="rebuild-log" aria-live="polite"></div>
       <div class="modal-footer">
         <button class="btn btn-save" id="rebuild-save-report" style="display:none">Save Error Report</button>
-        <button class="btn btn-reboot" id="rebuild-reboot-btn" style="display:none">Reboot</button>
+        <button class="btn btn-reboot" id="rebuild-reboot-btn" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="rebuild-close-btn" disabled>Close</button>
       </div>
     </div>
@@ -240,26 +241,61 @@
         You will need it to log in to your computer<br />and the Sovran Hub at <em>sovransystemsos.local</em>.
       </p>
       <button class="security-reset-reboot-btn" id="security-reset-reboot-btn" disabled>
-        I have written down my new password — Reboot now
+        I have written down my new password — Restart Entire System
       </button>
     </div>
   </div>
 
   <!-- Reboot overlay -->
   <div class="reboot-overlay" id="reboot-overlay">
-    <div class="reboot-card">
-      <div class="reboot-icon">↻</div>
-      <h2 class="reboot-title">System Rebooting</h2>
+    <!-- Normal restarting card -->
+    <div class="reboot-card" id="reboot-main-card">
+      <div class="reboot-icon" aria-hidden="true">↻</div>
+      <h2 class="reboot-title">Restarting Entire System</h2>
       <p class="reboot-message">
-        Sovran_SystemsOS is now restarting.<br />
-        This page will automatically reconnect once the system is back online.
+        The entire computer is restarting, including the desktop and all hosted services.<br />
+        This page will reconnect automatically when Sovran_SystemsOS is back online.
       </p>
-      <div class="reboot-dots">
+      <div class="reboot-dots" aria-hidden="true">
         <span class="reboot-dot"></span>
         <span class="reboot-dot"></span>
         <span class="reboot-dot"></span>
       </div>
-      <p class="reboot-submessage">Stay tuned…</p>
+      <p class="reboot-submessage" id="reboot-submessage" aria-live="polite">Sending restart request…</p>
+    </div>
+    <!-- Error card (shown if restart request fails definitively) -->
+    <div class="reboot-card" id="reboot-error-card" style="display:none">
+      <div class="reboot-icon" aria-hidden="true">⚠</div>
+      <h2 class="reboot-title">Restart could not be started</h2>
+      <p class="reboot-message">
+        The computer did not begin restarting. No services were intentionally stopped. Please try again.
+      </p>
+      <div class="reboot-error-actions">
+        <button class="btn btn-close-modal" id="reboot-error-close-btn">Close</button>
+        <button class="btn btn-restart-amber" id="reboot-error-retry-btn">Try Again</button>
+      </div>
+    </div>
+  </div>
+
+  <!-- Restart Confirm Dialog -->
+  <div class="modal-overlay" id="restart-confirm-modal" role="dialog" aria-modal="true" aria-labelledby="restart-confirm-title">
+    <div class="creds-dialog domain-narrow-dialog">
+      <div class="creds-header">
+        <span class="creds-title" id="restart-confirm-title">Restart the entire computer?</span>
+      </div>
+      <div class="creds-body">
+        <div id="restart-conflict-box" class="restart-conflict-box" style="display:none">
+          <p class="restart-conflict-title">The system cannot restart right now.</p>
+          <p class="restart-conflict-desc">A system update, rebuild, backup, restore, or security operation is currently running. Wait for it to finish, then try again.</p>
+        </div>
+        <p class="support-desc"><strong>This will reboot the physical machine running Sovran_SystemsOS — not just the Hub.</strong></p>
+        <p class="support-desc">The desktop and all hosted services will stop temporarily and restart with the computer. Anyone currently using these services will be disconnected.</p>
+        <p class="support-desc">The system usually returns within 1–3 minutes. This page will reconnect automatically.</p>
+        <div class="domain-field-actions">
+          <button class="btn btn-close-modal" id="restart-confirm-cancel-btn">Cancel</button>
+          <button class="btn btn-restart-amber" id="restart-confirm-ok-btn">Restart Entire System</button>
+        </div>
+      </div>
     </div>
   </div>
 

app/sovran_systemsos_web/templates/onboarding.html

@@ -128,9 +128,8 @@
           <h2 class="onboarding-step-title">Domain Configuration</h2>
           <p class="onboarding-step-desc">
             Sovran_SystemsOS uses <strong><a href="https://njal.la" target="_blank" style="color: var(--accent-color);">Njal.la</a></strong> for domains and Dynamic DNS.
-            First, create an account at <strong>Njal.la</strong> and purchase a new domain, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.
-            Then, in the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine's external IP address (shown below).
-            Finally, paste the DDNS curl command from your Njal.la dashboard for each service below.
+            Create an account at Njal.la, then for each service below, add a <strong>Dynamic</strong> record — no IP needed, it auto-populates once the DDNS curl command runs.
+            Paste the curl command from your Njal.la dashboard for each service.
           </p>
         </div>
         <div class="onboarding-card" id="step-3-body">
@@ -149,14 +148,14 @@
       <div class="onboarding-panel" id="step-4" style="display:none">
         <div class="onboarding-step-header">
           <span class="onboarding-step-icon">🔌</span>
-          <h2 class="onboarding-step-title">Port Forwarding Check</h2>
+          <h2 class="onboarding-step-title">Router Setup</h2>
           <p class="onboarding-step-desc">
-            Forward these ports on your router to this machine. Each port only needs to be opened once — they are shared across all your services.
-            <strong>Ports 80 and 443 must be open for SSL certificates to work.</strong>
+            Forward these ports in your router to this Sovran_SystemsOS computer. These rules let people reach your services from outside your home network.
+            <strong>Ports 80 and 443 are required for HTTPS and SSL certificates.</strong>
           </p>
         </div>
         <div class="onboarding-card" id="step-4-body">
-          <p class="onboarding-loading">Checking ports…</p>
+          <p class="onboarding-loading">Loading router setup…</p>
         </div>
         <div class="onboarding-footer">
           <button class="btn btn-close-modal onboarding-btn-back" data-prev="3">← Back</button>

app/tests/test_bip110_status.py

@@ -0,0 +1,166 @@
+import unittest
+from unittest.mock import patch
+from pathlib import Path
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+
+                return _decorator
+
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server
+
+
+class Bip110StatusTests(unittest.TestCase):
+    def _status(self, deploy_info, net_info):
+        with patch.object(server, "_get_bitcoin_deployment_info", return_value=deploy_info), patch.object(
+            server, "_get_bitcoin_version_info", return_value=net_info
+        ):
+            return server._get_bip110_status()
+
+    def test_started_reduced_data_reports_signaling(self):
+        deploy_info = {
+            "deployments": {
+                "reduced_data": {
+                    "type": "bip9",
+                    "active": False,
+                    "bip9": {
+                        "bit": 4,
+                        "status": "started",
+                        "statistics": {"elapsed": 833, "count": 4, "threshold": 1109},
+                        "signalling": "--#--",
+                    },
+                }
+            }
+        }
+
+        result = self._status(deploy_info, {"subversion": "/Satoshi:29.0.0/"})
+        self.assertEqual(
+            result,
+            {"supported": True, "signaling": True, "state": "signaling", "source": "getdeploymentinfo"},
+        )
+
+    def test_active_reduced_data_reports_active(self):
+        deploy_info = {
+            "deployments": {"reduced_data": {"active": True, "bip9": {"bit": 4, "status": "active"}}}
+        }
+
+        result = self._status(deploy_info, {"subversion": "/Satoshi:29.0.0/"})
+        self.assertEqual(result["state"], "active")
+        self.assertTrue(result["supported"])
+        self.assertTrue(result["signaling"])
+        self.assertEqual(result["source"], "getdeploymentinfo")
+
+    def test_locked_in_reduced_data_reports_locked_in(self):
+        deploy_info = {
+            "deployments": {"reduced_data": {"active": False, "bip9": {"bit": 4, "status": "locked_in"}}}
+        }
+
+        result = self._status(deploy_info, {"subversion": "/Satoshi:29.0.0/"})
+        self.assertEqual(result["state"], "locked_in")
+        self.assertTrue(result["supported"])
+        self.assertTrue(result["signaling"])
+        self.assertEqual(result["source"], "getdeploymentinfo")
+
+    def test_no_bip110_deployment_and_plain_subversion_reports_unsupported(self):
+        deploy_info = {
+            "deployments": {
+                "taproot": {"type": "bip9", "active": True, "bip9": {"bit": 2, "status": "active"}},
+            }
+        }
+        result = self._status(deploy_info, {"subversion": "/Satoshi:27.0.0/"})
+        self.assertEqual(
+            result,
+            {"supported": False, "signaling": False, "state": "unsupported", "source": "subversion"},
+        )
+
+    def test_node_unreachable_reports_unknown(self):
+        result = self._status(None, None)
+        self.assertEqual(result, {"supported": False, "signaling": False, "state": "unknown", "source": "none"})
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_hub_update_boot_staging.py

@@ -0,0 +1,44 @@
+import unittest
+from pathlib import Path
+
+
+HUB_NIX = Path(__file__).resolve().parents[2] / "modules" / "core" / "sovran-hub.nix"
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class HubUpdateBootStagingTests(unittest.TestCase):
+    def setUp(self):
+        self.source = HUB_NIX.read_text()
+        self.update_section = _section(
+            self.source,
+            'update-script = pkgs.writeShellScript "sovran-hub-update.sh" \'\'',
+            "# ── Rebuild wrapper script",
+        )
+        self.rebuild_section = _section(
+            self.source,
+            'rebuild-script = pkgs.writeShellScript "sovran-hub-rebuild.sh" \'\'',
+            "# ── Brave launcher wrapper",
+        )
+
+    def test_full_update_uses_boot_not_switch(self):
+        self.assertIn("nixos-rebuild boot --flake /etc/nixos", self.update_section)
+        self.assertNotIn("nixos-rebuild switch --flake /etc/nixos", self.update_section)
+
+    def test_full_update_marks_reboot_required(self):
+        self.assertIn('echo "REBOOT_REQUIRED" > "$STATUS"', self.update_section)
+
+    def test_rebuild_path_keeps_switch_semantics(self):
+        self.assertIn("nixos-rebuild switch --flake /etc/nixos", self.rebuild_section)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_local_domain_loopback_nix.py

@@ -0,0 +1,193 @@
+"""Structural regression tests for modules/core/local-domain-loopback.nix.
+
+Verifies that the sovran-hosts-update helper:
+  - is built as a pkgs.writeShellApplication with explicit runtimeInputs
+    (gawk, gnugrep, coreutils);
+  - uses ``lib.getExe hostsUpdateScript`` for both the systemd ExecStart and
+    the activation script so that both contexts share the same Nix-store
+    executable;
+  - does NOT point ExecStart at the raw /etc path;
+  - includes element-calling in the supported domain list;
+  - uses ``awk -v`` for safe marker-variable passing rather than interpolating
+    marker text directly into the awk program;
+  - retains the domain validation regex (idempotency / injection prevention);
+  - exposes /etc/sovran-hosts-update.sh as a symlink via ``source =`` (not a
+    second raw ``text =`` body);
+  - does NOT rely on environment.systemPackages for the helper's dependencies.
+"""
+
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+NIX_STRING_INDENT = 6
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FLAKE_SOURCE = (REPO_ROOT / "flake.nix").read_text()
+PRIMARY_NIXOS_CONFIGURATION_MATCH = re.search(
+    r"nixosConfigurations\.([A-Za-z0-9_-]+)\s*=",
+    FLAKE_SOURCE,
+)
+if PRIMARY_NIXOS_CONFIGURATION_MATCH is None:
+    raise RuntimeError("Could not determine the primary nixosConfigurations entry from flake.nix")
+PRIMARY_NIXOS_CONFIGURATION = PRIMARY_NIXOS_CONFIGURATION_MATCH.group(1)
+HELPER_BUILD_ATTR = (
+    f'.#nixosConfigurations.{PRIMARY_NIXOS_CONFIGURATION}.config.environment.etc.'
+    '"sovran-hosts-update.sh".source'
+)
+NIX_FILE = (
+    REPO_ROOT
+    / "modules"
+    / "core"
+    / "local-domain-loopback.nix"
+)
+
+
+class LocalDomainLoopbackNixStructureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = NIX_FILE.read_text()
+
+    def _helper_script(self) -> str:
+        start = self.source.index("text = ''") + len("text = ''")
+        end = self.source.index("    '';", start)
+        return "\n".join(
+            line[NIX_STRING_INDENT:]
+            if line.startswith(" " * NIX_STRING_INDENT)
+            else line
+            for line in self.source[start:end].splitlines()
+        ).lstrip("\n")
+
+    # ── writeShellApplication and explicit runtimeInputs ────────────────────
+
+    def test_uses_write_shell_application(self):
+        self.assertIn("pkgs.writeShellApplication", self.source)
+
+    def test_runtime_inputs_includes_coreutils(self):
+        self.assertIn("pkgs.coreutils", self.source)
+
+    def test_runtime_inputs_includes_gawk(self):
+        self.assertIn("pkgs.gawk", self.source)
+
+    def test_runtime_inputs_includes_gnugrep(self):
+        self.assertIn("pkgs.gnugrep", self.source)
+
+    def test_runtime_inputs_block_present(self):
+        self.assertIn("runtimeInputs", self.source)
+
+    # ── Both execution paths use lib.getExe ─────────────────────────────────
+
+    def test_exec_start_uses_lib_get_exe(self):
+        """systemd ExecStart must reference the Nix-store executable."""
+        self.assertIn("ExecStart = lib.getExe hostsUpdateScript", self.source)
+
+    def test_activation_script_uses_lib_get_exe(self):
+        """Activation text must call the same Nix-store executable."""
+        self.assertIn("${lib.getExe hostsUpdateScript}", self.source)
+
+    def test_exec_start_does_not_point_to_etc_path(self):
+        """ExecStart must NOT use the raw /etc path (which lacks a deterministic PATH)."""
+        self.assertNotIn('ExecStart = "/etc/sovran-hosts-update.sh"', self.source)
+
+    # ── /etc symlink uses source =, not a second text = body ────────────────
+
+    def test_etc_entry_uses_source_not_text(self):
+        """The /etc/sovran-hosts-update.sh entry must be a symlink (source =),
+        not a second raw script body (text =)."""
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source', self.source
+        )
+
+    def test_etc_source_points_to_get_exe(self):
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript',
+            self.source,
+        )
+
+    # ── element-calling domain is supported ─────────────────────────────────
+
+    def test_element_calling_domain_key_present(self):
+        self.assertIn("element-calling", self.source)
+
+    # ── Robust awk -v variable passing ──────────────────────────────────────
+
+    def test_awk_uses_dash_v_for_begin_marker(self):
+        """awk must receive the begin marker via -v, not by shell interpolation."""
+        self.assertIn('awk -v begin=', self.source)
+
+    def test_awk_uses_dash_v_for_end_marker(self):
+        self.assertIn('-v end=', self.source)
+
+    def test_awk_does_not_interpolate_marker_into_program(self):
+        """The old pattern interpolated $BEGIN_MARKER directly into the awk source."""
+        self.assertNotIn('/$BEGIN_MARKER', self.source)
+        self.assertNotIn('/$END_MARKER', self.source)
+
+    # ── Domain validation ────────────────────────────────────────────────────
+
+    def test_domain_validation_regex_present(self):
+        """The hostname validation regex must still be present for injection prevention."""
+        self.assertIn("grep -qE", self.source)
+        self.assertIn("[a-zA-Z0-9]", self.source)
+
+    def test_invalid_domain_warning_present(self):
+        self.assertIn("skipping invalid domain value", self.source)
+
+    # ── No environment.systemPackages reliance ───────────────────────────────
+
+    def test_no_environment_system_packages_for_helper(self):
+        """The helper's tools are declared via runtimeInputs; the module must
+        not add them to environment.systemPackages."""
+        self.assertNotIn("environment.systemPackages", self.source)
+
+    # ── Idempotency: existing Sovran block is removed before rewriting ───────
+
+    def test_existing_block_removal_logic_present(self):
+        """awk strip of the managed block must be present for idempotency."""
+        self.assertIn("skip=1", self.source)
+        self.assertIn("skip=0", self.source)
+
+    def test_managed_block_uses_grouped_append_redirect(self):
+        self.assertIn('} >> "$TMP"', self.source)
+        self.assertEqual(self.source.count('>> "$TMP"'), 1)
+
+    def test_helper_script_passes_shellcheck(self):
+        shellcheck = shutil.which("shellcheck")
+        if shellcheck is None:
+            self.skipTest("shellcheck is not installed")
+        proc = subprocess.run(
+            [shellcheck, "-s", "bash", "-"],
+            input=self._helper_script(),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        output = proc.stdout + proc.stderr
+        self.assertEqual(proc.returncode, 0, output)
+
+    def test_helper_derivation_builds_when_nix_available(self):
+        nix = shutil.which("nix")
+        if nix is None:
+            self.skipTest("nix is not installed")
+        proc = subprocess.run(
+            [
+                nix,
+                "build",
+                HELPER_BUILD_ATTR,
+                "--no-link",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+
+    # ── Activation script warns on failure rather than silently swallowing ───
+
+    def test_activation_script_emits_warning_on_failure(self):
+        self.assertIn("warning: sovran-hosts-update", self.source)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_loopback_diagnostics.py

@@ -0,0 +1,399 @@
+"""Tests for server-local loopback diagnostics and domain validation.
+
+Covers:
+- Domain value validation and injection prevention.
+- Loopback address detection (IPv4 and IPv6).
+- _resolve_all_addresses returning multiple addresses.
+- _check_domain_health_fast with loopback resolution.
+- _evaluate_domain_checklist with loopback override — no false dns_mismatch.
+- _evaluate_domain_checklist with genuine DNS mismatch — still reports error.
+- api_services health stays "healthy" when domain resolves to loopback.
+- api_services health stays "needs_attention" when DNS is genuinely wrong.
+- api_domains_check returns "local_override" for loopback-resolved domains.
+"""
+
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+# ---------------------------------------------------------------------------
+# Minimal stubs so server.py can be imported without the full FastAPI stack.
+# ---------------------------------------------------------------------------
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+                return _decorator
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server  # noqa: E402
+
+
+# ===========================================================================
+# Domain value validation
+# ===========================================================================
+
+class TestValidateDomainValue(unittest.TestCase):
+    """_validate_domain_value must reject anything that could corrupt /etc/hosts."""
+
+    def _v(self, value: str) -> bool:
+        return server._validate_domain_value(value)
+
+    # -- Valid values --------------------------------------------------------
+
+    def test_simple_domain_valid(self):
+        self.assertTrue(self._v("cloud.example.com"))
+
+    def test_subdomain_valid(self):
+        self.assertTrue(self._v("matrix.home.example.org"))
+
+    def test_single_label_with_tld_valid(self):
+        self.assertTrue(self._v("example.com"))
+
+    def test_hyphen_in_domain_valid(self):
+        self.assertTrue(self._v("my-nextcloud.example.com"))
+
+    # -- Injection / malformed values ----------------------------------------
+
+    def test_empty_string_invalid(self):
+        self.assertFalse(self._v(""))
+
+    def test_newline_injection_invalid(self):
+        self.assertFalse(self._v("evil.com\n127.0.0.1 other.host"))
+
+    def test_carriage_return_injection_invalid(self):
+        self.assertFalse(self._v("evil.com\r127.0.0.1 other.host"))
+
+    def test_space_injection_invalid(self):
+        self.assertFalse(self._v("evil.com 127.0.0.1"))
+
+    def test_hash_comment_injection_invalid(self):
+        self.assertFalse(self._v("evil.com# comment"))
+
+    def test_bare_hostname_no_dot_invalid(self):
+        self.assertFalse(self._v("localhost"))
+
+    def test_bare_ip_invalid(self):
+        self.assertFalse(self._v("192.168.1.1"))
+
+    def test_too_long_invalid(self):
+        self.assertFalse(self._v("a" * 254 + ".com"))
+
+    def test_leading_dot_invalid(self):
+        self.assertFalse(self._v(".example.com"))
+
+    def test_trailing_dot_invalid(self):
+        self.assertFalse(self._v("example.com."))
+
+
+# ===========================================================================
+# Loopback address detection
+# ===========================================================================
+
+class TestIsLoopbackAddress(unittest.TestCase):
+
+    def test_ipv4_loopback(self):
+        self.assertTrue(server._is_loopback_address("127.0.0.1"))
+
+    def test_ipv4_loopback_other(self):
+        self.assertTrue(server._is_loopback_address("127.0.0.2"))
+
+    def test_ipv4_loopback_high(self):
+        self.assertTrue(server._is_loopback_address("127.255.255.255"))
+
+    def test_ipv6_loopback(self):
+        self.assertTrue(server._is_loopback_address("::1"))
+
+    def test_public_ipv4_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("203.0.113.10"))
+
+    def test_private_ipv4_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("192.168.1.50"))
+
+    def test_ipv6_public_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("2001:db8::1"))
+
+    def test_invalid_string_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("not-an-ip"))
+
+
+# ===========================================================================
+# _check_domain_health_fast
+# ===========================================================================
+
+class TestCheckDomainHealthFast(unittest.TestCase):
+    """_check_domain_health_fast returns True when there is an issue,
+    False when everything looks fine."""
+
+    def _fast(self, domain, external_ip, resolved_addrs):
+        with patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs):
+            return server._check_domain_health_fast(domain, external_ip)
+
+    def test_no_domain_no_issue(self):
+        # None/empty domain: the fast check reports True (handled by checklist).
+        result = server._check_domain_health_fast(None, "203.0.113.10")
+        self.assertTrue(result)
+
+    def test_empty_domain_no_issue(self):
+        result = server._check_domain_health_fast("", "203.0.113.10")
+        self.assertTrue(result)
+
+    def test_loopback_ipv4_no_issue(self):
+        """Loopback override must not be flagged as a DNS mismatch."""
+        result = self._fast("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        self.assertFalse(result)
+
+    def test_loopback_ipv6_no_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["::1"])
+        self.assertFalse(result)
+
+    def test_matches_external_ip_no_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["203.0.113.10"])
+        self.assertFalse(result)
+
+    def test_mismatch_is_an_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["198.51.100.1"])
+        self.assertTrue(result)
+
+    def test_unavailable_external_ip_no_issue(self):
+        result = self._fast("cloud.example.com", "unavailable", ["198.51.100.1"])
+        self.assertFalse(result)
+
+    def test_multiple_addresses_one_matches_no_issue(self):
+        """If any resolved address matches external_ip the check should pass."""
+        result = self._fast(
+            "cloud.example.com", "203.0.113.10",
+            ["198.51.100.1", "203.0.113.10"],
+        )
+        self.assertFalse(result)
+
+
+# ===========================================================================
+# _evaluate_domain_checklist — loopback override path
+# ===========================================================================
+
+class TestEvaluateDomainChecklistLoopback(unittest.TestCase):
+
+    def _eval(self, domain, external_ip, resolved_addrs, reachable_result=None):
+        with (
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_check_domain_reachable",
+                         return_value=reachable_result or {"reachable": True, "status_code": 200}),
+        ):
+            return server._evaluate_domain_checklist(domain, external_ip)
+
+    def test_loopback_dns_step_is_ok_not_error(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        dns_step = next(s for s in result["domain_check_steps"] if s["step"] == 2)
+        self.assertEqual(dns_step["status"], "ok")
+        self.assertNotIn("mismatch", dns_step.get("detail", "").lower())
+
+    def test_loopback_domain_status_is_local_override(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        self.assertEqual(result["domain_status"]["status"], "local_override")
+
+    def test_loopback_has_no_issues_when_reachable(self):
+        result = self._eval(
+            "cloud.example.com", "203.0.113.10", ["127.0.0.1"],
+            reachable_result={"reachable": True, "status_code": 200},
+        )
+        self.assertFalse(result["has_issues"])
+
+    def test_loopback_has_issues_when_caddy_unreachable(self):
+        """A loopback override with Caddy down should still report an issue."""
+        result = self._eval(
+            "cloud.example.com", "203.0.113.10", ["127.0.0.1"],
+            reachable_result={"reachable": False, "error": "connection refused"},
+        )
+        self.assertTrue(result["has_issues"])
+
+    def test_ipv6_loopback_no_issue(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["::1"])
+        self.assertEqual(result["domain_status"]["status"], "local_override")
+        self.assertFalse(result["has_issues"])
+
+    def test_genuine_mismatch_still_reports_error(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["198.51.100.1"])
+        self.assertEqual(result["domain_status"]["status"], "dns_mismatch")
+        self.assertTrue(result["has_issues"])
+
+    def test_correct_public_dns_still_reports_ok(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["203.0.113.10"])
+        self.assertEqual(result["domain_status"]["status"], "connected")
+        self.assertFalse(result["has_issues"])
+
+    def test_no_domain_has_issues(self):
+        result = self._eval(None, "203.0.113.10", [])
+        self.assertTrue(result["has_issues"])
+
+
+# ===========================================================================
+# api_services — composite health with loopback
+# ===========================================================================
+
+class TestApiServicesLoopbackHealth(unittest.IsolatedAsyncioTestCase):
+
+    async def _get_health(self, resolved_addrs, cached_reachable):
+        """Return the health value for a single domain-requiring service."""
+        service_cfg = {
+            "services": [
+                {"unit": "caddy.service", "icon": "nextcloud", "enabled": True, "type": "system"}
+            ]
+        }
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"caddy.service": "nextcloud"}, clear=False),
+            patch("builtins.open", mock_open(read_data="cloud.example.com\n")),
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_is_domain_reachable_cached", return_value=cached_reachable),
+            patch.object(server, "_get_listening_ports",
+                         return_value={"tcp": {80, 443}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports",
+                         return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_cached_external_ip", "203.0.113.10"),
+        ):
+            results = await server.api_services()
+
+        return results[0]["health"]
+
+    async def test_loopback_and_reachable_is_healthy(self):
+        """Loopback override + Caddy reachable → healthy, not needs_attention."""
+        health = await self._get_health(["127.0.0.1"], cached_reachable=True)
+        self.assertEqual(health, "healthy")
+
+    async def test_loopback_and_caddy_down_is_needs_attention(self):
+        """Loopback override + Caddy unreachable → needs_attention (genuine issue)."""
+        health = await self._get_health(["127.0.0.1"], cached_reachable=False)
+        self.assertEqual(health, "needs_attention")
+
+    async def test_correct_dns_and_reachable_is_healthy(self):
+        health = await self._get_health(["203.0.113.10"], cached_reachable=True)
+        self.assertEqual(health, "healthy")
+
+    async def test_dns_mismatch_is_needs_attention(self):
+        health = await self._get_health(["198.51.100.1"], cached_reachable=True)
+        self.assertEqual(health, "needs_attention")
+
+
+# ===========================================================================
+# api_domains_check — loopback detection
+# ===========================================================================
+
+class TestApiDomainsCheckLoopback(unittest.IsolatedAsyncioTestCase):
+
+    async def _check(self, resolved_addrs, external_ip="203.0.113.10"):
+        with (
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_cached_external_ip", external_ip),
+        ):
+            result = await server.api_domains_check(
+                MagicMock(domains=["cloud.example.com"])
+            )
+        return result["domains"][0]
+
+    async def test_loopback_ipv4_returns_local_override(self):
+        result = await self._check(["127.0.0.1"])
+        self.assertEqual(result["status"], "local_override")
+
+    async def test_loopback_ipv6_returns_local_override(self):
+        result = await self._check(["::1"])
+        self.assertEqual(result["status"], "local_override")
+
+    async def test_correct_dns_returns_connected(self):
+        result = await self._check(["203.0.113.10"])
+        self.assertEqual(result["status"], "connected")
+
+    async def test_mismatch_returns_dns_mismatch(self):
+        result = await self._check(["198.51.100.1"])
+        self.assertEqual(result["status"], "dns_mismatch")
+
+    async def test_no_resolution_returns_unresolvable(self):
+        result = await self._check([])
+        self.assertEqual(result["status"], "unresolvable")
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_rdp_module_boot_setup.py

@@ -0,0 +1,122 @@
+import unittest
+from pathlib import Path
+
+
+RDP_NIX = Path(__file__).resolve().parents[2] / "modules" / "rdp.nix"
+USERNAME_READ = "USERNAME=\"$(tr -d '\\n' < \"$USERNAME_FILE\")\""
+USERNAME_LENGTH_GUARD = "if [ \"''${#USERNAME}\" -gt 32 ]; then"
+SHORT_PASSWORD_GUARD = 'if [ "\'\'${#PASSWORD}" -lt 8 ]; then'
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class RdpModuleBootSetupTests(unittest.TestCase):
+    def setUp(self):
+        self.source = RDP_NIX.read_text()
+        self.gnome_service = _section(
+            self.source,
+            "systemd.services.gnome-remote-desktop = {",
+            "systemd.tmpfiles.rules = [",
+        )
+        self.setup_service = _section(
+            self.source,
+            "systemd.services.gnome-remote-desktop-setup = {",
+            "};\n}",
+        )
+
+    def test_does_not_redeclare_gnome_remote_desktop_user(self):
+        self.assertNotIn("users.users.gnome-remote-desktop", self.source)
+        self.assertNotIn("createHome = true;", self.source)
+
+    def test_main_service_requires_setup_before_starting(self):
+        self.assertIn('wantedBy = [ "graphical.target" ];', self.gnome_service)
+        self.assertIn('after = [ "gnome-remote-desktop-setup.service" ];', self.gnome_service)
+        self.assertIn('requires = [ "gnome-remote-desktop-setup.service" ];', self.gnome_service)
+
+    def test_setup_waits_for_configuration_service_and_bounded_timeout(self):
+        self.assertIn('wantedBy = [ "graphical.target" ];', self.setup_service)
+        self.assertIn('before = [ "gnome-remote-desktop.service" ];', self.setup_service)
+        self.assertIn('"dbus.service"', self.setup_service)
+        self.assertIn('"gnome-remote-desktop-configuration.service"', self.setup_service)
+        self.assertNotIn("RemainAfterExit", self.setup_service)
+        self.assertIn('TimeoutStartSec = "2min";', self.setup_service)
+        self.assertIn('timeout --kill-after=5s 10s', self.setup_service)
+        self.assertIn('echo "grdctl command timed out: $*" >&2', self.setup_service)
+        self.assertIn('echo "grdctl command failed (exit $rc): $*" >&2', self.setup_service)
+
+    def test_setup_runs_grdctl_directly_as_root(self):
+        # The oneshot service runs as root; grdctl --system is called directly.
+        # GRD 50.x invokes pkexec internally, but the call itself is plain
+        # grdctl --system, not a manual pkexec invocation.
+        self.assertIn('grdctl --system "$@"', self.setup_service)
+        self.assertNotIn("runuser", self.setup_service)
+        self.assertNotIn("sudo", self.setup_service)
+        # No direct Nix-store pkexec invocation (pkgs.polkit}/bin/pkexec).
+        self.assertNotIn("pkgs.polkit}/bin/pkexec", self.setup_service)
+
+    def test_privilege_escalation_packages_absent_from_setup_path(self):
+        self.assertNotIn("pkgs.polkit", self.setup_service)
+        self.assertNotIn("pkgs.util-linux", self.setup_service)
+
+    def test_run_wrappers_bin_prepended_to_path(self):
+        # /run/wrappers/bin must be prepended to PATH before any grdctl_system
+        # invocation so that grdctl --system resolves the NixOS setuid pkexec.
+        path_export = 'export PATH="/run/wrappers/bin:$PATH"'
+        grdctl_marker = "grdctl_system"
+        script = self.setup_service
+        path_idx = script.find(path_export)
+        grdctl_idx = script.find(grdctl_marker)
+        self.assertGreater(path_idx, -1, f"{path_export!r} not found in setup script")
+        self.assertGreater(
+            grdctl_idx, path_idx,
+            "PATH export must appear before the first grdctl_system usage",
+        )
+
+    def test_pkexec_preflight_check(self):
+        # A preflight must confirm /run/wrappers/bin/pkexec is executable
+        # with a clear error message before any GRD configuration changes.
+        self.assertIn("test -x /run/wrappers/bin/pkexec", self.setup_service)
+        self.assertIn(
+            "/run/wrappers/bin/pkexec is absent or not executable",
+            self.setup_service,
+        )
+
+    def test_hub_files_are_the_source_of_truth_for_username_and_password(self):
+        self.assertIn('DEFAULT_USERNAME="sovran"', self.setup_service)
+        self.assertIn('if [ ! -f "$USERNAME_FILE" ]; then', self.setup_service)
+        self.assertIn(USERNAME_READ, self.setup_service)
+        self.assertIn(USERNAME_LENGTH_GUARD, self.setup_service)
+        self.assertIn('case "$USERNAME" in', self.setup_service)
+        self.assertIn('[A-Za-z_][A-Za-z0-9_-]*)', self.setup_service)
+        self.assertIn("RDP username is too long (''${#USERNAME} characters, maximum 32)", self.setup_service)
+        self.assertIn("RDP username must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens", self.setup_service)
+        self.assertIn('if [ ! -f "$PASSWORD_FILE" ]; then', self.setup_service)
+        self.assertIn("tr -d '\\n'", self.setup_service)
+        self.assertIn('"$PASSWORD_FILE"', self.setup_service)
+        self.assertIn(SHORT_PASSWORD_GUARD, self.setup_service)
+        self.assertIn("RDP password is too short (''${#PASSWORD} characters, minimum 8)", self.setup_service)
+        self.assertIn('grdctl_system rdp set-credentials "$USERNAME" "$PASSWORD"', self.setup_service)
+        self.assertNotIn('grdctl --system rdp set-credentials sovran "$PASSWORD"', self.setup_service)
+
+    def test_secure_permissions_are_enforced_for_state_and_secret_files(self):
+        self.assertIn('"d /var/lib/gnome-remote-desktop/tls 0700', self.source)
+        self.assertIn("chmod 700", self.setup_service)
+        self.assertIn('chmod 600 "$USERNAME_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$PASSWORD_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$CRED_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$TLS_DIR/rdp-tls.key"', self.setup_service)
+        self.assertIn('chmod 644 "$TLS_DIR/rdp-tls.crt"', self.setup_service)
+        self.assertIn('LOCAL_IP="$(hostname -I | awk \'{print $1}\')"', self.setup_service)
+        self.assertIn('LOCAL_IP="127.0.0.1"', self.setup_service)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_service_detail_router_wording.py

@@ -0,0 +1,171 @@
+import unittest
+from pathlib import Path
+from unittest.mock import mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+
+                return _decorator
+
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server
+
+
+class ServiceDetailRouterWordingTests(unittest.IsolatedAsyncioTestCase):
+    async def test_livekit_service_detail_includes_internal_ip(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": {7881}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["internal_ip"], "192.168.1.44")
+        self.assertEqual(result["extra_ports"][0]["status"], "listening")
+        self.assertEqual(result["domain_check_steps"][-1]["label"], "Router Setup Needed")
+
+    async def test_livekit_router_step_uses_not_ready_yet_wording(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["extra_ports"][0]["status"], "closed")
+        self.assertIn("Not ready yet", result["domain_check_steps"][-1]["detail"])
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_template_response_signature.py

@@ -0,0 +1,99 @@
+"""Regression test for Starlette 1.1.0+ TemplateResponse keyword-argument style.
+
+Prior to this fix, the three HTML routes called:
+    templates.TemplateResponse("name.html", {"request": request, ...})
+which passes the context dict as the second positional argument.  With the
+updated Starlette/FastAPI versions shipped in NixOS unstable (Starlette 1.1.0,
+FastAPI 0.136.3) that positional argument is the template name, causing Jinja2
+to receive a dict as a cache key and raise:
+    TypeError: unhashable type: 'dict'
+
+The fix updates every call to use keyword arguments:
+    templates.TemplateResponse(request=request, name="name.html", context={...})
+"""
+
+import ast
+import unittest
+from pathlib import Path
+
+
+SERVER_PY = Path(__file__).resolve().parents[1] / "sovran_systemsos_web" / "server.py"
+
+
+def _template_response_calls(source: str):
+    """Return a list of ast.Call nodes that are TemplateResponse calls."""
+    tree = ast.parse(source)
+    calls = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func = node.func
+        if isinstance(func, ast.Attribute) and func.attr == "TemplateResponse":
+            calls.append(node)
+    return calls
+
+
+class TemplateResponseSignatureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = SERVER_PY.read_text()
+        self.calls = _template_response_calls(self.source)
+
+    def test_at_least_one_template_response_call_found(self):
+        self.assertGreater(len(self.calls), 0, "No TemplateResponse calls found in server.py")
+
+    def test_no_old_style_positional_dict_context(self):
+        """No TemplateResponse call should pass a dict literal as its second positional arg.
+
+        The old style was:
+            templates.TemplateResponse("name.html", {"request": request, ...})
+        where args[0] is a string and args[1] is a Dict node.  That pattern
+        triggers the Starlette 1.1.0 bug.
+        """
+        for call in self.calls:
+            positional = call.args
+            if len(positional) >= 2 and isinstance(positional[1], ast.Dict):
+                self.fail(
+                    f"Found old-style TemplateResponse call at line {call.lineno}: "
+                    "second positional argument is a dict literal.  "
+                    "Use keyword arguments (request=, name=, context=) instead."
+                )
+
+    def test_request_not_duplicated_in_context(self):
+        """The 'request' key must not appear inside the context= dict when
+        request= is already passed as a dedicated keyword argument."""
+        for call in self.calls:
+            kw_dict = {kw.arg: kw.value for kw in call.keywords if isinstance(kw, ast.keyword)}
+
+            if "request" not in kw_dict:
+                continue  # no request= kwarg, nothing to check
+
+            context_node = kw_dict.get("context")
+            if not isinstance(context_node, ast.Dict):
+                continue
+
+            for key_node in context_node.keys:
+                if isinstance(key_node, ast.Constant) and key_node.value == "request":
+                    self.fail(
+                        f"TemplateResponse at line {call.lineno} passes 'request' both as "
+                        "request= keyword argument and inside the context dict."
+                    )
+
+    def test_all_calls_use_keyword_arguments(self):
+        """Every TemplateResponse call should use keyword arguments for request, name,
+        and context rather than relying on positional ordering."""
+        for call in self.calls:
+            kw_args = {kw.arg for kw in call.keywords if isinstance(kw, ast.keyword)}
+            self.assertIn(
+                "request",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'request='.",
+            )
+            self.assertIn(
+                "name",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'name='.",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

configuration.nix

@@ -145,12 +145,12 @@
     ranger fastfetch gedit openssl pwgen
     aspell aspellDicts.en lm_sensors
     hunspell hunspellDicts.en_US
-    synadm brave dua bitwarden-desktop
+    synadm brave dua
     gparted pv unzip parted screen zenity
     libargon2 gnome-terminal libreoffice-fresh
     dig firefox wp-cli axel
-    lk-jwt-service livekit-libwebrtc livekit-cli livekit
-    matrix-synapse age
+    lk-jwt-service livekit-libwebrtc livekit
+    matrix-synapse age onlyoffice-desktopeditors
   ];
 
   # ── Shell ──────────────────────────────────────────────────

flake.lock

@@ -1,33 +1,15 @@
 {
   "nodes": {
-    "bip110": {
+    "btc-clients": {
       "inputs": {
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1778967282,
-        "narHash": "sha256-0g9RvVCD6zxY2vy54GhbB1OeeEZdKuxTr9r0whcpRjQ=",
-        "owner": "emmanuelrosa",
-        "repo": "bitcoin-knots-bip-110-nix",
-        "rev": "8d23ed98940d70e42ee870d719677a073a0a5920",
-        "type": "github"
-      },
-      "original": {
-        "owner": "emmanuelrosa",
-        "repo": "bitcoin-knots-bip-110-nix",
-        "type": "github"
-      }
-    },
-    "btc-clients": {
-      "inputs": {
-        "nixpkgs": "nixpkgs_2"
-      },
-      "locked": {
-        "lastModified": 1780397635,
-        "narHash": "sha256-6WH7LKD6i91VLWoz4mEpoULtqVinCEZxG7ZjJPMSi3k=",
+        "lastModified": 1784932759,
+        "narHash": "sha256-44/iCx+wiYukHGhvPm65ppJZ3FZZbp6f9JE5isz8TsA=",
         "owner": "emmanuelrosa",
         "repo": "btc-clients-nix",
-        "rev": "feacd7684dc6bfcd49c57764944a2049bbd71924",
+        "rev": "8aab86c245ab9a2bea0d72175d6fd663a892af9f",
         "type": "github"
       },
       "original": {
@@ -70,11 +52,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1778716662,
-        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "lastModified": 1782949081,
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -105,7 +87,7 @@
       "inputs": {
         "extra-container": "extra-container",
         "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs_3",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-25_05": "nixpkgs-25_05",
         "nixpkgs-unstable": "nixpkgs-unstable"
       },
@@ -126,16 +108,15 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1777728799,
-        "narHash": "sha256-z7jjYQqhkFKab92VQ3duB7QVO7f7Y62qTFrJYXO/lyo=",
+        "lastModified": 1782911660,
+        "narHash": "sha256-PbR+tJ5E/Ux+01UtdFKqblccVA4/FgWbkym4ev3VHHQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4b2287113c2f9a2331c04899b2e2e5ab92dea9c5",
+        "rev": "cf720c15e108d432d29041cc5a185630809acefb",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "master",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -158,16 +139,16 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1751274312,
-        "narHash": "sha256-/bVBlRpECLVzjV19t5KMdMFWSwKLtb5RyXdjz3LJT+g=",
+        "lastModified": 1784856561,
+        "narHash": "sha256-J+Bx1Z6Oeoj2FgnBhRMKyUhhtDoOpTgXYaVLZpDjW4A=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "50ab793786d9de88ee30ec4e4c24fb4236fc2674",
+        "rev": "597283ad8aa0b331c788e97c4c262d58877074ef",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-24.11",
+        "ref": "nixos-26.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -189,21 +170,6 @@
       }
     },
     "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1780218263,
-        "narHash": "sha256-T/f0pPDrH3Qc1VXyQXbK7yfHWRn90l3xwplc/nsxin4=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "7fc393d1b46fa000d48ff14e8b6a3c9985f03af0",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_3": {
       "locked": {
         "lastModified": 1778737229,
         "narHash": "sha256-6xWoytx8jFW4PF1GjRm/i/53trbpKGfz6zjzQGBr4cI=",
@@ -219,13 +185,13 @@
         "type": "github"
       }
     },
-    "nixpkgs_4": {
+    "nixpkgs_3": {
       "locked": {
-        "lastModified": 1780243769,
-        "narHash": "sha256-x5UQuRsH3MqI0U9afaXSNqzTPSeZlRLvFAav2Ux1pNw=",
+        "lastModified": 1784796856,
+        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "331800de5053fcebacf6813adb5db9c9dca22a0c",
+        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
         "type": "github"
       },
       "original": {
@@ -235,13 +201,13 @@
         "type": "github"
       }
     },
-    "nixpkgs_5": {
+    "nixpkgs_4": {
       "locked": {
-        "lastModified": 1780030872,
-        "narHash": "sha256-u6WU/yd/o8iYQrHX3RAwO1hYa3LkoSL+WNQD0rJfJZQ=",
+        "lastModified": 1784555310,
+        "narHash": "sha256-/FCliTPgiuV1owejZFNx3Ch9irdvkOfOFl+HHZ+DrtM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e9a7635a57597d9754eccebdfc7045e6c8600e6b",
+        "rev": "421eebfd0ec7bccd4abe826ce62d7e6e83129493",
         "type": "github"
       },
       "original": {
@@ -254,15 +220,15 @@
     "nixvim": {
       "inputs": {
         "flake-parts": "flake-parts",
-        "nixpkgs": "nixpkgs_5",
+        "nixpkgs": "nixpkgs_4",
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1780235872,
-        "narHash": "sha256-/TTVFhJQ5StCOrRFO+5NfSkYPSbAAekwymovcQzxNgE=",
+        "lastModified": 1784814601,
+        "narHash": "sha256-T32JXjZ7kIbhBn8/Har171yGg6IBdl97cxAWameqZDE=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "e5c7b40dc569f5c97ba2182d409f0fb54c02d7c1",
+        "rev": "f316e949e0ed9df0e1e0bf645c6dce721d4e230e",
         "type": "github"
       },
       "original": {
@@ -273,10 +239,9 @@
     },
     "root": {
       "inputs": {
-        "bip110": "bip110",
         "btc-clients": "btc-clients",
         "nix-bitcoin": "nix-bitcoin",
-        "nixpkgs": "nixpkgs_4",
+        "nixpkgs": "nixpkgs_3",
         "nixpkgs-stable": "nixpkgs-stable",
         "nixvim": "nixvim"
       }

flake.nix

@@ -6,11 +6,10 @@
 		nix-bitcoin.url = "github:fort-nix/nix-bitcoin/release";
 		nixvim.url = "github:nix-community/nixvim";
 		btc-clients.url = "github:emmanuelrosa/btc-clients-nix";
-		nixpkgs-stable.url = "github:nixos/nixpkgs/nixos-24.11";
-		bip110.url = "github:emmanuelrosa/bitcoin-knots-bip-110-nix";
+		nixpkgs-stable.url = "github:nixos/nixpkgs/nixos-26.05";
 	};
 
-	outputs = { self, nixpkgs, nix-bitcoin, nixvim, btc-clients, nixpkgs-stable, bip110, ... }:
+	outputs = { self, nixpkgs, nix-bitcoin, nixvim, btc-clients, nixpkgs-stable, ... }:
 
 	let
 		overlay-stable = final: prev: {
@@ -56,7 +55,6 @@
 					btc-clients.packages.${pkgs.system}.bisq2
 					btc-clients.packages.${pkgs.system}.sparrow
 				];
-				sovran_systemsOS.packages.bip110 = bip110.packages.${pkgs.system}.bitcoind-knots-bip-110;
 			};
 		};
 	};

iso/common.nix

@@ -16,7 +16,6 @@ in
 {
   imports = [
     "${modulesPath}/installer/cd-dvd/installation-cd-graphical-gnome.nix"
-    ./branding.nix
   ];
 
   image.baseName = lib.mkForce "Sovran_SystemsOS";

iso/installer.py

@@ -1169,7 +1169,7 @@ class InstallerWindow(Adw.ApplicationWindow):
         btn_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=0)
         btn_box.set_halign(Gtk.Align.CENTER)
         btn_box.set_margin_bottom(32)
-        reboot_btn = Gtk.Button(label="I Have Written Down My Password — Reboot Now")
+        reboot_btn = Gtk.Button(label="I Have Written Down My Password — Restart Entire System")
         reboot_btn.add_css_class("suggested-action")
         reboot_btn.add_css_class("pill")
         reboot_btn.connect("clicked", lambda b: subprocess.run(["sudo", "reboot"]))

modules/bip110.nix

@@ -1,23 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  cfg = config.sovran_systemsOS;
-in
-{
-  options.sovran_systemsOS.packages.bip110 = lib.mkOption {
-    type = lib.types.nullOr lib.types.package;
-    default = null;
-    description = "BIP110 Bitcoin package";
-  };
-
-  config = lib.mkIf (
-    cfg.features.bip110 &&
-    cfg.packages.bip110 != null
-  ) {
-    services.bitcoind.package = lib.mkForce cfg.packages.bip110;
-
-    environment.systemPackages = [
-      cfg.packages.bip110
-    ];
-  };
-}

modules/bitcoinecosystem.nix

@@ -4,7 +4,7 @@ lib.mkIf config.sovran_systemsOS.services.bitcoin {
 
   services.bitcoind = {
     enable = true;
-    package = config.nix-bitcoin.pkgs.bitcoind-knots;
+    package = pkgs.bitcoind-knots;
     dataDir = "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node";
     txindex = true;
     tor.proxy = true;

modules/core/caddy.nix

@@ -16,7 +16,10 @@ let
 in
 {
   services.caddy = {
-    enable = true;
+    # Only enable Caddy when at least one domain-based service needs it or
+    # the operator has defined custom vhosts.  This prevents Caddy from
+    # running on Desktop Only installs that have no web services configured.
+    enable = needsHttpsPorts || extraVhosts != "";
     user = "caddy";
     group = "root";
   };
@@ -94,10 +97,10 @@ EOF
 $MATRIX {
   reverse_proxy /_matrix/* http://localhost:8008
   reverse_proxy /_synapse/client/* http://localhost:8008
-}
-
-$MATRIX:8448 {
-  reverse_proxy http://localhost:8008
+  handle /.well-known/matrix/server {
+    header Content-Type application/json
+    respond \`{"m.server":"$MATRIX:443"}\` 200
+  }
 }
 EOF
         fi

modules/core/local-domain-loopback.nix

@@ -0,0 +1,171 @@
+{ config, pkgs, lib, ... }:
+
+# ── Server-local domain loopback overrides ────────────────────────────────────
+#
+# Some routers (especially newer ISP-provided devices) do not support NAT
+# loopback (hairpin NAT).  When a request originates on this computer and
+# targets a public domain name that resolves to the router's WAN address, the
+# router may refuse to loop the connection back in — causing Nextcloud, WordPress
+# background jobs, and other server-side callbacks to fail even when the service
+# is fully operational from the internet.
+#
+# This module installs a one-shot systemd service,
+# ``sovran-hosts-update.service``, that reads the configured service domains
+# from ``/var/lib/domains/`` at boot (and whenever triggered by the Hub after a
+# domain is saved) and writes ``127.0.0.1`` entries for them into a dedicated
+# Sovran-managed block in ``/etc/hosts``.
+#
+# With those entries in place:
+#   • Requests originating on this computer resolve the public domain name to
+#     127.0.0.1, reach Caddy directly, and never touch the router.
+#   • Caddy still receives the correct public hostname via TLS SNI so virtual-
+#     host routing and certificate validation continue to work.
+#   • The Sovran Hub can verify Caddy reachability locally without needing NAT
+#     loopback.
+#
+# Limitation: this does not help other devices on your home network (phones,
+# laptops).  Those devices resolve domains via the router's DNS and still depend
+# on NAT loopback (or require manual router DNS overrides).  For now, only
+# server-originated requests benefit from this override.
+#
+# On NixOS, /etc/hosts is normally a symlink into the Nix store and is
+# regenerated by the system activation script.  The ``system.activationScripts``
+# hook below converts it to a writable file each time the system is activated
+# (i.e. after every ``nixos-rebuild switch``) and then injects the Sovran block.
+# The same wrapped Nix-store executable is reused by both the activation hook
+# and the ``sovran-hosts-update.service`` unit, ensuring a deterministic runtime
+# PATH in every execution context.
+
+let
+  # ── Wrapped Nix-store executable ──────────────────────────────────────────
+  # Built with pkgs.writeShellApplication so that all required runtime tools
+  # (awk, grep, coreutils) are declared explicitly and injected into PATH by
+  # Nix.  Both the systemd service and the activation hook reference this same
+  # store-path executable — there is no second raw script body.
+  hostsUpdateScript = pkgs.writeShellApplication {
+    name = "sovran-hosts-update";
+
+    # Declare every external command the script calls.  These packages are
+    # added to the script's runtime PATH by writeShellApplication; nothing from
+    # the system PATH is relied upon.
+    runtimeInputs = [
+      pkgs.coreutils   # readlink, cp, mv, chmod, mktemp, rm, tr, head
+      pkgs.gawk        # awk
+      pkgs.gnugrep     # grep
+    ];
+
+    text = ''
+      # Regenerate the Sovran-managed loopback block in /etc/hosts.
+      # Safe to run multiple times — idempotent.
+
+      DOMAINS_DIR="/var/lib/domains"
+      HOSTS_FILE="/etc/hosts"
+      BEGIN_MARKER="# Sovran managed begin — server-local loopback overrides"
+      END_MARKER="# Sovran managed end"
+
+      # ── Step 1: ensure /etc/hosts is a regular writable file ──────────────
+      # On NixOS /etc/hosts starts as a symlink to the Nix store.  We replace
+      # it with a copy so we can append our block without touching the store.
+      if [ -L "$HOSTS_FILE" ]; then
+        TARGET=$(readlink -f "$HOSTS_FILE")
+        cp --no-preserve=all "$TARGET" "$HOSTS_FILE.sovran-tmp"
+        mv "$HOSTS_FILE.sovran-tmp" "$HOSTS_FILE"
+        chmod 644 "$HOSTS_FILE"
+      fi
+
+      # ── Step 2: remove any existing Sovran block ──────────────────────────
+      # Use a temp file so the operation is atomic.
+      # awk -v passes marker strings safely without shell interpolation.
+      TMP=$(mktemp "$HOSTS_FILE.XXXXXX")
+      trap 'rm -f "$TMP"' EXIT
+      awk -v begin="$BEGIN_MARKER" -v end="$END_MARKER" '
+        $0 == begin { skip=1; next }
+        $0 == end   { skip=0; next }
+        !skip
+      ' "$HOSTS_FILE" > "$TMP"
+
+      # ── Step 3: collect valid configured service domains ──────────────────
+      # NOTE: The hostname validation regex below must stay in sync with
+      # _SAFE_DOMAIN_RE in app/sovran_systemsos_web/server.py.
+      ENTRIES=""
+      for KEY in matrix wordpress nextcloud btcpayserver vaultwarden haven element-calling; do
+        FILE="$DOMAINS_DIR/$KEY"
+        [ -f "$FILE" ] || continue
+        # Read the domain value (strip all whitespace, limit to 253 chars)
+        DOMAIN=$(tr -d '[:space:]' < "$FILE" | head -c 253)
+        [ -z "$DOMAIN" ] && continue
+        # Validate: must match a reasonable hostname pattern (no injection)
+        if ! printf '%s' "$DOMAIN" | grep -qE \
+          '^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$'; then
+          echo "sovran-hosts-update: skipping invalid domain value for $KEY: $DOMAIN" >&2
+          continue
+        fi
+        ENTRIES="$ENTRIES
+127.0.0.1 $DOMAIN
+::1       $DOMAIN"
+      done
+
+      # ── Step 4: append the Sovran block if there are any entries ──────────
+      if [ -n "$ENTRIES" ]; then
+        {
+          printf '\n%s\n' "$BEGIN_MARKER"
+          printf '%s\n' "# These entries route configured service domains to local Caddy."
+          printf '%s\n' "# They are managed automatically — do not edit this block."
+          printf '%s\n' "$ENTRIES"
+          printf '%s\n' "$END_MARKER"
+        } >> "$TMP"
+      fi
+
+      # ── Step 5: atomically replace /etc/hosts ─────────────────────────────
+      mv "$TMP" "$HOSTS_FILE"
+      chmod 644 "$HOSTS_FILE"
+    '';
+  };
+
+in
+{
+  # ── /etc/sovran-hosts-update.sh — operator discoverability symlink ─────────
+  # Retain the familiar /etc path so administrators can inspect or manually
+  # invoke the helper.  The target is the wrapped Nix-store executable, so
+  # there is no second raw script body to keep in sync.
+  environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript;
+
+  # ── Systemd service ────────────────────────────────────────────────────────
+
+  systemd.services.sovran-hosts-update = {
+    description = "Update /etc/hosts with Sovran server-local loopback overrides";
+    documentation = [ "https://github.com/naturallaw777/sovran-systems" ];
+
+    # Run before Caddy so loopback entries are ready when it starts.
+    before = [
+      "caddy.service"
+      "network-online.target"
+    ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      # Point directly at the wrapped Nix-store executable, not the /etc path.
+      ExecStart = lib.getExe hostsUpdateScript;
+    };
+  };
+
+  # ── Activation script (runs after every nixos-rebuild switch) ─────────────
+  # This ensures the loopback block survives rebuilds that restore the /etc/hosts
+  # symlink.  The "users" and "etc" scripts must complete first.
+  # The same wrapped Nix-store executable used by the systemd service is
+  # referenced here, guaranteeing identical runtime dependencies in both
+  # execution contexts.
+
+  system.activationScripts.sovranDomainLoopback = {
+    text = ''
+      if [ -d /var/lib/domains ]; then
+        if ! ${lib.getExe hostsUpdateScript}; then
+          echo "warning: sovran-hosts-update: failed to update /etc/hosts loopback entries" >&2
+        fi
+      fi
+    '';
+    deps = [ "etc" "users" ];
+  };
+}

modules/core/role-logic.nix

@@ -3,6 +3,13 @@
 {
   config = lib.mkMerge [
 
+    # nix-bitcoin is globally imported by the flake (nixosModules.Sovran_SystemsOS).
+    # This default satisfies nix-bitcoin's generateSecrets assertion so that Desktop
+    # Only systems can evaluate without enabling any Bitcoin services.
+    {
+      nix-bitcoin.generateSecrets = lib.mkDefault true;
+    }
+
     # ── Server+Desktop Role (default) ─────────────────────────
     (lib.mkIf config.sovran_systemsOS.roles.server_plus_desktop {
       sovran_systemsOS.web.btcpayserver = lib.mkDefault true;
@@ -12,19 +19,28 @@
     (lib.mkIf config.sovran_systemsOS.roles.desktop {
       services.desktopManager.gnome.enable = true;
 
+      # Force all server/node services and features off so they cannot be
+      # accidentally enabled via custom.nix or option defaults on Desktop Only.
       sovran_systemsOS.services = {
-        synapse = lib.mkDefault false;
-        bitcoin = lib.mkDefault false;
-        vaultwarden = lib.mkDefault false;
-        wordpress = lib.mkDefault false;
-        nextcloud = lib.mkDefault false;
+        synapse = lib.mkForce false;
+        bitcoin = lib.mkForce false;
+        vaultwarden = lib.mkForce false;
+        wordpress = lib.mkForce false;
+        nextcloud = lib.mkForce false;
       };
 
-      sovran_systemsOS.web.btcpayserver = lib.mkDefault false;
+      sovran_systemsOS.features = {
+        haven = lib.mkForce false;
+        mempool = lib.mkForce false;
+        element-calling = lib.mkForce false;
+        bitcoin-core = lib.mkForce false;
+      };
+
+      sovran_systemsOS.web.btcpayserver = lib.mkForce false;
     })
 
     # ── Bitcoin Node Only Role ────────────────────────────────
-    # Bitcoin ecosystem + mempool + bip110, BTCPay runs but not exposed via Caddy
+    # Bitcoin ecosystem + mempool, BTCPay runs but not exposed via Caddy
     (lib.mkIf config.sovran_systemsOS.roles.node {
       sovran_systemsOS.services = {
         bitcoin = lib.mkDefault true;
@@ -36,7 +52,6 @@
 
       sovran_systemsOS.features = {
         mempool = lib.mkDefault true;
-        bip110 = lib.mkDefault true;
       };
 
       sovran_systemsOS.web.btcpayserver = lib.mkDefault false;

modules/core/roles.nix

@@ -43,12 +43,24 @@
     # ── Features (default OFF — user can enable in custom.nix) ──
     features = {
       haven = lib.mkEnableOption "Haven NOSTR relay";
-      bip110 = lib.mkEnableOption "BIP-110 Bitcoin Better Money";
       mempool = lib.mkEnableOption "Bitcoin Mempool Explorer";
       element-calling = lib.mkEnableOption "Element Video and Audio Calling";
       bitcoin-core = lib.mkEnableOption "Bitcoin Core";
       rdp = lib.mkEnableOption "Gnome Remote Desktop";
       sshd = lib.mkEnableOption "SSH remote access";
+
+      # Deprecated: BIP-110 is now built into mainline Bitcoin Knots and is the
+      # default node. This option is retained ONLY so that existing machines with
+      # `sovran_systemsOS.features.bip110 = lib.mkForce true;` left in their local
+      # custom.nix continue to evaluate. It has no effect and will be removed in a
+      # future release once the Hub has cleaned up old custom.nix files.
+      bip110 = lib.mkOption {
+        type = lib.types.nullOr lib.types.bool;
+        default = null;
+        internal = true;
+        visible = false;
+        description = "(Deprecated, no-op) BIP-110 is now built into Bitcoin Knots.";
+      };
     };
 
     # ── Web exposure (controls Caddy vhosts) ──────────────────
@@ -89,4 +101,15 @@
       description = "Nostr public key (npub1...) for Haven relay";
     };
   };
+
+  config = lib.mkIf (config.sovran_systemsOS.features.bip110 != null) {
+    warnings = [
+      ''
+        sovran_systemsOS.features.bip110 is deprecated and has no effect:
+        BIP-110 is now built into mainline Bitcoin Knots, which is the default node.
+        You can safely remove the `sovran_systemsOS.features.bip110` line from
+        /etc/nixos/custom.nix. The Sovran Hub will also remove it automatically.
+      ''
+    ];
+  };
 }

modules/core/sovran-hub.nix

@@ -29,10 +29,7 @@ let
     ]
     # ── Bitcoin Base (node implementations) ────────────────────
     ++ lib.optionals cfg.services.bitcoin [
-      { name = "Bitcoin Knots + BIP110"; unit = "bitcoind.service"; type = "system"; icon = "bip110";        enabled = cfg.features.bip110;        category = "bitcoin-base"; credentials = [
-        { label = "Tor Address — Access from anywhere via Tor Browser"; file = "/var/lib/tor/onion/bitcoind/hostname"; prefix = "http://"; }
-      ]; }
-      { name = "Bitcoin Knots";          unit = "bitcoind.service"; type = "system"; icon = "bitcoind";      enabled = cfg.services.bitcoin && !cfg.features.bitcoin-core && !cfg.features.bip110; category = "bitcoin-base"; credentials = [
+      { name = "Bitcoin Knots + BIP110"; unit = "bitcoind.service"; type = "system"; icon = "bip110";        enabled = cfg.services.bitcoin && !cfg.features.bitcoin-core; category = "bitcoin-base"; credentials = [
         { label = "Tor Address — Access from anywhere via Tor Browser"; file = "/var/lib/tor/onion/bitcoind/hostname"; prefix = "http://"; }
       ]; }
       { name = "Bitcoin Core";           unit = "bitcoind.service"; type = "system"; icon = "bitcoin-core";  enabled = cfg.features.bitcoin-core;  category = "bitcoin-base"; credentials = [
@@ -149,33 +146,16 @@ let
     echo ""
 
     if [ "$RC" -eq 0 ]; then
-      echo "── Step 2/3: nixos-rebuild ──────────────────────────"
-      SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+      echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──"
+      BOOT_OUT=$(nixos-rebuild boot --flake /etc/nixos --print-build-logs \
         --option connect-timeout 10 \
         --option stalled-download-timeout 90 \
         --option download-attempts 7 \
         --option fallback true 2>&1)
-      SWITCH_RC=$?
-      echo "$SWITCH_OUT"
-      if [ "$SWITCH_RC" -eq 0 ]; then
-        echo "[OK] switch succeeded"
-      elif echo "$SWITCH_OUT" | grep -q "switchInhibitors\|Pre-switch checks failed"; then
-        echo ""
-        echo "  ✓ Build succeeded — a reboot is required to apply this update"
-        echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-        if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
-             --option connect-timeout 10 \
-             --option stalled-download-timeout 90 \
-             --option download-attempts 7 \
-             --option fallback true 2>&1; then
-          echo "REBOOT_REQUIRED" > "$STATUS"
-          exit 0
-        else
-          echo "[ERROR] nixos-rebuild boot also failed"
-          RC=1
-        fi
-      else
-        echo "[ERROR] nixos-rebuild switch failed"
+      BOOT_RC=$?
+      echo "$BOOT_OUT"
+      if [ "$BOOT_RC" -ne 0 ]; then
+        echo "[ERROR] nixos-rebuild boot failed"
         RC=1
       fi
       echo ""
@@ -191,9 +171,10 @@ let
 
     if [ "$RC" -eq 0 ]; then
       echo "══════════════════════════════════════════════════"
-      echo "  ✓ Update completed successfully"
+      echo "  ✓ Update staged successfully"
+      echo "  Reboot required to activate the new system"
       echo "══════════════════════════════════════════════════"
-      echo "SUCCESS" > "$STATUS"
+      echo "REBOOT_REQUIRED" > "$STATUS"
     else
       echo "══════════════════════════════════════════════════"
       echo "  ✗ Update failed — see errors above"
@@ -401,13 +382,25 @@ in
       };
 
       path = [
+        pkgs.bash
+        pkgs.gawk
         pkgs.qrencode
         pkgs.curl
         pkgs.iproute2
         pkgs.nftables
         pkgs.iptables
         pkgs.hostname
-      ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package;
+        pkgs.coreutils
+        pkgs.findutils
+        pkgs.gnugrep
+        pkgs.rsync
+        pkgs.acl
+        pkgs.util-linux
+      ]
+      ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
+      ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
+      ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
+      ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
     };
 
     systemd.services.sovran-hub-update = {

modules/core/ssh-bootstrap.nix

@@ -31,7 +31,7 @@ lib.mkIf userExists {
   };
 
   systemd.services.factory-ssh-keygen = {
-    description = "Generate factory SSH key for ${userName} if missing";
+    description = "Generate or repair factory SSH key for ${userName}";
     wantedBy = [ "multi-user.target" ];
     after = [ "ssh-passphrase-setup.service" ];
     requires = [ "ssh-passphrase-setup.service" ];
@@ -39,14 +39,47 @@ lib.mkIf userExists {
       Type = "oneshot";
       RemainAfterExit = true;
     };
-    path = [ pkgs.openssh pkgs.coreutils ];
+    path = [ pkgs.openssh pkgs.coreutils pkgs.util-linux ];
     script = ''
-      if [ ! -f "${keyPath}" ]; then
-        PASSPHRASE=$(cat /var/lib/secrets/ssh-passphrase)
+      set -eu
+
+      PASSPHRASE=$(cat /var/lib/secrets/ssh-passphrase)
+      lock_file="${keyPath}.lock"
+
+      exec 9>"$lock_file"
+
+      if ! flock -n 9; then
+        echo "Factory SSH key setup is already running." >&2
+        exit 1
+      fi
+
+      generate_factory_key() {
         ssh-keygen -q -N "$PASSPHRASE" -t ed25519 -f "${keyPath}"
         chown ${userName}:users "${keyPath}" "${keyPath}.pub"
         chmod 600 "${keyPath}"
         chmod 644 "${keyPath}.pub"
+      }
+
+      if [ ! -f "${keyPath}" ]; then
+        generate_factory_key
+      elif ! ssh-keygen -y -P "$PASSPHRASE" -f "${keyPath}" >/dev/null 2>&1; then
+        backup_suffix="$(date -u +%Y%m%d_%H%M%S)-$$"
+        backup_path="${keyPath}.bak-$backup_suffix"
+        backup_index=0
+
+        while [ -e "$backup_path" ] || [ -e "$backup_path.pub" ]; do
+          backup_index=$((backup_index + 1))
+          backup_path="${keyPath}.bak-$backup_suffix-$backup_index"
+        done
+
+        echo "Existing factory SSH key does not match current passphrase; backing it up to $backup_path and generating a replacement."
+        mv "${keyPath}" "$backup_path"
+
+        if [ -f "${keyPath}.pub" ]; then
+          mv "${keyPath}.pub" "$backup_path.pub"
+        fi
+
+        generate_factory_key
       fi
     '';
   };

modules/element-calling.nix

@@ -34,8 +34,8 @@ lib.mkIf config.sovran_systemsOS.features.element-calling {
   };
 
   ####### ENSURE SERVICES START AFTER KEY EXISTS #######
-  systemd.services.livekit.after = [ "livekit-key-setup.service" ];
-  systemd.services.livekit.wants = [ "livekit-key-setup.service" ];
+  systemd.services.livekit.after = [ "livekit-key-setup.service" "livekit-turn-setup.service" ];
+  systemd.services.livekit.wants = [ "livekit-key-setup.service" "livekit-turn-setup.service" ];
   systemd.services.lk-jwt-service.after = [ "livekit-key-setup.service" ];
   systemd.services.lk-jwt-service.wants = [ "livekit-key-setup.service" ];
 
@@ -68,35 +68,54 @@ $MATRIX {
   header /.well-known/matrix/* Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
   header /.well-known/matrix/* Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization"
   respond /.well-known/matrix/client \`{ "m.homeserver": {"base_url": "https://$MATRIX" }, "org.matrix.msc4143.rtc_foci": [{ "type":"livekit", "livekit_service_url":"https://$ELEMENT_CALLING/livekit/jwt" }] }\`
-}
-
-$MATRIX:8448 {
-  reverse_proxy http://localhost:8008
+  respond /.well-known/matrix/server \`{"m.server":"$MATRIX:443"}\`
 }
 
 $ELEMENT_CALLING {
-  handle /livekit/jwt/sfu/get {
+  # Route all current lk-jwt-service authorization endpoints to port 8073,
+  # stripping the /livekit/jwt prefix that Caddy adds on the public URL.
+  @lk_jwt path /livekit/jwt/sfu/get* /livekit/jwt/get_token* /livekit/jwt/healthz* /livekit/jwt/sfu_webhook* /livekit/jwt/delegate_delayed_leave*
+  handle @lk_jwt {
     uri strip_prefix /livekit/jwt
     reverse_proxy [::1]:8073 {
       header_up Host {host}
       header_up X-Forwarded-Server {host}
       header_up X-Real-IP {remote_host}
       header_up X-Forwarded-For {remote_host}
+      header_up X-Forwarded-Proto {scheme}
     }
   }
   handle {
-    reverse_proxy localhost:7880
+    reverse_proxy localhost:7880 {
+      header_up Host {host}
+      header_up X-Forwarded-Proto {scheme}
+      header_up X-Forwarded-For {remote_host}
+      header_up X-Real-IP {remote_host}
+      transport http {
+        read_timeout 300s
+        write_timeout 300s
+      }
+    }
   }
 }
 EOF
     '';
   };
 
-  ####### LIVEKIT RUNTIME CONFIG #######
-  systemd.services.livekit-runtime-config = {
-    description = "Generate LiveKit runtime config from domain files";
+  ####### LIVEKIT TURN SETUP (runtime cert + config) #######
+  # Replaces the old dead livekit-runtime-config.service. At runtime this:
+  #   * reads the matrix domain from /var/lib/domains/matrix (never hardcoded)
+  #   * copies Caddy's already-issued matrix cert/key into /var/lib/livekit
+  #     so LoadCredential can stage them for the (DynamicUser) livekit unit
+  #   * detects the primary network interface from the IPv4 default route so
+  #     LiveKit only advertises real ICE candidates — not VPN/container/private
+  #     addresses from interfaces like Tailscale or Docker bridges
+  #   * writes a complete LiveKit config (with turn.domain and interface
+  #     substituted) that the overridden ExecStart loads.
+  systemd.services.livekit-turn-setup = {
+    description = "Stage TURN cert and generate LiveKit runtime config from domain files";
+    after = [ "caddy.service" "livekit-key-setup.service" ];
     before = [ "livekit.service" ];
-    after = [ "livekit-key-setup.service" ];
     requiredBy = [ "livekit.service" ];
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
@@ -106,20 +125,63 @@ EOF
     unitConfig = {
       ConditionPathExists = "/var/lib/domains/element-calling";
     };
-    path = [ pkgs.coreutils ];
+    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ];
     script = ''
       MATRIX=$(cat /var/lib/domains/matrix)
 
       mkdir -p /run/livekit
 
-      cat > /run/livekit/runtime-config.yaml <<EOF
+      # Copy Caddy's already-issued matrix cert/key into LiveKit's state dir.
+      # The ACME CA hostname directory can vary, so glob for the domain dir.
+      CRT=$(find /var/lib/caddy -path "*/$MATRIX/$MATRIX.crt" | head -n1)
+      KEY=$(find /var/lib/caddy -path "*/$MATRIX/$MATRIX.key" | head -n1)
+      cp "$CRT" /var/lib/livekit/turn.crt
+      cp "$KEY" /var/lib/livekit/turn.key
+      chmod 640 /var/lib/livekit/turn.crt /var/lib/livekit/turn.key
+
+      # Detect the primary network interface from the IPv4 default route.
+      # Restricting LiveKit to this single interface prevents it from
+      # advertising VPN/container/private ICE candidates (e.g. Tailscale,
+      # Docker bridges) that remote peers cannot reach, which causes all
+      # ICE negotiation attempts to fail with responsesReceived: 0.
+      IFACE=$(ip -4 route show default | awk '/^default/ { for(i=1;i<=NF;i++) if($i=="dev" && (i+1)<=NF) { print $(i+1); exit } }')
+      if [ -z "$IFACE" ]; then
+        echo "ERROR: Could not detect a default-route network interface from 'ip -4 route show default'." >&2
+        echo "ERROR: Cannot generate a valid LiveKit config without a real interface to bind ICE candidates to." >&2
+        echo "ERROR: Ensure a default IPv4 route is configured, e.g.: ip route add default via <gateway> dev <interface>" >&2
+        echo "ERROR: Inspect the current routing table with: ip -4 route show" >&2
+        exit 1
+      fi
+      echo "Detected primary network interface: $IFACE"
+
+      # Generate the full LiveKit config the daemon will load. turn.domain and
+      # rtc.interfaces.includes are only known at runtime, so they are
+      # substituted here. The cert/key paths point at the LoadCredential-staged
+      # copies under /run/credentials.
+      cat > /run/livekit/livekit.yaml <<EOF
+port: 7880
+rtc:
+  use_external_ip: true
+  skip_external_ip_validation: true
+  tcp_port: 7881
+  udp_port: 7882
+  port_range_start: 30000
+  port_range_end: 40000
+  interfaces:
+    includes:
+      - $IFACE
+room:
+  auto_create: false
 turn:
+  enabled: true
   domain: $MATRIX
-  cert_file: /var/lib/livekit/$MATRIX.crt
-  key_file: /var/lib/livekit/$MATRIX.key
+  tls_port: 5349
+  udp_port: 3478
+  cert_file: /run/credentials/livekit.service/turn-cert
+  key_file: /run/credentials/livekit.service/turn-key
 EOF
 
-      chmod 640 /run/livekit/runtime-config.yaml
+      chmod 644 /run/livekit/livekit.yaml
     '';
   };
 
@@ -130,7 +192,11 @@ EOF
     keyFile = livekitKeyFile;
     settings = {
       rtc.use_external_ip = true;
+      rtc.skip_external_ip_validation = true;
+      rtc.tcp_port = 7881;
       rtc.udp_port = 7882;
+      rtc.port_range_start = 30000;
+      rtc.port_range_end = 40000;
       room.auto_create = false;
       turn = {
         enabled = true;
@@ -140,15 +206,29 @@ EOF
     };
   };
 
+  # Override ExecStart to load the runtime-generated config (which carries the
+  # runtime-only turn.domain), mirroring the Caddy ExecStart override pattern in
+  # modules/core/caddy.nix. Deliver the TURN cert/key via LoadCredential so they
+  # are readable under the upstream unit's DynamicUser=true sandbox without
+  # weakening it. Everything else about the standard unit is left intact.
+  systemd.services.livekit.serviceConfig.ExecStart = lib.mkForce [
+    ""
+    "${pkgs.livekit}/bin/livekit-server --config /run/credentials/livekit.service/livekit-config --key-file /run/credentials/livekit.service/livekit-secrets"
+  ];
+
+  systemd.services.livekit.serviceConfig.LoadCredential = [
+    "livekit-config:/run/livekit/livekit.yaml"
+    "livekit-secrets:${livekitKeyFile}"
+    "turn-cert:/var/lib/livekit/turn.crt"
+    "turn-key:/var/lib/livekit/turn.key"
+  ];
+
   networking.firewall.allowedTCPPorts = [ 5349 7881 ];
   networking.firewall.allowedUDPPorts = [ 3478 7882 ];
   networking.firewall.allowedUDPPortRanges = [
-    { from = 30000; to = 40000; }
+    { from = 30000; to = 40000; } # LiveKit internal TURN relay range
   ];
-  networking.firewall.allowedTCPPortRanges = [
-    { from = 30000; to = 40000; }
-  ];
-  
+
   ####### JWT SERVICE RUNTIME CONFIG #######
   systemd.services.lk-jwt-service-runtime-config = {
     description = "Generate lk-jwt-service runtime config from domain files";
@@ -166,11 +246,13 @@ EOF
     path = [ pkgs.coreutils ];
     script = ''
       ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
+      MATRIX=$(cat /var/lib/domains/matrix)
 
       mkdir -p /run/lk-jwt-service
 
       cat > /run/lk-jwt-service/env <<EOF
 LIVEKIT_URL=wss://$ELEMENT_CALLING
+LIVEKIT_FULL_ACCESS_HOMESERVERS=$MATRIX
 EOF
 
       chmod 640 /run/lk-jwt-service/env

modules/modules.nix

@@ -16,6 +16,7 @@
     ./core/remote-deploy.nix
     ./core/no-sleep.nix
     ./core/cpu-performance.nix
+    ./core/local-domain-loopback.nix
 
     # ── Always on (no flag) ───────────────────────────────────
     ./php.nix
@@ -31,7 +32,6 @@
 
     # ── Features (default OFF — enable in custom.nix) ─────────
     ./haven.nix
-    ./bip110.nix
     ./element-calling.nix
     ./mempool.nix
     ./bitcoin-core.nix

modules/rdp.nix

@@ -2,70 +2,104 @@
 
 lib.mkIf config.sovran_systemsOS.features.rdp {
 
-  users.users.gnome-remote-desktop = {
-    isSystemUser = true;
-    group = "gnome-remote-desktop";
-    home = "/var/lib/gnome-remote-desktop";
-    createHome = true;
-  };
-  users.groups.gnome-remote-desktop = {};
-
   # Enable the GNOME Remote Desktop service at the system level
   services.gnome.gnome-remote-desktop.enable = true;
 
   # Open RDP port in the firewall
   networking.firewall.allowedTCPPorts = [ 3389 ];
 
-  # Ensure the service actually starts and waits for setup to complete
+  # Ensure the service only starts after setup succeeds
   systemd.services.gnome-remote-desktop = {
     wantedBy = [ "graphical.target" ];
     after = [ "gnome-remote-desktop-setup.service" ];
-    wants = [ "gnome-remote-desktop-setup.service" ];
+    requires = [ "gnome-remote-desktop-setup.service" ];
   };
 
   systemd.tmpfiles.rules = [
-    "d /var/lib/gnome-remote-desktop 0750 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local 0750 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local/share 0750 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop 0750 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local 0700 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local/share 0700 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop 0700 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/tls 0700 gnome-remote-desktop gnome-remote-desktop -"
   ];
 
   systemd.services.gnome-remote-desktop-setup = {
     description = "Configure GNOME Remote Desktop RDP";
-    wantedBy = [ "multi-user.target" ];
+    wantedBy = [ "graphical.target" ];
     before = [ "gnome-remote-desktop.service" ];
-    after = [ "systemd-tmpfiles-setup.service" "network-online.target" ];
-    wants = [ "network-online.target" ];
+    after = [
+      "dbus.service"
+      "systemd-tmpfiles-setup.service"
+      "network-online.target"
+      "gnome-remote-desktop-configuration.service"
+    ];
+    wants = [
+      "network-online.target"
+      "gnome-remote-desktop-configuration.service"
+    ];
     serviceConfig = {
       Type = "oneshot";
-      RemainAfterExit = true;
+      TimeoutStartSec = "2min";
     };
     path = [
-      pkgs.gnome-remote-desktop
-      pkgs.polkit
-      pkgs.openssl
-      pkgs.hostname
+      pkgs.coreutils
       pkgs.gawk
+      pkgs.gnome-remote-desktop
+      pkgs.hostname
+      pkgs.openssl
+      pkgs.systemd
     ];
     script = ''
-      # Ensure directory structure exists
-      mkdir -p /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop
-      chown -R gnome-remote-desktop:gnome-remote-desktop /var/lib/gnome-remote-desktop
+      set -euo pipefail
 
-      TLS_DIR="/var/lib/gnome-remote-desktop/tls"
-      CRED_FILE="/var/lib/gnome-remote-desktop/rdp-credentials"
+      # GRD 50.x invokes pkexec internally for every grdctl --system call, even
+      # when the caller is root.  NixOS exposes the required setuid wrapper at
+      # /run/wrappers/bin/pkexec; the Nix-store polkit binary is not setuid and
+      # must not shadow it.  Prepend the wrapper directory so every subsequent
+      # grdctl --system resolves the correct binary.
+      export PATH="/run/wrappers/bin:$PATH"
+
+      STATE_DIR="/var/lib/gnome-remote-desktop"
+      TLS_DIR="$STATE_DIR/tls"
+      USERNAME_FILE="$STATE_DIR/rdp-username"
+      PASSWORD_FILE="$STATE_DIR/rdp-password"
+      CRED_FILE="$STATE_DIR/rdp-credentials"
+      DEFAULT_USERNAME="sovran"
+
+      grdctl_system() {
+        local rc=0
+
+        if timeout --kill-after=5s 10s \
+          grdctl --system "$@"; then
+          return 0
+        else
+          rc=$?
+        fi
+
+        if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
+          echo "grdctl command timed out: $*" >&2
+        fi
+        echo "grdctl command failed (exit $rc): $*" >&2
+
+        return "$rc"
+      }
+
+      mkdir -p "$STATE_DIR/.local/share/gnome-remote-desktop" "$TLS_DIR"
+      chown -R gnome-remote-desktop:gnome-remote-desktop "$STATE_DIR"
+      chmod 700 \
+        "$STATE_DIR" \
+        "$STATE_DIR/.local" \
+        "$STATE_DIR/.local/share" \
+        "$STATE_DIR/.local/share/gnome-remote-desktop" \
+        "$TLS_DIR"
 
-      # Regenerate TLS certificate if missing OR if ownership is wrong
-      # (disable/re-enable cycle can break ownership or grdctl state)
       NEED_REGEN=0
       if [ ! -f "$TLS_DIR/rdp-tls.crt" ] || [ ! -f "$TLS_DIR/rdp-tls.key" ]; then
         NEED_REGEN=1
-      elif [ "$(stat -c '%U' "$TLS_DIR/rdp-tls.key" 2>/dev/null)" != "gnome-remote-desktop" ]; then
+      elif [ "$(stat -c '%U:%G' "$TLS_DIR/rdp-tls.key" 2>/dev/null)" != "gnome-remote-desktop:gnome-remote-desktop" ]; then
         NEED_REGEN=1
       fi
 
       if [ "$NEED_REGEN" = "1" ]; then
-        mkdir -p "$TLS_DIR"
         rm -f "$TLS_DIR/rdp-tls.key" "$TLS_DIR/rdp-tls.crt"
         openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \
           -sha256 -nodes -days 3650 \
@@ -75,39 +109,59 @@ lib.mkIf config.sovran_systemsOS.features.rdp {
         echo "Generated new RDP TLS certificate"
       fi
 
-      # Always fix ownership and permissions (handles re-enable after disable)
-      chown -R gnome-remote-desktop:gnome-remote-desktop "$TLS_DIR"
+      chown gnome-remote-desktop:gnome-remote-desktop "$TLS_DIR/rdp-tls.key" "$TLS_DIR/rdp-tls.crt"
       chmod 600 "$TLS_DIR/rdp-tls.key"
       chmod 644 "$TLS_DIR/rdp-tls.crt"
 
-      # Configure TLS certificate
-      grdctl --system rdp set-tls-cert "$TLS_DIR/rdp-tls.crt"
-      grdctl --system rdp set-tls-key "$TLS_DIR/rdp-tls.key"
+      if [ ! -f "$USERNAME_FILE" ]; then
+        printf '%s\n' "$DEFAULT_USERNAME" > "$USERNAME_FILE"
+      fi
+      USERNAME="$(tr -d '\n' < "$USERNAME_FILE")"
+      if [ -z "$USERNAME" ]; then
+        USERNAME="$DEFAULT_USERNAME"
+        printf '%s\n' "$USERNAME" > "$USERNAME_FILE"
+      fi
+      if [ "''${#USERNAME}" -gt 32 ]; then
+        echo "RDP username is too long (''${#USERNAME} characters, maximum 32): $USERNAME from $USERNAME_FILE" >&2
+        exit 1
+      fi
+      case "$USERNAME" in
+        [A-Za-z_][A-Za-z0-9_-]*)
+          ;;
+        *)
+          echo "RDP username must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens: $USERNAME from $USERNAME_FILE" >&2
+          exit 1
+          ;;
+      esac
+      chown gnome-remote-desktop:gnome-remote-desktop "$USERNAME_FILE"
+      chmod 600 "$USERNAME_FILE"
 
-      # Generate password on first boot only
-      PASSWORD=""
-      if [ ! -f /var/lib/gnome-remote-desktop/rdp-password ]; then
-        PASSWORD=$(openssl rand -base64 16)
-        echo "$PASSWORD" > /var/lib/gnome-remote-desktop/rdp-password
-        chmod 600 /var/lib/gnome-remote-desktop/rdp-password
-      else
-        PASSWORD=$(cat /var/lib/gnome-remote-desktop/rdp-password)
+      if [ ! -f "$PASSWORD_FILE" ]; then
+        openssl rand -base64 16 > "$PASSWORD_FILE"
+      fi
+      PASSWORD="$(tr -d '\n' < "$PASSWORD_FILE")"
+      if [ -z "$PASSWORD" ]; then
+        echo "RDP password file is empty: $PASSWORD_FILE" >&2
+        exit 1
+      fi
+      if [ "''${#PASSWORD}" -lt 8 ]; then
+        echo "RDP password is too short (''${#PASSWORD} characters, minimum 8): $PASSWORD_FILE" >&2
+        exit 1
+      fi
+      chown gnome-remote-desktop:gnome-remote-desktop "$PASSWORD_FILE"
+      chmod 600 "$PASSWORD_FILE"
+
+      LOCAL_IP="$(hostname -I | awk '{print $1}')"
+      if [ -z "$LOCAL_IP" ]; then
+        LOCAL_IP="127.0.0.1"
       fi
 
-      # Write username to a separate file for the hub
-      echo "sovran" > /var/lib/gnome-remote-desktop/rdp-username
-      chmod 600 /var/lib/gnome-remote-desktop/rdp-username
-
-      # Get current IP address
-      LOCAL_IP=$(hostname -I | awk '{print $1}')
-
-      # Always rewrite the credentials file with the current IP
       cat > "$CRED_FILE" <<EOF
       ========================================
        GNOME Remote Desktop (RDP) Credentials
       ========================================
 
-       Username: sovran
+       Username: $USERNAME
        Password: $PASSWORD
 
        Connect from any RDP client to:
@@ -116,11 +170,22 @@ lib.mkIf config.sovran_systemsOS.features.rdp {
       ========================================
       EOF
 
+      chown gnome-remote-desktop:gnome-remote-desktop "$CRED_FILE"
       chmod 600 "$CRED_FILE"
 
-      # Enable RDP backend and set credentials
-      grdctl --system rdp enable
-      grdctl --system rdp set-credentials sovran "$PASSWORD"
+      # Preflight: the NixOS setuid pkexec wrapper must be present and executable
+      # before any grdctl --system call.  Absence means the system was booted
+      # without security.wrappers or the wrapper directory is not mounted yet.
+      if ! test -x /run/wrappers/bin/pkexec; then
+        echo "Preflight check failed: /run/wrappers/bin/pkexec is absent or not executable." >&2
+        echo "GNOME Remote Desktop system configuration requires the NixOS setuid pkexec wrapper at /run/wrappers/bin/pkexec." >&2
+        exit 1
+      fi
+
+      grdctl_system rdp enable
+      grdctl_system rdp set-tls-cert "$TLS_DIR/rdp-tls.crt"
+      grdctl_system rdp set-tls-key "$TLS_DIR/rdp-tls.key"
+      grdctl_system rdp set-credentials "$USERNAME" "$PASSWORD"
 
       echo "GNOME Remote Desktop RDP configured successfully"
     '';

modules/synapse.nix

@@ -250,9 +250,6 @@ CREDS
     '';
   };
 
-  networking.firewall.allowedTCPPorts = [ 8448 ];
-  networking.firewall.allowedUDPPorts = [ 8448 ];
-
   sovran_systemsOS.domainRequirements = [
     { name = "matrix"; label = "Matrix Synapse"; example = "matrix.yourdomain.com"; }
   ];