Merge pull request #333 from naturallaw777/copilot/replace-manual-backup-implementation

Replace Manual Backup: tar+DB+LND → ext4+rsync mirror

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ── Sovran Hub External Backup Script ────────────────────────────
-# Backs up Sovran_SystemsOS data to an external USB hard drive.
+# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
 # Designed for the Hub web UI (no GUI dependencies).
 #
 # Your Sovran Pro already backs up your data automatically to its
@@ -8,6 +8,15 @@
 # This script creates an additional copy on an external USB drive —
 # storing your data in a third location for maximum protection.
 #
+# The external drive must be formatted as ext4. Files are stored as
+# directly browsable files under Sovran_SystemsOS_Backup/current/.
+# Later runs update the same mirror and only transfer changed or new
+# files, making repeat backups fast.
+#
+# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
+# blockchain and Electrs index data are NOT included (they live on
+# the internal second drive).
+#
 # Usage:
 #   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
 #   (or run with no env var to auto-detect the first external USB drive)
@@ -17,13 +26,22 @@ set -euo pipefail
 BACKUP_LOG="/var/log/sovran-hub-backup.log"
 BACKUP_STATUS="/var/log/sovran-hub-backup.status"
 MEDIA_ROOT="/run/media"
-MIN_FREE_GB=10
 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
 ROLE_STATE_NIX="/etc/nixos/role-state.nix"
+SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
+SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))
 
 # ── Internal drive labels/paths to NEVER use as backup targets ───
 INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
-INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/")
+INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
+
+FAILED_ALREADY=0
+BACKUP_COMPLETE=0
+RSYNC_WARNINGS=()
+
+# Stable rsync mirror sub-path under the target drive. Not timestamped
+# so later runs update the same destination and only transfer new or changed files.
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
 
 # ── Logging helpers ──────────────────────────────────────────────
 
@@ -37,16 +55,45 @@ set_status() {
 }
 
 fail() {
+  FAILED_ALREADY=1
   log "ERROR: $*"
   set_status "FAILED"
   exit 1
 }
 
+cleanup() {
+  local rc=$?
+
+  # Release the concurrency lock file descriptor if it was opened
+  [[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true
+
+  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
+    return
+  fi
+
+  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
+    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
+    set_status "FAILED"
+  fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
+}
+
+trap cleanup EXIT
+trap 'exit 1' INT TERM
+
+require_cmd() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
+}
+
 # ── Check whether a mount point is an internal drive ────────────
 
 is_internal() {
   local mnt="$1"
-  # Reject known internal mount points and their subdirectories
   for internal in "${INTERNAL_MOUNTS[@]}"; do
     if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
       return 0
@@ -59,22 +106,18 @@ is_internal() {
 
 find_external_drive() {
   local target=""
-  # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
+
-  if command -v lsblk &>/dev/null; then
   while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
-      # Must be a partition or disk, and be removable/hotplug
     [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
     [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
     [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal labels
     local skip=0
     for lbl in "${INTERNAL_LABELS[@]}"; do
       [[ "$label" == "$lbl" ]] && skip=1 && break
     done
     [[ "$skip" -eq 1 ]] && continue
 
-      # Filter out internal mount points
     is_internal "$mountpoint" && continue
 
     if mountpoint -q "$mountpoint" 2>/dev/null; then
@@ -84,14 +127,16 @@ find_external_drive() {
   done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
     python3 -c "
 import sys, json
-data = json.load(sys.stdin)
+
 def flatten(devs):
     for d in devs:
         yield d
         for c in d.get('children', []):
             yield from flatten([c])
+
+data = json.load(sys.stdin)
 for d in flatten(data.get('blockdevices', [])):
-    print('\t'.join([
+    print('\\t'.join([
         d.get('type') or '',
         str(d.get('hotplug') or '0'),
         str(d.get('rm') or '0'),
@@ -99,24 +144,10 @@ for d in flatten(data.get('blockdevices', [])):
         d.get('mountpoint') or '',
     ]))
 " 2>/dev/null || true)
-  fi
 
-  # Fallback: walk /run/media/ if lsblk produced nothing
   if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
     while IFS= read -r -d '' mnt; do
       is_internal "$mnt" && continue
-      # Check label via lsblk on the device backing this mount
-      local dev
-      dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
-      if [[ -n "$dev" ]]; then
-        local lbl
-        lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
-        local skip=0
-        for internal_lbl in "${INTERNAL_LABELS[@]}"; do
-          [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
-        done
-        [[ "$skip" -eq 1 ]] && continue
-      fi
       if mountpoint -q "$mnt" 2>/dev/null; then
         target="$mnt"
         break
@@ -128,16 +159,10 @@ for d in flatten(data.get('blockdevices', [])):
 }
 
 # ── Detect the configured system role ───────────────────────────
-#
-# Priority:
-#   1. Hub config JSON  (/var/lib/sovran-hub/config.json)  — "role" key
-#   2. role-state.nix   (/etc/nixos/role-state.nix)        — grep for true flag
-#   3. Default: server_plus_desktop
 
 detect_role() {
   local role="server_plus_desktop"
 
-  # 1. Try the Hub config JSON
   if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
     local r
     r=$(python3 -c \
@@ -149,7 +174,6 @@ detect_role() {
     fi
   fi
 
-  # 2. Fall back to parsing role-state.nix
   if [[ -f "$ROLE_STATE_NIX" ]]; then
     if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
       role="desktop"
@@ -161,6 +185,85 @@ detect_role() {
   echo "$role"
 }
 
+validate_target_mount() {
+  local target="$1"
+  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
+  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
+  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
+
+  local fstype=""
+  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
+  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
+
+  if [[ "$fstype" != "ext4" ]]; then
+    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
+  fi
+
+  local write_test
+  write_test="$target/.sovran-write-test-$$"
+  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
+    fail "Target '$target' is not writable."
+  fi
+
+  log "Verified backup target filesystem: $fstype"
+}
+
+estimate_path_bytes() {
+  local path="$1"
+  shift || true
+  [[ -e "$path" ]] || {
+    echo 0
+    return
+  }
+
+  local size
+  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
+  [[ -n "$size" ]] || size=0
+  echo "$size"
+}
+
+# ── Run rsync for one source tree ────────────────────────────────
+# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
+# For /home only — files may disappear while the desktop is active.
+# All other nonzero exit codes are always fatal.
+run_rsync() {
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+  # Remaining args are passed directly to rsync (filters + source + dest).
+
+  local rsync_err_tmp
+  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
+
+  local rc=0
+  rsync \
+    --archive \
+    --acls \
+    --xattrs \
+    --hard-links \
+    --numeric-ids \
+    --one-file-system \
+    --partial \
+    "$@" 2>"$rsync_err_tmp" || rc=$?
+
+  if [[ -s "$rsync_err_tmp" ]]; then
+    while IFS= read -r rline; do
+      log "rsync: $rline"
+    done < "$rsync_err_tmp"
+  fi
+  rm -f "$rsync_err_tmp"
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
+  fi
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -169,6 +272,30 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"
 
+# ── Acquire exclusive run lock ────────────────────────────────────
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd rsync
+require_cmd findmnt
+require_cmd lsblk
+require_cmd mountpoint
+require_cmd df
+require_cmd du
+require_cmd awk
+require_cmd find
+require_cmd hostname
+require_cmd date
+require_cmd python3
+require_cmd flock
+
 # ── Detect system role ───────────────────────────────────────────
 
 ROLE="$(detect_role)"
@@ -184,7 +311,6 @@ log "Detected role: $ROLE_LABEL"
 
 if [[ -n "${BACKUP_TARGET:-}" ]]; then
   TARGET="$BACKUP_TARGET"
-  # Safety: never allow internal drives even if explicitly passed
   if is_internal "$TARGET"; then
     fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
   fi
@@ -193,39 +319,70 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. " \
+    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
-         "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
-# ── Verify mount point ───────────────────────────────────────────
+validate_target_mount "$TARGET"
 
-[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist."
+# ── Set up stable backup destination ────────────────────────────
-mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
+# Subsequent runs update the same mirror, transferring only new or changed files.
 
-# ── Check free disk space (require ≥ 10 GB) ──────────────────────
+BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
-
-FREE_KB=$(df -k --output=avail "$TARGET" | tail -1)
-FREE_GB=$(( FREE_KB / 1024 / 1024 ))
-log "Free space on drive: ${FREE_GB} GB"
-(( FREE_GB >= MIN_FREE_GB )) || \
-  fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)."
-
-# ── Create timestamped backup directory ─────────────────────────
-
-TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
-BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
 mkdir -p "$BACKUP_DIR"
+
+# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
+# after all rsync stages and manifest write succeed. Failed or interrupted
+# runs keep this marker so they are clearly identifiable.
+touch "$BACKUP_DIR/INCOMPLETE"
 log "Backup destination: $BACKUP_DIR"
 
+# ── Estimate required free space ─────────────────────────────────
+# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
+# from the estimate to avoid inflating the required size.
+
+ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
+HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
+SECRETS_BYTES=0
+if [[ "$ROLE" != "desktop" ]]; then
+  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
+fi
+
+VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='postgresql' \
+  --exclude='mysql' \
+  --exclude='mariadb' \
+  --exclude='bitcoind' \
+  --exclude='electrs' \
+  --exclude='*/log' \
+  --exclude='*/logs' \
+  --exclude='*/cache' \
+  --exclude='*/tmp')
+
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
+# Require 20% growth headroom plus a fixed 1 GiB safety margin.
+# Later incremental runs need far less space, but a conservative first-run
+# check protects against running out of space mid-backup.
+REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
+
+FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
+FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
+REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
+
+log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
+log "Required free space (with safety margin): ${REQUIRED_GB} GB"
+log "Free space on drive: ${FREE_GB} GB"
+
+(( FREE_BYTES >= REQUIRED_BYTES )) || \
+  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
+
 # ── Stage 1/4: NixOS configuration ──────────────────────────────
 
 log ""
 log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \
+  run_rsync "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
-    fail "Stage 1 failed while copying /etc/nixos"
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
@@ -234,64 +391,66 @@ fi
 # ── Stage 2/4: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/4: Secrets ───────────────────────────────────────"
+log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
-mkdir -p "$BACKUP_DIR/secrets"
-
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
+elif [[ -e /etc/nix-bitcoin-secrets ]]; then
+  run_rsync "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
 else
-  if [[ -e /etc/nix-bitcoin-secrets ]]; then
+  log "(not found: /etc/nix-bitcoin-secrets — skipping)"
-    rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
-  else
-    log "  (not found: /etc/nix-bitcoin-secrets — skipping)"
-  fi
 fi
-
 log "Stage 2 complete."
 
-# ── Stage 3/4: Home directory ────────────────────────────────────
+# ── Stage 3/4: Home directory ───────────────────────────────────
+# Rsync exit code 24 (vanished source files) is treated as nonfatal here
+# because the desktop may be active and files can disappear between the
+# directory scan and the copy. All other nonzero exit codes remain fatal.
 
 log ""
 log "── Stage 3/4: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  rsync -a --info=progress2 \
+  run_rsync "/home" yes \
     --exclude='.cache/' \
     --exclude='.local/share/Trash/' \
-    --exclude='*/Trash/' \
+    --exclude='Trash/' \
-    /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \
+    --exclude='.mozilla/firefox/*/cache2/' \
-    fail "Stage 3 failed while copying /home"
+    --exclude='.mozilla/firefox/*/startupCache/' \
+    --exclude='.mozilla/firefox/*/thumbnails/' \
+    --exclude='.config/google-chrome/*/Cache/' \
+    --exclude='.config/google-chrome/*/Code Cache/' \
+    --exclude='.config/chromium/*/Cache/' \
+    --exclude='.config/chromium/*/Code Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
+    --exclude='.local/share/baloo/' \
+    --exclude='.thumbnails/' \
+    --exclude='.xsession-errors' \
+    --exclude='.xsession-errors.old' \
+    /home/ "$BACKUP_DIR/home/"
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/4: System data ───────────────────────────────────────
+# ── Stage 4/4: System data ──────────────────────────────────────
+# PostgreSQL/MariaDB raw database directories are excluded. Application
+# databases must be backed up separately with native database tools.
+# Bitcoin/Electrs data are excluded; they live on the internal second drive.
 
 log ""
-log "── Stage 4/4: System data (/var/lib) ────────────────────────"
+log "── Stage 4/4: System data (/var/lib) ───────────────────────"
-if [[ "$ROLE" == "desktop" ]]; then
+if [[ -d /var/lib ]]; then
-  if [[ -d /var/lib ]]; then
+  run_rsync "/var/lib" no \
-    rsync -a --info=progress2 \
+    --exclude='postgresql/' \
-      --filter='- /lnd/***' \
+    --exclude='mysql/' \
-      --exclude='logs/' \
+    --exclude='mariadb/' \
-      --exclude='log/' \
+    --exclude='bitcoind/' \
-      --exclude='*/logs/' \
+    --exclude='electrs/' \
     --exclude='*/log/' \
-      /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      fail "Stage 4 failed while copying /var/lib for Desktop Only role"
-    log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)."
-  else
-    log "WARNING: /var/lib not found — skipping."
-  fi
-elif [[ -d /var/lib ]]; then
-  rsync -a --info=progress2 \
-    --exclude='logs/' \
-    --exclude='log/' \
     --exclude='*/logs/' \
-    --exclude='*/log/' \
+    --exclude='*/cache/' \
-    /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
+    --exclude='*/tmp/' \
-    fail "Stage 4 failed while copying /var/lib"
+    /var/lib/ "$BACKUP_DIR/var/lib/"
   log "Stage 4 complete."
 else
   log "WARNING: /var/lib not found — skipping."
@@ -301,21 +460,91 @@ fi
 
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
+MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
+
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date)"
+  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Contents:"
+  echo "Backup type: Live rsync mirror (directly browsable files)"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort
+  echo "Location:    ${BACKUP_DIR}"
-} > "$BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo ""
-log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo "Source paths mirrored:"
+  echo "- /etc/nixos  →  current/etc/nixos/"
+  if [[ "$ROLE" != "desktop" ]]; then
+    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
+  fi
+  echo "- /home  →  current/home/"
+  echo "- /var/lib  →  current/var/lib/"
+  echo ""
+  echo "Exclusions:"
+  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
+  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
+  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
+  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
+  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
+  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
+  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
+  echo ""
+  echo "Important limitations:"
+  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
+  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
+  echo "  applications, their data must be backed up separately using native tools."
+  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
+  echo "  reconstructable or stored on the internal second drive."
+  echo "- This is a live file-level mirror, not a transactional database backup."
+  echo "  Files being written during the backup may be in an inconsistent state."
+  echo ""
+  echo "Restore guidance:"
+  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
+  echo "- To restore a directory:"
+  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
+  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
+  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
+  echo "- To copy individual files:"
+  echo "    sudo cp -a current/home/username/ /home/username/"
+  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
+  echo "  hardware-configuration.nix for the new hardware before rebuilding."
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for warning in "${RSYNC_WARNINGS[@]}"; do
+      echo "- $warning"
+    done
+  fi
+  echo ""
+  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "from manual external backup because they already live on the internal second drive"
+  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
+} > "$MANIFEST_FILE"
+
+log "Manifest written to $MANIFEST_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
+if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${RSYNC_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that"
+  log "vanished during backup, which is normal on an active desktop."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
+log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
 log "Please eject the drive safely before removing it from your Sovran Pro."
+
+# Remove incomplete marker and write completion marker only after all work succeeds.
+# A later successful run will update the same mirror and replace any INCOMPLETE state.
+rm -f "$BACKUP_DIR/INCOMPLETE"
+echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE"
+
+BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/server.py

@@ -8,6 +8,7 @@ import contextlib
 import glob
 import hashlib
 import hmac
+import ipaddress
 import json
 import logging
 import os
@@ -80,6 +81,15 @@ DOMAINS_DIR = "/var/lib/domains"
 NOSTR_NPUB_FILE   = "/var/lib/secrets/nostr_npub"
 NJALLA_SCRIPT     = "/var/lib/njalla/njalla.sh"
 
+# Systemd service that rewrites the Sovran-managed /etc/hosts loopback block
+SOVRAN_HOSTS_SERVICE = "sovran-hosts-update.service"
+
+# Domain keys that produce a public HTTPS virtual host via Caddy
+_SERVICE_DOMAIN_KEYS = frozenset([
+    "matrix", "wordpress", "nextcloud", "btcpayserver",
+    "vaultwarden", "haven", "element-calling",
+])
+
 INTERNAL_IP_FILE = "/var/lib/secrets/internal-ip"
 ZEUS_CONNECT_FILE = "/var/lib/secrets/zeus-connect-url"
 
@@ -964,18 +974,94 @@ def _check_port_status(
     return "closed"
 
 
+
+# Regex for validating domain values written into /etc/hosts.  Rejects anything
+# containing whitespace, newlines, or characters that could escape a hosts entry.
+# NOTE: The equivalent pattern in modules/core/local-domain-loopback.nix (shell
+# grep -E) must be kept in sync with this Python regex.
+_SAFE_DOMAIN_RE = re.compile(
+    r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'
+)
+
+
+def _validate_domain_value(domain: str) -> bool:
+    """Return True if *domain* is a valid hostname safe to write into /etc/hosts.
+
+    Rejects values containing whitespace, newlines, or other characters that
+    could inject additional entries or corrupt the hosts file.
+    """
+    if not domain or len(domain) > 253:
+        return False
+    # Guard against newline / whitespace injection before regex check.
+    if any(c in domain for c in ('\n', '\r', ' ', '\t', '#')):
+        return False
+    return bool(_SAFE_DOMAIN_RE.match(domain))
+
+
+def _is_loopback_address(ip: str) -> bool:
+    """Return True if *ip* is a loopback address (127.0.0.0/8 or ::1)."""
+    try:
+        return ipaddress.ip_address(ip).is_loopback
+    except ValueError:
+        return False
+
+
+def _resolve_all_addresses(domain: str) -> list[str]:
+    """Return all unique IP addresses that *domain* resolves to, or an empty list.
+
+    The first element is the address that the system resolver would normally
+    use for a connection.  All elements are checked when determining whether
+    any address matches the expected public IP or is a loopback address.
+    """
+    try:
+        results = socket.getaddrinfo(domain, None)
+        unique_addresses: list[str] = []
+        for r in results:
+            addr = r[4][0]
+            if addr not in unique_addresses:
+                unique_addresses.append(addr)
+        return unique_addresses
+    except Exception:
+        return []
+
+
+def _trigger_hosts_update() -> None:
+    """Start the sovran-hosts-update systemd service (best-effort, no-op if unavailable)."""
+    try:
+        subprocess.run(
+            ["systemctl", "start", SOVRAN_HOSTS_SERVICE],
+            timeout=30,
+            check=False,
+            capture_output=True,
+        )
+    except Exception:
+        pass
+
+
 def _check_domain_reachable(domain: str) -> dict:
-    """Curl the domain to verify end-to-end HTTPS reachability."""
+    """Check HTTPS reachability for *domain* via local Caddy (loopback).
+
+    Using ``--resolve`` ensures the request reaches Caddy on this computer
+    without depending on router NAT loopback or the public DNS result.
+    A successful local check is sufficient to confirm that Caddy and the
+    virtual-host configuration are working correctly.
+    """
     try:
         result = subprocess.run(
-            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "10", f"https://{domain}"],
+            [
+                "curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}",
+                "--max-time", "10",
+                "--resolve", f"{domain}:443:127.0.0.1",
+                "--resolve", f"{domain}:80:127.0.0.1",
+                f"https://{domain}",
+            ],
             capture_output=True,
             text=True,
             timeout=15,
         )
         status_code = result.stdout.strip()
         if status_code and status_code.isdigit() and int(status_code) > 0:
-            return {"reachable": True, "status_code": int(status_code)}
+            return {"reachable": True, "status_code": int(status_code), "via_loopback": True}
         return {"reachable": False, "error": result.stderr.strip() or "No response"}
     except subprocess.TimeoutExpired:
         return {"reachable": False, "error": "timeout"}
@@ -984,25 +1070,28 @@ def _check_domain_reachable(domain: str) -> dict:
 
 
 def _check_domain_health_fast(domain: str | None, external_ip: str) -> bool:
-    """Fast domain issue check for tile health (no curl/subprocess calls)."""
+    """Fast domain issue check for tile health (no curl/subprocess calls).
+
+    Returns ``True`` when a domain issue is detected that warrants
+    ``needs_attention``, ``False`` otherwise.
+    Loopback resolution is treated as an intentional server-local override,
+    not a DNS mismatch.
+    """
     if not domain:
         return True
 
-    resolved_ip: str | None = None
+    addrs = _resolve_all_addresses(domain)
-    try:
+    if not addrs:
-        results = socket.getaddrinfo(domain, None)
-        if results:
-            resolved_ip = results[0][4][0]
-    except socket.gaierror:
-        resolved_ip = None
-    except Exception:
-        resolved_ip = None
-
-    if not resolved_ip:
         return True
+
+    # If every resolved address is loopback the intentional /etc/hosts
+    # override is in place — this is healthy, not a mismatch.
+    if all(_is_loopback_address(a) for a in addrs):
+        return False
+
     if external_ip == "unavailable":
         return False
-    return resolved_ip != external_ip
+    return not any(a == external_ip for a in addrs)
 
 
 def _is_domain_reachable_cached(domain: str) -> bool | None:
@@ -1070,15 +1159,8 @@ def _evaluate_domain_checklist(
         "detail": domain,
     })
 
-    resolved_ip: str | None = None
+    addrs = _resolve_all_addresses(domain)
-    try:
+    resolved_ip: str | None = addrs[0] if addrs else None
-        results = socket.getaddrinfo(domain, None)
-        if results:
-            resolved_ip = results[0][4][0]
-    except socket.gaierror:
-        resolved_ip = None
-    except Exception:
-        resolved_ip = None
 
     if not resolved_ip:
         domain_status = {
@@ -1105,7 +1187,31 @@ def _evaluate_domain_checklist(
             "has_issues": True,
         }
 
-    if external_ip == "unavailable":
+    # Detect intentional server-local loopback override from /etc/hosts.
+    # When all addresses are loopback the public DNS is not checked via the
+    # system resolver (which would always return the override).  We proceed
+    # to the reachability check so Caddy health can still be verified.
+    loopback_override = all(_is_loopback_address(a) for a in addrs)
+
+    if loopback_override:
+        domain_status = {
+            "status": "local_override",
+            "resolved_ip": resolved_ip,
+            "expected_ip": external_ip,
+        }
+        steps.append({
+            "step": 2,
+            "label": "DNS / Local Override",
+            "status": "ok",
+            "detail": (
+                "Server-local loopback override is active — this computer routes the domain "
+                "directly to Caddy without going through the router. "
+                "Public DNS cannot be verified from this computer while the override is in place. "
+                "To check your public DNS from outside, use a tool such as "
+                "https://dnschecker.org or run: dig @1.1.1.1 " + domain
+            ),
+        })
+    elif external_ip == "unavailable":
         domain_status = {
             "status": "error",
             "resolved_ip": resolved_ip,
@@ -1117,7 +1223,7 @@ def _evaluate_domain_checklist(
             "status": "warning",
             "detail": f"Resolves to {resolved_ip} (external IP unavailable for comparison)",
         })
-    elif resolved_ip != external_ip:
+    elif not any(a == external_ip for a in addrs):
         domain_status = {
             "status": "dns_mismatch",
             "resolved_ip": resolved_ip,
@@ -1219,7 +1325,7 @@ def _generate_qr_base64(data: str) -> str | None:
 # ── Update helpers (file-based, no systemctl) ────────────────────
 
 def _read_update_status() -> str:
-    """Read the status file. Returns RUNNING, SUCCESS, FAILED, or IDLE."""
+    """Read the status file. Returns RUNNING, SUCCESS, REBOOT_REQUIRED, FAILED, or IDLE."""
     try:
         with open(UPDATE_STATUS, "r") as f:
             return f.read().strip()
@@ -1345,6 +1451,12 @@ def _read_backup_status() -> str:
         return "IDLE"
 
 
+def _write_backup_status(value: str) -> None:
+    """Write backup status file."""
+    with open(BACKUP_STATUS, "w") as f:
+        f.write(value)
+
+
 def _read_backup_log(offset: int = 0) -> tuple[str, int]:
     """Read the backup log file from the given byte offset.
     Returns (new_text, new_offset)."""
@@ -1361,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]:
         return "", 0
 
 
+def _append_backup_log(line: str) -> None:
+    """Append one line to backup log."""
+    with open(BACKUP_LOG, "a") as f:
+        f.write(line.rstrip("\n") + "\n")
+
+
 _INTERNAL_LABELS  = {"BTCEcoandBackup", "sovran_systemsos"}
 _INTERNAL_MOUNTS  = {"/", "/boot/efi"}
 _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive"
@@ -1375,6 +1493,16 @@ def _is_internal_mount(mnt: str) -> bool:
     return False
 
 
+def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
+    """Return whether the target filesystem type is supported for manual backup.
+
+    Manual Backup requires ext4 for Linux metadata preservation (ACLs, xattrs,
+    hard links). exFAT, FAT32, NTFS, and other filesystems are not supported.
+    """
+    fstype = (fstype or "").lower()
+    return fstype == "ext4"
+
+
 def _detect_external_drives() -> list[dict]:
     """Scan for mounted external USB drives.
 
@@ -1384,7 +1512,8 @@ def _detect_external_drives() -> list[dict]:
     /run/media/ directly if lsblk is unavailable, applying the same
     label/path filters.
 
-    Returns a list of dicts with name, path, free_gb, total_gb.
+    Returns:
+        list[dict]: Each dict contains name, path, free_gb, total_gb, fstype.
     """
     import json as _json
     import subprocess as _subprocess
@@ -1395,7 +1524,7 @@ def _detect_external_drives() -> list[dict]:
     # ── Primary path: lsblk JSON ────────────────────────────────
     try:
         result = _subprocess.run(
-            ["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"],
+            ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"],
             capture_output=True, text=True, timeout=10
         )
         if result.returncode == 0:
@@ -1413,6 +1542,7 @@ def _detect_external_drives() -> list[dict]:
                 hotplug    = str(dev.get("hotplug", "0"))
                 rm         = str(dev.get("rm", "0"))
                 label      = dev.get("label") or ""
+                fstype     = (dev.get("fstype") or "").lower()
                 mountpoint = dev.get("mountpoint") or ""
 
                 if dev_type not in ("part", "disk"):
@@ -1438,6 +1568,7 @@ def _detect_external_drives() -> list[dict]:
                         "path":     mountpoint,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(mountpoint)
                 except OSError:
@@ -1471,11 +1602,20 @@ def _detect_external_drives() -> list[dict]:
                     st = os.statvfs(drive_path)
                     total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1)
                     free_gb  = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1)
+                    fstype = ""
+                    try:
+                        fstype = _subprocess.run(
+                            ["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path],
+                            capture_output=True, text=True, timeout=5
+                        ).stdout.strip().lower()
+                    except Exception:
+                        fstype = ""
                     drives.append({
                         "name":     drive_name,
                         "path":     drive_path,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(drive_path)
                 except OSError:
@@ -1950,10 +2090,13 @@ def _verify_support_removed() -> bool:
 
 @app.get("/login", response_class=HTMLResponse)
 async def login_page(request: Request):
-    return templates.TemplateResponse("login.html", {
+    return templates.TemplateResponse(
-        "request": request,
+        request=request,
+        name="login.html",
+        context={
             "asset_version": ASSET_VERSION,
-    })
+        },
+    )
 
 
 @app.get("/auto-login")
@@ -2018,20 +2161,26 @@ async def api_logout(request: Request):
 
 @app.get("/", response_class=HTMLResponse)
 async def index(request: Request):
-    return templates.TemplateResponse("index.html", {
+    return templates.TemplateResponse(
-        "request": request,
+        request=request,
+        name="index.html",
+        context={
             "asset_version": ASSET_VERSION,
-    })
+        },
+    )
 
 
 @app.get("/onboarding", response_class=HTMLResponse)
 async def onboarding(request: Request):
     _ensure_onboarding_reopened_for_migration()
-    return templates.TemplateResponse("onboarding.html", {
+    return templates.TemplateResponse(
-        "request": request,
+        request=request,
+        name="onboarding.html",
+        context={
             "asset_version": ASSET_VERSION,
             "onboarding_js_hash": _ONBOARDING_JS_HASH,
-    })
+        },
+    )
 
 
 @app.get("/api/onboarding/status")
@@ -2740,20 +2889,18 @@ async def api_services():
                     break
         has_domain_issues = False
         if needs_domain and domain and enabled:
+            addrs = _resolve_all_addresses(domain)
             dns_ok = True
-            try:
+            if not addrs:
-                results = socket.getaddrinfo(domain, None)
+                dns_ok = False
-                if results:
+            elif all(_is_loopback_address(a) for a in addrs):
-                    resolved_ip = results[0][4][0]
+                # Intentional server-local /etc/hosts override — not a mismatch.
-                    if (
+                dns_ok = True
+            elif (
                 _cached_external_ip != "unavailable"
-                        and resolved_ip != _cached_external_ip
+                and not any(a == _cached_external_ip for a in addrs)
             ):
                 dns_ok = False
-                else:
-                    dns_ok = False
-            except (socket.gaierror, Exception):
-                dns_ok = False
 
             if not dns_ok:
                 has_domain_issues = True
@@ -2986,18 +3133,22 @@ async def api_service_detail(unit: str, icon: str | None = None):
         if has_domain_issues:
             domain_check_steps.append({
                 "step": 4,
-                "label": "Additional Ports Required",
+                "label": "Router Setup Needed",
                 "status": "skipped",
-                "detail": "Skipped until Steps 1-3 are complete",
+                "detail": "Finish the domain steps first, then forward the Element Call ports in your router.",
             })
         else:
-            extra_open = all(p["status"] != "closed" for p in extra_ports)
+            # These checks are local-only (listening/firewall state on this computer),
+            # not an outside-in verification of router/NAT forwarding.
+            all_local_ready = all(p["status"] != "closed" for p in extra_ports)
             domain_check_steps.append({
                 "step": 4,
-                "label": "Additional Ports Required",
+                "label": "Router Setup Needed" if all_local_ready else "Sovran_SystemsOS Port Setup Needed",
-                "status": "ok" if extra_open else "error",
+                "status": "warning" if all_local_ready else "error",
                 "detail": (
-                    "Element-Call/LiveKit requires additional forwarded ports for WebRTC and TURN traffic."
+                    "Sovran_SystemsOS is ready to use these ports on this computer. Now forward them in your router so Element Call can work from outside your home network."
+                    if all_local_ready
+                    else "Sovran_SystemsOS is not ready to use all required Element Call ports on this computer yet. Fix the ports marked “Not ready yet” below, then forward them in your router."
                 ),
             })
 
@@ -3565,6 +3716,37 @@ async def api_backup_drives():
     return {"drives": drives}
 
 
+async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
+    """Drain stderr, then mark status FAILED if backup subprocess exits unexpectedly."""
+    stderr_chunks: list[bytes] = []
+
+    async def _drain_stderr() -> None:
+        if proc.stderr is not None:
+            async for line in proc.stderr:
+                stderr_chunks.append(line)
+
+    drain_task = asyncio.create_task(_drain_stderr())
+    rc = await proc.wait()
+    await drain_task
+
+    if rc == 0:
+        return
+
+    loop = asyncio.get_event_loop()
+    status = await loop.run_in_executor(None, _read_backup_status)
+    if status in {"SUCCESS", "FAILED"}:
+        return
+
+    detail = ""
+    if stderr_chunks:
+        stderr_text = b"".join(stderr_chunks).decode("utf-8", errors="replace").strip()
+        if stderr_text:
+            detail = f" — stderr: {stderr_text}"
+    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc}).{detail}"
+    await loop.run_in_executor(None, _append_backup_log, msg)
+    await loop.run_in_executor(None, _write_backup_status, "FAILED")
+
+
 @app.post("/api/backup/run")
 async def api_backup_run(target: str = ""):
     """Start the backup script as a background subprocess.
@@ -3575,6 +3757,26 @@ async def api_backup_run(target: str = ""):
     if status == "RUNNING":
         return {"ok": True, "status": "already_running"}
 
+    drives = await loop.run_in_executor(None, _detect_external_drives)
+    if not drives:
+        raise HTTPException(status_code=400, detail="No external backup drive detected.")
+
+    drive_map = {d.get("path", ""): d for d in drives if d.get("path")}
+    if target:
+        if target not in drive_map:
+            raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.")
+        selected = drive_map[target]
+    else:
+        selected = drives[0]
+
+    selected_target = selected.get("path", "")
+    selected_fstype = (selected.get("fstype") or "").lower()
+    if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup. Manual Backup requires an ext4-formatted drive.",
+        )
+
     # Clear stale log before starting
     try:
         with open(BACKUP_LOG, "w") as f:
@@ -3582,21 +3784,48 @@ async def api_backup_run(target: str = ""):
     except OSError:
         pass
 
-    env = dict(os.environ)
+    try:
-    if target:
+        await loop.run_in_executor(None, _write_backup_status, "RUNNING")
-        env["BACKUP_TARGET"] = target
+    except OSError as exc:
+        raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}")
 
-    # Fire-and-forget: the script writes its own status/log files.
+    await loop.run_in_executor(
-    # Progress is read by the client via /api/backup/status (same pattern
+        None,
-    # as /api/updates/run and the rebuild feature).
+        _append_backup_log,
-    await asyncio.create_subprocess_exec(
+        f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…",
-        "/usr/bin/env", "bash", BACKUP_SCRIPT,
-        stdout=asyncio.subprocess.DEVNULL,
-        stderr=asyncio.subprocess.DEVNULL,
-        env=env,
     )
 
-    return {"ok": True, "status": "started"}
+    env = dict(os.environ)
+    env["BACKUP_TARGET"] = selected_target
+
+    bash_path = shutil.which("bash")
+    if bash_path is None:
+        no_bash_msg = (
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Cannot start backup:"
+            " interpreter 'bash' not found on PATH."
+            " Ensure pkgs.bash is in the sovran-hub-web service PATH."
+        )
+        await loop.run_in_executor(None, _append_backup_log, no_bash_msg)
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(
+            status_code=500,
+            detail="Backup interpreter (bash) not available. Check service PATH configuration.",
+        )
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bash_path, BACKUP_SCRIPT,
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except Exception as exc:
+        await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}")
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(status_code=500, detail="Failed to launch backup process.")
+
+    asyncio.create_task(_monitor_backup_subprocess(proc))
+    return {"ok": True, "status": "started", "target": selected_target}
 
 
 # ── Feature Manager endpoints ─────────────────────────────────────
@@ -3873,6 +4102,12 @@ async def api_domains_set(req: DomainSetRequest):
         except Exception:
             pass
 
+    # Regenerate the server-local /etc/hosts loopback entries so the newly
+    # saved domain is immediately reachable on this computer without NAT
+    # loopback support on the router.
+    if req.domain_name in _SERVICE_DOMAIN_KEYS:
+        _trigger_hosts_update()
+
     return {"ok": True}
 
 
@@ -3920,20 +4155,27 @@ async def api_domains_check(req: DomainCheckRequest):
     external_ip = _cached_external_ip
 
     def check_domain(domain: str) -> dict:
-        try:
+        addrs = _resolve_all_addresses(domain)
-            results = socket.getaddrinfo(domain, None)
+        if not addrs:
-            if not results:
             return {
                 "domain": domain, "status": "unresolvable",
                 "resolved_ip": None, "expected_ip": external_ip,
             }
-            resolved_ip = results[0][4][0]
+        resolved_ip = addrs[0]
+        # Server-local /etc/hosts loopback override — report as such rather
+        # than as a DNS mismatch.  Public DNS cannot be verified from this
+        # computer when the override is active.
+        if all(_is_loopback_address(a) for a in addrs):
+            return {
+                "domain": domain, "status": "local_override",
+                "resolved_ip": resolved_ip, "expected_ip": external_ip,
+            }
         if external_ip == "unavailable":
             return {
                 "domain": domain, "status": "error",
                 "resolved_ip": resolved_ip, "expected_ip": external_ip,
             }
-            if resolved_ip == external_ip:
+        if any(a == external_ip for a in addrs):
             return {
                 "domain": domain, "status": "connected",
                 "resolved_ip": resolved_ip, "expected_ip": external_ip,
@@ -3942,16 +4184,6 @@ async def api_domains_check(req: DomainCheckRequest):
             "domain": domain, "status": "dns_mismatch",
             "resolved_ip": resolved_ip, "expected_ip": external_ip,
         }
-        except socket.gaierror:
-            return {
-                "domain": domain, "status": "unresolvable",
-                "resolved_ip": None, "expected_ip": external_ip,
-            }
-        except Exception:
-            return {
-                "domain": domain, "status": "error",
-                "resolved_ip": None, "expected_ip": external_ip,
-            }
 
     check_results = await asyncio.gather(*[
         loop.run_in_executor(None, check_domain, d) for d in req.domains
@@ -4742,17 +4974,21 @@ def _recover_stale_status(status_file: str, log_file: str, unit_name: str) -> bo
     except Exception:
         pass
 
-    new_status = "SUCCESS" if unit_result == "success" else "FAILED"
+    if unit_result == "success":
+        new_status = "REBOOT_REQUIRED" if unit_name == UPDATE_UNIT else "SUCCESS"
+    else:
+        new_status = "FAILED"
     try:
         with open(status_file, "w") as f:
             f.write(new_status)
     except OSError:
         pass
-    msg = (
+    if new_status == "REBOOT_REQUIRED":
-        "\n[Update completed successfully while the server was restarting.]\n"
+        msg = "\n[Update staged successfully while the server was restarting. Reboot required.]\n"
-        if new_status == "SUCCESS"
+    elif new_status == "SUCCESS":
-        else "\n[Update encountered an error. See log above for details.]\n"
+        msg = "\n[Update completed successfully while the server was restarting.]\n"
-    )
+    else:
+        msg = "\n[Update encountered an error. See log above for details.]\n"
     try:
         with open(log_file, "a") as f:
             f.write(msg)

app/sovran_systemsos_web/static/css/header.css

@@ -91,3 +91,30 @@
   border-color: var(--accent-color);
   color: var(--accent-color);
 }
+
+/* ── Header reboot button ───────────────────────────────────────── */
+
+.btn-header-reboot {
+  background: transparent;
+  border: 1px solid rgba(184, 125, 0, 0.35);
+  color: #c98d08;
+  font-size: 0.78rem;
+  font-weight: 600;
+  padding: 4px 12px;
+  border-radius: var(--radius-btn);
+  cursor: pointer;
+  transition: border-color 0.15s, color 0.15s, background-color 0.15s;
+}
+
+.btn-header-reboot:hover {
+  border-color: #b87d00;
+  color: #e0a010;
+  background-color: rgba(184, 125, 0, 0.1);
+}
+
+@media (max-width: 480px) {
+  .btn-header-reboot {
+    padding: 4px 8px;
+    font-size: 0.72rem;
+  }
+}

app/sovran_systemsos_web/static/css/modals.css

@@ -102,6 +102,48 @@ button.btn-reboot:hover:not(:disabled) {
   background-color: #529E7E;
 }
 
+/* Restart = AMBER (manual restart action) */
+.btn-restart-amber {
+  background-color: #b87d00;
+  color: #fff;
+}
+
+.btn-restart-amber:hover:not(:disabled) {
+  background-color: #9a6800;
+}
+
+/* Restart conflict warning box */
+.restart-conflict-box {
+  background-color: rgba(180, 100, 0, 0.12);
+  border-left: 3px solid #c97a00;
+  border-radius: 6px;
+  padding: 12px 14px;
+  margin-bottom: 14px;
+}
+
+.restart-conflict-title {
+  font-size: 0.88rem;
+  font-weight: 700;
+  color: #e69000;
+  margin: 0 0 6px 0;
+}
+
+.restart-conflict-desc {
+  font-size: 0.83rem;
+  color: var(--text-secondary);
+  line-height: 1.5;
+  margin: 0;
+}
+
+/* Reboot error card actions row */
+.reboot-error-actions {
+  display: flex;
+  gap: 12px;
+  justify-content: center;
+  flex-wrap: wrap;
+  margin-top: 20px;
+}
+
 .btn-save {
   background-color: var(--yellow);
   color: #0A1A10;

app/sovran_systemsos_web/static/js/events.js

@@ -44,6 +44,28 @@ if ($upgradeCloseBtn) $upgradeCloseBtn.addEventListener("click", closeUpgradeMod
 if ($upgradeCancelBtn) $upgradeCancelBtn.addEventListener("click", closeUpgradeModal);
 if ($upgradeModal) $upgradeModal.addEventListener("click", function(e) { if (e.target === $upgradeModal) closeUpgradeModal(); });
 
+// Restart confirm dialog
+if ($restartConfirmCancel) $restartConfirmCancel.addEventListener("click", closeRestartConfirmDialog);
+if ($restartConfirmModal) $restartConfirmModal.addEventListener("click", function(e) { if (e.target === $restartConfirmModal) closeRestartConfirmDialog(); });
+if ($restartConfirmModal) $restartConfirmModal.addEventListener("keydown", function(e) { if (e.key === "Escape") closeRestartConfirmDialog(); });
+
+// Header Reboot button
+if ($headerRebootBtn) $headerRebootBtn.addEventListener("click", function() { openRestartConfirmDialog(); });
+if ($restartConfirmOk) $restartConfirmOk.addEventListener("click", function() {
+  if ($restartConfirmOk.disabled) return;
+  $restartConfirmOk.disabled = true;
+  closeRestartConfirmDialog();
+  doReboot();
+});
+
+// Reboot error card buttons
+var $rebootErrorCloseBtn = document.getElementById("reboot-error-close-btn");
+var $rebootErrorRetryBtn = document.getElementById("reboot-error-retry-btn");
+if ($rebootErrorCloseBtn) $rebootErrorCloseBtn.addEventListener("click", function() {
+  if ($rebootOverlay) $rebootOverlay.classList.remove("visible");
+});
+if ($rebootErrorRetryBtn) $rebootErrorRetryBtn.addEventListener("click", doReboot);
+
 // ── Upgrade modal functions ───────────────────────────────────────
 
 function openUpgradeModal() {
@@ -54,6 +76,37 @@ function closeUpgradeModal() {
   if ($upgradeModal) $upgradeModal.classList.remove("open");
 }
 
+// ── Restart confirm dialog functions ─────────────────────────────
+
+var _restartDialogOpener = null;
+
+function openRestartConfirmDialog() {
+  if (!$restartConfirmModal) return;
+  _restartDialogOpener = document.activeElement;
+
+  // Detect conflicting operations
+  var isOperationInProgress = !!_updatePollTimer || !!_rebuildPollTimer;
+  if ($restartConflictBox) $restartConflictBox.style.display = isOperationInProgress ? "" : "none";
+  if ($restartConfirmOk) $restartConfirmOk.disabled = isOperationInProgress;
+
+  $restartConfirmModal.classList.add("open");
+
+  // Focus Cancel initially for safety
+  var cancelBtn = document.getElementById("restart-confirm-cancel-btn");
+  if (cancelBtn) setTimeout(function() { cancelBtn.focus(); }, 50);
+}
+
+function closeRestartConfirmDialog() {
+  if ($restartConfirmModal) $restartConfirmModal.classList.remove("open");
+  // Re-enable confirm button for next open
+  if ($restartConfirmOk) $restartConfirmOk.disabled = false;
+  // Return focus to the element that opened the dialog
+  if (_restartDialogOpener && _restartDialogOpener.focus) {
+    try { _restartDialogOpener.focus(); } catch (_) {}
+    _restartDialogOpener = null;
+  }
+}
+
 async function doUpgradeToServer() {
   var confirmBtn = $upgradeConfirmBtn;
   if (confirmBtn) { confirmBtn.disabled = true; confirmBtn.textContent = "Upgrading…"; }

app/sovran_systemsos_web/static/js/features.js

@@ -73,22 +73,47 @@ function openDomainSetupModal(feat, onSaved) {
     npubField = '<div class="domain-field-group"><label class="domain-field-label" for="domain-npub-input">Nostr Public Key (npub1...):</label><input class="domain-field-input" type="text" id="domain-npub-input" placeholder="npub1..." value="' + escHtml(currentNpub) + '" /></div>';
   }
 
-  var externalIp = _cachedExternalIp || "your external IP";
+  var introHtml;
+  if (_currentRole === "node") {
+    introHtml =
+      '<p>To enable <strong>' + escHtml(feat.name) + '</strong>, it needs its own domain from Njal.la.</p>' +
+      '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+      '<li>Create an account at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">njal.la</a>.</li>' +
+      '<li>Set up a domain for it — either a free subdomain or a separate domain. Pick one option:</li>' +
+      '</ol>';
+  } else {
+    introHtml =
+      '<p>To enable <strong>' + escHtml(feat.name) + '</strong>, it needs its own domain from Njal.la. ' +
+      'In your Njal.la account, set up a domain for it — either a free subdomain or a separate domain. Pick one option:</p>';
+  }
 
   $domainSetupBody.innerHTML =
     '<div class="domain-setup-intro">' +
-    '<p><strong>Before continuing:</strong></p>' +
+    introHtml +
-    '<ol>' +
+    '<details style="margin-top:10px;">' +
-    '<li>Create an account at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">https://njal.la</a></li>' +
+    '<summary style="cursor:pointer;font-weight:600;">Option A — Free subdomain (recommended)</summary>' +
-    '<li>Purchase a new domain on Njal.la, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.</li>' +
+    '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
-    '<li>In the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine\'s external IP address:<br>' +
+    '<li>In Njal.la, open a domain you own and click &quot;Add record&quot;.</li>' +
-    '<span style="display:inline-block;margin-top:4px;padding:4px 10px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>' +
+    '<li>Set record type to <strong>Dynamic</strong>.</li>' +
-    '<li>Njal.la will give you a curl command like:<br>' +
+    '<li>In the <strong>Name</strong> field, type ONLY the host part — the word before your domain.<br>' +
-    '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=sub.domain.com&amp;k=abc123&amp;auto&quot;</code></li>' +
+    '(Example only, your choice — for &quot;call.yourdomain.com&quot; you&apos;d type just: &nbsp;<code>call</code>)<br>' +
-    '<li>Enter the subdomain and paste that curl command below</li>' +
+    '&#9888; Do NOT type the full domain here — Njal.la adds it automatically.</li>' +
+    '<li>A Dynamic record has NO IP field — the IP auto-fills after the rebuild/reboot.</li>' +
+    '<li>Copy the curl command Njal.la gives you, e.g.:<br>' +
+    '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=call.yourdomain.com&amp;k=abc123&amp;auto&quot;</code></li>' +
     '</ol>' +
+    '</details>' +
+    '<details style="margin-top:6px;">' +
+    '<summary style="cursor:pointer;font-weight:600;">Option B — Separate / new domain</summary>' +
+    '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+    '<li>In Njal.la, buy the domain you want.</li>' +
+    '<li>Add a Dynamic record as in Option A. If this domain is dedicated to the service, leave the Name field blank or use <code>@</code>.</li>' +
+    '<li>Copy the curl command Njal.la gives you.</li>' +
+    '</ol>' +
+    '</details>' +
+    '<p style="margin-top:10px;">Below, enter the full domain for this service — a subdomain (e.g. call.yourdomain.com) or a separate domain (e.g. call.com) — and paste its curl command.</p>' +
     '</div>' +
-    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Subdomain (e.g. myservice.example.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" /></div>' +
+    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Service domain (e.g. call.yourdomain.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" /></div>' +
     '<div class="domain-field-group"><label class="domain-field-label" for="domain-ddns-input">Njal.la Dynamic DNS Update Command:</label><input class="domain-field-input" type="text" id="domain-ddns-input" placeholder="curl &quot;https://njal.la/update/?h=myservice.example.com&amp;k=abc123&amp;auto&quot;" /><p class="domain-field-hint">ℹ Paste the full curl command from your Njal.la dashboard\'s Dynamic record</p></div>' +
     npubField +
     '<div class="domain-field-actions"><button class="btn btn-close-modal" id="domain-setup-cancel-btn">Cancel</button><button class="btn btn-primary" id="domain-setup-save-btn">Save &amp; Enable</button></div>';
@@ -103,7 +128,7 @@ function openDomainSetupModal(feat, onSaved) {
     ddnsUrl   = ddnsUrl.trim();
     npub      = npub.trim();
 
-    if (!subdomain) { alert("Please enter a subdomain."); return; }
+    if (!subdomain) { alert("Please enter a domain."); return; }
     if (feat.id === "haven" && !npub) { alert("Please enter your Nostr public key."); return; }
 
     var saveBtn = document.getElementById("domain-setup-save-btn");
@@ -159,14 +184,14 @@ function openDomainReconfigureModal(feat, existingDomain, onSaved) {
     '<p><strong>Troubleshooting steps:</strong></p>' +
     '<ol>' +
     '<li>Log into your Njal.la dashboard at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">https://njal.la</a></li>' +
-    '<li>Find the DNS record for <strong>' + escHtml(currentDomain || "your domain") + '</strong></li>' +
+    '<li>Find the DNS record for <strong>' + escHtml(currentDomain || "your domain") + '</strong>. In Njal.la\'s Name field, note that only the host part is stored (the word before the domain) — not the full domain.</li>' +
     '<li>Verify it has a <strong>Dynamic</strong> record pointing to your current external IP:<br>' +
     '<span style="display:inline-block;margin-top:4px;padding:4px 10px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>' +
     '<li>If the IP is wrong or the record is missing, update it</li>' +
     '<li>If you changed the DDNS curl command, paste the updated one below</li>' +
     '</ol>' +
     '</div>' +
-    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Subdomain (e.g. myservice.example.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" value="' + escHtml(currentDomain) + '" /></div>' +
+    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Service domain (e.g. call.yourdomain.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" value="' + escHtml(currentDomain) + '" /></div>' +
     '<div class="domain-field-group"><label class="domain-field-label" for="domain-ddns-input">Njal.la Dynamic DNS Update Command:</label><input class="domain-field-input" type="text" id="domain-ddns-input" placeholder="curl &quot;https://njal.la/update/?h=myservice.example.com&amp;k=abc123&amp;auto&quot;" /><p class="domain-field-hint">ℹ Paste the full curl command from your Njal.la dashboard\'s Dynamic record</p></div>' +
     npubField +
     '<div class="domain-field-actions"><button class="btn btn-close-modal" id="domain-setup-cancel-btn">Cancel</button><button class="btn btn-primary" id="domain-setup-save-btn">Save &amp; Update</button></div>';

app/sovran_systemsos_web/static/js/rebuild.js

@@ -69,7 +69,7 @@ function onRebuildDone(result) {
     // Auto-reload the page after a short delay so tiles and toggles reflect the new state
     setTimeout(function() { window.location.reload(); }, 1200);
   } else if (result === "reboot_required") {
-    if ($rebuildStatus) $rebuildStatus.textContent = "✓ Done — reboot required";
+    if ($rebuildStatus) $rebuildStatus.textContent = "✓ Done — restart required";
     if ($rebuildReboot) $rebuildReboot.style.display = "inline-flex";
   } else {
     if ($rebuildStatus) $rebuildStatus.textContent = "✗ Something went wrong";

app/sovran_systemsos_web/static/js/security.js

@@ -145,28 +145,25 @@ function openSecurityModal() {
           if (rebootBtn) {
             // Keep button disabled for 5 seconds to prevent accidental clicks
             var countdown = 5;
-            rebootBtn.textContent = "I have written down my new password \u2014 Reboot now (" + countdown + ")";
+            rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System (" + countdown + ")";
             var timer = setInterval(function() {
               countdown--;
               if (countdown <= 0) {
                 clearInterval(timer);
                 rebootBtn.disabled = false;
-                rebootBtn.textContent = "I have written down my new password \u2014 Reboot now";
+                rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System";
               } else {
-                rebootBtn.textContent = "I have written down my new password \u2014 Reboot now (" + countdown + ")";
+                rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System (" + countdown + ")";
               }
             }, 1000);
 
             rebootBtn.addEventListener("click", function() {
               rebootBtn.disabled = true;
-              rebootBtn.textContent = "Rebooting\u2026";
+              rebootBtn.textContent = "Restarting\u2026";
-              if ($rebootOverlay) $rebootOverlay.classList.add("visible");
+              // Hide the security reset overlay so the shared reboot overlay is visible
-              _rebootStartTime = Date.now();
+              var $secResetOverlay2 = document.getElementById("security-reset-overlay");
-              _serverWentDown = false;
+              if ($secResetOverlay2) $secResetOverlay2.classList.remove("visible");
-              setTimeout(waitForServerReboot, REBOOT_INITIAL_DELAY);
+              doReboot();
-              var rebootCtrl = new AbortController();
-              setTimeout(function() { rebootCtrl.abort(); }, REBOOT_REQUEST_TIMEOUT);
-              fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal }).catch(function() {});
             }, { once: true });
           }
         } catch (err) {

app/sovran_systemsos_web/static/js/service-detail.js

@@ -154,20 +154,36 @@ async function openServiceDetailModal(unit, name, icon) {
         '</div>';
 
       if (unit === "livekit.service" && data.extra_ports && data.extra_ports.length > 0) {
+        var trimmedInternalIp = data.internal_ip ? String(data.internal_ip).trim() : "";
+        var internalIp = trimmedInternalIp || "";
+        var internalIpHtml = internalIp ? escHtml(internalIp) : "Could not detect";
+        var routerIpHelp = internalIp
+          ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+          : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+        var routerNextStep = internalIp
+          ? 'Next step: Log in to your router and create forwarding rules for the ports above. Set the destination/internal IP to <strong>' + internalIpHtml + '</strong>.'
+          : 'Next step: Log in to your router and create forwarding rules for the ports above. Use this computer’s internal IP as the destination/internal IP.';
+        var domainConfigured = !!(data.domain && String(data.domain).trim());
         var extraRows = "";
         data.extra_ports.forEach(function(p) {
           var statusIcon, statusClass2;
-          if (p.status === "listening") {
+          if (!effectiveEnabled) {
-            statusIcon = "✅ Open";
+            statusIcon = "⚠ Configure Element Call first";
+            statusClass2 = "port-status-open";
+          } else if (!domainConfigured) {
+            statusIcon = "⚠ Configure domain first";
+            statusClass2 = "port-status-open";
+          } else if (p.status === "listening") {
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-listening";
           } else if (p.status === "firewall_open") {
-            statusIcon = "🟡 Firewall open";
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-open";
           } else if (p.status === "closed") {
-            statusIcon = "❌ Closed";
+            statusIcon = "❌ Not ready yet";
             statusClass2 = "port-status-closed";
           } else {
-            statusIcon = "— Unknown";
+            statusIcon = "— Could not check";
             statusClass2 = "port-status-unknown";
           }
           extraRows += '<tr>' +
@@ -178,11 +194,16 @@ async function openServiceDetailModal(unit, name, icon) {
             '</tr>';
         });
         html += '<div class="svc-detail-section">' +
-          '<div class="svc-detail-section-title">Step 4: Additional Ports</div>' +
+          '<div class="svc-detail-section-title">Ports to Forward in Your Router</div>' +
+          '<div class="svc-detail-port-note">Forward these ports in your router to this Sovran_SystemsOS computer.</div>' +
+          '<div class="svc-detail-port-note"><strong>Router Forward-To IP:</strong> ' + internalIpHtml + '</div>' +
+          '<div class="svc-detail-port-note">' + routerIpHelp + '</div>' +
           '<table class="svc-detail-port-table">' +
-            '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+            '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
             '<tbody>' + extraRows + '</tbody>' +
           '</table>' +
+          '<div class="svc-detail-port-note">The Hub can check whether Sovran_SystemsOS is ready on this computer, but full public port verification requires an outside internet check.</div>' +
+          '<div class="svc-detail-port-note">' + routerNextStep + '</div>' +
           '</div>';
       }
     } else if (data.port_statuses && data.port_statuses.length > 0) {
@@ -191,16 +212,16 @@ async function openServiceDetailModal(unit, name, icon) {
       data.port_statuses.forEach(function(p) {
         var statusIcon, statusClass2;
         if (p.status === "listening") {
-          statusIcon = "✅ Open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-listening";
         } else if (p.status === "firewall_open") {
-          statusIcon = "🟡 Firewall open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-open";
         } else if (p.status === "closed") {
-          statusIcon = "🔴 Closed";
+          statusIcon = "❌ Not ready";
           statusClass2 = "port-status-closed";
         } else {
-          statusIcon = "— Unknown";
+          statusIcon = "— Could not check";
           statusClass2 = "port-status-unknown";
         }
         portTableRows += '<tr>' +
@@ -211,9 +232,10 @@ async function openServiceDetailModal(unit, name, icon) {
           '</tr>';
       });
       html += '<div class="svc-detail-section">' +
-        '<div class="svc-detail-section-title">Port Status</div>' +
+        '<div class="svc-detail-section-title">Port Requirements</div>' +
+        '<div class="svc-detail-port-note">This shows whether Sovran_SystemsOS is ready to use this port on this computer. If you need access from outside your home network, forward this port in your router.</div>' +
         '<table class="svc-detail-port-table">' +
-          '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+          '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
           '<tbody>' + portTableRows + '</tbody>' +
         '</table>' +
         '</div>';

app/sovran_systemsos_web/static/js/state.js

@@ -49,6 +49,9 @@ const $btnSave        = document.getElementById("btn-save-report");
 const $btnCloseModal  = document.getElementById("btn-close-modal");
 
 const $rebootOverlay  = document.getElementById("reboot-overlay");
+const $rebootMainCard = document.getElementById("reboot-main-card");
+const $rebootErrorCard = document.getElementById("reboot-error-card");
+const $rebootSubmessage = document.getElementById("reboot-submessage");
 
 const $credsModal     = document.getElementById("creds-modal");
 const $credsTitle     = document.getElementById("creds-modal-title");
@@ -101,5 +104,14 @@ const $upgradeConfirmBtn  = document.getElementById("upgrade-confirm-btn");
 const $upgradeCancelBtn   = document.getElementById("upgrade-cancel-btn");
 const $upgradeCloseBtn    = document.getElementById("upgrade-close-btn");
 
+// Restart confirm dialog
+const $restartConfirmModal  = document.getElementById("restart-confirm-modal");
+const $restartConfirmOk     = document.getElementById("restart-confirm-ok-btn");
+const $restartConfirmCancel = document.getElementById("restart-confirm-cancel-btn");
+const $restartConflictBox   = document.getElementById("restart-conflict-box");
+
+// Header reboot button
+const $headerRebootBtn = document.getElementById("btn-header-reboot");
+
 // System status banner
 // (removed — health is now shown per-tile via the composite health field)

app/sovran_systemsos_web/static/js/support.js

@@ -491,8 +491,9 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>At least 500 GB of free space on the drive</li>',
+        '<li>Enough free space for your data (the backup checks this before starting)</li>',
-        '<li>Drive must be formatted as <strong>exFAT</strong></li>',
+        '<li>Drive must be formatted as <strong>ext4</strong> (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or <code>mkfs.ext4</code> — note that formatting erases all data on the drive.</li>',
+        '<li>The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.</li>',
       '</ol>',
     '</div>',
 
@@ -501,17 +502,25 @@ function renderBackupReady(drives) {
       '<ol class="support-backup-steps">',
         '<li>NixOS configuration (<code>/etc/nixos</code>)</li>',
         '<li>nix-bitcoin secrets (<code>/etc/nix-bitcoin-secrets</code>)</li>',
-        '<li>System service data (<code>/var/lib</code>) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets</li>',
+        '<li>System service data (<code>/var/lib</code>) — excluding databases and blockchain data (see note below)</li>',
         '<li>Home directory (<code>/home</code>)</li>',
       '</ol>',
     '</div>',
 
     '<div class="support-wallet-box support-wallet-warning">',
+      '<div class="support-wallet-header">',
+        '<span class="support-wallet-icon">\u2139\ufe0f</span>',
+        '<span class="support-wallet-title">Database and Blockchain Data</span>',
+      '</div>',
+      '<p class="support-wallet-desc">Application databases stored in PostgreSQL or MariaDB/MySQL are <strong>not included</strong> in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.</p>',
+    '</div>',
+
+    '<div class="support-wallet-box support-wallet-protected">',
       '<div class="support-wallet-header">',
         '<span class="support-wallet-icon">\u23f1\ufe0f</span>',
         '<span class="support-wallet-title">Time Estimate</span>',
       '</div>',
-      '<p class="support-wallet-desc">This backup can take <strong>up to 4 hours</strong> depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026</p>',
+      '<p class="support-wallet-desc">The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.</p>',
     '</div>',
 
     driveSelector,
@@ -584,9 +593,10 @@ async function pollBackupStatus() {
       logDiv.scrollTop = logDiv.scrollHeight;
     }
     _backupLogOffset = data.offset;
-    if (!data.running) {
+    const result = (data.result || "").toLowerCase();
+    if (result === "success" || result === "failed") {
       stopBackupPoll();
-      renderBackupDone(data.result === "success");
+      renderBackupDone(result === "success");
     }
   } catch (_) {}
 }
@@ -618,7 +628,7 @@ function renderBackupDone(success) {
       '<div class="support-section">',
       '<div class="support-icon-big">\u26a0\ufe0f</div>',
       '<h3 class="support-heading">Backup Failed</h3>',
-      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.</p>',
+      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.</p>',
       '<div class="modal-log" id="backup-log-fail" style="text-align:left;"></div>',
       '<button class="btn support-btn-done" id="btn-backup-close">Close</button>',
       '</div>',

app/sovran_systemsos_web/static/js/update.js

@@ -154,7 +154,7 @@ function onUpdateDone(result) {
     if ($modalStatus) $modalStatus.textContent = "✓ Update complete";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else if (result === "reboot_required") {
-    if ($modalStatus) $modalStatus.textContent = "✓ Update complete — reboot required";
+    if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else {
     if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
@@ -179,23 +179,50 @@ function saveErrorReport() {
 
 var _rebootStartTime = 0;
 var _serverWentDown = false;
+var _rebootFailed = false;
+
+function _setRebootStatus(msg) {
+  if ($rebootSubmessage) $rebootSubmessage.textContent = msg;
+}
 
 function doReboot() {
   if ($modal) $modal.classList.remove("open");
   if ($rebuildModal) $rebuildModal.classList.remove("open");
   stopUpdatePoll();
   stopRebuildPoll();
+  // Reset overlay to main card
+  if ($rebootMainCard) $rebootMainCard.style.display = "";
+  if ($rebootErrorCard) $rebootErrorCard.style.display = "none";
+  _setRebootStatus("Sending restart request\u2026");
   if ($rebootOverlay) $rebootOverlay.classList.add("visible");
   _rebootStartTime = Date.now();
   _serverWentDown = false;
+  _rebootFailed = false;
   var rebootCtrl = new AbortController();
   setTimeout(function() { rebootCtrl.abort(); }, REBOOT_REQUEST_TIMEOUT);
-  fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal }).catch(function() {});
+  fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal })
+    .then(function(res) {
+      if (!res.ok) {
+        // Definitive HTTP error — server rejected the request before going down
+        _rebootFailed = true;
+        if ($rebootMainCard) $rebootMainCard.style.display = "none";
+        if ($rebootErrorCard) $rebootErrorCard.style.display = "";
+        // Leave overlay visible so the error card is shown
+      }
+      // HTTP 2xx: request accepted, proceed with polling
+    })
+    .catch(function() {
+      // Connection dropped or request aborted — the server is likely already going
+      // down as part of the restart. Treat as success and continue polling.
+    });
   // Wait before the first check — NixOS shutdown after an update can take 20-40s
   setTimeout(waitForServerReboot, REBOOT_INITIAL_DELAY);
 }
 
 function waitForServerReboot() {
+  if (_rebootFailed) return;
+  // Update status on first check (server hasn't gone down yet)
+  if (!_serverWentDown) _setRebootStatus("Waiting for the computer to shut down\u2026");
   var controller = new AbortController();
   var timeoutId = setTimeout(function() { controller.abort(); }, REBOOT_FETCH_TIMEOUT);
 
@@ -205,18 +232,23 @@ function waitForServerReboot() {
       if (_serverWentDown) {
         // Server is responding after having been down — reboot is complete.
         // Any response (even 401/500) means the server process is back.
+        _setRebootStatus("System is back online. Reconnecting\u2026");
         window.location.reload();
       } else if ((Date.now() - _rebootStartTime) < 90000) {
         // Server still responding but hasn't gone down yet — keep waiting
         setTimeout(waitForServerReboot, REBOOT_CHECK_INTERVAL);
       } else {
         // Been over 90 seconds and server is responding — just reload
+        _setRebootStatus("System is back online. Reconnecting\u2026");
         window.location.reload();
       }
     })
     .catch(function() {
       clearTimeout(timeoutId);
+      if (!_serverWentDown) {
         _serverWentDown = true;
+        _setRebootStatus("The computer is restarting\u2026");
+      }
       setTimeout(waitForServerReboot, REBOOT_CHECK_INTERVAL);
     });
 }

app/sovran_systemsos_web/static/onboarding.js

@@ -333,8 +333,6 @@ async function loadStep3() {
     return;
   }
 
-  var externalIp = (networkData && networkData.external_ip) || "Unknown (could not retrieve)";
-
   // Build set of enabled service units
   var enabledUnits = new Set();
   (_servicesData || []).forEach(function(svc) {
@@ -352,18 +350,24 @@ async function loadStep3() {
     html += '<p class="onboarding-body-text">No domain-based services are enabled for your role. You can skip this step.</p>';
   } else {
     html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
-      + '<strong>Before you continue:</strong>'
+      + '<p style="margin:0 0 8px;"><strong>Sovran_SystemsOS uses Njal.la for domains and Dynamic DNS.</strong></p>'
       + '<ol style="margin:8px 0 0 16px; padding:0; line-height:1.7;">'
-      + '<li>Create an account at <a href="https://njal.la" target="_blank" style="color:var(--accent-color);">https://njal.la</a></li>'
+      + '<li>Create an account at <a href="https://njal.la" target="_blank" style="color:var(--accent-color);">https://njal.la</a>.</li>'
-      + '<li>Purchase a new domain on Njal.la, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you like.</li>'
+      + '<li>Buy at least one domain. Each service below needs its own domain — you can either give each service its own subdomain of a single domain you buy (subdomains are free, and one domain can have many), OR use a separate domain for each. Your choice.</li>'
-      + '<li>In the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine\'s external IP address:<br>'
+      + '<li>For each service, add a <strong>Dynamic</strong> record in Njal.la:'
-      + '<span style="display:inline-block;margin-top:4px;padding:4px 12px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1.1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>'
+      + '<ul style="margin:4px 0 0 16px;padding:0;line-height:1.7;">'
-      + '<li>Njal.la will give you a curl command like:<br>'
+      + '<li>In the Njal.la <strong>Name</strong> field, type ONLY the host part — the word before your domain.<br>'
-      + '<code style="font-size:0.8em;">curl "https://njal.la/update/?h=sub.domain.com&amp;k=abc123&amp;auto"</code></li>'
+      + '(Example only, your choice — for &quot;call.yourdomain.com&quot; you&apos;d type just: <code>call</code>.)<br>'
-      + '<li>Enter the subdomain and paste that curl command below for each service</li>'
+      + 'If you bought a whole separate domain just for this service, leave Name blank or use <code>@</code>.<br>'
+      + '&#9888; Do NOT type the full domain in the Name field — Njal.la adds it automatically.</li>'
+      + '<li>A Dynamic record has NO IP field. You don&apos;t enter an IP anywhere — it auto-fills once Sovran_SystemsOS updates it (on save, and again after reboot).</li>'
+      + '</ul>'
+      + '</li>'
+      + '<li>Njal.la gives you a curl command like:<br>'
+      + '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=call.yourdomain.com&amp;k=abc123&amp;auto&quot;</code></li>'
       + '</ol>'
       + '</div>';
-    html += '<p class="onboarding-hint">Enter each fully-qualified subdomain (e.g. <code>matrix.yourdomain.com</code>) and its Njal.la DDNS curl command.</p>';
+    html += '<p class="onboarding-hint">Enter each service\'s full domain — a subdomain (e.g. <code>call.yourdomain.com</code>) or a separate domain (e.g. <code>call.com</code>) — and its Njal.la DDNS curl command.</p>';
     relevantDomains.forEach(function(d) {
       var currentVal = (_domainsData && _domainsData[d.name]) || "";
       html += '<div class="onboarding-domain-group">';
@@ -512,7 +516,7 @@ async function saveStep3() {
 async function loadStep4() {
   var body = document.getElementById("step-4-body");
   if (!body) return;
-  body.innerHTML = '<p class="onboarding-loading">Checking ports…</p>';
+  body.innerHTML = '<p class="onboarding-loading">Loading router setup…</p>';
 
   var networkData = null;
 
@@ -523,51 +527,59 @@ async function loadStep4() {
     return;
   }
 
-  var internalIp = (networkData && networkData.internal_ip) || "unknown";
+  var trimmedInternalIp = (networkData && networkData.internal_ip) ? String(networkData.internal_ip).trim() : "";
-
+  var internalIp = trimmedInternalIp || "";
-  var ip = escHtml(internalIp);
+  var hasInternalIp = !!internalIp;
+  var ip = escHtml(internalIp || "Could not detect");
+  var routerIpHelp = hasInternalIp
+    ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+    : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+  var destinationInstruction = hasInternalIp
+    ? 'Set the destination/internal IP to <strong>' + ip + '</strong>'
+    : 'Use this computer’s internal IP as the destination/internal IP';
 
   var html = '<p class="onboarding-port-note" style="margin-bottom:14px;">'
     + '⚠ <strong>Each port only needs to be forwarded once — all services share the same ports.</strong>'
     + '</p>';
 
   html += '<div class="onboarding-port-ip">';
-  html += '  <span class="onboarding-port-ip-label">Forward ports to this machine\'s internal IP:</span>';
+  html += '  <span class="onboarding-port-ip-label">Forward router traffic to this Sovran_SystemsOS computer:</span>';
   html += '  <span class="port-req-internal-ip">' + ip + '</span>';
   html += '</div>';
+  html += '<div class="onboarding-port-note" style="margin:8px 0 16px;">' + routerIpHelp + '</div>';
 
   // Required ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Ports — open these on your router:</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Router Rules</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
-  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP</td></tr>';
+  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP / SSL setup</td></tr>';
   html += '<tr><td class="port-req-port">443</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTPS</td></tr>';
-  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">SSH Remote Access</td></tr>';
+  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">Remote SSH access</td></tr>';
   html += '</tbody></table>';
   html += '</div>';
 
   // Optional ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Optional — Only needed if you enable Element Calling:</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Element Call Router Rules</div>';
-  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">These 5 additional port openings are required on top of the 3 required ports above.</div>';
+  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">Only add these if you enable Element Call. These ports help video and audio calls connect reliably.</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
   html += '<tr><td class="port-req-port">7881</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit WebRTC signalling</td></tr>';
   html += '<tr><td class="port-req-port">7882</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit media (UDP mux)</td></tr>';
   html += '<tr><td class="port-req-port">5349</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN over TLS</td></tr>';
   html += '<tr><td class="port-req-port">3478</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN (STUN/relay)</td></tr>';
-  html += '<tr><td class="port-req-port">30000–40000</td><td class="port-req-proto">TCP &amp; UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
+  html += '<tr><td class="port-req-port">30000-40000</td><td class="port-req-proto">TCP &amp; UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
   html += '</tbody></table>';
-  html += '<div style="font-size:0.85em;margin-top:6px;color:var(--color-text-muted,#888);">ℹ The <strong>30000–40000</strong> range is a single forwarding rule — just set its protocol to <strong>both TCP and UDP</strong> (often shown as "Both" or "TCP/UDP" on your router).</div>';
+  html += '<div style="font-size:0.85em;margin-top:6px;color:var(--color-text-muted,#888);">ℹ The <strong>30000-40000</strong> range is a single forwarding rule — just set its protocol to <strong>both TCP and UDP</strong> (often shown as "Both" or "TCP/UDP" on your router).</div>';
   html += '</div>';
 
   // Totals
   html += '<div class="onboarding-port-totals">';
-  html += '<strong>Total port openings: 3</strong> (without Element Calling)<br>';
+  html += '<strong>Total port openings: 3</strong> (without Element Call)<br>';
-  html += '<strong>Total port openings: 8</strong> (with Element Calling — 3 required + 5 optional)';
+  html += '<strong>Total port openings: 8</strong> (with Element Call — 3 required + 5 optional)';
   html += '</div>';
 
   html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
@@ -582,12 +594,16 @@ async function loadStep4() {
     + '<li>Open your router\'s admin panel — usually <code>http://192.168.1.1</code> or <code>http://192.168.0.1</code></li>'
     + '<li>Look for <strong>"Port Forwarding"</strong>, <strong>"NAT"</strong>, or <strong>"Virtual Server"</strong> in the settings</li>'
     + '<li>Create a new rule for each port listed above</li>'
-    + '<li>Set the destination/internal IP to <strong>' + ip + '</strong></li>'
+    + '<li>' + destinationInstruction + '</li>'
     + '<li>Set both internal and external port to the same number</li>'
     + '<li>Save and apply changes</li>'
     + '</ol>'
     + '</details>';
 
+  html += '<div class="onboarding-port-note" style="margin-top:12px;">'
+    + '<strong>Important:</strong> The Hub can show which ports Sovran_SystemsOS needs, but it cannot fully confirm router forwarding from inside your home network. Full public port verification requires an outside internet check.'
+    + '</div>';
+
   body.innerHTML = html;
 }
 

app/sovran_systemsos_web/templates/index.html

@@ -24,6 +24,7 @@
     <span class="title">Sovran_SystemsOS Hub</span>
     <div class="header-buttons">
       <span class="role-badge" id="role-badge">Loading…</span>
+      <button class="btn btn-header-reboot" id="btn-header-reboot" title="Restart the entire computer">Reboot</button>
       <button class="btn btn-logout" id="btn-logout" title="Sign out">Sign Out</button>
     </div>
   </header>
@@ -61,7 +62,7 @@
       <div class="modal-log" id="modal-log" aria-live="polite"></div>
       <div class="modal-footer">
         <button class="btn btn-save" id="btn-save-report" style="display:none">Save Error Report</button>
-        <button class="btn btn-reboot" id="btn-reboot" style="display:none">Reboot</button>
+        <button class="btn btn-reboot" id="btn-reboot" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="btn-close-modal" disabled>Close</button>
       </div>
     </div>
@@ -164,7 +165,7 @@
       <div class="modal-log" id="rebuild-log" aria-live="polite"></div>
       <div class="modal-footer">
         <button class="btn btn-save" id="rebuild-save-report" style="display:none">Save Error Report</button>
-        <button class="btn btn-reboot" id="rebuild-reboot-btn" style="display:none">Reboot</button>
+        <button class="btn btn-reboot" id="rebuild-reboot-btn" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="rebuild-close-btn" disabled>Close</button>
       </div>
     </div>
@@ -240,26 +241,61 @@
         You will need it to log in to your computer<br />and the Sovran Hub at <em>sovransystemsos.local</em>.
       </p>
       <button class="security-reset-reboot-btn" id="security-reset-reboot-btn" disabled>
-        I have written down my new password — Reboot now
+        I have written down my new password — Restart Entire System
       </button>
     </div>
   </div>
 
   <!-- Reboot overlay -->
   <div class="reboot-overlay" id="reboot-overlay">
-    <div class="reboot-card">
+    <!-- Normal restarting card -->
-      <div class="reboot-icon">↻</div>
+    <div class="reboot-card" id="reboot-main-card">
-      <h2 class="reboot-title">System Rebooting</h2>
+      <div class="reboot-icon" aria-hidden="true">↻</div>
+      <h2 class="reboot-title">Restarting Entire System</h2>
       <p class="reboot-message">
-        Sovran_SystemsOS is now restarting.<br />
+        The entire computer is restarting, including the desktop and all hosted services.<br />
-        This page will automatically reconnect once the system is back online.
+        This page will reconnect automatically when Sovran_SystemsOS is back online.
       </p>
-      <div class="reboot-dots">
+      <div class="reboot-dots" aria-hidden="true">
         <span class="reboot-dot"></span>
         <span class="reboot-dot"></span>
         <span class="reboot-dot"></span>
       </div>
-      <p class="reboot-submessage">Stay tuned…</p>
+      <p class="reboot-submessage" id="reboot-submessage" aria-live="polite">Sending restart request…</p>
+    </div>
+    <!-- Error card (shown if restart request fails definitively) -->
+    <div class="reboot-card" id="reboot-error-card" style="display:none">
+      <div class="reboot-icon" aria-hidden="true">⚠</div>
+      <h2 class="reboot-title">Restart could not be started</h2>
+      <p class="reboot-message">
+        The computer did not begin restarting. No services were intentionally stopped. Please try again.
+      </p>
+      <div class="reboot-error-actions">
+        <button class="btn btn-close-modal" id="reboot-error-close-btn">Close</button>
+        <button class="btn btn-restart-amber" id="reboot-error-retry-btn">Try Again</button>
+      </div>
+    </div>
+  </div>
+
+  <!-- Restart Confirm Dialog -->
+  <div class="modal-overlay" id="restart-confirm-modal" role="dialog" aria-modal="true" aria-labelledby="restart-confirm-title">
+    <div class="creds-dialog domain-narrow-dialog">
+      <div class="creds-header">
+        <span class="creds-title" id="restart-confirm-title">Restart the entire computer?</span>
+      </div>
+      <div class="creds-body">
+        <div id="restart-conflict-box" class="restart-conflict-box" style="display:none">
+          <p class="restart-conflict-title">The system cannot restart right now.</p>
+          <p class="restart-conflict-desc">A system update, rebuild, backup, restore, or security operation is currently running. Wait for it to finish, then try again.</p>
+        </div>
+        <p class="support-desc"><strong>This will reboot the physical machine running Sovran_SystemsOS — not just the Hub.</strong></p>
+        <p class="support-desc">The desktop and all hosted services will stop temporarily and restart with the computer. Anyone currently using these services will be disconnected.</p>
+        <p class="support-desc">The system usually returns within 1–3 minutes. This page will reconnect automatically.</p>
+        <div class="domain-field-actions">
+          <button class="btn btn-close-modal" id="restart-confirm-cancel-btn">Cancel</button>
+          <button class="btn btn-restart-amber" id="restart-confirm-ok-btn">Restart Entire System</button>
+        </div>
+      </div>
     </div>
   </div>
 

app/sovran_systemsos_web/templates/onboarding.html

@@ -128,9 +128,8 @@
           <h2 class="onboarding-step-title">Domain Configuration</h2>
           <p class="onboarding-step-desc">
             Sovran_SystemsOS uses <strong><a href="https://njal.la" target="_blank" style="color: var(--accent-color);">Njal.la</a></strong> for domains and Dynamic DNS.
-            First, create an account at <strong>Njal.la</strong> and purchase a new domain, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.
+            Create an account at Njal.la, then for each service below, add a <strong>Dynamic</strong> record — no IP needed, it auto-populates once the DDNS curl command runs.
-            Then, in the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine's external IP address (shown below).
+            Paste the curl command from your Njal.la dashboard for each service.
-            Finally, paste the DDNS curl command from your Njal.la dashboard for each service below.
           </p>
         </div>
         <div class="onboarding-card" id="step-3-body">
@@ -149,14 +148,14 @@
       <div class="onboarding-panel" id="step-4" style="display:none">
         <div class="onboarding-step-header">
           <span class="onboarding-step-icon">🔌</span>
-          <h2 class="onboarding-step-title">Port Forwarding Check</h2>
+          <h2 class="onboarding-step-title">Router Setup</h2>
           <p class="onboarding-step-desc">
-            Forward these ports on your router to this machine. Each port only needs to be opened once — they are shared across all your services.
+            Forward these ports in your router to this Sovran_SystemsOS computer. These rules let people reach your services from outside your home network.
-            <strong>Ports 80 and 443 must be open for SSL certificates to work.</strong>
+            <strong>Ports 80 and 443 are required for HTTPS and SSL certificates.</strong>
           </p>
         </div>
         <div class="onboarding-card" id="step-4-body">
-          <p class="onboarding-loading">Checking ports…</p>
+          <p class="onboarding-loading">Loading router setup…</p>
         </div>
         <div class="onboarding-footer">
           <button class="btn btn-close-modal onboarding-btn-back" data-prev="3">← Back</button>

app/tests/test_hub_update_boot_staging.py

@@ -0,0 +1,44 @@
+import unittest
+from pathlib import Path
+
+
+HUB_NIX = Path(__file__).resolve().parents[2] / "modules" / "core" / "sovran-hub.nix"
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class HubUpdateBootStagingTests(unittest.TestCase):
+    def setUp(self):
+        self.source = HUB_NIX.read_text()
+        self.update_section = _section(
+            self.source,
+            'update-script = pkgs.writeShellScript "sovran-hub-update.sh" \'\'',
+            "# ── Rebuild wrapper script",
+        )
+        self.rebuild_section = _section(
+            self.source,
+            'rebuild-script = pkgs.writeShellScript "sovran-hub-rebuild.sh" \'\'',
+            "# ── Brave launcher wrapper",
+        )
+
+    def test_full_update_uses_boot_not_switch(self):
+        self.assertIn("nixos-rebuild boot --flake /etc/nixos", self.update_section)
+        self.assertNotIn("nixos-rebuild switch --flake /etc/nixos", self.update_section)
+
+    def test_full_update_marks_reboot_required(self):
+        self.assertIn('echo "REBOOT_REQUIRED" > "$STATUS"', self.update_section)
+
+    def test_rebuild_path_keeps_switch_semantics(self):
+        self.assertIn("nixos-rebuild switch --flake /etc/nixos", self.rebuild_section)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_local_domain_loopback_nix.py

@@ -0,0 +1,193 @@
+"""Structural regression tests for modules/core/local-domain-loopback.nix.
+
+Verifies that the sovran-hosts-update helper:
+  - is built as a pkgs.writeShellApplication with explicit runtimeInputs
+    (gawk, gnugrep, coreutils);
+  - uses ``lib.getExe hostsUpdateScript`` for both the systemd ExecStart and
+    the activation script so that both contexts share the same Nix-store
+    executable;
+  - does NOT point ExecStart at the raw /etc path;
+  - includes element-calling in the supported domain list;
+  - uses ``awk -v`` for safe marker-variable passing rather than interpolating
+    marker text directly into the awk program;
+  - retains the domain validation regex (idempotency / injection prevention);
+  - exposes /etc/sovran-hosts-update.sh as a symlink via ``source =`` (not a
+    second raw ``text =`` body);
+  - does NOT rely on environment.systemPackages for the helper's dependencies.
+"""
+
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+NIX_STRING_INDENT = 6
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FLAKE_SOURCE = (REPO_ROOT / "flake.nix").read_text()
+PRIMARY_NIXOS_CONFIGURATION_MATCH = re.search(
+    r"nixosConfigurations\.([A-Za-z0-9_-]+)\s*=",
+    FLAKE_SOURCE,
+)
+if PRIMARY_NIXOS_CONFIGURATION_MATCH is None:
+    raise RuntimeError("Could not determine the primary nixosConfigurations entry from flake.nix")
+PRIMARY_NIXOS_CONFIGURATION = PRIMARY_NIXOS_CONFIGURATION_MATCH.group(1)
+HELPER_BUILD_ATTR = (
+    f'.#nixosConfigurations.{PRIMARY_NIXOS_CONFIGURATION}.config.environment.etc.'
+    '"sovran-hosts-update.sh".source'
+)
+NIX_FILE = (
+    REPO_ROOT
+    / "modules"
+    / "core"
+    / "local-domain-loopback.nix"
+)
+
+
+class LocalDomainLoopbackNixStructureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = NIX_FILE.read_text()
+
+    def _helper_script(self) -> str:
+        start = self.source.index("text = ''") + len("text = ''")
+        end = self.source.index("    '';", start)
+        return "\n".join(
+            line[NIX_STRING_INDENT:]
+            if line.startswith(" " * NIX_STRING_INDENT)
+            else line
+            for line in self.source[start:end].splitlines()
+        ).lstrip("\n")
+
+    # ── writeShellApplication and explicit runtimeInputs ────────────────────
+
+    def test_uses_write_shell_application(self):
+        self.assertIn("pkgs.writeShellApplication", self.source)
+
+    def test_runtime_inputs_includes_coreutils(self):
+        self.assertIn("pkgs.coreutils", self.source)
+
+    def test_runtime_inputs_includes_gawk(self):
+        self.assertIn("pkgs.gawk", self.source)
+
+    def test_runtime_inputs_includes_gnugrep(self):
+        self.assertIn("pkgs.gnugrep", self.source)
+
+    def test_runtime_inputs_block_present(self):
+        self.assertIn("runtimeInputs", self.source)
+
+    # ── Both execution paths use lib.getExe ─────────────────────────────────
+
+    def test_exec_start_uses_lib_get_exe(self):
+        """systemd ExecStart must reference the Nix-store executable."""
+        self.assertIn("ExecStart = lib.getExe hostsUpdateScript", self.source)
+
+    def test_activation_script_uses_lib_get_exe(self):
+        """Activation text must call the same Nix-store executable."""
+        self.assertIn("${lib.getExe hostsUpdateScript}", self.source)
+
+    def test_exec_start_does_not_point_to_etc_path(self):
+        """ExecStart must NOT use the raw /etc path (which lacks a deterministic PATH)."""
+        self.assertNotIn('ExecStart = "/etc/sovran-hosts-update.sh"', self.source)
+
+    # ── /etc symlink uses source =, not a second text = body ────────────────
+
+    def test_etc_entry_uses_source_not_text(self):
+        """The /etc/sovran-hosts-update.sh entry must be a symlink (source =),
+        not a second raw script body (text =)."""
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source', self.source
+        )
+
+    def test_etc_source_points_to_get_exe(self):
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript',
+            self.source,
+        )
+
+    # ── element-calling domain is supported ─────────────────────────────────
+
+    def test_element_calling_domain_key_present(self):
+        self.assertIn("element-calling", self.source)
+
+    # ── Robust awk -v variable passing ──────────────────────────────────────
+
+    def test_awk_uses_dash_v_for_begin_marker(self):
+        """awk must receive the begin marker via -v, not by shell interpolation."""
+        self.assertIn('awk -v begin=', self.source)
+
+    def test_awk_uses_dash_v_for_end_marker(self):
+        self.assertIn('-v end=', self.source)
+
+    def test_awk_does_not_interpolate_marker_into_program(self):
+        """The old pattern interpolated $BEGIN_MARKER directly into the awk source."""
+        self.assertNotIn('/$BEGIN_MARKER', self.source)
+        self.assertNotIn('/$END_MARKER', self.source)
+
+    # ── Domain validation ────────────────────────────────────────────────────
+
+    def test_domain_validation_regex_present(self):
+        """The hostname validation regex must still be present for injection prevention."""
+        self.assertIn("grep -qE", self.source)
+        self.assertIn("[a-zA-Z0-9]", self.source)
+
+    def test_invalid_domain_warning_present(self):
+        self.assertIn("skipping invalid domain value", self.source)
+
+    # ── No environment.systemPackages reliance ───────────────────────────────
+
+    def test_no_environment_system_packages_for_helper(self):
+        """The helper's tools are declared via runtimeInputs; the module must
+        not add them to environment.systemPackages."""
+        self.assertNotIn("environment.systemPackages", self.source)
+
+    # ── Idempotency: existing Sovran block is removed before rewriting ───────
+
+    def test_existing_block_removal_logic_present(self):
+        """awk strip of the managed block must be present for idempotency."""
+        self.assertIn("skip=1", self.source)
+        self.assertIn("skip=0", self.source)
+
+    def test_managed_block_uses_grouped_append_redirect(self):
+        self.assertIn('} >> "$TMP"', self.source)
+        self.assertEqual(self.source.count('>> "$TMP"'), 1)
+
+    def test_helper_script_passes_shellcheck(self):
+        shellcheck = shutil.which("shellcheck")
+        if shellcheck is None:
+            self.skipTest("shellcheck is not installed")
+        proc = subprocess.run(
+            [shellcheck, "-s", "bash", "-"],
+            input=self._helper_script(),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        output = proc.stdout + proc.stderr
+        self.assertEqual(proc.returncode, 0, output)
+
+    def test_helper_derivation_builds_when_nix_available(self):
+        nix = shutil.which("nix")
+        if nix is None:
+            self.skipTest("nix is not installed")
+        proc = subprocess.run(
+            [
+                nix,
+                "build",
+                HELPER_BUILD_ATTR,
+                "--no-link",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+
+    # ── Activation script warns on failure rather than silently swallowing ───
+
+    def test_activation_script_emits_warning_on_failure(self):
+        self.assertIn("warning: sovran-hosts-update", self.source)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_loopback_diagnostics.py

@@ -0,0 +1,399 @@
+"""Tests for server-local loopback diagnostics and domain validation.
+
+Covers:
+- Domain value validation and injection prevention.
+- Loopback address detection (IPv4 and IPv6).
+- _resolve_all_addresses returning multiple addresses.
+- _check_domain_health_fast with loopback resolution.
+- _evaluate_domain_checklist with loopback override — no false dns_mismatch.
+- _evaluate_domain_checklist with genuine DNS mismatch — still reports error.
+- api_services health stays "healthy" when domain resolves to loopback.
+- api_services health stays "needs_attention" when DNS is genuinely wrong.
+- api_domains_check returns "local_override" for loopback-resolved domains.
+"""
+
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+# ---------------------------------------------------------------------------
+# Minimal stubs so server.py can be imported without the full FastAPI stack.
+# ---------------------------------------------------------------------------
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+                return _decorator
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server  # noqa: E402
+
+
+# ===========================================================================
+# Domain value validation
+# ===========================================================================
+
+class TestValidateDomainValue(unittest.TestCase):
+    """_validate_domain_value must reject anything that could corrupt /etc/hosts."""
+
+    def _v(self, value: str) -> bool:
+        return server._validate_domain_value(value)
+
+    # -- Valid values --------------------------------------------------------
+
+    def test_simple_domain_valid(self):
+        self.assertTrue(self._v("cloud.example.com"))
+
+    def test_subdomain_valid(self):
+        self.assertTrue(self._v("matrix.home.example.org"))
+
+    def test_single_label_with_tld_valid(self):
+        self.assertTrue(self._v("example.com"))
+
+    def test_hyphen_in_domain_valid(self):
+        self.assertTrue(self._v("my-nextcloud.example.com"))
+
+    # -- Injection / malformed values ----------------------------------------
+
+    def test_empty_string_invalid(self):
+        self.assertFalse(self._v(""))
+
+    def test_newline_injection_invalid(self):
+        self.assertFalse(self._v("evil.com\n127.0.0.1 other.host"))
+
+    def test_carriage_return_injection_invalid(self):
+        self.assertFalse(self._v("evil.com\r127.0.0.1 other.host"))
+
+    def test_space_injection_invalid(self):
+        self.assertFalse(self._v("evil.com 127.0.0.1"))
+
+    def test_hash_comment_injection_invalid(self):
+        self.assertFalse(self._v("evil.com# comment"))
+
+    def test_bare_hostname_no_dot_invalid(self):
+        self.assertFalse(self._v("localhost"))
+
+    def test_bare_ip_invalid(self):
+        self.assertFalse(self._v("192.168.1.1"))
+
+    def test_too_long_invalid(self):
+        self.assertFalse(self._v("a" * 254 + ".com"))
+
+    def test_leading_dot_invalid(self):
+        self.assertFalse(self._v(".example.com"))
+
+    def test_trailing_dot_invalid(self):
+        self.assertFalse(self._v("example.com."))
+
+
+# ===========================================================================
+# Loopback address detection
+# ===========================================================================
+
+class TestIsLoopbackAddress(unittest.TestCase):
+
+    def test_ipv4_loopback(self):
+        self.assertTrue(server._is_loopback_address("127.0.0.1"))
+
+    def test_ipv4_loopback_other(self):
+        self.assertTrue(server._is_loopback_address("127.0.0.2"))
+
+    def test_ipv4_loopback_high(self):
+        self.assertTrue(server._is_loopback_address("127.255.255.255"))
+
+    def test_ipv6_loopback(self):
+        self.assertTrue(server._is_loopback_address("::1"))
+
+    def test_public_ipv4_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("203.0.113.10"))
+
+    def test_private_ipv4_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("192.168.1.50"))
+
+    def test_ipv6_public_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("2001:db8::1"))
+
+    def test_invalid_string_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("not-an-ip"))
+
+
+# ===========================================================================
+# _check_domain_health_fast
+# ===========================================================================
+
+class TestCheckDomainHealthFast(unittest.TestCase):
+    """_check_domain_health_fast returns True when there is an issue,
+    False when everything looks fine."""
+
+    def _fast(self, domain, external_ip, resolved_addrs):
+        with patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs):
+            return server._check_domain_health_fast(domain, external_ip)
+
+    def test_no_domain_no_issue(self):
+        # None/empty domain: the fast check reports True (handled by checklist).
+        result = server._check_domain_health_fast(None, "203.0.113.10")
+        self.assertTrue(result)
+
+    def test_empty_domain_no_issue(self):
+        result = server._check_domain_health_fast("", "203.0.113.10")
+        self.assertTrue(result)
+
+    def test_loopback_ipv4_no_issue(self):
+        """Loopback override must not be flagged as a DNS mismatch."""
+        result = self._fast("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        self.assertFalse(result)
+
+    def test_loopback_ipv6_no_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["::1"])
+        self.assertFalse(result)
+
+    def test_matches_external_ip_no_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["203.0.113.10"])
+        self.assertFalse(result)
+
+    def test_mismatch_is_an_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["198.51.100.1"])
+        self.assertTrue(result)
+
+    def test_unavailable_external_ip_no_issue(self):
+        result = self._fast("cloud.example.com", "unavailable", ["198.51.100.1"])
+        self.assertFalse(result)
+
+    def test_multiple_addresses_one_matches_no_issue(self):
+        """If any resolved address matches external_ip the check should pass."""
+        result = self._fast(
+            "cloud.example.com", "203.0.113.10",
+            ["198.51.100.1", "203.0.113.10"],
+        )
+        self.assertFalse(result)
+
+
+# ===========================================================================
+# _evaluate_domain_checklist — loopback override path
+# ===========================================================================
+
+class TestEvaluateDomainChecklistLoopback(unittest.TestCase):
+
+    def _eval(self, domain, external_ip, resolved_addrs, reachable_result=None):
+        with (
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_check_domain_reachable",
+                         return_value=reachable_result or {"reachable": True, "status_code": 200}),
+        ):
+            return server._evaluate_domain_checklist(domain, external_ip)
+
+    def test_loopback_dns_step_is_ok_not_error(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        dns_step = next(s for s in result["domain_check_steps"] if s["step"] == 2)
+        self.assertEqual(dns_step["status"], "ok")
+        self.assertNotIn("mismatch", dns_step.get("detail", "").lower())
+
+    def test_loopback_domain_status_is_local_override(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        self.assertEqual(result["domain_status"]["status"], "local_override")
+
+    def test_loopback_has_no_issues_when_reachable(self):
+        result = self._eval(
+            "cloud.example.com", "203.0.113.10", ["127.0.0.1"],
+            reachable_result={"reachable": True, "status_code": 200},
+        )
+        self.assertFalse(result["has_issues"])
+
+    def test_loopback_has_issues_when_caddy_unreachable(self):
+        """A loopback override with Caddy down should still report an issue."""
+        result = self._eval(
+            "cloud.example.com", "203.0.113.10", ["127.0.0.1"],
+            reachable_result={"reachable": False, "error": "connection refused"},
+        )
+        self.assertTrue(result["has_issues"])
+
+    def test_ipv6_loopback_no_issue(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["::1"])
+        self.assertEqual(result["domain_status"]["status"], "local_override")
+        self.assertFalse(result["has_issues"])
+
+    def test_genuine_mismatch_still_reports_error(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["198.51.100.1"])
+        self.assertEqual(result["domain_status"]["status"], "dns_mismatch")
+        self.assertTrue(result["has_issues"])
+
+    def test_correct_public_dns_still_reports_ok(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["203.0.113.10"])
+        self.assertEqual(result["domain_status"]["status"], "connected")
+        self.assertFalse(result["has_issues"])
+
+    def test_no_domain_has_issues(self):
+        result = self._eval(None, "203.0.113.10", [])
+        self.assertTrue(result["has_issues"])
+
+
+# ===========================================================================
+# api_services — composite health with loopback
+# ===========================================================================
+
+class TestApiServicesLoopbackHealth(unittest.IsolatedAsyncioTestCase):
+
+    async def _get_health(self, resolved_addrs, cached_reachable):
+        """Return the health value for a single domain-requiring service."""
+        service_cfg = {
+            "services": [
+                {"unit": "caddy.service", "icon": "nextcloud", "enabled": True, "type": "system"}
+            ]
+        }
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"caddy.service": "nextcloud"}, clear=False),
+            patch("builtins.open", mock_open(read_data="cloud.example.com\n")),
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_is_domain_reachable_cached", return_value=cached_reachable),
+            patch.object(server, "_get_listening_ports",
+                         return_value={"tcp": {80, 443}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports",
+                         return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_cached_external_ip", "203.0.113.10"),
+        ):
+            results = await server.api_services()
+
+        return results[0]["health"]
+
+    async def test_loopback_and_reachable_is_healthy(self):
+        """Loopback override + Caddy reachable → healthy, not needs_attention."""
+        health = await self._get_health(["127.0.0.1"], cached_reachable=True)
+        self.assertEqual(health, "healthy")
+
+    async def test_loopback_and_caddy_down_is_needs_attention(self):
+        """Loopback override + Caddy unreachable → needs_attention (genuine issue)."""
+        health = await self._get_health(["127.0.0.1"], cached_reachable=False)
+        self.assertEqual(health, "needs_attention")
+
+    async def test_correct_dns_and_reachable_is_healthy(self):
+        health = await self._get_health(["203.0.113.10"], cached_reachable=True)
+        self.assertEqual(health, "healthy")
+
+    async def test_dns_mismatch_is_needs_attention(self):
+        health = await self._get_health(["198.51.100.1"], cached_reachable=True)
+        self.assertEqual(health, "needs_attention")
+
+
+# ===========================================================================
+# api_domains_check — loopback detection
+# ===========================================================================
+
+class TestApiDomainsCheckLoopback(unittest.IsolatedAsyncioTestCase):
+
+    async def _check(self, resolved_addrs, external_ip="203.0.113.10"):
+        with (
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_cached_external_ip", external_ip),
+        ):
+            result = await server.api_domains_check(
+                MagicMock(domains=["cloud.example.com"])
+            )
+        return result["domains"][0]
+
+    async def test_loopback_ipv4_returns_local_override(self):
+        result = await self._check(["127.0.0.1"])
+        self.assertEqual(result["status"], "local_override")
+
+    async def test_loopback_ipv6_returns_local_override(self):
+        result = await self._check(["::1"])
+        self.assertEqual(result["status"], "local_override")
+
+    async def test_correct_dns_returns_connected(self):
+        result = await self._check(["203.0.113.10"])
+        self.assertEqual(result["status"], "connected")
+
+    async def test_mismatch_returns_dns_mismatch(self):
+        result = await self._check(["198.51.100.1"])
+        self.assertEqual(result["status"], "dns_mismatch")
+
+    async def test_no_resolution_returns_unresolvable(self):
+        result = await self._check([])
+        self.assertEqual(result["status"], "unresolvable")
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_manual_backup_workflow.py

@@ -0,0 +1,748 @@
+import asyncio
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BACKUP_SCRIPT = REPO_ROOT / "app" / "sovran_systemsos_web" / "scripts" / "sovran-hub-backup.sh"
+SERVER_FILE = REPO_ROOT / "app" / "sovran_systemsos_web" / "server.py"
+SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "support.js"
+NIX_HUB_FILE = REPO_ROOT / "modules" / "core" / "sovran-hub.nix"
+
+
+class ManualBackupWorkflowTests(unittest.TestCase):
+    # ── Core design: rsync, no tar, no DB, no LND ─────────────────────────────
+
+    def test_backup_script_uses_rsync_not_tar(self):
+        """New design: backup must use rsync, not tar archives."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("rsync", source, "backup script must use rsync")
+        self.assertIn("--archive", source, "rsync must be called with --archive flag")
+        self.assertNotIn("tar --create", source, "backup script must not create tar archives")
+        self.assertNotIn("--file ", source, "backup script must not write tar --file output")
+
+    def test_backup_script_has_no_database_dump_functions(self):
+        """Removed: PostgreSQL and MariaDB dump functions must not exist."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("pg_dump", source, "backup script must not contain pg_dump")
+        self.assertNotIn("pg_dumpall", source, "backup script must not contain pg_dumpall")
+        self.assertNotIn("mariadb-dump", source, "backup script must not contain mariadb-dump")
+        self.assertNotIn("mysqldump", source, "backup script must not contain mysqldump")
+        self.assertNotIn("export_postgresql_dumps", source)
+        self.assertNotIn("export_mariadb_dumps", source)
+
+    def test_backup_script_has_no_lnd_service_orchestration(self):
+        """Removed: LND stop/restart and SCB export must not exist."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("export_lnd_scb_if_possible", source)
+        self.assertNotIn("stop_lnd_stack_if_needed", source)
+        self.assertNotIn("lncli", source, "backup script must not call lncli")
+        self.assertNotIn("LND_STOPPED", source, "backup script must not track LND_STOPPED state")
+        self.assertNotIn("LND_UNITS_TO_RESTART", source)
+
+    def test_backup_script_has_no_tar_dependencies(self):
+        """Removed: sha256sum checksums for tar artifacts must not exist."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("sha256sum", source, "backup script must not generate tar checksums")
+        self.assertNotIn("SHA256SUMS", source, "backup script must not write SHA256SUMS.txt")
+
+    # ── Rsync options ──────────────────────────────────────────────────────────
+
+    def test_backup_script_rsync_uses_metadata_preserving_options(self):
+        """Rsync must be called with Linux metadata-preserving options."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--archive", source)
+        self.assertIn("--acls", source)
+        self.assertIn("--xattrs", source)
+        self.assertIn("--hard-links", source)
+        self.assertIn("--numeric-ids", source)
+        self.assertIn("--one-file-system", source)
+
+    def test_backup_script_rsync_no_delete(self):
+        """Rsync must NOT use --delete or --delete-delay.
+        Accidental source deletion must not silently wipe the backup copy."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("--delete", source,
+            "rsync must not use --delete; accidental source deletion must not erase backup")
+
+    def test_backup_script_uses_stable_current_mirror_path(self):
+        """Backup must write to a stable 'current/' path, not timestamped directories.
+        Later runs should update the same mirror."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("Sovran_SystemsOS_Backup/current", source,
+            "backup must use a stable 'current' mirror path")
+        self.assertIn("BACKUP_SUBPATH", source,
+            "stable sub-path must be defined in BACKUP_SUBPATH variable")
+        # Must not create new timestamped directories per run
+        self.assertNotIn(
+            "date '+%Y%m%d_%H%M%S'",
+            source,
+            "backup must not create timestamped per-run directories",
+        )
+
+    def test_backup_script_source_destination_mapping(self):
+        """Each source tree must be synced to a matching destination path."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("/etc/nixos/", source)
+        self.assertIn('"$BACKUP_DIR/etc/nixos/"', source)
+        self.assertIn("/home/", source)
+        self.assertIn('"$BACKUP_DIR/home/"', source)
+        self.assertIn("/var/lib/", source)
+        self.assertIn('"$BACKUP_DIR/var/lib/"', source)
+
+    # ── ext4 filesystem validation ─────────────────────────────────────────────
+
+    def test_backup_script_requires_ext4_filesystem(self):
+        """Backup script must reject non-ext4 filesystems and require ext4."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'fstype" != "ext4"',
+            source,
+            "backup script must check for ext4 filesystem",
+        )
+        self.assertIn("ext4", source, "backup script must reference ext4")
+        self.assertNotIn(
+            '"exfat"',
+            source,
+            "backup script must not accept exFAT (old requirement)",
+        )
+        self.assertNotIn(
+            '"fuseblk"',
+            source,
+            "backup script must not accept fuseblk/NTFS",
+        )
+
+    def test_backup_script_rejects_unsupported_filesystems_in_error_message(self):
+        """The ext4 rejection message must mention the unsupported filesystem types."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "exFAT, FAT32, and NTFS are not supported",
+            source,
+            "error message must clearly list unsupported filesystem types",
+        )
+
+    def test_backend_accepts_ext4_rejects_exfat(self):
+        """Backend _is_supported_backup_fstype must accept ext4 and reject exFAT."""
+        server_source = SERVER_FILE.read_text()
+        # Must accept ext4
+        self.assertIn(
+            'return fstype == "ext4"',
+            server_source,
+            "backend must accept only ext4",
+        )
+        # Must not accept exFAT
+        self.assertNotIn(
+            'fstype == "exfat"',
+            server_source,
+            "backend must not accept exFAT",
+        )
+        self.assertNotIn(
+            'fuseblk',
+            server_source.split("_is_supported_backup_fstype")[1][:500],
+            "backend must not accept fuseblk",
+        )
+
+    def test_frontend_requires_ext4_not_exfat(self):
+        """Frontend UI copy must require ext4 and not mention exFAT as the requirement."""
+        support_source = SUPPORT_JS.read_text()
+        self.assertIn(
+            "ext4",
+            support_source,
+            "support.js must mention ext4 as the required filesystem",
+        )
+        # The word exFAT must not appear as a requirement (it may appear in a
+        # note about unsupported formats, but the requirement itself must say ext4)
+        requirements_section = re.search(
+            r"Requirements.*?What gets backed up",
+            support_source,
+            re.DOTALL,
+        )
+        if requirements_section:
+            req_text = requirements_section.group(0)
+            self.assertNotIn(
+                "Drive must be formatted as <strong>exFAT</strong>",
+                req_text,
+                "Requirements section must not say 'formatted as exFAT'",
+            )
+            self.assertIn(
+                "ext4",
+                req_text,
+                "Requirements section must say ext4",
+            )
+
+    def test_frontend_failure_message_says_ext4(self):
+        """Failure message in renderBackupDone must say ext4, not exFAT."""
+        support_source = SUPPORT_JS.read_text()
+        self.assertNotIn(
+            "formatted as exFAT",
+            support_source,
+            "failure message must not say 'formatted as exFAT'",
+        )
+        self.assertIn(
+            "formatted as ext4",
+            support_source,
+            "failure message must say 'formatted as ext4'",
+        )
+
+    # ── Database exclusions ────────────────────────────────────────────────────
+
+    def test_backup_script_excludes_postgresql_and_mariadb(self):
+        """PostgreSQL and MariaDB raw database directories must be excluded."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='postgresql/'", source,
+            "rsync must exclude postgresql directory")
+        self.assertIn("--exclude='mysql/'", source,
+            "rsync must exclude mysql directory")
+        self.assertIn("--exclude='mariadb/'", source,
+            "rsync must exclude mariadb directory")
+
+    def test_backup_script_excludes_bitcoin_and_electrs(self):
+        """Bitcoin blockchain and Electrs index data must be excluded."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='bitcoind/'", source,
+            "rsync must exclude bitcoind directory")
+        self.assertIn("--exclude='electrs/'", source,
+            "rsync must exclude electrs directory")
+
+    def test_manifest_states_database_exclusion(self):
+        """The manifest must explicitly state that PostgreSQL and MariaDB are excluded."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "PostgreSQL and MariaDB/MySQL application databases are NOT included",
+            source,
+            "manifest must state that databases are not included",
+        )
+        self.assertIn(
+            "Bitcoin blockchain data and Electrs indexes are NOT included",
+            source,
+            "manifest must state that blockchain data is not included",
+        )
+
+    def test_manifest_has_no_tar_restore_instructions(self):
+        """Manifest must not mention tar extraction, pg_restore, or LND SCB."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn(
+            "tar --acls --xattrs",
+            source,
+            "manifest must not give tar restore instructions",
+        )
+        self.assertNotIn(
+            "pg_restore",
+            source,
+            "manifest must not reference pg_restore",
+        )
+        self.assertNotIn(
+            "lnd-static-channel-backup.scb",
+            source,
+            "manifest must not reference LND SCB restore procedures",
+        )
+
+    def test_manifest_has_rsync_restore_guidance(self):
+        """Manifest must provide rsync-based restore guidance."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "rsync -aAXH --numeric-ids",
+            source,
+            "manifest must show rsync restore command",
+        )
+
+    # ── Exit code 24 (vanished files) for /home only ──────────────────────────
+
+    def test_backup_script_exit_code_24_nonfatal_for_home(self):
+        """Rsync exit code 24 (vanished files) must be nonfatal for /home only."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            '"$rc" -eq 24',
+            source,
+            "backup script must handle rsync exit code 24",
+        )
+        self.assertIn(
+            'allow_vanished" == "yes"',
+            source,
+            "exit 24 acceptance must be gated on allow_vanished flag",
+        )
+
+    def test_backup_script_home_stage_allows_vanished(self):
+        """Stage 3 (/home) must call run_rsync with allow_vanished=yes."""
+        source = BACKUP_SCRIPT.read_text()
+        # The /home call must pass "yes" as the allow_vanished argument
+        self.assertIn(
+            'run_rsync "/home" yes',
+            source,
+            "/home stage must pass allow_vanished=yes to run_rsync",
+        )
+
+    def test_backup_script_other_stages_disallow_vanished(self):
+        """Stages other than /home must call run_rsync with allow_vanished=no."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'run_rsync "/etc/nixos" no',
+            source,
+            "/etc/nixos stage must use allow_vanished=no",
+        )
+        self.assertIn(
+            'run_rsync "/var/lib" no',
+            source,
+            "/var/lib stage must use allow_vanished=no",
+        )
+
+    # ── INCOMPLETE / BACKUP_COMPLETE markers ──────────────────────────────────
+
+    def test_backup_script_writes_incomplete_marker(self):
+        """A backup directory must be marked INCOMPLETE immediately after
+        creation, so interrupted or failed runs are identifiable."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'touch "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must create INCOMPLETE marker after mkdir",
+        )
+
+    def test_backup_script_writes_backup_complete_marker(self):
+        """A BACKUP_COMPLETE file must be written only after all work succeeds."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'BACKUP_COMPLETE"',
+            source,
+            "backup script must write BACKUP_COMPLETE file on success",
+        )
+        self.assertIn(
+            'rm -f "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must remove INCOMPLETE marker on success",
+        )
+
+    # ── Concurrency lock ──────────────────────────────────────────────────────
+
+    def test_backup_script_uses_flock_concurrency_lock(self):
+        """The script must acquire an exclusive flock before starting work."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("flock", source, "backup script must use flock")
+        self.assertIn("LOCK_FILE", source, "backup script must define LOCK_FILE")
+
+    # ── Status states ─────────────────────────────────────────────────────────
+
+    def test_backup_script_uses_running_success_failed_states(self):
+        """Status values must be RUNNING, SUCCESS, and FAILED."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn('set_status "RUNNING"', source)
+        self.assertIn('set_status "SUCCESS"', source)
+        self.assertIn('set_status "FAILED"', source)
+
+    def test_backend_and_frontend_use_explicit_backup_terminal_states(self):
+        """Backend and frontend must use consistent terminal state handling."""
+        server_source = SERVER_FILE.read_text()
+        support_source = SUPPORT_JS.read_text()
+
+        self.assertIn("_write_backup_status, \"RUNNING\"", server_source)
+        self.assertIn("_monitor_backup_subprocess", server_source)
+        self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source)
+        self.assertIn("result === \"success\" || result === \"failed\"", support_source)
+
+    # ── Nix service PATH ──────────────────────────────────────────────────────
+
+    def test_nix_service_path_includes_rsync_and_acl(self):
+        """pkgs.rsync and pkgs.acl must appear in the sovran-hub-web service path
+        so that rsync with ACL/xattr support is available at runtime."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertIn(
+            "pkgs.rsync",
+            nix_source,
+            "pkgs.rsync must be declared in the sovran-hub-web service path",
+        )
+        self.assertIn(
+            "pkgs.acl",
+            nix_source,
+            "pkgs.acl must be declared in the sovran-hub-web service path",
+        )
+
+    def test_nix_service_path_includes_bash_and_gawk(self):
+        """Regression: pkgs.bash and pkgs.gawk must appear in the service path."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertIn("pkgs.bash", nix_source,
+            "pkgs.bash must be declared in the sovran-hub-web service path")
+        self.assertIn("pkgs.gawk", nix_source,
+            "pkgs.gawk must be declared in the sovran-hub-web service path")
+
+    def test_nix_service_path_has_no_gnutar(self):
+        """pkgs.gnutar must be removed from the service path (no longer needed)."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertNotIn(
+            "pkgs.gnutar",
+            nix_source,
+            "pkgs.gnutar must be removed from the service path (tar is no longer used)",
+        )
+
+    # ── Regression tests for exit-code-127 / missing-interpreter bug ──────────
+
+    def test_server_resolves_bash_via_shutil_which(self):
+        """Regression: server must locate bash with shutil.which()."""
+        source = SERVER_FILE.read_text()
+        self.assertIn('shutil.which("bash")', source)
+        self.assertNotIn('"/usr/bin/env", "bash"', source)
+
+    def test_server_logs_actionable_message_when_bash_missing(self):
+        """Regression: when bash is absent the server must write an actionable log."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("bash_path is None", source)
+        self.assertIn("interpreter", source)
+
+    def test_server_captures_stderr_from_backup_subprocess(self):
+        """Regression: backup subprocess must use stderr=PIPE."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("stderr=asyncio.subprocess.PIPE", source)
+        self.assertIn("stderr_text", source)
+
+    def test_monitor_includes_stderr_detail_in_failed_message(self):
+        """Regression: _monitor_backup_subprocess must append captured stderr."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("stderr_chunks", source)
+        self.assertIn("stderr_text", source)
+        self.assertIn('detail = f" — stderr: {stderr_text}"', source)
+
+    def test_exit_code_127_subprocess_stderr_drain(self):
+        """Behavioral regression: a subprocess that exits 127 drains stderr without deadlock."""
+        async def _run():
+            proc = await asyncio.create_subprocess_exec(
+                "bash", "-c", "echo 'bash: command not found' >&2; exit 127",
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            chunks: list[bytes] = []
+
+            async def _drain():
+                async for line in proc.stderr:
+                    chunks.append(line)
+
+            drain_task = asyncio.create_task(_drain())
+            rc = await proc.wait()
+            await drain_task
+            return rc, b"".join(chunks).decode()
+
+        rc, stderr_out = asyncio.run(_run())
+        self.assertEqual(rc, 127)
+        self.assertIn("bash: command not found", stderr_out)
+
+    # ── Rsync exit code 24 behavioral tests ──────────────────────────────────
+
+    def test_run_rsync_exit_24_nonfatal_for_home(self):
+        """Behavioral: run_rsync with allow_vanished=yes treats exit 24 as a
+        nonfatal warning (recorded in RSYNC_WARNINGS) and does not fail."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+RSYNC_WARNINGS=()
+
+log() {{ echo "$*"; }}
+fail() {{ echo "FAILED: $*" >&2; exit 1; }}
+
+run_rsync() {{
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+
+  local rc=24  # simulate rsync exit 24
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop)."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc)"
+  fi
+}}
+
+run_rsync "/home" yes /home/ "{dest}/home/"
+echo "WARNINGS: ${{#RSYNC_WARNINGS[@]}}"
+echo "SUCCESS"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("SUCCESS", result.stdout)
+            self.assertIn("WARNINGS: 1", result.stdout)
+
+    def test_run_rsync_exit_24_fatal_for_non_home(self):
+        """Behavioral: run_rsync with allow_vanished=no treats exit 24 as fatal."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+RSYNC_WARNINGS=()
+
+log() {{ echo "$*"; }}
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+
+run_rsync() {{
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+
+  local rc=24  # simulate rsync exit 24
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    RSYNC_WARNINGS+=("$label: vanished")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc)"
+  fi
+}}
+
+run_rsync "/etc/nixos" no /etc/nixos/ "{dest}/etc/nixos/"
+echo "SHOULD_NOT_REACH_HERE"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1)
+            self.assertIn("CORRECTLY_FAILED", result.stdout)
+            self.assertNotIn("SHOULD_NOT_REACH_HERE", result.stdout)
+
+    def test_run_rsync_other_nonzero_codes_always_fatal(self):
+        """Behavioral: rsync exit codes other than 0 and 24 (for home) are fatal."""
+        for rc in [1, 10, 11, 12, 23, 25]:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                dest = os.path.join(tmpdir, "backup")
+                os.makedirs(dest, exist_ok=True)
+
+                script = f"""#!/usr/bin/env bash
+RSYNC_WARNINGS=()
+
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+log() {{ echo "$*"; }}
+
+run_rsync() {{
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+
+  local rc={rc}  # simulated exit code
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    RSYNC_WARNINGS+=("$label: vanished")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc)"
+  fi
+}}
+
+run_rsync "/home" yes /home/ "{dest}/home/"
+echo "SHOULD_NOT_REACH_HERE"
+"""
+                script_file = os.path.join(tmpdir, "test.sh")
+                with open(script_file, "w") as sf:
+                    sf.write(script)
+                result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+                self.assertEqual(result.returncode, 1,
+                    f"Exit code {rc} should fail: {result.stdout}")
+                self.assertIn("CORRECTLY_FAILED", result.stdout)
+
+    # ── Behavioral: repeated runs target same 'current' mirror ──────────────
+
+    def test_repeated_runs_use_same_current_directory(self):
+        """Behavioral: a second call to the backup logic must sync to the same
+        BACKUP_DIR (not create a new timestamped directory) so only changed
+        files are transferred."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            target = os.path.join(tmpdir, "drive")
+            os.makedirs(target, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+TARGET="{target}"
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
+BACKUP_DIR="${{TARGET}}/${{BACKUP_SUBPATH}}"
+mkdir -p "$BACKUP_DIR"
+echo "$BACKUP_DIR"
+"""
+            for _ in range(2):
+                script_file = os.path.join(tmpdir, "test.sh")
+                with open(script_file, "w") as sf:
+                    sf.write(script)
+                result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+                self.assertEqual(result.returncode, 0)
+                backup_dir = result.stdout.strip()
+                self.assertEqual(
+                    backup_dir,
+                    os.path.join(target, "Sovran_SystemsOS_Backup", "current"),
+                )
+
+    # ── Behavioral: INCOMPLETE/BACKUP_COMPLETE marker lifecycle ─────────────
+
+    def test_incomplete_marker_written_before_work_starts(self):
+        """Behavioral: INCOMPLETE marker must exist during backup and be removed on success."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            backup_dir = os.path.join(tmpdir, "current")
+            os.makedirs(backup_dir, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+BACKUP_DIR="{backup_dir}"
+BACKUP_COMPLETE=0
+
+touch "$BACKUP_DIR/INCOMPLETE"
+[[ -f "$BACKUP_DIR/INCOMPLETE" ]] && echo "INCOMPLETE_EXISTS"
+
+# Simulate successful completion
+rm -f "$BACKUP_DIR/INCOMPLETE"
+echo "done" > "$BACKUP_DIR/BACKUP_COMPLETE"
+BACKUP_COMPLETE=1
+
+[[ ! -f "$BACKUP_DIR/INCOMPLETE" ]] && echo "INCOMPLETE_REMOVED"
+[[ -f "$BACKUP_DIR/BACKUP_COMPLETE" ]] && echo "COMPLETE_EXISTS"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("INCOMPLETE_EXISTS", result.stdout)
+            self.assertIn("INCOMPLETE_REMOVED", result.stdout)
+            self.assertIn("COMPLETE_EXISTS", result.stdout)
+
+    # ── Behavioral: actual rsync with real source/dest trees ─────────────────
+
+    def test_rsync_mirrors_source_to_dest(self):
+        """Behavioral: rsync correctly mirrors a source tree to a destination."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "etc", "nixos")
+            dest = os.path.join(tmpdir, "backup", "etc", "nixos")
+            os.makedirs(src, exist_ok=True)
+            os.makedirs(os.path.join(tmpdir, "backup", "etc"), exist_ok=True)
+
+            # Write test files
+            with open(os.path.join(src, "configuration.nix"), "w") as f:
+                f.write("{ ... }: {}\n")
+            with open(os.path.join(src, "custom.nix"), "w") as f:
+                f.write("{ ... }: {}\n")
+
+            result = subprocess.run(
+                ["rsync", "--archive", "--one-file-system",
+                 src + "/", dest + "/"],
+                capture_output=True, text=True,
+            )
+            self.assertEqual(result.returncode, 0, f"rsync failed: {result.stderr}")
+            self.assertTrue(os.path.exists(os.path.join(dest, "configuration.nix")))
+            self.assertTrue(os.path.exists(os.path.join(dest, "custom.nix")))
+
+    def test_rsync_second_run_only_transfers_changes(self):
+        """Behavioral: a second rsync run to the same dest only copies changed files."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "source")
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(src)
+            os.makedirs(dest)
+
+            with open(os.path.join(src, "file.txt"), "w") as f:
+                f.write("initial content\n")
+
+            # First run
+            r1 = subprocess.run(
+                ["rsync", "--archive", "--one-file-system", src + "/", dest + "/"],
+                capture_output=True, text=True,
+            )
+            self.assertEqual(r1.returncode, 0)
+
+            # Modify one file and add another; advance mtime so rsync detects the change
+            with open(os.path.join(src, "file.txt"), "w") as f:
+                f.write("updated content\n")
+            import time as _time
+            future_mtime = _time.time() + 2
+            os.utime(os.path.join(src, "file.txt"), (future_mtime, future_mtime))
+
+            with open(os.path.join(src, "new.txt"), "w") as f:
+                f.write("new file\n")
+
+            # Second run
+            r2 = subprocess.run(
+                ["rsync", "--archive", "--one-file-system", src + "/", dest + "/"],
+                capture_output=True, text=True,
+            )
+            self.assertEqual(r2.returncode, 0)
+
+            # Both files exist in dest
+            with open(os.path.join(dest, "file.txt")) as fh:
+                self.assertEqual(fh.read(), "updated content\n")
+            self.assertTrue(os.path.exists(os.path.join(dest, "new.txt")))
+
+    # ── Script syntax check ──────────────────────────────────────────────────
+
+    def test_backup_script_passes_bash_syntax_check(self):
+        """bash -n must report no syntax errors in the backup script."""
+        result = subprocess.run(
+            ["bash", "-n", str(BACKUP_SCRIPT)],
+            capture_output=True, text=True,
+        )
+        self.assertEqual(
+            result.returncode, 0,
+            f"bash -n failed:\n{result.stderr}",
+        )
+
+    # ── Home exclusions ──────────────────────────────────────────────────────
+
+    def test_backup_script_home_excludes_browser_caches(self):
+        """Home sync must exclude browser disk caches."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='.mozilla/firefox/*/cache2/'", source)
+        self.assertIn("--exclude='.config/google-chrome/*/Cache/'", source)
+        self.assertIn("--exclude='.config/chromium/*/Cache/'", source)
+        self.assertIn("--exclude='.config/BraveSoftware/Brave-Browser/*/Cache/'", source)
+
+    def test_backup_script_home_excludes_other_volatile_caches(self):
+        """Home sync must exclude baloo, thumbnails, and X session error logs."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='.local/share/baloo/'", source)
+        self.assertIn("--exclude='.thumbnails/'", source)
+        self.assertIn("--exclude='.xsession-errors'", source)
+
+    # ── require_cmd ──────────────────────────────────────────────────────────
+
+    def test_backup_script_requires_rsync_and_flock(self):
+        """Script must require rsync and flock via require_cmd."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("require_cmd rsync", source)
+        self.assertIn("require_cmd flock", source)
+
+    def test_backup_script_does_not_require_tar_or_sha256sum(self):
+        """Script must not require tar or sha256sum (no longer used)."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("require_cmd tar", source)
+        self.assertNotIn("require_cmd sha256sum", source)
+
+    # ── Frontend database limitation notice ──────────────────────────────────
+
+    def test_frontend_mentions_database_limitation(self):
+        """Frontend must explain that PostgreSQL/MariaDB databases are excluded."""
+        support_source = SUPPORT_JS.read_text()
+        self.assertIn(
+            "PostgreSQL",
+            support_source,
+            "UI must mention PostgreSQL exclusion",
+        )
+        self.assertIn(
+            "not included",
+            support_source,
+            "UI must explain that databases are not included",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_rdp_module_boot_setup.py

@@ -0,0 +1,122 @@
+import unittest
+from pathlib import Path
+
+
+RDP_NIX = Path(__file__).resolve().parents[2] / "modules" / "rdp.nix"
+USERNAME_READ = "USERNAME=\"$(tr -d '\\n' < \"$USERNAME_FILE\")\""
+USERNAME_LENGTH_GUARD = "if [ \"''${#USERNAME}\" -gt 32 ]; then"
+SHORT_PASSWORD_GUARD = 'if [ "\'\'${#PASSWORD}" -lt 8 ]; then'
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class RdpModuleBootSetupTests(unittest.TestCase):
+    def setUp(self):
+        self.source = RDP_NIX.read_text()
+        self.gnome_service = _section(
+            self.source,
+            "systemd.services.gnome-remote-desktop = {",
+            "systemd.tmpfiles.rules = [",
+        )
+        self.setup_service = _section(
+            self.source,
+            "systemd.services.gnome-remote-desktop-setup = {",
+            "};\n}",
+        )
+
+    def test_does_not_redeclare_gnome_remote_desktop_user(self):
+        self.assertNotIn("users.users.gnome-remote-desktop", self.source)
+        self.assertNotIn("createHome = true;", self.source)
+
+    def test_main_service_requires_setup_before_starting(self):
+        self.assertIn('wantedBy = [ "graphical.target" ];', self.gnome_service)
+        self.assertIn('after = [ "gnome-remote-desktop-setup.service" ];', self.gnome_service)
+        self.assertIn('requires = [ "gnome-remote-desktop-setup.service" ];', self.gnome_service)
+
+    def test_setup_waits_for_configuration_service_and_bounded_timeout(self):
+        self.assertIn('wantedBy = [ "graphical.target" ];', self.setup_service)
+        self.assertIn('before = [ "gnome-remote-desktop.service" ];', self.setup_service)
+        self.assertIn('"dbus.service"', self.setup_service)
+        self.assertIn('"gnome-remote-desktop-configuration.service"', self.setup_service)
+        self.assertNotIn("RemainAfterExit", self.setup_service)
+        self.assertIn('TimeoutStartSec = "2min";', self.setup_service)
+        self.assertIn('timeout --kill-after=5s 10s', self.setup_service)
+        self.assertIn('echo "grdctl command timed out: $*" >&2', self.setup_service)
+        self.assertIn('echo "grdctl command failed (exit $rc): $*" >&2', self.setup_service)
+
+    def test_setup_runs_grdctl_directly_as_root(self):
+        # The oneshot service runs as root; grdctl --system is called directly.
+        # GRD 50.x invokes pkexec internally, but the call itself is plain
+        # grdctl --system, not a manual pkexec invocation.
+        self.assertIn('grdctl --system "$@"', self.setup_service)
+        self.assertNotIn("runuser", self.setup_service)
+        self.assertNotIn("sudo", self.setup_service)
+        # No direct Nix-store pkexec invocation (pkgs.polkit}/bin/pkexec).
+        self.assertNotIn("pkgs.polkit}/bin/pkexec", self.setup_service)
+
+    def test_privilege_escalation_packages_absent_from_setup_path(self):
+        self.assertNotIn("pkgs.polkit", self.setup_service)
+        self.assertNotIn("pkgs.util-linux", self.setup_service)
+
+    def test_run_wrappers_bin_prepended_to_path(self):
+        # /run/wrappers/bin must be prepended to PATH before any grdctl_system
+        # invocation so that grdctl --system resolves the NixOS setuid pkexec.
+        path_export = 'export PATH="/run/wrappers/bin:$PATH"'
+        grdctl_marker = "grdctl_system"
+        script = self.setup_service
+        path_idx = script.find(path_export)
+        grdctl_idx = script.find(grdctl_marker)
+        self.assertGreater(path_idx, -1, f"{path_export!r} not found in setup script")
+        self.assertGreater(
+            grdctl_idx, path_idx,
+            "PATH export must appear before the first grdctl_system usage",
+        )
+
+    def test_pkexec_preflight_check(self):
+        # A preflight must confirm /run/wrappers/bin/pkexec is executable
+        # with a clear error message before any GRD configuration changes.
+        self.assertIn("test -x /run/wrappers/bin/pkexec", self.setup_service)
+        self.assertIn(
+            "/run/wrappers/bin/pkexec is absent or not executable",
+            self.setup_service,
+        )
+
+    def test_hub_files_are_the_source_of_truth_for_username_and_password(self):
+        self.assertIn('DEFAULT_USERNAME="sovran"', self.setup_service)
+        self.assertIn('if [ ! -f "$USERNAME_FILE" ]; then', self.setup_service)
+        self.assertIn(USERNAME_READ, self.setup_service)
+        self.assertIn(USERNAME_LENGTH_GUARD, self.setup_service)
+        self.assertIn('case "$USERNAME" in', self.setup_service)
+        self.assertIn('[A-Za-z_][A-Za-z0-9_-]*)', self.setup_service)
+        self.assertIn("RDP username is too long (''${#USERNAME} characters, maximum 32)", self.setup_service)
+        self.assertIn("RDP username must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens", self.setup_service)
+        self.assertIn('if [ ! -f "$PASSWORD_FILE" ]; then', self.setup_service)
+        self.assertIn("tr -d '\\n'", self.setup_service)
+        self.assertIn('"$PASSWORD_FILE"', self.setup_service)
+        self.assertIn(SHORT_PASSWORD_GUARD, self.setup_service)
+        self.assertIn("RDP password is too short (''${#PASSWORD} characters, minimum 8)", self.setup_service)
+        self.assertIn('grdctl_system rdp set-credentials "$USERNAME" "$PASSWORD"', self.setup_service)
+        self.assertNotIn('grdctl --system rdp set-credentials sovran "$PASSWORD"', self.setup_service)
+
+    def test_secure_permissions_are_enforced_for_state_and_secret_files(self):
+        self.assertIn('"d /var/lib/gnome-remote-desktop/tls 0700', self.source)
+        self.assertIn("chmod 700", self.setup_service)
+        self.assertIn('chmod 600 "$USERNAME_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$PASSWORD_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$CRED_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$TLS_DIR/rdp-tls.key"', self.setup_service)
+        self.assertIn('chmod 644 "$TLS_DIR/rdp-tls.crt"', self.setup_service)
+        self.assertIn('LOCAL_IP="$(hostname -I | awk \'{print $1}\')"', self.setup_service)
+        self.assertIn('LOCAL_IP="127.0.0.1"', self.setup_service)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_service_detail_router_wording.py

@@ -0,0 +1,171 @@
+import unittest
+from pathlib import Path
+from unittest.mock import mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+
+                return _decorator
+
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server
+
+
+class ServiceDetailRouterWordingTests(unittest.IsolatedAsyncioTestCase):
+    async def test_livekit_service_detail_includes_internal_ip(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": {7881}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["internal_ip"], "192.168.1.44")
+        self.assertEqual(result["extra_ports"][0]["status"], "listening")
+        self.assertEqual(result["domain_check_steps"][-1]["label"], "Router Setup Needed")
+
+    async def test_livekit_router_step_uses_not_ready_yet_wording(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["extra_ports"][0]["status"], "closed")
+        self.assertIn("Not ready yet", result["domain_check_steps"][-1]["detail"])
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_template_response_signature.py

@@ -0,0 +1,99 @@
+"""Regression test for Starlette 1.1.0+ TemplateResponse keyword-argument style.
+
+Prior to this fix, the three HTML routes called:
+    templates.TemplateResponse("name.html", {"request": request, ...})
+which passes the context dict as the second positional argument.  With the
+updated Starlette/FastAPI versions shipped in NixOS unstable (Starlette 1.1.0,
+FastAPI 0.136.3) that positional argument is the template name, causing Jinja2
+to receive a dict as a cache key and raise:
+    TypeError: unhashable type: 'dict'
+
+The fix updates every call to use keyword arguments:
+    templates.TemplateResponse(request=request, name="name.html", context={...})
+"""
+
+import ast
+import unittest
+from pathlib import Path
+
+
+SERVER_PY = Path(__file__).resolve().parents[1] / "sovran_systemsos_web" / "server.py"
+
+
+def _template_response_calls(source: str):
+    """Return a list of ast.Call nodes that are TemplateResponse calls."""
+    tree = ast.parse(source)
+    calls = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func = node.func
+        if isinstance(func, ast.Attribute) and func.attr == "TemplateResponse":
+            calls.append(node)
+    return calls
+
+
+class TemplateResponseSignatureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = SERVER_PY.read_text()
+        self.calls = _template_response_calls(self.source)
+
+    def test_at_least_one_template_response_call_found(self):
+        self.assertGreater(len(self.calls), 0, "No TemplateResponse calls found in server.py")
+
+    def test_no_old_style_positional_dict_context(self):
+        """No TemplateResponse call should pass a dict literal as its second positional arg.
+
+        The old style was:
+            templates.TemplateResponse("name.html", {"request": request, ...})
+        where args[0] is a string and args[1] is a Dict node.  That pattern
+        triggers the Starlette 1.1.0 bug.
+        """
+        for call in self.calls:
+            positional = call.args
+            if len(positional) >= 2 and isinstance(positional[1], ast.Dict):
+                self.fail(
+                    f"Found old-style TemplateResponse call at line {call.lineno}: "
+                    "second positional argument is a dict literal.  "
+                    "Use keyword arguments (request=, name=, context=) instead."
+                )
+
+    def test_request_not_duplicated_in_context(self):
+        """The 'request' key must not appear inside the context= dict when
+        request= is already passed as a dedicated keyword argument."""
+        for call in self.calls:
+            kw_dict = {kw.arg: kw.value for kw in call.keywords if isinstance(kw, ast.keyword)}
+
+            if "request" not in kw_dict:
+                continue  # no request= kwarg, nothing to check
+
+            context_node = kw_dict.get("context")
+            if not isinstance(context_node, ast.Dict):
+                continue
+
+            for key_node in context_node.keys:
+                if isinstance(key_node, ast.Constant) and key_node.value == "request":
+                    self.fail(
+                        f"TemplateResponse at line {call.lineno} passes 'request' both as "
+                        "request= keyword argument and inside the context dict."
+                    )
+
+    def test_all_calls_use_keyword_arguments(self):
+        """Every TemplateResponse call should use keyword arguments for request, name,
+        and context rather than relying on positional ordering."""
+        for call in self.calls:
+            kw_args = {kw.arg for kw in call.keywords if isinstance(kw, ast.keyword)}
+            self.assertIn(
+                "request",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'request='.",
+            )
+            self.assertIn(
+                "name",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'name='.",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

configuration.nix

@@ -149,8 +149,8 @@
     gparted pv unzip parted screen zenity
     libargon2 gnome-terminal libreoffice-fresh
     dig firefox wp-cli axel
-    lk-jwt-service livekit-libwebrtc livekit-cli livekit
+    lk-jwt-service livekit-libwebrtc livekit
-    matrix-synapse age
+    matrix-synapse age onlyoffice-desktopeditors
   ];
 
   # ── Shell ──────────────────────────────────────────────────

flake.lock

@@ -5,11 +5,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1781013869,
+        "lastModified": 1783519926,
-        "narHash": "sha256-XlEUtL+8M6kbPdmIh4sQQ7G02/1CwHQEk1RPvIMEWOs=",
+        "narHash": "sha256-2zwAN4lNitHFrHVnRZG3YcvpdtWOoF0cOBstxMeB1KI=",
         "owner": "emmanuelrosa",
         "repo": "btc-clients-nix",
-        "rev": "9a6c78204dc8961840375b110bca595b1f6f084c",
+        "rev": "731a1e11c2fefb14f0aa4b1f03cfa85c19c28d71",
         "type": "github"
       },
       "original": {
@@ -52,11 +52,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1778716662,
+        "lastModified": 1782949081,
-        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1780218263,
+        "lastModified": 1782911660,
-        "narHash": "sha256-T/f0pPDrH3Qc1VXyQXbK7yfHWRn90l3xwplc/nsxin4=",
+        "narHash": "sha256-PbR+tJ5E/Ux+01UtdFKqblccVA4/FgWbkym4ev3VHHQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7fc393d1b46fa000d48ff14e8b6a3c9985f03af0",
+        "rev": "cf720c15e108d432d29041cc5a185630809acefb",
         "type": "github"
       },
       "original": {
@@ -139,11 +139,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1780902259,
+        "lastModified": 1784432872,
-        "narHash": "sha256-q8yYEC5f1mFlQO9RGna4LTc9QrcvWunX6FYp83munkQ=",
+        "narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "bd0ff2d3eac24699c3664d5966b9ef36f388e2ca",
+        "rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
         "type": "github"
       },
       "original": {
@@ -187,11 +187,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1780749050,
+        "lastModified": 1784356753,
-        "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=",
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
         "type": "github"
       },
       "original": {
@@ -203,11 +203,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1780336545,
+        "lastModified": 1783915482,
-        "narHash": "sha256-vhVhuXzFrIOfcssC/9hDHx7MHzDKjF3keHuREOQqQiQ=",
+        "narHash": "sha256-FmieJB8/OUvNxbkboi7+IGfIuSXY3nF/hZQm8kD0r50=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4df1b885d76a54e1aa1a318f8d16fd6005b6401f",
+        "rev": "6cdc7fc76e8bf7fde9fa43a849fcaaa70e230dee",
         "type": "github"
       },
       "original": {
@@ -224,11 +224,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1780995253,
+        "lastModified": 1784057377,
-        "narHash": "sha256-6Lsoyw2XPvY8YNMCtPnsyw0JVVtHsXP2xtrFJBBTAOQ=",
+        "narHash": "sha256-yycNej5//EsRbV10moBoh+/63vXEwZD1ZFEiRm6C9rQ=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "43a7e6f82978ac975c3bba6728869b231e7a1ba0",
+        "rev": "07180a087e4a00720dc0731cbcd8dec796974381",
         "type": "github"
       },
       "original": {

iso/installer.py

@@ -1169,7 +1169,7 @@ class InstallerWindow(Adw.ApplicationWindow):
         btn_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=0)
         btn_box.set_halign(Gtk.Align.CENTER)
         btn_box.set_margin_bottom(32)
-        reboot_btn = Gtk.Button(label="I Have Written Down My Password — Reboot Now")
+        reboot_btn = Gtk.Button(label="I Have Written Down My Password — Restart Entire System")
         reboot_btn.add_css_class("suggested-action")
         reboot_btn.add_css_class("pill")
         reboot_btn.connect("clicked", lambda b: subprocess.run(["sudo", "reboot"]))

modules/core/caddy.nix

@@ -16,7 +16,10 @@ let
 in
 {
   services.caddy = {
-    enable = true;
+    # Only enable Caddy when at least one domain-based service needs it or
+    # the operator has defined custom vhosts.  This prevents Caddy from
+    # running on Desktop Only installs that have no web services configured.
+    enable = needsHttpsPorts || extraVhosts != "";
     user = "caddy";
     group = "root";
   };

modules/core/local-domain-loopback.nix

@@ -0,0 +1,171 @@
+{ config, pkgs, lib, ... }:
+
+# ── Server-local domain loopback overrides ────────────────────────────────────
+#
+# Some routers (especially newer ISP-provided devices) do not support NAT
+# loopback (hairpin NAT).  When a request originates on this computer and
+# targets a public domain name that resolves to the router's WAN address, the
+# router may refuse to loop the connection back in — causing Nextcloud, WordPress
+# background jobs, and other server-side callbacks to fail even when the service
+# is fully operational from the internet.
+#
+# This module installs a one-shot systemd service,
+# ``sovran-hosts-update.service``, that reads the configured service domains
+# from ``/var/lib/domains/`` at boot (and whenever triggered by the Hub after a
+# domain is saved) and writes ``127.0.0.1`` entries for them into a dedicated
+# Sovran-managed block in ``/etc/hosts``.
+#
+# With those entries in place:
+#   • Requests originating on this computer resolve the public domain name to
+#     127.0.0.1, reach Caddy directly, and never touch the router.
+#   • Caddy still receives the correct public hostname via TLS SNI so virtual-
+#     host routing and certificate validation continue to work.
+#   • The Sovran Hub can verify Caddy reachability locally without needing NAT
+#     loopback.
+#
+# Limitation: this does not help other devices on your home network (phones,
+# laptops).  Those devices resolve domains via the router's DNS and still depend
+# on NAT loopback (or require manual router DNS overrides).  For now, only
+# server-originated requests benefit from this override.
+#
+# On NixOS, /etc/hosts is normally a symlink into the Nix store and is
+# regenerated by the system activation script.  The ``system.activationScripts``
+# hook below converts it to a writable file each time the system is activated
+# (i.e. after every ``nixos-rebuild switch``) and then injects the Sovran block.
+# The same wrapped Nix-store executable is reused by both the activation hook
+# and the ``sovran-hosts-update.service`` unit, ensuring a deterministic runtime
+# PATH in every execution context.
+
+let
+  # ── Wrapped Nix-store executable ──────────────────────────────────────────
+  # Built with pkgs.writeShellApplication so that all required runtime tools
+  # (awk, grep, coreutils) are declared explicitly and injected into PATH by
+  # Nix.  Both the systemd service and the activation hook reference this same
+  # store-path executable — there is no second raw script body.
+  hostsUpdateScript = pkgs.writeShellApplication {
+    name = "sovran-hosts-update";
+
+    # Declare every external command the script calls.  These packages are
+    # added to the script's runtime PATH by writeShellApplication; nothing from
+    # the system PATH is relied upon.
+    runtimeInputs = [
+      pkgs.coreutils   # readlink, cp, mv, chmod, mktemp, rm, tr, head
+      pkgs.gawk        # awk
+      pkgs.gnugrep     # grep
+    ];
+
+    text = ''
+      # Regenerate the Sovran-managed loopback block in /etc/hosts.
+      # Safe to run multiple times — idempotent.
+
+      DOMAINS_DIR="/var/lib/domains"
+      HOSTS_FILE="/etc/hosts"
+      BEGIN_MARKER="# Sovran managed begin — server-local loopback overrides"
+      END_MARKER="# Sovran managed end"
+
+      # ── Step 1: ensure /etc/hosts is a regular writable file ──────────────
+      # On NixOS /etc/hosts starts as a symlink to the Nix store.  We replace
+      # it with a copy so we can append our block without touching the store.
+      if [ -L "$HOSTS_FILE" ]; then
+        TARGET=$(readlink -f "$HOSTS_FILE")
+        cp --no-preserve=all "$TARGET" "$HOSTS_FILE.sovran-tmp"
+        mv "$HOSTS_FILE.sovran-tmp" "$HOSTS_FILE"
+        chmod 644 "$HOSTS_FILE"
+      fi
+
+      # ── Step 2: remove any existing Sovran block ──────────────────────────
+      # Use a temp file so the operation is atomic.
+      # awk -v passes marker strings safely without shell interpolation.
+      TMP=$(mktemp "$HOSTS_FILE.XXXXXX")
+      trap 'rm -f "$TMP"' EXIT
+      awk -v begin="$BEGIN_MARKER" -v end="$END_MARKER" '
+        $0 == begin { skip=1; next }
+        $0 == end   { skip=0; next }
+        !skip
+      ' "$HOSTS_FILE" > "$TMP"
+
+      # ── Step 3: collect valid configured service domains ──────────────────
+      # NOTE: The hostname validation regex below must stay in sync with
+      # _SAFE_DOMAIN_RE in app/sovran_systemsos_web/server.py.
+      ENTRIES=""
+      for KEY in matrix wordpress nextcloud btcpayserver vaultwarden haven element-calling; do
+        FILE="$DOMAINS_DIR/$KEY"
+        [ -f "$FILE" ] || continue
+        # Read the domain value (strip all whitespace, limit to 253 chars)
+        DOMAIN=$(tr -d '[:space:]' < "$FILE" | head -c 253)
+        [ -z "$DOMAIN" ] && continue
+        # Validate: must match a reasonable hostname pattern (no injection)
+        if ! printf '%s' "$DOMAIN" | grep -qE \
+          '^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$'; then
+          echo "sovran-hosts-update: skipping invalid domain value for $KEY: $DOMAIN" >&2
+          continue
+        fi
+        ENTRIES="$ENTRIES
+127.0.0.1 $DOMAIN
+::1       $DOMAIN"
+      done
+
+      # ── Step 4: append the Sovran block if there are any entries ──────────
+      if [ -n "$ENTRIES" ]; then
+        {
+          printf '\n%s\n' "$BEGIN_MARKER"
+          printf '%s\n' "# These entries route configured service domains to local Caddy."
+          printf '%s\n' "# They are managed automatically — do not edit this block."
+          printf '%s\n' "$ENTRIES"
+          printf '%s\n' "$END_MARKER"
+        } >> "$TMP"
+      fi
+
+      # ── Step 5: atomically replace /etc/hosts ─────────────────────────────
+      mv "$TMP" "$HOSTS_FILE"
+      chmod 644 "$HOSTS_FILE"
+    '';
+  };
+
+in
+{
+  # ── /etc/sovran-hosts-update.sh — operator discoverability symlink ─────────
+  # Retain the familiar /etc path so administrators can inspect or manually
+  # invoke the helper.  The target is the wrapped Nix-store executable, so
+  # there is no second raw script body to keep in sync.
+  environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript;
+
+  # ── Systemd service ────────────────────────────────────────────────────────
+
+  systemd.services.sovran-hosts-update = {
+    description = "Update /etc/hosts with Sovran server-local loopback overrides";
+    documentation = [ "https://github.com/naturallaw777/sovran-systems" ];
+
+    # Run before Caddy so loopback entries are ready when it starts.
+    before = [
+      "caddy.service"
+      "network-online.target"
+    ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      # Point directly at the wrapped Nix-store executable, not the /etc path.
+      ExecStart = lib.getExe hostsUpdateScript;
+    };
+  };
+
+  # ── Activation script (runs after every nixos-rebuild switch) ─────────────
+  # This ensures the loopback block survives rebuilds that restore the /etc/hosts
+  # symlink.  The "users" and "etc" scripts must complete first.
+  # The same wrapped Nix-store executable used by the systemd service is
+  # referenced here, guaranteeing identical runtime dependencies in both
+  # execution contexts.
+
+  system.activationScripts.sovranDomainLoopback = {
+    text = ''
+      if [ -d /var/lib/domains ]; then
+        if ! ${lib.getExe hostsUpdateScript}; then
+          echo "warning: sovran-hosts-update: failed to update /etc/hosts loopback entries" >&2
+        fi
+      fi
+    '';
+    deps = [ "etc" "users" ];
+  };
+}

modules/core/role-logic.nix

@@ -3,6 +3,13 @@
 {
   config = lib.mkMerge [
 
+    # nix-bitcoin is globally imported by the flake (nixosModules.Sovran_SystemsOS).
+    # This default satisfies nix-bitcoin's generateSecrets assertion so that Desktop
+    # Only systems can evaluate without enabling any Bitcoin services.
+    {
+      nix-bitcoin.generateSecrets = lib.mkDefault true;
+    }
+
     # ── Server+Desktop Role (default) ─────────────────────────
     (lib.mkIf config.sovran_systemsOS.roles.server_plus_desktop {
       sovran_systemsOS.web.btcpayserver = lib.mkDefault true;
@@ -12,15 +19,24 @@
     (lib.mkIf config.sovran_systemsOS.roles.desktop {
       services.desktopManager.gnome.enable = true;
 
+      # Force all server/node services and features off so they cannot be
+      # accidentally enabled via custom.nix or option defaults on Desktop Only.
       sovran_systemsOS.services = {
-        synapse = lib.mkDefault false;
+        synapse = lib.mkForce false;
-        bitcoin = lib.mkDefault false;
+        bitcoin = lib.mkForce false;
-        vaultwarden = lib.mkDefault false;
+        vaultwarden = lib.mkForce false;
-        wordpress = lib.mkDefault false;
+        wordpress = lib.mkForce false;
-        nextcloud = lib.mkDefault false;
+        nextcloud = lib.mkForce false;
       };
 
-      sovran_systemsOS.web.btcpayserver = lib.mkDefault false;
+      sovran_systemsOS.features = {
+        haven = lib.mkForce false;
+        mempool = lib.mkForce false;
+        element-calling = lib.mkForce false;
+        bitcoin-core = lib.mkForce false;
+      };
+
+      sovran_systemsOS.web.btcpayserver = lib.mkForce false;
     })
 
     # ── Bitcoin Node Only Role ────────────────────────────────

modules/core/sovran-hub.nix

@@ -146,33 +146,16 @@ let
     echo ""
 
     if [ "$RC" -eq 0 ]; then
-      echo "── Step 2/3: nixos-rebuild ──────────────────────────"
+      echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──"
-      SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+      BOOT_OUT=$(nixos-rebuild boot --flake /etc/nixos --print-build-logs \
         --option connect-timeout 10 \
         --option stalled-download-timeout 90 \
         --option download-attempts 7 \
         --option fallback true 2>&1)
-      SWITCH_RC=$?
+      BOOT_RC=$?
-      echo "$SWITCH_OUT"
+      echo "$BOOT_OUT"
-      if [ "$SWITCH_RC" -eq 0 ]; then
+      if [ "$BOOT_RC" -ne 0 ]; then
-        echo "[OK] switch succeeded"
+        echo "[ERROR] nixos-rebuild boot failed"
-      elif echo "$SWITCH_OUT" | grep -q "switchInhibitors\|Pre-switch checks failed"; then
-        echo ""
-        echo "  ✓ Build succeeded — a reboot is required to apply this update"
-        echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-        if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
-             --option connect-timeout 10 \
-             --option stalled-download-timeout 90 \
-             --option download-attempts 7 \
-             --option fallback true 2>&1; then
-          echo "REBOOT_REQUIRED" > "$STATUS"
-          exit 0
-        else
-          echo "[ERROR] nixos-rebuild boot also failed"
-          RC=1
-        fi
-      else
-        echo "[ERROR] nixos-rebuild switch failed"
         RC=1
       fi
       echo ""
@@ -188,9 +171,10 @@ let
 
     if [ "$RC" -eq 0 ]; then
       echo "══════════════════════════════════════════════════"
-      echo "  ✓ Update completed successfully"
+      echo "  ✓ Update staged successfully"
+      echo "  Reboot required to activate the new system"
       echo "══════════════════════════════════════════════════"
-      echo "SUCCESS" > "$STATUS"
+      echo "REBOOT_REQUIRED" > "$STATUS"
     else
       echo "══════════════════════════════════════════════════"
       echo "  ✗ Update failed — see errors above"
@@ -398,13 +382,25 @@ in
       };
 
       path = [
+        pkgs.bash
+        pkgs.gawk
         pkgs.qrencode
         pkgs.curl
         pkgs.iproute2
         pkgs.nftables
         pkgs.iptables
         pkgs.hostname
-      ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package;
+        pkgs.coreutils
+        pkgs.findutils
+        pkgs.gnugrep
+        pkgs.rsync
+        pkgs.acl
+        pkgs.util-linux
+      ]
+      ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
+      ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
+      ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
+      ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
     };
 
     systemd.services.sovran-hub-update = {

modules/core/ssh-bootstrap.nix

@@ -31,7 +31,7 @@ lib.mkIf userExists {
   };
 
   systemd.services.factory-ssh-keygen = {
-    description = "Generate factory SSH key for ${userName} if missing";
+    description = "Generate or repair factory SSH key for ${userName}";
     wantedBy = [ "multi-user.target" ];
     after = [ "ssh-passphrase-setup.service" ];
     requires = [ "ssh-passphrase-setup.service" ];
@@ -39,14 +39,47 @@ lib.mkIf userExists {
       Type = "oneshot";
       RemainAfterExit = true;
     };
-    path = [ pkgs.openssh pkgs.coreutils ];
+    path = [ pkgs.openssh pkgs.coreutils pkgs.util-linux ];
     script = ''
-      if [ ! -f "${keyPath}" ]; then
+      set -eu
+
       PASSPHRASE=$(cat /var/lib/secrets/ssh-passphrase)
+      lock_file="${keyPath}.lock"
+
+      exec 9>"$lock_file"
+
+      if ! flock -n 9; then
+        echo "Factory SSH key setup is already running." >&2
+        exit 1
+      fi
+
+      generate_factory_key() {
         ssh-keygen -q -N "$PASSPHRASE" -t ed25519 -f "${keyPath}"
         chown ${userName}:users "${keyPath}" "${keyPath}.pub"
         chmod 600 "${keyPath}"
         chmod 644 "${keyPath}.pub"
+      }
+
+      if [ ! -f "${keyPath}" ]; then
+        generate_factory_key
+      elif ! ssh-keygen -y -P "$PASSPHRASE" -f "${keyPath}" >/dev/null 2>&1; then
+        backup_suffix="$(date -u +%Y%m%d_%H%M%S)-$$"
+        backup_path="${keyPath}.bak-$backup_suffix"
+        backup_index=0
+
+        while [ -e "$backup_path" ] || [ -e "$backup_path.pub" ]; do
+          backup_index=$((backup_index + 1))
+          backup_path="${keyPath}.bak-$backup_suffix-$backup_index"
+        done
+
+        echo "Existing factory SSH key does not match current passphrase; backing it up to $backup_path and generating a replacement."
+        mv "${keyPath}" "$backup_path"
+
+        if [ -f "${keyPath}.pub" ]; then
+          mv "${keyPath}.pub" "$backup_path.pub"
+        fi
+
+        generate_factory_key
       fi
     '';
   };

modules/element-calling.nix

@@ -34,8 +34,8 @@ lib.mkIf config.sovran_systemsOS.features.element-calling {
   };
 
   ####### ENSURE SERVICES START AFTER KEY EXISTS #######
-  systemd.services.livekit.after = [ "livekit-key-setup.service" ];
+  systemd.services.livekit.after = [ "livekit-key-setup.service" "livekit-turn-setup.service" ];
-  systemd.services.livekit.wants = [ "livekit-key-setup.service" ];
+  systemd.services.livekit.wants = [ "livekit-key-setup.service" "livekit-turn-setup.service" ];
   systemd.services.lk-jwt-service.after = [ "livekit-key-setup.service" ];
   systemd.services.lk-jwt-service.wants = [ "livekit-key-setup.service" ];
 
@@ -72,28 +72,50 @@ $MATRIX {
 }
 
 $ELEMENT_CALLING {
-  handle /livekit/jwt/sfu/get {
+  # Route all current lk-jwt-service authorization endpoints to port 8073,
+  # stripping the /livekit/jwt prefix that Caddy adds on the public URL.
+  @lk_jwt path /livekit/jwt/sfu/get* /livekit/jwt/get_token* /livekit/jwt/healthz* /livekit/jwt/sfu_webhook* /livekit/jwt/delegate_delayed_leave*
+  handle @lk_jwt {
     uri strip_prefix /livekit/jwt
     reverse_proxy [::1]:8073 {
       header_up Host {host}
       header_up X-Forwarded-Server {host}
       header_up X-Real-IP {remote_host}
       header_up X-Forwarded-For {remote_host}
+      header_up X-Forwarded-Proto {scheme}
     }
   }
   handle {
-    reverse_proxy localhost:7880
+    reverse_proxy localhost:7880 {
+      header_up Host {host}
+      header_up X-Forwarded-Proto {scheme}
+      header_up X-Forwarded-For {remote_host}
+      header_up X-Real-IP {remote_host}
+      transport http {
+        read_timeout 300s
+        write_timeout 300s
+      }
+    }
   }
 }
 EOF
     '';
   };
 
-  ####### LIVEKIT RUNTIME CONFIG #######
+  ####### LIVEKIT TURN SETUP (runtime cert + config) #######
-  systemd.services.livekit-runtime-config = {
+  # Replaces the old dead livekit-runtime-config.service. At runtime this:
-    description = "Generate LiveKit runtime config from domain files";
+  #   * reads the matrix domain from /var/lib/domains/matrix (never hardcoded)
+  #   * copies Caddy's already-issued matrix cert/key into /var/lib/livekit
+  #     so LoadCredential can stage them for the (DynamicUser) livekit unit
+  #   * detects the primary network interface from the IPv4 default route so
+  #     LiveKit only advertises real ICE candidates — not VPN/container/private
+  #     addresses from interfaces like Tailscale or Docker bridges
+  #   * writes a complete LiveKit config (with turn.domain and interface
+  #     substituted) that the overridden ExecStart loads.
+  systemd.services.livekit-turn-setup = {
+    description = "Stage TURN cert and generate LiveKit runtime config from domain files";
+    after = [ "caddy.service" "livekit-key-setup.service" ];
     before = [ "livekit.service" ];
-    after = [ "livekit-key-setup.service" ];
     requiredBy = [ "livekit.service" ];
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
@@ -103,20 +125,63 @@ EOF
     unitConfig = {
       ConditionPathExists = "/var/lib/domains/element-calling";
     };
-    path = [ pkgs.coreutils ];
+    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ];
     script = ''
       MATRIX=$(cat /var/lib/domains/matrix)
 
       mkdir -p /run/livekit
 
-      cat > /run/livekit/runtime-config.yaml <<EOF
+      # Copy Caddy's already-issued matrix cert/key into LiveKit's state dir.
+      # The ACME CA hostname directory can vary, so glob for the domain dir.
+      CRT=$(find /var/lib/caddy -path "*/$MATRIX/$MATRIX.crt" | head -n1)
+      KEY=$(find /var/lib/caddy -path "*/$MATRIX/$MATRIX.key" | head -n1)
+      cp "$CRT" /var/lib/livekit/turn.crt
+      cp "$KEY" /var/lib/livekit/turn.key
+      chmod 640 /var/lib/livekit/turn.crt /var/lib/livekit/turn.key
+
+      # Detect the primary network interface from the IPv4 default route.
+      # Restricting LiveKit to this single interface prevents it from
+      # advertising VPN/container/private ICE candidates (e.g. Tailscale,
+      # Docker bridges) that remote peers cannot reach, which causes all
+      # ICE negotiation attempts to fail with responsesReceived: 0.
+      IFACE=$(ip -4 route show default | awk '/^default/ { for(i=1;i<=NF;i++) if($i=="dev" && (i+1)<=NF) { print $(i+1); exit } }')
+      if [ -z "$IFACE" ]; then
+        echo "ERROR: Could not detect a default-route network interface from 'ip -4 route show default'." >&2
+        echo "ERROR: Cannot generate a valid LiveKit config without a real interface to bind ICE candidates to." >&2
+        echo "ERROR: Ensure a default IPv4 route is configured, e.g.: ip route add default via <gateway> dev <interface>" >&2
+        echo "ERROR: Inspect the current routing table with: ip -4 route show" >&2
+        exit 1
+      fi
+      echo "Detected primary network interface: $IFACE"
+
+      # Generate the full LiveKit config the daemon will load. turn.domain and
+      # rtc.interfaces.includes are only known at runtime, so they are
+      # substituted here. The cert/key paths point at the LoadCredential-staged
+      # copies under /run/credentials.
+      cat > /run/livekit/livekit.yaml <<EOF
+port: 7880
+rtc:
+  use_external_ip: true
+  skip_external_ip_validation: true
+  tcp_port: 7881
+  udp_port: 7882
+  port_range_start: 30000
+  port_range_end: 40000
+  interfaces:
+    includes:
+      - $IFACE
+room:
+  auto_create: false
 turn:
+  enabled: true
   domain: $MATRIX
-  cert_file: /var/lib/livekit/$MATRIX.crt
+  tls_port: 5349
-  key_file: /var/lib/livekit/$MATRIX.key
+  udp_port: 3478
+  cert_file: /run/credentials/livekit.service/turn-cert
+  key_file: /run/credentials/livekit.service/turn-key
 EOF
 
-      chmod 640 /run/livekit/runtime-config.yaml
+      chmod 644 /run/livekit/livekit.yaml
     '';
   };
 
@@ -127,7 +192,11 @@ EOF
     keyFile = livekitKeyFile;
     settings = {
       rtc.use_external_ip = true;
+      rtc.skip_external_ip_validation = true;
+      rtc.tcp_port = 7881;
       rtc.udp_port = 7882;
+      rtc.port_range_start = 30000;
+      rtc.port_range_end = 40000;
       room.auto_create = false;
       turn = {
         enabled = true;
@@ -137,13 +206,27 @@ EOF
     };
   };
 
+  # Override ExecStart to load the runtime-generated config (which carries the
+  # runtime-only turn.domain), mirroring the Caddy ExecStart override pattern in
+  # modules/core/caddy.nix. Deliver the TURN cert/key via LoadCredential so they
+  # are readable under the upstream unit's DynamicUser=true sandbox without
+  # weakening it. Everything else about the standard unit is left intact.
+  systemd.services.livekit.serviceConfig.ExecStart = lib.mkForce [
+    ""
+    "${pkgs.livekit}/bin/livekit-server --config /run/credentials/livekit.service/livekit-config --key-file /run/credentials/livekit.service/livekit-secrets"
+  ];
+
+  systemd.services.livekit.serviceConfig.LoadCredential = [
+    "livekit-config:/run/livekit/livekit.yaml"
+    "livekit-secrets:${livekitKeyFile}"
+    "turn-cert:/var/lib/livekit/turn.crt"
+    "turn-key:/var/lib/livekit/turn.key"
+  ];
+
   networking.firewall.allowedTCPPorts = [ 5349 7881 ];
   networking.firewall.allowedUDPPorts = [ 3478 7882 ];
   networking.firewall.allowedUDPPortRanges = [
-    { from = 30000; to = 40000; }
+    { from = 30000; to = 40000; } # LiveKit internal TURN relay range
-  ];
-  networking.firewall.allowedTCPPortRanges = [
-    { from = 30000; to = 40000; }
   ];
 
   ####### JWT SERVICE RUNTIME CONFIG #######
@@ -163,11 +246,13 @@ EOF
     path = [ pkgs.coreutils ];
     script = ''
       ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
+      MATRIX=$(cat /var/lib/domains/matrix)
 
       mkdir -p /run/lk-jwt-service
 
       cat > /run/lk-jwt-service/env <<EOF
 LIVEKIT_URL=wss://$ELEMENT_CALLING
+LIVEKIT_FULL_ACCESS_HOMESERVERS=$MATRIX
 EOF
 
       chmod 640 /run/lk-jwt-service/env

modules/modules.nix

@@ -16,6 +16,7 @@
     ./core/remote-deploy.nix
     ./core/no-sleep.nix
     ./core/cpu-performance.nix
+    ./core/local-domain-loopback.nix
 
     # ── Always on (no flag) ───────────────────────────────────
     ./php.nix

modules/rdp.nix

@@ -2,70 +2,104 @@
 
 lib.mkIf config.sovran_systemsOS.features.rdp {
 
-  users.users.gnome-remote-desktop = {
-    isSystemUser = true;
-    group = "gnome-remote-desktop";
-    home = "/var/lib/gnome-remote-desktop";
-    createHome = true;
-  };
-  users.groups.gnome-remote-desktop = {};
-
   # Enable the GNOME Remote Desktop service at the system level
   services.gnome.gnome-remote-desktop.enable = true;
 
   # Open RDP port in the firewall
   networking.firewall.allowedTCPPorts = [ 3389 ];
 
-  # Ensure the service actually starts and waits for setup to complete
+  # Ensure the service only starts after setup succeeds
   systemd.services.gnome-remote-desktop = {
     wantedBy = [ "graphical.target" ];
     after = [ "gnome-remote-desktop-setup.service" ];
-    wants = [ "gnome-remote-desktop-setup.service" ];
+    requires = [ "gnome-remote-desktop-setup.service" ];
   };
 
   systemd.tmpfiles.rules = [
-    "d /var/lib/gnome-remote-desktop 0750 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local 0700 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local 0750 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local/share 0700 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local/share 0750 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop 0700 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop 0750 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/tls 0700 gnome-remote-desktop gnome-remote-desktop -"
   ];
 
   systemd.services.gnome-remote-desktop-setup = {
     description = "Configure GNOME Remote Desktop RDP";
-    wantedBy = [ "multi-user.target" ];
+    wantedBy = [ "graphical.target" ];
     before = [ "gnome-remote-desktop.service" ];
-    after = [ "systemd-tmpfiles-setup.service" "network-online.target" ];
+    after = [
-    wants = [ "network-online.target" ];
+      "dbus.service"
+      "systemd-tmpfiles-setup.service"
+      "network-online.target"
+      "gnome-remote-desktop-configuration.service"
+    ];
+    wants = [
+      "network-online.target"
+      "gnome-remote-desktop-configuration.service"
+    ];
     serviceConfig = {
       Type = "oneshot";
-      RemainAfterExit = true;
+      TimeoutStartSec = "2min";
     };
     path = [
-      pkgs.gnome-remote-desktop
+      pkgs.coreutils
-      pkgs.polkit
-      pkgs.openssl
-      pkgs.hostname
       pkgs.gawk
+      pkgs.gnome-remote-desktop
+      pkgs.hostname
+      pkgs.openssl
+      pkgs.systemd
     ];
     script = ''
-      # Ensure directory structure exists
+      set -euo pipefail
-      mkdir -p /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop
-      chown -R gnome-remote-desktop:gnome-remote-desktop /var/lib/gnome-remote-desktop
 
-      TLS_DIR="/var/lib/gnome-remote-desktop/tls"
+      # GRD 50.x invokes pkexec internally for every grdctl --system call, even
-      CRED_FILE="/var/lib/gnome-remote-desktop/rdp-credentials"
+      # when the caller is root.  NixOS exposes the required setuid wrapper at
+      # /run/wrappers/bin/pkexec; the Nix-store polkit binary is not setuid and
+      # must not shadow it.  Prepend the wrapper directory so every subsequent
+      # grdctl --system resolves the correct binary.
+      export PATH="/run/wrappers/bin:$PATH"
+
+      STATE_DIR="/var/lib/gnome-remote-desktop"
+      TLS_DIR="$STATE_DIR/tls"
+      USERNAME_FILE="$STATE_DIR/rdp-username"
+      PASSWORD_FILE="$STATE_DIR/rdp-password"
+      CRED_FILE="$STATE_DIR/rdp-credentials"
+      DEFAULT_USERNAME="sovran"
+
+      grdctl_system() {
+        local rc=0
+
+        if timeout --kill-after=5s 10s \
+          grdctl --system "$@"; then
+          return 0
+        else
+          rc=$?
+        fi
+
+        if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
+          echo "grdctl command timed out: $*" >&2
+        fi
+        echo "grdctl command failed (exit $rc): $*" >&2
+
+        return "$rc"
+      }
+
+      mkdir -p "$STATE_DIR/.local/share/gnome-remote-desktop" "$TLS_DIR"
+      chown -R gnome-remote-desktop:gnome-remote-desktop "$STATE_DIR"
+      chmod 700 \
+        "$STATE_DIR" \
+        "$STATE_DIR/.local" \
+        "$STATE_DIR/.local/share" \
+        "$STATE_DIR/.local/share/gnome-remote-desktop" \
+        "$TLS_DIR"
 
-      # Regenerate TLS certificate if missing OR if ownership is wrong
-      # (disable/re-enable cycle can break ownership or grdctl state)
       NEED_REGEN=0
       if [ ! -f "$TLS_DIR/rdp-tls.crt" ] || [ ! -f "$TLS_DIR/rdp-tls.key" ]; then
         NEED_REGEN=1
-      elif [ "$(stat -c '%U' "$TLS_DIR/rdp-tls.key" 2>/dev/null)" != "gnome-remote-desktop" ]; then
+      elif [ "$(stat -c '%U:%G' "$TLS_DIR/rdp-tls.key" 2>/dev/null)" != "gnome-remote-desktop:gnome-remote-desktop" ]; then
         NEED_REGEN=1
       fi
 
       if [ "$NEED_REGEN" = "1" ]; then
-        mkdir -p "$TLS_DIR"
         rm -f "$TLS_DIR/rdp-tls.key" "$TLS_DIR/rdp-tls.crt"
         openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \
           -sha256 -nodes -days 3650 \
@@ -75,39 +109,59 @@ lib.mkIf config.sovran_systemsOS.features.rdp {
         echo "Generated new RDP TLS certificate"
       fi
 
-      # Always fix ownership and permissions (handles re-enable after disable)
+      chown gnome-remote-desktop:gnome-remote-desktop "$TLS_DIR/rdp-tls.key" "$TLS_DIR/rdp-tls.crt"
-      chown -R gnome-remote-desktop:gnome-remote-desktop "$TLS_DIR"
       chmod 600 "$TLS_DIR/rdp-tls.key"
       chmod 644 "$TLS_DIR/rdp-tls.crt"
 
-      # Configure TLS certificate
+      if [ ! -f "$USERNAME_FILE" ]; then
-      grdctl --system rdp set-tls-cert "$TLS_DIR/rdp-tls.crt"
+        printf '%s\n' "$DEFAULT_USERNAME" > "$USERNAME_FILE"
-      grdctl --system rdp set-tls-key "$TLS_DIR/rdp-tls.key"
+      fi
+      USERNAME="$(tr -d '\n' < "$USERNAME_FILE")"
+      if [ -z "$USERNAME" ]; then
+        USERNAME="$DEFAULT_USERNAME"
+        printf '%s\n' "$USERNAME" > "$USERNAME_FILE"
+      fi
+      if [ "''${#USERNAME}" -gt 32 ]; then
+        echo "RDP username is too long (''${#USERNAME} characters, maximum 32): $USERNAME from $USERNAME_FILE" >&2
+        exit 1
+      fi
+      case "$USERNAME" in
+        [A-Za-z_][A-Za-z0-9_-]*)
+          ;;
+        *)
+          echo "RDP username must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens: $USERNAME from $USERNAME_FILE" >&2
+          exit 1
+          ;;
+      esac
+      chown gnome-remote-desktop:gnome-remote-desktop "$USERNAME_FILE"
+      chmod 600 "$USERNAME_FILE"
 
-      # Generate password on first boot only
+      if [ ! -f "$PASSWORD_FILE" ]; then
-      PASSWORD=""
+        openssl rand -base64 16 > "$PASSWORD_FILE"
-      if [ ! -f /var/lib/gnome-remote-desktop/rdp-password ]; then
+      fi
-        PASSWORD=$(openssl rand -base64 16)
+      PASSWORD="$(tr -d '\n' < "$PASSWORD_FILE")"
-        echo "$PASSWORD" > /var/lib/gnome-remote-desktop/rdp-password
+      if [ -z "$PASSWORD" ]; then
-        chmod 600 /var/lib/gnome-remote-desktop/rdp-password
+        echo "RDP password file is empty: $PASSWORD_FILE" >&2
-      else
+        exit 1
-        PASSWORD=$(cat /var/lib/gnome-remote-desktop/rdp-password)
+      fi
+      if [ "''${#PASSWORD}" -lt 8 ]; then
+        echo "RDP password is too short (''${#PASSWORD} characters, minimum 8): $PASSWORD_FILE" >&2
+        exit 1
+      fi
+      chown gnome-remote-desktop:gnome-remote-desktop "$PASSWORD_FILE"
+      chmod 600 "$PASSWORD_FILE"
+
+      LOCAL_IP="$(hostname -I | awk '{print $1}')"
+      if [ -z "$LOCAL_IP" ]; then
+        LOCAL_IP="127.0.0.1"
       fi
 
-      # Write username to a separate file for the hub
-      echo "sovran" > /var/lib/gnome-remote-desktop/rdp-username
-      chmod 600 /var/lib/gnome-remote-desktop/rdp-username
-
-      # Get current IP address
-      LOCAL_IP=$(hostname -I | awk '{print $1}')
-
-      # Always rewrite the credentials file with the current IP
       cat > "$CRED_FILE" <<EOF
       ========================================
        GNOME Remote Desktop (RDP) Credentials
       ========================================
 
-       Username: sovran
+       Username: $USERNAME
        Password: $PASSWORD
 
        Connect from any RDP client to:
@@ -116,11 +170,22 @@ lib.mkIf config.sovran_systemsOS.features.rdp {
       ========================================
       EOF
 
+      chown gnome-remote-desktop:gnome-remote-desktop "$CRED_FILE"
       chmod 600 "$CRED_FILE"
 
-      # Enable RDP backend and set credentials
+      # Preflight: the NixOS setuid pkexec wrapper must be present and executable
-      grdctl --system rdp enable
+      # before any grdctl --system call.  Absence means the system was booted
-      grdctl --system rdp set-credentials sovran "$PASSWORD"
+      # without security.wrappers or the wrapper directory is not mounted yet.
+      if ! test -x /run/wrappers/bin/pkexec; then
+        echo "Preflight check failed: /run/wrappers/bin/pkexec is absent or not executable." >&2
+        echo "GNOME Remote Desktop system configuration requires the NixOS setuid pkexec wrapper at /run/wrappers/bin/pkexec." >&2
+        exit 1
+      fi
+
+      grdctl_system rdp enable
+      grdctl_system rdp set-tls-cert "$TLS_DIR/rdp-tls.crt"
+      grdctl_system rdp set-tls-key "$TLS_DIR/rdp-tls.key"
+      grdctl_system rdp set-credentials "$USERNAME" "$PASSWORD"
 
       echo "GNOME Remote Desktop RDP configured successfully"
     '';