nixpkgs update

Fix home.tar exit-1 failure on active desktop; harden full backup workflow

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -17,13 +17,26 @@ set -euo pipefail
 BACKUP_LOG="/var/log/sovran-hub-backup.log"
 BACKUP_STATUS="/var/log/sovran-hub-backup.status"
 MEDIA_ROOT="/run/media"
-MIN_FREE_GB=10
 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
 ROLE_STATE_NIX="/etc/nixos/role-state.nix"
+SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
+SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))
 
 # ── Internal drive labels/paths to NEVER use as backup targets ───
 INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
-INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/")
+INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
+
+FAILED_ALREADY=0
+BACKUP_COMPLETE=0
+LND_STOPPED=0
+LND_UNITS_TO_RESTART=()
+
+ARCHIVE_FILES=()
+ARCHIVE_WARNINGS=()
+PARTIAL_FILES=()
+DB_DUMP_FILES=()
+MANIFEST_EXCLUDES=()
+LND_BACKUP_NOTES=()
 
 # ── Logging helpers ──────────────────────────────────────────────
 
@@ -37,16 +50,74 @@ set_status() {
 }
 
 fail() {
+  FAILED_ALREADY=1
   log "ERROR: $*"
   set_status "FAILED"
   exit 1
 }
 
+cleanup() {
+  local rc=$?
+  local restart_failed=0
+
+  # Remove any partial archive files or temporary diagnostic files
+  if [[ "${#PARTIAL_FILES[@]}" -gt 0 ]]; then
+    local partial
+    for partial in "${PARTIAL_FILES[@]}"; do
+      [[ -f "$partial" ]] && rm -f "$partial" || true
+    done
+  fi
+
+  # Release the concurrency lock file descriptor if it was opened
+  [[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true
+
+  if [[ "$LND_STOPPED" -eq 1 ]]; then
+    log "Restarting previously active LND-related services…"
+    for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do
+      local unit="${LND_UNITS_TO_RESTART[$idx]}"
+      if systemctl start "$unit"; then
+        log "Started $unit"
+      else
+        log "ERROR: Failed to start $unit"
+        restart_failed=1
+      fi
+    done
+    LND_STOPPED=0
+  fi
+
+  if [[ "$restart_failed" -eq 1 ]]; then
+    rc=1
+    FAILED_ALREADY=1
+    set_status "FAILED"
+  fi
+
+  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
+    return
+  fi
+
+  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
+    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
+    set_status "FAILED"
+  fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
+}
+
+trap cleanup EXIT
+trap 'exit 1' INT TERM
+
+require_cmd() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
+}
+
 # ── Check whether a mount point is an internal drive ────────────
 
 is_internal() {
   local mnt="$1"
-  # Reject known internal mount points and their subdirectories
   for internal in "${INTERNAL_MOUNTS[@]}"; do
     if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
       return 0
@@ -59,39 +130,37 @@ is_internal() {
 
 find_external_drive() {
   local target=""
-  # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
-  if command -v lsblk &>/dev/null; then
-    while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
-      # Must be a partition or disk, and be removable/hotplug
-      [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
-      [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
-      [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal labels
-      local skip=0
-      for lbl in "${INTERNAL_LABELS[@]}"; do
-        [[ "$label" == "$lbl" ]] && skip=1 && break
-      done
-      [[ "$skip" -eq 1 ]] && continue
+  while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
+    [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
+    [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
+    [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal mount points
-      is_internal "$mountpoint" && continue
+    local skip=0
+    for lbl in "${INTERNAL_LABELS[@]}"; do
+      [[ "$label" == "$lbl" ]] && skip=1 && break
+    done
+    [[ "$skip" -eq 1 ]] && continue
 
-      if mountpoint -q "$mountpoint" 2>/dev/null; then
-        target="$mountpoint"
-        break
-      fi
-    done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
-      python3 -c "
+    is_internal "$mountpoint" && continue
+
+    if mountpoint -q "$mountpoint" 2>/dev/null; then
+      target="$mountpoint"
+      break
+    fi
+  done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
+    python3 -c "
 import sys, json
-data = json.load(sys.stdin)
+
 def flatten(devs):
     for d in devs:
         yield d
         for c in d.get('children', []):
             yield from flatten([c])
+
+data = json.load(sys.stdin)
 for d in flatten(data.get('blockdevices', [])):
-    print('\t'.join([
+    print('\\t'.join([
         d.get('type') or '',
         str(d.get('hotplug') or '0'),
         str(d.get('rm') or '0'),
@@ -99,24 +168,10 @@ for d in flatten(data.get('blockdevices', [])):
         d.get('mountpoint') or '',
     ]))
 " 2>/dev/null || true)
-  fi
 
-  # Fallback: walk /run/media/ if lsblk produced nothing
   if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
     while IFS= read -r -d '' mnt; do
       is_internal "$mnt" && continue
-      # Check label via lsblk on the device backing this mount
-      local dev
-      dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
-      if [[ -n "$dev" ]]; then
-        local lbl
-        lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
-        local skip=0
-        for internal_lbl in "${INTERNAL_LABELS[@]}"; do
-          [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
-        done
-        [[ "$skip" -eq 1 ]] && continue
-      fi
       if mountpoint -q "$mnt" 2>/dev/null; then
         target="$mnt"
         break
@@ -128,16 +183,10 @@ for d in flatten(data.get('blockdevices', [])):
 }
 
 # ── Detect the configured system role ───────────────────────────
-#
-# Priority:
-#   1. Hub config JSON  (/var/lib/sovran-hub/config.json)  — "role" key
-#   2. role-state.nix   (/etc/nixos/role-state.nix)        — grep for true flag
-#   3. Default: server_plus_desktop
 
 detect_role() {
   local role="server_plus_desktop"
 
-  # 1. Try the Hub config JSON
   if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
     local r
     r=$(python3 -c \
@@ -149,7 +198,6 @@ detect_role() {
     fi
   fi
 
-  # 2. Fall back to parsing role-state.nix
   if [[ -f "$ROLE_STATE_NIX" ]]; then
     if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
       role="desktop"
@@ -161,6 +209,64 @@ detect_role() {
   echo "$role"
 }
 
+validate_target_mount() {
+  local target="$1"
+  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
+  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
+  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
+
+  local fstype=""
+  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
+  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
+
+  if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then
+    fail "Target '$target' must be exFAT (detected filesystem: $fstype)."
+  fi
+
+  if [[ "$fstype" == "fuseblk" ]]; then
+    local src_dev blk_type
+    src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true)
+    blk_type=""
+    if [[ -n "$src_dev" ]]; then
+      blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true)
+      [[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true)
+    fi
+    if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then
+      fail "Target '$target' is fuseblk but not identified as exFAT-compatible."
+    fi
+  fi
+
+  local write_test
+  write_test="$target/.sovran-write-test-$$"
+  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
+    fail "Target '$target' is not writable."
+  fi
+
+  log "Verified backup target filesystem: $fstype"
+}
+
+has_unit() {
+  systemctl cat "$1" >/dev/null 2>&1
+}
+
+is_unit_active() {
+  systemctl is-active --quiet "$1"
+}
+
+estimate_path_bytes() {
+  local path="$1"
+  shift || true
+  [[ -e "$path" ]] || {
+    echo 0
+    return
+  }
+
+  local size
+  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
+  [[ -n "$size" ]] || size=0
+  echo "$size"
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -169,14 +275,43 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"
 
+# ── Acquire exclusive run lock ────────────────────────────────────
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd tar
+require_cmd sha256sum
+require_cmd findmnt
+require_cmd lsblk
+require_cmd mountpoint
+require_cmd df
+require_cmd du
+require_cmd awk
+require_cmd sort
+require_cmd find
+require_cmd systemctl
+require_cmd hostname
+require_cmd date
+require_cmd python3
+require_cmd runuser
+require_cmd mktemp
+require_cmd flock
+
 # ── Detect system role ───────────────────────────────────────────
 
 ROLE="$(detect_role)"
 case "$ROLE" in
-  desktop)            ROLE_LABEL="Desktop Only" ;;
-  node)               ROLE_LABEL="Node (Bitcoin-only)" ;;
-  server_plus_desktop) ROLE_LABEL="Server + Desktop" ;;
-  *)                  ROLE_LABEL="$ROLE" ;;
+  desktop)              ROLE_LABEL="Desktop Only" ;;
+  node)                 ROLE_LABEL="Node (Bitcoin-only)" ;;
+  server_plus_desktop)  ROLE_LABEL="Server + Desktop" ;;
+  *)                    ROLE_LABEL="$ROLE" ;;
 esac
 log "Detected role: $ROLE_LABEL"
 
@@ -184,7 +319,6 @@ log "Detected role: $ROLE_LABEL"
 
 if [[ -n "${BACKUP_TARGET:-}" ]]; then
   TARGET="$BACKUP_TARGET"
-  # Safety: never allow internal drives even if explicitly passed
   if is_internal "$TARGET"; then
     fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
   fi
@@ -193,106 +327,472 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. " \
-         "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
+    fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
-# ── Verify mount point ───────────────────────────────────────────
+validate_target_mount "$TARGET"
 
-[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist."
-mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
+# ── Plan role-aware source scope and exclusions ─────────────────
 
-# ── Check free disk space (require ≥ 10 GB) ──────────────────────
+LND_AVAILABLE=0
+if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then
+  LND_AVAILABLE=1
+fi
 
-FREE_KB=$(df -k --output=avail "$TARGET" | tail -1)
-FREE_GB=$(( FREE_KB / 1024 / 1024 ))
+if [[ "$ROLE" == "desktop" ]]; then
+  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)")
+else
+  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent")
+fi
+
+MANIFEST_EXCLUDES+=(
+  "/run/media/Second_Drive (never traversed)"
+  "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)"
+  "/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)"
+  "/var/lib/bitcoind (excluded from manual backup)"
+  "/var/lib/electrs (excluded from manual backup)"
+  "/var/lib/*/log and /var/lib/*/logs"
+  "/var/lib/*/cache and /var/lib/*/tmp"
+  "/home/*/.cache (system and application disk caches)"
+  "/home/*/.local/share/Trash and /home/*/Trash (trash directories)"
+  "/home/*/.mozilla/firefox/*/cache2 and */startupCache (Firefox volatile cache — profile data is kept)"
+  "/home/*/.config/google-chrome/*/Cache (Chrome disk cache — profile data is kept)"
+  "/home/*/.config/chromium/*/Cache (Chromium disk cache — profile data is kept)"
+  "/home/*/.config/BraveSoftware/Brave-Browser/*/Cache (Brave disk cache — profile data is kept)"
+  "/home/*/.local/share/baloo (KDE file indexer — rebuilt automatically)"
+  "/home/*/.thumbnails (thumbnail cache — rebuilt automatically)"
+  "/home/*/.xsession-errors and .xsession-errors.old (X session error logs)"
+)
+
+if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
+  MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive")
+fi
+
+# ── Estimate required free space ─────────────────────────────────
+
+ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
+HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
+SECRETS_BYTES=0
+if [[ "$ROLE" != "desktop" ]]; then
+  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
+fi
+
+VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='bitcoind' \
+  --exclude='electrs' \
+  --exclude='lnd' \
+  --exclude='*/log' \
+  --exclude='*/logs' \
+  --exclude='*/cache' \
+  --exclude='*/tmp')
+
+LND_BYTES=0
+if [[ "$LND_AVAILABLE" -eq 1 ]]; then
+  LND_BYTES=$(estimate_path_bytes /var/lib/lnd)
+fi
+
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES ))
+# Require 20% growth headroom plus an additional fixed 1 GiB safety margin.
+REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
+
+FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
+FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
+REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
+
+log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
+log "Required free space (with safety margin): ${REQUIRED_GB} GB"
 log "Free space on drive: ${FREE_GB} GB"
-(( FREE_GB >= MIN_FREE_GB )) || \
-  fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)."
+
+(( FREE_BYTES >= REQUIRED_BYTES )) || \
+  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
 
 # ── Create timestamped backup directory ─────────────────────────
 
 TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
 BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
-mkdir -p "$BACKUP_DIR"
+DB_DUMP_DIR="$BACKUP_DIR/database-dumps"
+mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR"
+# Write an INCOMPLETE marker immediately; it is removed only on successful completion.
+# Failed runs keep this marker so they are easily identifiable and not confused with
+# complete backups during restore selection.
+touch "$BACKUP_DIR/INCOMPLETE"
 log "Backup destination: $BACKUP_DIR"
 
-# ── Stage 1/4: NixOS configuration ──────────────────────────────
+create_tar_archive() {
+  _create_archive_impl STRICT "$@"
+}
+
+# ── Helper: classify a GNU tar (LC_ALL=C) diagnostic line ────────
+# Returns 0 (true) if the message is an allowlisted transient condition
+# that is safe to ignore for /home (live desktop file changes).
+# Only two verified GNU tar messages qualify; all others are fatal.
+_is_home_warning_allowlisted() {
+  local msg="$1"
+  case "$msg" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+# ── Internal archive builder ──────────────────────────────────────
+# mode: STRICT — any tar error fails.
+#       HOME   — tar exit 1 is accepted when every diagnostic is an
+#                allowlisted transient condition (live file changes).
+_create_archive_impl() {
+  local mode="$1"; shift
+  local archive_name="$1"; shift
+  local archive_path="$BACKUP_DIR/$archive_name"
+  local partial_path="${archive_path}.partial"
+  local diag_tmp
+  # mktemp creates files with mode 0600 (owner-only) by default, so tar
+  # diagnostics (which may include file paths) are not readable by other users.
+  diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)"
+  PARTIAL_FILES+=("$partial_path" "$diag_tmp")
+
+  log "Creating $archive_name …"
+
+  local tar_rc=0
+  LC_ALL=C tar \
+    --create \
+    --file "$partial_path" \
+    --numeric-owner \
+    --acls \
+    --xattrs \
+    --sparse \
+    --one-file-system \
+    "$@" 2>"$diag_tmp" || tar_rc=$?
+
+  # Log every tar diagnostic to the backup log so it appears in the Hub UI
+  local has_fatal_diag=0
+  if [[ -s "$diag_tmp" ]]; then
+    while IFS= read -r diag_line; do
+      [[ -n "$diag_line" ]] || continue
+      log "tar: $diag_line"
+      if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then
+        has_fatal_diag=1
+      fi
+    done < "$diag_tmp"
+  fi
+
+  local accept=0
+  if [[ "$tar_rc" -eq 0 ]]; then
+    accept=1
+  elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+    accept=1
+    log "NOTE: $archive_name completed with nonfatal warnings (live files changed during backup -- this is normal on an active desktop and does not affect the safety of your backup)."
+    ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings -- live files changed during backup (normal on an active desktop)")
+  fi
+
+  if [[ "$accept" -eq 0 ]]; then
+    rm -f "$partial_path" "$diag_tmp"
+    if [[ "$tar_rc" -gt 1 ]]; then
+      fail "tar exited with fatal code $tar_rc while creating $archive_name."
+    elif [[ "$mode" == "HOME" ]]; then
+      fail "tar exited with code $tar_rc and unrecognized diagnostics while creating $archive_name."
+    else
+      fail "tar exited with code $tar_rc while creating $archive_name."
+    fi
+  fi
+
+  # Verify the partial archive is non-empty and readable (spot-check first entry)
+  if [[ ! -s "$partial_path" ]]; then
+    rm -f "$partial_path" "$diag_tmp"
+    fail "Archive $archive_name is empty after creation."
+  fi
+
+  # Fast readability check: read only the first tar entry header (~512 bytes).
+  # head -1 closes the pipe after one line, sending SIGPIPE to tar which then
+  # exits early — so this is O(1) regardless of archive size.
+  local spot_entry
+  spot_entry="$(LC_ALL=C tar --list --file "$partial_path" 2>/dev/null | head -1 || true)"
+  if [[ -z "$spot_entry" ]]; then
+    rm -f "$partial_path" "$diag_tmp"
+    fail "Archive $archive_name failed readability check."
+  fi
+
+  # Atomic publish: rename partial to final path only after acceptance
+  mv "$partial_path" "$archive_path"
+  rm -f "$diag_tmp"
+  ARCHIVE_FILES+=("$archive_name")
+  log "Created archive: $archive_name"
+}
+
+# ── Home archive: tolerates allowlisted live-file warnings ───────
+create_home_tar_archive() {
+  _create_archive_impl HOME "$@"
+}
+
+export_postgresql_dumps() {
+  if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then
+    log "PostgreSQL tools/service not available — skipping PostgreSQL exports."
+    return
+  fi
+
+  if ! is_unit_active "postgresql.service"; then
+    log "PostgreSQL service is not active — skipping PostgreSQL exports."
+    return
+  fi
+
+  log "Exporting PostgreSQL globals and databases…"
+  local globals_file="$DB_DUMP_DIR/postgresql_globals.sql"
+  runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \
+    fail "Failed to export PostgreSQL globals."
+  DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql")
+
+  local dbs
+  dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true)
+
+  if [[ -z "$dbs" ]]; then
+    log "No non-template PostgreSQL application databases found."
+    return
+  fi
+
+  while IFS= read -r db; do
+    [[ -n "$db" ]] || continue
+    local safe_db
+    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
+    local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump"
+    runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \
+      fail "Failed to export PostgreSQL database '$db'."
+    DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump")
+  done <<< "$dbs"
+}
+
+export_mariadb_dumps() {
+  local dump_cmd=""
+  local query_cmd=""
+  local mariadb_unit=""
+
+  if command -v mariadb-dump >/dev/null 2>&1; then
+    dump_cmd="mariadb-dump"
+  elif command -v mysqldump >/dev/null 2>&1; then
+    dump_cmd="mysqldump"
+  fi
+
+  if command -v mariadb >/dev/null 2>&1; then
+    query_cmd="mariadb"
+  elif command -v mysql >/dev/null 2>&1; then
+    query_cmd="mysql"
+  fi
+
+  if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then
+    log "MariaDB dump/query tools not available — skipping MariaDB exports."
+    return
+  fi
+
+  if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then
+    mariadb_unit="mariadb.service"
+  elif has_unit "mysql.service" && is_unit_active "mysql.service"; then
+    mariadb_unit="mysql.service"
+  else
+    log "MariaDB service is not active — skipping MariaDB exports."
+    return
+  fi
+
+  log "Exporting MariaDB databases from ${mariadb_unit}…"
+
+  local dbs
+  dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true)
+  if [[ -z "$dbs" ]]; then
+    log "No MariaDB databases found."
+    return
+  fi
+
+  while IFS= read -r db; do
+    [[ -n "$db" ]] || continue
+    case "$db" in
+      information_schema|performance_schema|mysql|sys) continue ;;
+    esac
+
+    local safe_db out_file
+    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
+    out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql"
+
+    $dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \
+      fail "Failed to export MariaDB database '$db'."
+
+    DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql")
+  done <<< "$dbs"
+}
+
+export_lnd_scb_if_possible() {
+  [[ "$LND_AVAILABLE" -eq 1 ]] || return
+
+  local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb"
+  local attempts=(
+    "lncli exportchanbackup --all --output_file $scb_file"
+    "lncli -n mainnet exportchanbackup --all --output_file $scb_file"
+    "runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file"
+    "runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file"
+  )
+
+  if ! command -v lncli >/dev/null 2>&1; then
+    log "lncli not available — skipping Static Channel Backup export."
+    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)")
+    return
+  fi
+
+  if ! is_unit_active "lnd.service"; then
+    log "LND service is not active — skipping Static Channel Backup export."
+    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)")
+    return
+  fi
+
+  log "Exporting LND Static Channel Backup…"
+  local attempt
+  for attempt in "${attempts[@]}"; do
+    if eval "$attempt" >/dev/null 2>&1; then
+      DB_DUMP_FILES+=("lnd-static-channel-backup.scb")
+      LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli")
+      log "LND Static Channel Backup exported."
+      return
+    fi
+  done
+
+  log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations."
+  LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)")
+}
+
+capture_active_lnd_dependents() {
+  [[ "$LND_AVAILABLE" -eq 1 ]] || return
+
+  LND_UNITS_TO_RESTART=()
+  local raw_units=""
+  raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true)
+
+  while IFS= read -r unit; do
+    [[ -n "$unit" ]] || continue
+    if is_unit_active "$unit"; then
+      LND_UNITS_TO_RESTART+=("$unit")
+    fi
+  done <<< "$raw_units"
+
+  if is_unit_active "lnd.service"; then
+    LND_UNITS_TO_RESTART+=("lnd.service")
+  fi
+}
+
+stop_lnd_stack_if_needed() {
+  [[ "$LND_AVAILABLE" -eq 1 ]] || return
+
+  capture_active_lnd_dependents
+
+  if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then
+    log "No active LND-related services needed stopping."
+    return
+  fi
+
+  log "Stopping active services that depend on LND for clean /var/lib/lnd archive…"
+
+  local unit
+  for unit in "${LND_UNITS_TO_RESTART[@]}"; do
+    if [[ "$unit" == "lnd.service" ]]; then
+      continue
+    fi
+    systemctl stop "$unit" || fail "Failed to stop dependent service: $unit"
+    log "Stopped $unit"
+  done
+
+  if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then
+    systemctl stop lnd.service || fail "Failed to stop lnd.service"
+    log "Stopped lnd.service"
+  fi
+
+  LND_STOPPED=1
+}
+
+# ── Stage 1/5: NixOS configuration ──────────────────────────────
 
 log ""
-log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
+log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 1 failed while copying /etc/nixos"
+  create_tar_archive "etc-nixos.tar" -C / etc/nixos
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
 fi
 
-# ── Stage 2/4: Secrets ──────────────────────────────────────────
+# ── Stage 2/5: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/4: Secrets ───────────────────────────────────────"
-mkdir -p "$BACKUP_DIR/secrets"
-
+log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
+elif [[ -e /etc/nix-bitcoin-secrets ]]; then
+  create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets
 else
-  if [[ -e /etc/nix-bitcoin-secrets ]]; then
-    rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
-  else
-    log "  (not found: /etc/nix-bitcoin-secrets — skipping)"
-  fi
+  log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
-
 log "Stage 2 complete."
 
-# ── Stage 3/4: Home directory ────────────────────────────────────
+# ── Stage 3/5: Home directory ───────────────────────────────────
 
 log ""
-log "── Stage 3/4: Home directory (/home) ───────────────────────"
+log "── Stage 3/5: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  rsync -a --info=progress2 \
-    --exclude='.cache/' \
-    --exclude='.local/share/Trash/' \
-    --exclude='*/Trash/' \
-    /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 3 failed while copying /home"
+  create_home_tar_archive "home.tar" \
+    -C / \
+    --exclude='home/*/.cache' \
+    --exclude='home/*/.local/share/Trash' \
+    --exclude='home/*/Trash' \
+    --exclude='home/*/.mozilla/firefox/*/cache2' \
+    --exclude='home/*/.mozilla/firefox/*/startupCache' \
+    --exclude='home/*/.mozilla/firefox/*/thumbnails' \
+    --exclude='home/*/.config/google-chrome/*/Cache' \
+    --exclude='home/*/.config/google-chrome/*/Code Cache' \
+    --exclude='home/*/.config/chromium/*/Cache' \
+    --exclude='home/*/.config/chromium/*/Code Cache' \
+    --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache' \
+    --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Code Cache' \
+    --exclude='home/*/.local/share/baloo' \
+    --exclude='home/*/.thumbnails' \
+    --exclude='home/*/.xsession-errors' \
+    --exclude='home/*/.xsession-errors.old' \
+    home
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/4: System data ───────────────────────────────────────
+# ── Stage 4/5: Database exports + LND artifacts ────────────────
 
 log ""
-log "── Stage 4/4: System data (/var/lib) ────────────────────────"
-if [[ "$ROLE" == "desktop" ]]; then
-  if [[ -d /var/lib ]]; then
-    rsync -a --info=progress2 \
-      --filter='- /lnd/***' \
-      --exclude='logs/' \
-      --exclude='log/' \
-      --exclude='*/logs/' \
-      --exclude='*/log/' \
-      /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      fail "Stage 4 failed while copying /var/lib for Desktop Only role"
-    log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)."
-  else
-    log "WARNING: /var/lib not found — skipping."
+log "── Stage 4/5: Database and LND consistency exports ─────────"
+export_postgresql_dumps
+export_mariadb_dumps
+export_lnd_scb_if_possible
+
+if [[ "$LND_AVAILABLE" -eq 1 ]]; then
+  stop_lnd_stack_if_needed
+  create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd
+  LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop")
+fi
+
+log "Stage 4 complete."
+
+# ── Stage 5/5: System data ──────────────────────────────────────
+
+log ""
+log "── Stage 5/5: System data (/var/lib) ───────────────────────"
+if [[ -d /var/lib ]]; then
+  VAR_LIB_EXCLUDES=(
+    --exclude='var/lib/bitcoind'
+    --exclude='var/lib/electrs'
+    --exclude='var/lib/*/log'
+    --exclude='var/lib/*/logs'
+    --exclude='var/lib/*/cache'
+    --exclude='var/lib/*/tmp'
+  )
+
+  if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
+    VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd')
   fi
-elif [[ -d /var/lib ]]; then
-  rsync -a --info=progress2 \
-    --exclude='logs/' \
-    --exclude='log/' \
-    --exclude='*/logs/' \
-    --exclude='*/log/' \
-    /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 4 failed while copying /var/lib"
-  log "Stage 4 complete."
+
+  create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib
+  log "Stage 5 complete."
 else
   log "WARNING: /var/lib not found — skipping."
 fi
@@ -301,21 +801,110 @@ fi
 
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
+MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
+CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt"
+
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date)"
+  echo "Generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+  echo "Timestamp: $TIMESTAMP"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Contents:"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort
-} > "$BACKUP_DIR/BACKUP_MANIFEST.txt"
-log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo "Source paths included:"
+  echo "- /etc/nixos"
+  echo "- /home"
+  if [[ "$ROLE" != "desktop" ]]; then
+    echo "- /etc/nix-bitcoin-secrets (when present)"
+  fi
+  echo "- /var/lib"
+  echo ""
+  echo "Exclusions:"
+  for ex in "${MANIFEST_EXCLUDES[@]}"; do
+    echo "- $ex"
+  done
+  echo ""
+  echo "Archives:"
+  for archive in "${ARCHIVE_FILES[@]}"; do
+    echo "- $archive"
+  done
+  echo ""
+  echo "Database and LND exports:"
+  if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for dump in "${DB_DUMP_FILES[@]}"; do
+      echo "- $dump"
+    done
+    for note in "${LND_BACKUP_NOTES[@]}"; do
+      echo "- $note"
+    done
+  fi
+  echo ""
+  echo "Restore guidance:"
+  echo "- Verify artifacts: cd <backup_dir> && sha256sum -c SHA256SUMS.txt"
+  echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf <archive>.tar -C /"
+  echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql"
+  echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_<db>.dump"
+  echo "- MariaDB DB dump: mariadb <db_name> < database-dumps/mariadb_<db>.sql"
+  echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures"
+  echo "- Note: when restoring /var/lib, exclude raw DB directories (var/lib/postgresql, var/lib/mysql)"
+  echo "  and restore from native dumps instead for PostgreSQL and MariaDB"
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#ARCHIVE_WARNINGS[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for warning in "${ARCHIVE_WARNINGS[@]}"; do
+      echo "- $warning"
+    done
+  fi
+  echo ""
+  echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "from manual external backup because they already live on the internal second drive"
+  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
+  echo ""
+  echo "Artifact listing:"
+  # INCOMPLETE exists at this point (created at backup start); BACKUP_COMPLETE does not
+  # exist yet (written after checksums). Excluding both marker files here is intentional:
+  # INCOMPLETE is excluded so it doesn't appear as a data artifact, and BACKUP_COMPLETE
+  # is excluded defensively for consistency should the ordering ever change.
+  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f \
+    ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z | tr '\0' '\n'
+} > "$MANIFEST_FILE"
+
+# ── Generate checksums for all backup artifacts ─────────────────
+
+log "Generating SHA-256 checksums …"
+(
+  cd "$BACKUP_DIR"
+  while IFS= read -r -d '' file; do
+    sha256sum "$file"
+  done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z)
+) > "$CHECKSUM_FILE"
+
+log "Manifest written to $MANIFEST_FILE"
+log "Checksums written to $CHECKSUM_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
+if [[ "${#ARCHIVE_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${ARCHIVE_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that changed"
+  log "during backup, which is normal on an active desktop and does not affect your backup."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
 log "Please eject the drive safely before removing it from your Sovran Pro."
+
+# Remove incomplete marker and write completion marker only after all work succeeds
+rm -f "$BACKUP_DIR/INCOMPLETE"
+echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE"
+
+BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/server.py

@@ -8,6 +8,7 @@ import contextlib
 import glob
 import hashlib
 import hmac
+import ipaddress
 import json
 import logging
 import os
@@ -80,6 +81,15 @@ DOMAINS_DIR = "/var/lib/domains"
 NOSTR_NPUB_FILE   = "/var/lib/secrets/nostr_npub"
 NJALLA_SCRIPT     = "/var/lib/njalla/njalla.sh"
 
+# Systemd service that rewrites the Sovran-managed /etc/hosts loopback block
+SOVRAN_HOSTS_SERVICE = "sovran-hosts-update.service"
+
+# Domain keys that produce a public HTTPS virtual host via Caddy
+_SERVICE_DOMAIN_KEYS = frozenset([
+    "matrix", "wordpress", "nextcloud", "btcpayserver",
+    "vaultwarden", "haven", "element-calling",
+])
+
 INTERNAL_IP_FILE = "/var/lib/secrets/internal-ip"
 ZEUS_CONNECT_FILE = "/var/lib/secrets/zeus-connect-url"
 
@@ -281,9 +291,6 @@ FEATURE_SERVICE_MAP = {
 }
 
 # Port requirements for service tiles (keyed by unit name or icon)
-_PORTS_MATRIX_FEDERATION = [
-    {"port": "8448", "protocol": "TCP", "description": "Matrix server-to-server federation"},
-]
 _PORTS_ELEMENT_CALLING = [
     {"port": "7881",        "protocol": "TCP",     "description": "LiveKit WebRTC signalling"},
     {"port": "7882",        "protocol": "UDP",     "description": "LiveKit media (UDP mux)"},
@@ -296,7 +303,7 @@ SERVICE_PORT_REQUIREMENTS: dict[str, list[dict]] = {
     # Infrastructure
     "caddy.service":                    [],
     # Communication
-    "matrix-synapse.service":           _PORTS_MATRIX_FEDERATION,
+    "matrix-synapse.service":           [],
     "livekit.service":                  _PORTS_ELEMENT_CALLING,
     # Domain-based apps (80/443 handled by end-to-end domain reachability checks)
     "btcpayserver.service":             [],
@@ -967,18 +974,94 @@ def _check_port_status(
     return "closed"
 
 
+
+# Regex for validating domain values written into /etc/hosts.  Rejects anything
+# containing whitespace, newlines, or characters that could escape a hosts entry.
+# NOTE: The equivalent pattern in modules/core/local-domain-loopback.nix (shell
+# grep -E) must be kept in sync with this Python regex.
+_SAFE_DOMAIN_RE = re.compile(
+    r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'
+)
+
+
+def _validate_domain_value(domain: str) -> bool:
+    """Return True if *domain* is a valid hostname safe to write into /etc/hosts.
+
+    Rejects values containing whitespace, newlines, or other characters that
+    could inject additional entries or corrupt the hosts file.
+    """
+    if not domain or len(domain) > 253:
+        return False
+    # Guard against newline / whitespace injection before regex check.
+    if any(c in domain for c in ('\n', '\r', ' ', '\t', '#')):
+        return False
+    return bool(_SAFE_DOMAIN_RE.match(domain))
+
+
+def _is_loopback_address(ip: str) -> bool:
+    """Return True if *ip* is a loopback address (127.0.0.0/8 or ::1)."""
+    try:
+        return ipaddress.ip_address(ip).is_loopback
+    except ValueError:
+        return False
+
+
+def _resolve_all_addresses(domain: str) -> list[str]:
+    """Return all unique IP addresses that *domain* resolves to, or an empty list.
+
+    The first element is the address that the system resolver would normally
+    use for a connection.  All elements are checked when determining whether
+    any address matches the expected public IP or is a loopback address.
+    """
+    try:
+        results = socket.getaddrinfo(domain, None)
+        unique_addresses: list[str] = []
+        for r in results:
+            addr = r[4][0]
+            if addr not in unique_addresses:
+                unique_addresses.append(addr)
+        return unique_addresses
+    except Exception:
+        return []
+
+
+def _trigger_hosts_update() -> None:
+    """Start the sovran-hosts-update systemd service (best-effort, no-op if unavailable)."""
+    try:
+        subprocess.run(
+            ["systemctl", "start", SOVRAN_HOSTS_SERVICE],
+            timeout=30,
+            check=False,
+            capture_output=True,
+        )
+    except Exception:
+        pass
+
+
 def _check_domain_reachable(domain: str) -> dict:
-    """Curl the domain to verify end-to-end HTTPS reachability."""
+    """Check HTTPS reachability for *domain* via local Caddy (loopback).
+
+    Using ``--resolve`` ensures the request reaches Caddy on this computer
+    without depending on router NAT loopback or the public DNS result.
+    A successful local check is sufficient to confirm that Caddy and the
+    virtual-host configuration are working correctly.
+    """
     try:
         result = subprocess.run(
-            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "10", f"https://{domain}"],
+            [
+                "curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}",
+                "--max-time", "10",
+                "--resolve", f"{domain}:443:127.0.0.1",
+                "--resolve", f"{domain}:80:127.0.0.1",
+                f"https://{domain}",
+            ],
             capture_output=True,
             text=True,
             timeout=15,
         )
         status_code = result.stdout.strip()
         if status_code and status_code.isdigit() and int(status_code) > 0:
-            return {"reachable": True, "status_code": int(status_code)}
+            return {"reachable": True, "status_code": int(status_code), "via_loopback": True}
         return {"reachable": False, "error": result.stderr.strip() or "No response"}
     except subprocess.TimeoutExpired:
         return {"reachable": False, "error": "timeout"}
@@ -987,25 +1070,28 @@ def _check_domain_reachable(domain: str) -> dict:
 
 
 def _check_domain_health_fast(domain: str | None, external_ip: str) -> bool:
-    """Fast domain issue check for tile health (no curl/subprocess calls)."""
+    """Fast domain issue check for tile health (no curl/subprocess calls).
+
+    Returns ``True`` when a domain issue is detected that warrants
+    ``needs_attention``, ``False`` otherwise.
+    Loopback resolution is treated as an intentional server-local override,
+    not a DNS mismatch.
+    """
     if not domain:
         return True
 
-    resolved_ip: str | None = None
-    try:
-        results = socket.getaddrinfo(domain, None)
-        if results:
-            resolved_ip = results[0][4][0]
-    except socket.gaierror:
-        resolved_ip = None
-    except Exception:
-        resolved_ip = None
-
-    if not resolved_ip:
+    addrs = _resolve_all_addresses(domain)
+    if not addrs:
         return True
+
+    # If every resolved address is loopback the intentional /etc/hosts
+    # override is in place — this is healthy, not a mismatch.
+    if all(_is_loopback_address(a) for a in addrs):
+        return False
+
     if external_ip == "unavailable":
         return False
-    return resolved_ip != external_ip
+    return not any(a == external_ip for a in addrs)
 
 
 def _is_domain_reachable_cached(domain: str) -> bool | None:
@@ -1073,15 +1159,8 @@ def _evaluate_domain_checklist(
         "detail": domain,
     })
 
-    resolved_ip: str | None = None
-    try:
-        results = socket.getaddrinfo(domain, None)
-        if results:
-            resolved_ip = results[0][4][0]
-    except socket.gaierror:
-        resolved_ip = None
-    except Exception:
-        resolved_ip = None
+    addrs = _resolve_all_addresses(domain)
+    resolved_ip: str | None = addrs[0] if addrs else None
 
     if not resolved_ip:
         domain_status = {
@@ -1108,7 +1187,31 @@ def _evaluate_domain_checklist(
             "has_issues": True,
         }
 
-    if external_ip == "unavailable":
+    # Detect intentional server-local loopback override from /etc/hosts.
+    # When all addresses are loopback the public DNS is not checked via the
+    # system resolver (which would always return the override).  We proceed
+    # to the reachability check so Caddy health can still be verified.
+    loopback_override = all(_is_loopback_address(a) for a in addrs)
+
+    if loopback_override:
+        domain_status = {
+            "status": "local_override",
+            "resolved_ip": resolved_ip,
+            "expected_ip": external_ip,
+        }
+        steps.append({
+            "step": 2,
+            "label": "DNS / Local Override",
+            "status": "ok",
+            "detail": (
+                "Server-local loopback override is active — this computer routes the domain "
+                "directly to Caddy without going through the router. "
+                "Public DNS cannot be verified from this computer while the override is in place. "
+                "To check your public DNS from outside, use a tool such as "
+                "https://dnschecker.org or run: dig @1.1.1.1 " + domain
+            ),
+        })
+    elif external_ip == "unavailable":
         domain_status = {
             "status": "error",
             "resolved_ip": resolved_ip,
@@ -1120,7 +1223,7 @@ def _evaluate_domain_checklist(
             "status": "warning",
             "detail": f"Resolves to {resolved_ip} (external IP unavailable for comparison)",
         })
-    elif resolved_ip != external_ip:
+    elif not any(a == external_ip for a in addrs):
         domain_status = {
             "status": "dns_mismatch",
             "resolved_ip": resolved_ip,
@@ -1222,7 +1325,7 @@ def _generate_qr_base64(data: str) -> str | None:
 # ── Update helpers (file-based, no systemctl) ────────────────────
 
 def _read_update_status() -> str:
-    """Read the status file. Returns RUNNING, SUCCESS, FAILED, or IDLE."""
+    """Read the status file. Returns RUNNING, SUCCESS, REBOOT_REQUIRED, FAILED, or IDLE."""
     try:
         with open(UPDATE_STATUS, "r") as f:
             return f.read().strip()
@@ -1348,6 +1451,12 @@ def _read_backup_status() -> str:
         return "IDLE"
 
 
+def _write_backup_status(value: str) -> None:
+    """Write backup status file."""
+    with open(BACKUP_STATUS, "w") as f:
+        f.write(value)
+
+
 def _read_backup_log(offset: int = 0) -> tuple[str, int]:
     """Read the backup log file from the given byte offset.
     Returns (new_text, new_offset)."""
@@ -1364,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]:
         return "", 0
 
 
+def _append_backup_log(line: str) -> None:
+    """Append one line to backup log."""
+    with open(BACKUP_LOG, "a") as f:
+        f.write(line.rstrip("\n") + "\n")
+
+
 _INTERNAL_LABELS  = {"BTCEcoandBackup", "sovran_systemsos"}
 _INTERNAL_MOUNTS  = {"/", "/boot/efi"}
 _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive"
@@ -1378,6 +1493,41 @@ def _is_internal_mount(mnt: str) -> bool:
     return False
 
 
+def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
+    """Return whether the target filesystem type is supported for manual backup."""
+    fstype = (fstype or "").lower()
+    if fstype == "exfat":
+        return True
+    if fstype != "fuseblk":
+        return False
+
+    src_dev = ""
+    try:
+        result = subprocess.run(
+            ["findmnt", "-n", "-o", "SOURCE", "-T", path],
+            capture_output=True, text=True, timeout=5,
+        )
+        if result.returncode == 0:
+            src_dev = result.stdout.strip()
+    except Exception:
+        src_dev = ""
+
+    if not src_dev:
+        return False
+
+    for cmd in (
+        ["lsblk", "-no", "FSTYPE", src_dev],
+        ["blkid", "-o", "value", "-s", "TYPE", src_dev],
+    ):
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
+            if result.returncode == 0 and result.stdout.strip().lower() in {"exfat", "fuseblk"}:
+                return True
+        except Exception:
+            continue
+    return False
+
+
 def _detect_external_drives() -> list[dict]:
     """Scan for mounted external USB drives.
 
@@ -1387,7 +1537,8 @@ def _detect_external_drives() -> list[dict]:
     /run/media/ directly if lsblk is unavailable, applying the same
     label/path filters.
 
-    Returns a list of dicts with name, path, free_gb, total_gb.
+    Returns:
+        list[dict]: Each dict contains name, path, free_gb, total_gb, fstype.
     """
     import json as _json
     import subprocess as _subprocess
@@ -1398,7 +1549,7 @@ def _detect_external_drives() -> list[dict]:
     # ── Primary path: lsblk JSON ────────────────────────────────
     try:
         result = _subprocess.run(
-            ["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"],
+            ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"],
             capture_output=True, text=True, timeout=10
         )
         if result.returncode == 0:
@@ -1416,6 +1567,7 @@ def _detect_external_drives() -> list[dict]:
                 hotplug    = str(dev.get("hotplug", "0"))
                 rm         = str(dev.get("rm", "0"))
                 label      = dev.get("label") or ""
+                fstype     = (dev.get("fstype") or "").lower()
                 mountpoint = dev.get("mountpoint") or ""
 
                 if dev_type not in ("part", "disk"):
@@ -1441,6 +1593,7 @@ def _detect_external_drives() -> list[dict]:
                         "path":     mountpoint,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(mountpoint)
                 except OSError:
@@ -1474,11 +1627,20 @@ def _detect_external_drives() -> list[dict]:
                     st = os.statvfs(drive_path)
                     total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1)
                     free_gb  = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1)
+                    fstype = ""
+                    try:
+                        fstype = _subprocess.run(
+                            ["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path],
+                            capture_output=True, text=True, timeout=5
+                        ).stdout.strip().lower()
+                    except Exception:
+                        fstype = ""
                     drives.append({
                         "name":     drive_name,
                         "path":     drive_path,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(drive_path)
                 except OSError:
@@ -1953,10 +2115,13 @@ def _verify_support_removed() -> bool:
 
 @app.get("/login", response_class=HTMLResponse)
 async def login_page(request: Request):
-    return templates.TemplateResponse("login.html", {
-        "request": request,
-        "asset_version": ASSET_VERSION,
-    })
+    return templates.TemplateResponse(
+        request=request,
+        name="login.html",
+        context={
+            "asset_version": ASSET_VERSION,
+        },
+    )
 
 
 @app.get("/auto-login")
@@ -2021,20 +2186,26 @@ async def api_logout(request: Request):
 
 @app.get("/", response_class=HTMLResponse)
 async def index(request: Request):
-    return templates.TemplateResponse("index.html", {
-        "request": request,
-        "asset_version": ASSET_VERSION,
-    })
+    return templates.TemplateResponse(
+        request=request,
+        name="index.html",
+        context={
+            "asset_version": ASSET_VERSION,
+        },
+    )
 
 
 @app.get("/onboarding", response_class=HTMLResponse)
 async def onboarding(request: Request):
     _ensure_onboarding_reopened_for_migration()
-    return templates.TemplateResponse("onboarding.html", {
-        "request": request,
-        "asset_version": ASSET_VERSION,
-        "onboarding_js_hash": _ONBOARDING_JS_HASH,
-    })
+    return templates.TemplateResponse(
+        request=request,
+        name="onboarding.html",
+        context={
+            "asset_version": ASSET_VERSION,
+            "onboarding_js_hash": _ONBOARDING_JS_HASH,
+        },
+    )
 
 
 @app.get("/api/onboarding/status")
@@ -2743,19 +2914,17 @@ async def api_services():
                     break
         has_domain_issues = False
         if needs_domain and domain and enabled:
+            addrs = _resolve_all_addresses(domain)
             dns_ok = True
-            try:
-                results = socket.getaddrinfo(domain, None)
-                if results:
-                    resolved_ip = results[0][4][0]
-                    if (
-                        _cached_external_ip != "unavailable"
-                        and resolved_ip != _cached_external_ip
-                    ):
-                        dns_ok = False
-                else:
-                    dns_ok = False
-            except (socket.gaierror, Exception):
+            if not addrs:
+                dns_ok = False
+            elif all(_is_loopback_address(a) for a in addrs):
+                # Intentional server-local /etc/hosts override — not a mismatch.
+                dns_ok = True
+            elif (
+                _cached_external_ip != "unavailable"
+                and not any(a == _cached_external_ip for a in addrs)
+            ):
                 dns_ok = False
 
             if not dns_ok:
@@ -2983,36 +3152,28 @@ async def api_service_detail(unit: str, icon: str | None = None):
                 "status": ps,
                 "description": p.get("description", ""),
             })
-    extra_ports = port_statuses if unit in ("matrix-synapse.service", "livekit.service") else []
+    extra_ports = port_statuses if unit == "livekit.service" else []
 
-    if needs_domain and unit in ("matrix-synapse.service", "livekit.service"):
+    if needs_domain and unit == "livekit.service":
         if has_domain_issues:
             domain_check_steps.append({
                 "step": 4,
-                "label": "Federation Port" if unit == "matrix-synapse.service" else "Additional Ports Required",
+                "label": "Router Setup Needed",
                 "status": "skipped",
-                "detail": "Skipped until Steps 1-3 are complete",
+                "detail": "Finish the domain steps first, then forward the Element Call ports in your router.",
             })
-        elif unit == "matrix-synapse.service":
-            if extra_ports:
-                matrix_open = extra_ports[0]["status"] != "closed"
-                domain_check_steps.append({
-                    "step": 4,
-                    "label": "Federation Port",
-                    "status": "ok" if matrix_open else "error",
-                    "detail": (
-                        f"Matrix federation port 8448 (TCP) is {'open' if matrix_open else 'closed'}.\n"
-                        f"Matrix federation requires port 8448 (TCP) forwarded to {internal_ip}"
-                    ),
-                })
         else:
-            extra_open = all(p["status"] != "closed" for p in extra_ports)
+            # These checks are local-only (listening/firewall state on this computer),
+            # not an outside-in verification of router/NAT forwarding.
+            all_local_ready = all(p["status"] != "closed" for p in extra_ports)
             domain_check_steps.append({
                 "step": 4,
-                "label": "Additional Ports Required",
-                "status": "ok" if extra_open else "error",
+                "label": "Router Setup Needed" if all_local_ready else "Sovran_SystemsOS Port Setup Needed",
+                "status": "warning" if all_local_ready else "error",
                 "detail": (
-                    "Element-Call/LiveKit requires additional forwarded ports for WebRTC and TURN traffic."
+                    "Sovran_SystemsOS is ready to use these ports on this computer. Now forward them in your router so Element Call can work from outside your home network."
+                    if all_local_ready
+                    else "Sovran_SystemsOS is not ready to use all required Element Call ports on this computer yet. Fix the ports marked “Not ready yet” below, then forward them in your router."
                 ),
             })
 
@@ -3580,6 +3741,37 @@ async def api_backup_drives():
     return {"drives": drives}
 
 
+async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
+    """Drain stderr, then mark status FAILED if backup subprocess exits unexpectedly."""
+    stderr_chunks: list[bytes] = []
+
+    async def _drain_stderr() -> None:
+        if proc.stderr is not None:
+            async for line in proc.stderr:
+                stderr_chunks.append(line)
+
+    drain_task = asyncio.create_task(_drain_stderr())
+    rc = await proc.wait()
+    await drain_task
+
+    if rc == 0:
+        return
+
+    loop = asyncio.get_event_loop()
+    status = await loop.run_in_executor(None, _read_backup_status)
+    if status in {"SUCCESS", "FAILED"}:
+        return
+
+    detail = ""
+    if stderr_chunks:
+        stderr_text = b"".join(stderr_chunks).decode("utf-8", errors="replace").strip()
+        if stderr_text:
+            detail = f" — stderr: {stderr_text}"
+    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc}).{detail}"
+    await loop.run_in_executor(None, _append_backup_log, msg)
+    await loop.run_in_executor(None, _write_backup_status, "FAILED")
+
+
 @app.post("/api/backup/run")
 async def api_backup_run(target: str = ""):
     """Start the backup script as a background subprocess.
@@ -3590,6 +3782,26 @@ async def api_backup_run(target: str = ""):
     if status == "RUNNING":
         return {"ok": True, "status": "already_running"}
 
+    drives = await loop.run_in_executor(None, _detect_external_drives)
+    if not drives:
+        raise HTTPException(status_code=400, detail="No external backup drive detected.")
+
+    drive_map = {d.get("path", ""): d for d in drives if d.get("path")}
+    if target:
+        if target not in drive_map:
+            raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.")
+        selected = drive_map[target]
+    else:
+        selected = drives[0]
+
+    selected_target = selected.get("path", "")
+    selected_fstype = (selected.get("fstype") or "").lower()
+    if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.",
+        )
+
     # Clear stale log before starting
     try:
         with open(BACKUP_LOG, "w") as f:
@@ -3597,21 +3809,48 @@ async def api_backup_run(target: str = ""):
     except OSError:
         pass
 
-    env = dict(os.environ)
-    if target:
-        env["BACKUP_TARGET"] = target
+    try:
+        await loop.run_in_executor(None, _write_backup_status, "RUNNING")
+    except OSError as exc:
+        raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}")
 
-    # Fire-and-forget: the script writes its own status/log files.
-    # Progress is read by the client via /api/backup/status (same pattern
-    # as /api/updates/run and the rebuild feature).
-    await asyncio.create_subprocess_exec(
-        "/usr/bin/env", "bash", BACKUP_SCRIPT,
-        stdout=asyncio.subprocess.DEVNULL,
-        stderr=asyncio.subprocess.DEVNULL,
-        env=env,
+    await loop.run_in_executor(
+        None,
+        _append_backup_log,
+        f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…",
     )
 
-    return {"ok": True, "status": "started"}
+    env = dict(os.environ)
+    env["BACKUP_TARGET"] = selected_target
+
+    bash_path = shutil.which("bash")
+    if bash_path is None:
+        no_bash_msg = (
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Cannot start backup:"
+            " interpreter 'bash' not found on PATH."
+            " Ensure pkgs.bash is in the sovran-hub-web service PATH."
+        )
+        await loop.run_in_executor(None, _append_backup_log, no_bash_msg)
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(
+            status_code=500,
+            detail="Backup interpreter (bash) not available. Check service PATH configuration.",
+        )
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bash_path, BACKUP_SCRIPT,
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except Exception as exc:
+        await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}")
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(status_code=500, detail="Failed to launch backup process.")
+
+    asyncio.create_task(_monitor_backup_subprocess(proc))
+    return {"ok": True, "status": "started", "target": selected_target}
 
 
 # ── Feature Manager endpoints ─────────────────────────────────────
@@ -3888,6 +4127,12 @@ async def api_domains_set(req: DomainSetRequest):
         except Exception:
             pass
 
+    # Regenerate the server-local /etc/hosts loopback entries so the newly
+    # saved domain is immediately reachable on this computer without NAT
+    # loopback support on the router.
+    if req.domain_name in _SERVICE_DOMAIN_KEYS:
+        _trigger_hosts_update()
+
     return {"ok": True}
 
 
@@ -3935,38 +4180,35 @@ async def api_domains_check(req: DomainCheckRequest):
     external_ip = _cached_external_ip
 
     def check_domain(domain: str) -> dict:
-        try:
-            results = socket.getaddrinfo(domain, None)
-            if not results:
-                return {
-                    "domain": domain, "status": "unresolvable",
-                    "resolved_ip": None, "expected_ip": external_ip,
-                }
-            resolved_ip = results[0][4][0]
-            if external_ip == "unavailable":
-                return {
-                    "domain": domain, "status": "error",
-                    "resolved_ip": resolved_ip, "expected_ip": external_ip,
-                }
-            if resolved_ip == external_ip:
-                return {
-                    "domain": domain, "status": "connected",
-                    "resolved_ip": resolved_ip, "expected_ip": external_ip,
-                }
-            return {
-                "domain": domain, "status": "dns_mismatch",
-                "resolved_ip": resolved_ip, "expected_ip": external_ip,
-            }
-        except socket.gaierror:
+        addrs = _resolve_all_addresses(domain)
+        if not addrs:
             return {
                 "domain": domain, "status": "unresolvable",
                 "resolved_ip": None, "expected_ip": external_ip,
             }
-        except Exception:
+        resolved_ip = addrs[0]
+        # Server-local /etc/hosts loopback override — report as such rather
+        # than as a DNS mismatch.  Public DNS cannot be verified from this
+        # computer when the override is active.
+        if all(_is_loopback_address(a) for a in addrs):
+            return {
+                "domain": domain, "status": "local_override",
+                "resolved_ip": resolved_ip, "expected_ip": external_ip,
+            }
+        if external_ip == "unavailable":
             return {
                 "domain": domain, "status": "error",
-                "resolved_ip": None, "expected_ip": external_ip,
+                "resolved_ip": resolved_ip, "expected_ip": external_ip,
             }
+        if any(a == external_ip for a in addrs):
+            return {
+                "domain": domain, "status": "connected",
+                "resolved_ip": resolved_ip, "expected_ip": external_ip,
+            }
+        return {
+            "domain": domain, "status": "dns_mismatch",
+            "resolved_ip": resolved_ip, "expected_ip": external_ip,
+        }
 
     check_results = await asyncio.gather(*[
         loop.run_in_executor(None, check_domain, d) for d in req.domains
@@ -4757,17 +4999,21 @@ def _recover_stale_status(status_file: str, log_file: str, unit_name: str) -> bo
     except Exception:
         pass
 
-    new_status = "SUCCESS" if unit_result == "success" else "FAILED"
+    if unit_result == "success":
+        new_status = "REBOOT_REQUIRED" if unit_name == UPDATE_UNIT else "SUCCESS"
+    else:
+        new_status = "FAILED"
     try:
         with open(status_file, "w") as f:
             f.write(new_status)
     except OSError:
         pass
-    msg = (
-        "\n[Update completed successfully while the server was restarting.]\n"
-        if new_status == "SUCCESS"
-        else "\n[Update encountered an error. See log above for details.]\n"
-    )
+    if new_status == "REBOOT_REQUIRED":
+        msg = "\n[Update staged successfully while the server was restarting. Reboot required.]\n"
+    elif new_status == "SUCCESS":
+        msg = "\n[Update completed successfully while the server was restarting.]\n"
+    else:
+        msg = "\n[Update encountered an error. See log above for details.]\n"
     try:
         with open(log_file, "a") as f:
             f.write(msg)

app/sovran_systemsos_web/static/css/header.css

@@ -91,3 +91,30 @@
   border-color: var(--accent-color);
   color: var(--accent-color);
 }
+
+/* ── Header reboot button ───────────────────────────────────────── */
+
+.btn-header-reboot {
+  background: transparent;
+  border: 1px solid rgba(184, 125, 0, 0.35);
+  color: #c98d08;
+  font-size: 0.78rem;
+  font-weight: 600;
+  padding: 4px 12px;
+  border-radius: var(--radius-btn);
+  cursor: pointer;
+  transition: border-color 0.15s, color 0.15s, background-color 0.15s;
+}
+
+.btn-header-reboot:hover {
+  border-color: #b87d00;
+  color: #e0a010;
+  background-color: rgba(184, 125, 0, 0.1);
+}
+
+@media (max-width: 480px) {
+  .btn-header-reboot {
+    padding: 4px 8px;
+    font-size: 0.72rem;
+  }
+}

app/sovran_systemsos_web/static/css/modals.css

@@ -102,6 +102,48 @@ button.btn-reboot:hover:not(:disabled) {
   background-color: #529E7E;
 }
 
+/* Restart = AMBER (manual restart action) */
+.btn-restart-amber {
+  background-color: #b87d00;
+  color: #fff;
+}
+
+.btn-restart-amber:hover:not(:disabled) {
+  background-color: #9a6800;
+}
+
+/* Restart conflict warning box */
+.restart-conflict-box {
+  background-color: rgba(180, 100, 0, 0.12);
+  border-left: 3px solid #c97a00;
+  border-radius: 6px;
+  padding: 12px 14px;
+  margin-bottom: 14px;
+}
+
+.restart-conflict-title {
+  font-size: 0.88rem;
+  font-weight: 700;
+  color: #e69000;
+  margin: 0 0 6px 0;
+}
+
+.restart-conflict-desc {
+  font-size: 0.83rem;
+  color: var(--text-secondary);
+  line-height: 1.5;
+  margin: 0;
+}
+
+/* Reboot error card actions row */
+.reboot-error-actions {
+  display: flex;
+  gap: 12px;
+  justify-content: center;
+  flex-wrap: wrap;
+  margin-top: 20px;
+}
+
 .btn-save {
   background-color: var(--yellow);
   color: #0A1A10;

app/sovran_systemsos_web/static/js/events.js

@@ -44,6 +44,28 @@ if ($upgradeCloseBtn) $upgradeCloseBtn.addEventListener("click", closeUpgradeMod
 if ($upgradeCancelBtn) $upgradeCancelBtn.addEventListener("click", closeUpgradeModal);
 if ($upgradeModal) $upgradeModal.addEventListener("click", function(e) { if (e.target === $upgradeModal) closeUpgradeModal(); });
 
+// Restart confirm dialog
+if ($restartConfirmCancel) $restartConfirmCancel.addEventListener("click", closeRestartConfirmDialog);
+if ($restartConfirmModal) $restartConfirmModal.addEventListener("click", function(e) { if (e.target === $restartConfirmModal) closeRestartConfirmDialog(); });
+if ($restartConfirmModal) $restartConfirmModal.addEventListener("keydown", function(e) { if (e.key === "Escape") closeRestartConfirmDialog(); });
+
+// Header Reboot button
+if ($headerRebootBtn) $headerRebootBtn.addEventListener("click", function() { openRestartConfirmDialog(); });
+if ($restartConfirmOk) $restartConfirmOk.addEventListener("click", function() {
+  if ($restartConfirmOk.disabled) return;
+  $restartConfirmOk.disabled = true;
+  closeRestartConfirmDialog();
+  doReboot();
+});
+
+// Reboot error card buttons
+var $rebootErrorCloseBtn = document.getElementById("reboot-error-close-btn");
+var $rebootErrorRetryBtn = document.getElementById("reboot-error-retry-btn");
+if ($rebootErrorCloseBtn) $rebootErrorCloseBtn.addEventListener("click", function() {
+  if ($rebootOverlay) $rebootOverlay.classList.remove("visible");
+});
+if ($rebootErrorRetryBtn) $rebootErrorRetryBtn.addEventListener("click", doReboot);
+
 // ── Upgrade modal functions ───────────────────────────────────────
 
 function openUpgradeModal() {
@@ -54,6 +76,37 @@ function closeUpgradeModal() {
   if ($upgradeModal) $upgradeModal.classList.remove("open");
 }
 
+// ── Restart confirm dialog functions ─────────────────────────────
+
+var _restartDialogOpener = null;
+
+function openRestartConfirmDialog() {
+  if (!$restartConfirmModal) return;
+  _restartDialogOpener = document.activeElement;
+
+  // Detect conflicting operations
+  var isOperationInProgress = !!_updatePollTimer || !!_rebuildPollTimer;
+  if ($restartConflictBox) $restartConflictBox.style.display = isOperationInProgress ? "" : "none";
+  if ($restartConfirmOk) $restartConfirmOk.disabled = isOperationInProgress;
+
+  $restartConfirmModal.classList.add("open");
+
+  // Focus Cancel initially for safety
+  var cancelBtn = document.getElementById("restart-confirm-cancel-btn");
+  if (cancelBtn) setTimeout(function() { cancelBtn.focus(); }, 50);
+}
+
+function closeRestartConfirmDialog() {
+  if ($restartConfirmModal) $restartConfirmModal.classList.remove("open");
+  // Re-enable confirm button for next open
+  if ($restartConfirmOk) $restartConfirmOk.disabled = false;
+  // Return focus to the element that opened the dialog
+  if (_restartDialogOpener && _restartDialogOpener.focus) {
+    try { _restartDialogOpener.focus(); } catch (_) {}
+    _restartDialogOpener = null;
+  }
+}
+
 async function doUpgradeToServer() {
   var confirmBtn = $upgradeConfirmBtn;
   if (confirmBtn) { confirmBtn.disabled = true; confirmBtn.textContent = "Upgrading…"; }

app/sovran_systemsos_web/static/js/features.js

@@ -73,22 +73,47 @@ function openDomainSetupModal(feat, onSaved) {
     npubField = '<div class="domain-field-group"><label class="domain-field-label" for="domain-npub-input">Nostr Public Key (npub1...):</label><input class="domain-field-input" type="text" id="domain-npub-input" placeholder="npub1..." value="' + escHtml(currentNpub) + '" /></div>';
   }
 
-  var externalIp = _cachedExternalIp || "your external IP";
+  var introHtml;
+  if (_currentRole === "node") {
+    introHtml =
+      '<p>To enable <strong>' + escHtml(feat.name) + '</strong>, it needs its own domain from Njal.la.</p>' +
+      '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+      '<li>Create an account at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">njal.la</a>.</li>' +
+      '<li>Set up a domain for it — either a free subdomain or a separate domain. Pick one option:</li>' +
+      '</ol>';
+  } else {
+    introHtml =
+      '<p>To enable <strong>' + escHtml(feat.name) + '</strong>, it needs its own domain from Njal.la. ' +
+      'In your Njal.la account, set up a domain for it — either a free subdomain or a separate domain. Pick one option:</p>';
+  }
 
   $domainSetupBody.innerHTML =
     '<div class="domain-setup-intro">' +
-    '<p><strong>Before continuing:</strong></p>' +
-    '<ol>' +
-    '<li>Create an account at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">https://njal.la</a></li>' +
-    '<li>Purchase a new domain on Njal.la, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.</li>' +
-    '<li>In the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine\'s external IP address:<br>' +
-    '<span style="display:inline-block;margin-top:4px;padding:4px 10px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>' +
-    '<li>Njal.la will give you a curl command like:<br>' +
-    '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=sub.domain.com&amp;k=abc123&amp;auto&quot;</code></li>' +
-    '<li>Enter the subdomain and paste that curl command below</li>' +
+    introHtml +
+    '<details style="margin-top:10px;">' +
+    '<summary style="cursor:pointer;font-weight:600;">Option A — Free subdomain (recommended)</summary>' +
+    '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+    '<li>In Njal.la, open a domain you own and click &quot;Add record&quot;.</li>' +
+    '<li>Set record type to <strong>Dynamic</strong>.</li>' +
+    '<li>In the <strong>Name</strong> field, type ONLY the host part — the word before your domain.<br>' +
+    '(Example only, your choice — for &quot;call.yourdomain.com&quot; you&apos;d type just: &nbsp;<code>call</code>)<br>' +
+    '&#9888; Do NOT type the full domain here — Njal.la adds it automatically.</li>' +
+    '<li>A Dynamic record has NO IP field — the IP auto-fills after the rebuild/reboot.</li>' +
+    '<li>Copy the curl command Njal.la gives you, e.g.:<br>' +
+    '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=call.yourdomain.com&amp;k=abc123&amp;auto&quot;</code></li>' +
     '</ol>' +
+    '</details>' +
+    '<details style="margin-top:6px;">' +
+    '<summary style="cursor:pointer;font-weight:600;">Option B — Separate / new domain</summary>' +
+    '<ol style="margin:8px 0 0 16px;padding:0;line-height:1.7;">' +
+    '<li>In Njal.la, buy the domain you want.</li>' +
+    '<li>Add a Dynamic record as in Option A. If this domain is dedicated to the service, leave the Name field blank or use <code>@</code>.</li>' +
+    '<li>Copy the curl command Njal.la gives you.</li>' +
+    '</ol>' +
+    '</details>' +
+    '<p style="margin-top:10px;">Below, enter the full domain for this service — a subdomain (e.g. call.yourdomain.com) or a separate domain (e.g. call.com) — and paste its curl command.</p>' +
     '</div>' +
-    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Subdomain (e.g. myservice.example.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" /></div>' +
+    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Service domain (e.g. call.yourdomain.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" /></div>' +
     '<div class="domain-field-group"><label class="domain-field-label" for="domain-ddns-input">Njal.la Dynamic DNS Update Command:</label><input class="domain-field-input" type="text" id="domain-ddns-input" placeholder="curl &quot;https://njal.la/update/?h=myservice.example.com&amp;k=abc123&amp;auto&quot;" /><p class="domain-field-hint">ℹ Paste the full curl command from your Njal.la dashboard\'s Dynamic record</p></div>' +
     npubField +
     '<div class="domain-field-actions"><button class="btn btn-close-modal" id="domain-setup-cancel-btn">Cancel</button><button class="btn btn-primary" id="domain-setup-save-btn">Save &amp; Enable</button></div>';
@@ -103,7 +128,7 @@ function openDomainSetupModal(feat, onSaved) {
     ddnsUrl   = ddnsUrl.trim();
     npub      = npub.trim();
 
-    if (!subdomain) { alert("Please enter a subdomain."); return; }
+    if (!subdomain) { alert("Please enter a domain."); return; }
     if (feat.id === "haven" && !npub) { alert("Please enter your Nostr public key."); return; }
 
     var saveBtn = document.getElementById("domain-setup-save-btn");
@@ -159,14 +184,14 @@ function openDomainReconfigureModal(feat, existingDomain, onSaved) {
     '<p><strong>Troubleshooting steps:</strong></p>' +
     '<ol>' +
     '<li>Log into your Njal.la dashboard at <a href="https://njal.la" target="_blank" rel="noopener noreferrer" style="color:var(--accent-color);">https://njal.la</a></li>' +
-    '<li>Find the DNS record for <strong>' + escHtml(currentDomain || "your domain") + '</strong></li>' +
+    '<li>Find the DNS record for <strong>' + escHtml(currentDomain || "your domain") + '</strong>. In Njal.la\'s Name field, note that only the host part is stored (the word before the domain) — not the full domain.</li>' +
     '<li>Verify it has a <strong>Dynamic</strong> record pointing to your current external IP:<br>' +
     '<span style="display:inline-block;margin-top:4px;padding:4px 10px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1em;font-weight:700;">' + escHtml(externalIp) + '</span></li>' +
     '<li>If the IP is wrong or the record is missing, update it</li>' +
     '<li>If you changed the DDNS curl command, paste the updated one below</li>' +
     '</ol>' +
     '</div>' +
-    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Subdomain (e.g. myservice.example.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" value="' + escHtml(currentDomain) + '" /></div>' +
+    '<div class="domain-field-group"><label class="domain-field-label" for="domain-subdomain-input">Service domain (e.g. call.yourdomain.com):</label><input class="domain-field-input" type="text" id="domain-subdomain-input" placeholder="myservice.example.com" value="' + escHtml(currentDomain) + '" /></div>' +
     '<div class="domain-field-group"><label class="domain-field-label" for="domain-ddns-input">Njal.la Dynamic DNS Update Command:</label><input class="domain-field-input" type="text" id="domain-ddns-input" placeholder="curl &quot;https://njal.la/update/?h=myservice.example.com&amp;k=abc123&amp;auto&quot;" /><p class="domain-field-hint">ℹ Paste the full curl command from your Njal.la dashboard\'s Dynamic record</p></div>' +
     npubField +
     '<div class="domain-field-actions"><button class="btn btn-close-modal" id="domain-setup-cancel-btn">Cancel</button><button class="btn btn-primary" id="domain-setup-save-btn">Save &amp; Update</button></div>';

app/sovran_systemsos_web/static/js/rebuild.js

@@ -69,7 +69,7 @@ function onRebuildDone(result) {
     // Auto-reload the page after a short delay so tiles and toggles reflect the new state
     setTimeout(function() { window.location.reload(); }, 1200);
   } else if (result === "reboot_required") {
-    if ($rebuildStatus) $rebuildStatus.textContent = "✓ Done — reboot required";
+    if ($rebuildStatus) $rebuildStatus.textContent = "✓ Done — restart required";
     if ($rebuildReboot) $rebuildReboot.style.display = "inline-flex";
   } else {
     if ($rebuildStatus) $rebuildStatus.textContent = "✗ Something went wrong";

app/sovran_systemsos_web/static/js/security.js

@@ -145,28 +145,25 @@ function openSecurityModal() {
           if (rebootBtn) {
             // Keep button disabled for 5 seconds to prevent accidental clicks
             var countdown = 5;
-            rebootBtn.textContent = "I have written down my new password \u2014 Reboot now (" + countdown + ")";
+            rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System (" + countdown + ")";
             var timer = setInterval(function() {
               countdown--;
               if (countdown <= 0) {
                 clearInterval(timer);
                 rebootBtn.disabled = false;
-                rebootBtn.textContent = "I have written down my new password \u2014 Reboot now";
+                rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System";
               } else {
-                rebootBtn.textContent = "I have written down my new password \u2014 Reboot now (" + countdown + ")";
+                rebootBtn.textContent = "I have written down my new password \u2014 Restart Entire System (" + countdown + ")";
               }
             }, 1000);
 
             rebootBtn.addEventListener("click", function() {
               rebootBtn.disabled = true;
-              rebootBtn.textContent = "Rebooting\u2026";
-              if ($rebootOverlay) $rebootOverlay.classList.add("visible");
-              _rebootStartTime = Date.now();
-              _serverWentDown = false;
-              setTimeout(waitForServerReboot, REBOOT_INITIAL_DELAY);
-              var rebootCtrl = new AbortController();
-              setTimeout(function() { rebootCtrl.abort(); }, REBOOT_REQUEST_TIMEOUT);
-              fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal }).catch(function() {});
+              rebootBtn.textContent = "Restarting\u2026";
+              // Hide the security reset overlay so the shared reboot overlay is visible
+              var $secResetOverlay2 = document.getElementById("security-reset-overlay");
+              if ($secResetOverlay2) $secResetOverlay2.classList.remove("visible");
+              doReboot();
             }, { once: true });
           }
         } catch (err) {

app/sovran_systemsos_web/static/js/service-detail.js

@@ -154,20 +154,36 @@ async function openServiceDetailModal(unit, name, icon) {
         '</div>';
 
       if (unit === "livekit.service" && data.extra_ports && data.extra_ports.length > 0) {
+        var trimmedInternalIp = data.internal_ip ? String(data.internal_ip).trim() : "";
+        var internalIp = trimmedInternalIp || "";
+        var internalIpHtml = internalIp ? escHtml(internalIp) : "Could not detect";
+        var routerIpHelp = internalIp
+          ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+          : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+        var routerNextStep = internalIp
+          ? 'Next step: Log in to your router and create forwarding rules for the ports above. Set the destination/internal IP to <strong>' + internalIpHtml + '</strong>.'
+          : 'Next step: Log in to your router and create forwarding rules for the ports above. Use this computer’s internal IP as the destination/internal IP.';
+        var domainConfigured = !!(data.domain && String(data.domain).trim());
         var extraRows = "";
         data.extra_ports.forEach(function(p) {
           var statusIcon, statusClass2;
-          if (p.status === "listening") {
-            statusIcon = "✅ Open";
+          if (!effectiveEnabled) {
+            statusIcon = "⚠ Configure Element Call first";
+            statusClass2 = "port-status-open";
+          } else if (!domainConfigured) {
+            statusIcon = "⚠ Configure domain first";
+            statusClass2 = "port-status-open";
+          } else if (p.status === "listening") {
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-listening";
           } else if (p.status === "firewall_open") {
-            statusIcon = "🟡 Firewall open";
+            statusIcon = "✅ Ready";
             statusClass2 = "port-status-open";
           } else if (p.status === "closed") {
-            statusIcon = "❌ Closed";
+            statusIcon = "❌ Not ready yet";
             statusClass2 = "port-status-closed";
           } else {
-            statusIcon = "— Unknown";
+            statusIcon = "— Could not check";
             statusClass2 = "port-status-unknown";
           }
           extraRows += '<tr>' +
@@ -178,11 +194,16 @@ async function openServiceDetailModal(unit, name, icon) {
             '</tr>';
         });
         html += '<div class="svc-detail-section">' +
-          '<div class="svc-detail-section-title">Step 4: Additional Ports</div>' +
+          '<div class="svc-detail-section-title">Ports to Forward in Your Router</div>' +
+          '<div class="svc-detail-port-note">Forward these ports in your router to this Sovran_SystemsOS computer.</div>' +
+          '<div class="svc-detail-port-note"><strong>Router Forward-To IP:</strong> ' + internalIpHtml + '</div>' +
+          '<div class="svc-detail-port-note">' + routerIpHelp + '</div>' +
           '<table class="svc-detail-port-table">' +
-            '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+            '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
             '<tbody>' + extraRows + '</tbody>' +
           '</table>' +
+          '<div class="svc-detail-port-note">The Hub can check whether Sovran_SystemsOS is ready on this computer, but full public port verification requires an outside internet check.</div>' +
+          '<div class="svc-detail-port-note">' + routerNextStep + '</div>' +
           '</div>';
       }
     } else if (data.port_statuses && data.port_statuses.length > 0) {
@@ -191,16 +212,16 @@ async function openServiceDetailModal(unit, name, icon) {
       data.port_statuses.forEach(function(p) {
         var statusIcon, statusClass2;
         if (p.status === "listening") {
-          statusIcon = "✅ Open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-listening";
         } else if (p.status === "firewall_open") {
-          statusIcon = "🟡 Firewall open";
+          statusIcon = "✅ Ready";
           statusClass2 = "port-status-open";
         } else if (p.status === "closed") {
-          statusIcon = "🔴 Closed";
+          statusIcon = "❌ Not ready";
           statusClass2 = "port-status-closed";
         } else {
-          statusIcon = "— Unknown";
+          statusIcon = "— Could not check";
           statusClass2 = "port-status-unknown";
         }
         portTableRows += '<tr>' +
@@ -211,9 +232,10 @@ async function openServiceDetailModal(unit, name, icon) {
           '</tr>';
       });
       html += '<div class="svc-detail-section">' +
-        '<div class="svc-detail-section-title">Port Status</div>' +
+        '<div class="svc-detail-section-title">Port Requirements</div>' +
+        '<div class="svc-detail-port-note">This shows whether Sovran_SystemsOS is ready to use this port on this computer. If you need access from outside your home network, forward this port in your router.</div>' +
         '<table class="svc-detail-port-table">' +
-          '<thead><tr><th>Port</th><th>Protocol</th><th>Description</th><th>Status</th></tr></thead>' +
+          '<thead><tr><th>Port</th><th>Protocol</th><th>Used For</th><th>Sovran_SystemsOS Status</th></tr></thead>' +
           '<tbody>' + portTableRows + '</tbody>' +
         '</table>' +
         '</div>';

app/sovran_systemsos_web/static/js/state.js

@@ -49,6 +49,9 @@ const $btnSave        = document.getElementById("btn-save-report");
 const $btnCloseModal  = document.getElementById("btn-close-modal");
 
 const $rebootOverlay  = document.getElementById("reboot-overlay");
+const $rebootMainCard = document.getElementById("reboot-main-card");
+const $rebootErrorCard = document.getElementById("reboot-error-card");
+const $rebootSubmessage = document.getElementById("reboot-submessage");
 
 const $credsModal     = document.getElementById("creds-modal");
 const $credsTitle     = document.getElementById("creds-modal-title");
@@ -101,5 +104,14 @@ const $upgradeConfirmBtn  = document.getElementById("upgrade-confirm-btn");
 const $upgradeCancelBtn   = document.getElementById("upgrade-cancel-btn");
 const $upgradeCloseBtn    = document.getElementById("upgrade-close-btn");
 
+// Restart confirm dialog
+const $restartConfirmModal  = document.getElementById("restart-confirm-modal");
+const $restartConfirmOk     = document.getElementById("restart-confirm-ok-btn");
+const $restartConfirmCancel = document.getElementById("restart-confirm-cancel-btn");
+const $restartConflictBox   = document.getElementById("restart-conflict-box");
+
+// Header reboot button
+const $headerRebootBtn = document.getElementById("btn-header-reboot");
+
 // System status banner
 // (removed — health is now shown per-tile via the composite health field)

app/sovran_systemsos_web/static/js/support.js

@@ -491,7 +491,7 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>At least 500 GB of free space on the drive</li>',
+        '<li>Enough free space for your selected backup data (the backup checks this before starting)</li>',
         '<li>Drive must be formatted as <strong>exFAT</strong></li>',
       '</ol>',
     '</div>',
@@ -584,9 +584,10 @@ async function pollBackupStatus() {
       logDiv.scrollTop = logDiv.scrollHeight;
     }
     _backupLogOffset = data.offset;
-    if (!data.running) {
+    const result = (data.result || "").toLowerCase();
+    if (result === "success" || result === "failed") {
       stopBackupPoll();
-      renderBackupDone(data.result === "success");
+      renderBackupDone(result === "success");
     }
   } catch (_) {}
 }

app/sovran_systemsos_web/static/js/update.js

@@ -154,7 +154,7 @@ function onUpdateDone(result) {
     if ($modalStatus) $modalStatus.textContent = "✓ Update complete";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else if (result === "reboot_required") {
-    if ($modalStatus) $modalStatus.textContent = "✓ Update complete — reboot required";
+    if ($modalStatus) $modalStatus.textContent = "✓ Update complete — restart required";
     if ($btnReboot) $btnReboot.style.display = "inline-flex";
   } else {
     if ($modalStatus) $modalStatus.textContent = "✗ Update failed";
@@ -179,23 +179,50 @@ function saveErrorReport() {
 
 var _rebootStartTime = 0;
 var _serverWentDown = false;
+var _rebootFailed = false;
+
+function _setRebootStatus(msg) {
+  if ($rebootSubmessage) $rebootSubmessage.textContent = msg;
+}
 
 function doReboot() {
   if ($modal) $modal.classList.remove("open");
   if ($rebuildModal) $rebuildModal.classList.remove("open");
   stopUpdatePoll();
   stopRebuildPoll();
+  // Reset overlay to main card
+  if ($rebootMainCard) $rebootMainCard.style.display = "";
+  if ($rebootErrorCard) $rebootErrorCard.style.display = "none";
+  _setRebootStatus("Sending restart request\u2026");
   if ($rebootOverlay) $rebootOverlay.classList.add("visible");
   _rebootStartTime = Date.now();
   _serverWentDown = false;
+  _rebootFailed = false;
   var rebootCtrl = new AbortController();
   setTimeout(function() { rebootCtrl.abort(); }, REBOOT_REQUEST_TIMEOUT);
-  fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal }).catch(function() {});
+  fetch("/api/reboot", { method: "POST", signal: rebootCtrl.signal })
+    .then(function(res) {
+      if (!res.ok) {
+        // Definitive HTTP error — server rejected the request before going down
+        _rebootFailed = true;
+        if ($rebootMainCard) $rebootMainCard.style.display = "none";
+        if ($rebootErrorCard) $rebootErrorCard.style.display = "";
+        // Leave overlay visible so the error card is shown
+      }
+      // HTTP 2xx: request accepted, proceed with polling
+    })
+    .catch(function() {
+      // Connection dropped or request aborted — the server is likely already going
+      // down as part of the restart. Treat as success and continue polling.
+    });
   // Wait before the first check — NixOS shutdown after an update can take 20-40s
   setTimeout(waitForServerReboot, REBOOT_INITIAL_DELAY);
 }
 
 function waitForServerReboot() {
+  if (_rebootFailed) return;
+  // Update status on first check (server hasn't gone down yet)
+  if (!_serverWentDown) _setRebootStatus("Waiting for the computer to shut down\u2026");
   var controller = new AbortController();
   var timeoutId = setTimeout(function() { controller.abort(); }, REBOOT_FETCH_TIMEOUT);
 
@@ -205,18 +232,23 @@ function waitForServerReboot() {
       if (_serverWentDown) {
         // Server is responding after having been down — reboot is complete.
         // Any response (even 401/500) means the server process is back.
+        _setRebootStatus("System is back online. Reconnecting\u2026");
         window.location.reload();
       } else if ((Date.now() - _rebootStartTime) < 90000) {
         // Server still responding but hasn't gone down yet — keep waiting
         setTimeout(waitForServerReboot, REBOOT_CHECK_INTERVAL);
       } else {
         // Been over 90 seconds and server is responding — just reload
+        _setRebootStatus("System is back online. Reconnecting\u2026");
         window.location.reload();
       }
     })
     .catch(function() {
       clearTimeout(timeoutId);
-      _serverWentDown = true;
+      if (!_serverWentDown) {
+        _serverWentDown = true;
+        _setRebootStatus("The computer is restarting\u2026");
+      }
       setTimeout(waitForServerReboot, REBOOT_CHECK_INTERVAL);
     });
 }

app/sovran_systemsos_web/static/onboarding.js

@@ -333,8 +333,6 @@ async function loadStep3() {
     return;
   }
 
-  var externalIp = (networkData && networkData.external_ip) || "Unknown (could not retrieve)";
-
   // Build set of enabled service units
   var enabledUnits = new Set();
   (_servicesData || []).forEach(function(svc) {
@@ -352,25 +350,31 @@ async function loadStep3() {
     html += '<p class="onboarding-body-text">No domain-based services are enabled for your role. You can skip this step.</p>';
   } else {
     html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
-      + '<strong>Before you continue:</strong>'
+      + '<p style="margin:0 0 8px;"><strong>Sovran_SystemsOS uses Njal.la for domains and Dynamic DNS.</strong></p>'
       + '<ol style="margin:8px 0 0 16px; padding:0; line-height:1.7;">'
-      + '<li>Create an account at <a href="https://njal.la" target="_blank" style="color:var(--accent-color);">https://njal.la</a></li>'
-      + '<li>Purchase a new domain on Njal.la, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.</li>'
-      + '<li>In the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine\'s external IP address:<br>'
-      + '<span style="display:inline-block;margin-top:4px;padding:4px 12px;background:var(--card-color);border:1px solid var(--border-color);border-radius:6px;font-family:monospace;font-size:1.1em;font-weight:700;letter-spacing:0.03em;">' + escHtml(externalIp) + '</span></li>'
-      + '<li>Njal.la will give you a curl command like:<br>'
-      + '<code style="font-size:0.8em;">curl "https://njal.la/update/?h=sub.domain.com&amp;k=abc123&amp;auto"</code></li>'
-      + '<li>Enter the subdomain and paste that curl command below for each service</li>'
+      + '<li>Create an account at <a href="https://njal.la" target="_blank" style="color:var(--accent-color);">https://njal.la</a>.</li>'
+      + '<li>Buy at least one domain. Each service below needs its own domain — you can either give each service its own subdomain of a single domain you buy (subdomains are free, and one domain can have many), OR use a separate domain for each. Your choice.</li>'
+      + '<li>For each service, add a <strong>Dynamic</strong> record in Njal.la:'
+      + '<ul style="margin:4px 0 0 16px;padding:0;line-height:1.7;">'
+      + '<li>In the Njal.la <strong>Name</strong> field, type ONLY the host part — the word before your domain.<br>'
+      + '(Example only, your choice — for &quot;call.yourdomain.com&quot; you&apos;d type just: <code>call</code>.)<br>'
+      + 'If you bought a whole separate domain just for this service, leave Name blank or use <code>@</code>.<br>'
+      + '&#9888; Do NOT type the full domain in the Name field — Njal.la adds it automatically.</li>'
+      + '<li>A Dynamic record has NO IP field. You don&apos;t enter an IP anywhere — it auto-fills once Sovran_SystemsOS updates it (on save, and again after reboot).</li>'
+      + '</ul>'
+      + '</li>'
+      + '<li>Njal.la gives you a curl command like:<br>'
+      + '<code style="font-size:0.8em;">curl &quot;https://njal.la/update/?h=call.yourdomain.com&amp;k=abc123&amp;auto&quot;</code></li>'
       + '</ol>'
       + '</div>';
-    html += '<p class="onboarding-hint">Enter each fully-qualified subdomain (e.g. <code>matrix.yourdomain.com</code>) and its Njal.la DDNS curl command.</p>';
+    html += '<p class="onboarding-hint">Enter each service\'s full domain — a subdomain (e.g. <code>call.yourdomain.com</code>) or a separate domain (e.g. <code>call.com</code>) — and its Njal.la DDNS curl command.</p>';
     relevantDomains.forEach(function(d) {
       var currentVal = (_domainsData && _domainsData[d.name]) || "";
       html += '<div class="onboarding-domain-group">';
       html += '<label class="onboarding-domain-label">' + escHtml(d.label) + '</label>';
       html += '<input class="onboarding-domain-input domain-field-input" type="text" id="domain-input-' + escHtml(d.name) + '" data-domain="' + escHtml(d.name) + '" placeholder="e.g. ' + escHtml(d.name) + '.yourdomain.com" value="' + escHtml(currentVal) + '" />';
       html += '<label class="onboarding-domain-label onboarding-domain-label--sub">Njal.la DDNS Curl Command</label>';
-      html += '<input class="onboarding-domain-input domain-field-input" type="text" id="ddns-input-' + escHtml(d.name) + '" data-ddns="' + escHtml(d.name) + '" placeholder="curl &quot;https://njal.la/update/?h=' + escHtml(d.name) + '.yourdomain.com&amp;k=abc123&amp;auto&quot;" />';
+      html += '<input class="onboarding-domain-input domain-field-input" type="text" id="ddns-input-' + escHtml(d.name) + '" data-ddns="' + escHtml(d.name) + '" placeholder="curl &quot;https://njal.la/update/?h=...&amp;k=...&amp;auto&quot;" />';
       html += '<p class="onboarding-hint" style="margin-top:4px;">ℹ Paste the curl URL from your Njal.la dashboard\'s Dynamic record</p>';
       html += '<button type="button" class="btn btn-primary onboarding-domain-save-btn" data-save-domain="' + escHtml(d.name) + '" style="align-self:flex-start;margin-top:8px;font-size:0.82rem;padding:6px 16px;">Save</button>';
       html += '<span class="onboarding-domain-save-status" id="domain-save-status-' + escHtml(d.name) + '" style="font-size:0.82rem;min-height:1.2em;"></span>';
@@ -512,7 +516,7 @@ async function saveStep3() {
 async function loadStep4() {
   var body = document.getElementById("step-4-body");
   if (!body) return;
-  body.innerHTML = '<p class="onboarding-loading">Checking ports…</p>';
+  body.innerHTML = '<p class="onboarding-loading">Loading router setup…</p>';
 
   var networkData = null;
 
@@ -523,51 +527,59 @@ async function loadStep4() {
     return;
   }
 
-  var internalIp = (networkData && networkData.internal_ip) || "unknown";
-
-  var ip = escHtml(internalIp);
+  var trimmedInternalIp = (networkData && networkData.internal_ip) ? String(networkData.internal_ip).trim() : "";
+  var internalIp = trimmedInternalIp || "";
+  var hasInternalIp = !!internalIp;
+  var ip = escHtml(internalIp || "Could not detect");
+  var routerIpHelp = hasInternalIp
+    ? "Use this IP address as the destination/internal IP when creating each router forwarding rule."
+    : "Use this computer’s internal IP as the destination/internal IP when creating each router forwarding rule.";
+  var destinationInstruction = hasInternalIp
+    ? 'Set the destination/internal IP to <strong>' + ip + '</strong>'
+    : 'Use this computer’s internal IP as the destination/internal IP';
 
   var html = '<p class="onboarding-port-note" style="margin-bottom:14px;">'
     + '⚠ <strong>Each port only needs to be forwarded once — all services share the same ports.</strong>'
     + '</p>';
 
   html += '<div class="onboarding-port-ip">';
-  html += '  <span class="onboarding-port-ip-label">Forward ports to this machine\'s internal IP:</span>';
+  html += '  <span class="onboarding-port-ip-label">Forward router traffic to this Sovran_SystemsOS computer:</span>';
   html += '  <span class="port-req-internal-ip">' + ip + '</span>';
   html += '</div>';
+  html += '<div class="onboarding-port-note" style="margin:8px 0 16px;">' + routerIpHelp + '</div>';
 
   // Required ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Ports — open these on your router:</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:8px;">Required Router Rules</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
-  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP</td></tr>';
+  html += '<tr><td class="port-req-port">80</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTP / SSL setup</td></tr>';
   html += '<tr><td class="port-req-port">443</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">HTTPS</td></tr>';
-  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">SSH Remote Access</td></tr>';
-  html += '<tr><td class="port-req-port">8448</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">Matrix Federation</td></tr>';
+  html += '<tr><td class="port-req-port">22</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">Remote SSH access</td></tr>';
   html += '</tbody></table>';
   html += '</div>';
 
   // Optional ports table
   html += '<div class="onboarding-port-section" style="margin-bottom:20px;">';
-  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Optional — Only needed if you enable Element Calling:</div>';
-  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">These 5 additional port openings are required on top of the 4 required ports above.</div>';
+  html += '<div class="onboarding-port-section-title" style="font-weight:700;margin-bottom:4px;">Element Call Router Rules</div>';
+  html += '<div style="font-size:0.88em;margin-bottom:8px;color:var(--color-text-muted,#888);">Only add these if you enable Element Call. These ports help video and audio calls connect reliably.</div>';
   html += '<table class="onboarding-port-table">';
-  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;to</th><th>Purpose</th></tr></thead>';
+  html += '<thead><tr><th>Port</th><th>Protocol</th><th>Forward&nbsp;To</th><th>Used For</th></tr></thead>';
   html += '<tbody>';
   html += '<tr><td class="port-req-port">7881</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit WebRTC signalling</td></tr>';
   html += '<tr><td class="port-req-port">7882</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">LiveKit media (UDP mux)</td></tr>';
   html += '<tr><td class="port-req-port">5349</td><td class="port-req-proto">TCP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN over TLS</td></tr>';
   html += '<tr><td class="port-req-port">3478</td><td class="port-req-proto">UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN (STUN/relay)</td></tr>';
-  html += '<tr><td class="port-req-port">30000–40000</td><td class="port-req-proto">TCP/UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
+  html += '<tr><td class="port-req-port">30000-40000</td><td class="port-req-proto">TCP &amp; UDP</td><td class="port-req-internal-ip">' + ip + '</td><td class="port-req-desc">TURN relay (WebRTC)</td></tr>';
   html += '</tbody></table>';
+  html += '<div style="font-size:0.85em;margin-top:6px;color:var(--color-text-muted,#888);">ℹ The <strong>30000-40000</strong> range is a single forwarding rule — just set its protocol to <strong>both TCP and UDP</strong> (often shown as "Both" or "TCP/UDP" on your router).</div>';
   html += '</div>';
 
   // Totals
   html += '<div class="onboarding-port-totals">';
-  html += '<strong>Total port openings: 4</strong> (without Element Calling)<br>';
-  html += '<strong>Total port openings: 9</strong> (with Element Calling — 4 required + 5 optional)';
+  html += '<strong>Total port openings: 3</strong> (without Element Call)<br>';
+  html += '<strong>Total port openings: 8</strong> (with Element Call — 3 required + 5 optional)';
   html += '</div>';
 
   html += '<div class="onboarding-port-warn" style="margin-bottom:16px;">'
@@ -582,12 +594,16 @@ async function loadStep4() {
     + '<li>Open your router\'s admin panel — usually <code>http://192.168.1.1</code> or <code>http://192.168.0.1</code></li>'
     + '<li>Look for <strong>"Port Forwarding"</strong>, <strong>"NAT"</strong>, or <strong>"Virtual Server"</strong> in the settings</li>'
     + '<li>Create a new rule for each port listed above</li>'
-    + '<li>Set the destination/internal IP to <strong>' + ip + '</strong></li>'
+    + '<li>' + destinationInstruction + '</li>'
     + '<li>Set both internal and external port to the same number</li>'
     + '<li>Save and apply changes</li>'
     + '</ol>'
     + '</details>';
 
+  html += '<div class="onboarding-port-note" style="margin-top:12px;">'
+    + '<strong>Important:</strong> The Hub can show which ports Sovran_SystemsOS needs, but it cannot fully confirm router forwarding from inside your home network. Full public port verification requires an outside internet check.'
+    + '</div>';
+
   body.innerHTML = html;
 }
 

app/sovran_systemsos_web/templates/index.html

@@ -24,6 +24,7 @@
     <span class="title">Sovran_SystemsOS Hub</span>
     <div class="header-buttons">
       <span class="role-badge" id="role-badge">Loading…</span>
+      <button class="btn btn-header-reboot" id="btn-header-reboot" title="Restart the entire computer">Reboot</button>
       <button class="btn btn-logout" id="btn-logout" title="Sign out">Sign Out</button>
     </div>
   </header>
@@ -61,7 +62,7 @@
       <div class="modal-log" id="modal-log" aria-live="polite"></div>
       <div class="modal-footer">
         <button class="btn btn-save" id="btn-save-report" style="display:none">Save Error Report</button>
-        <button class="btn btn-reboot" id="btn-reboot" style="display:none">Reboot</button>
+        <button class="btn btn-reboot" id="btn-reboot" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="btn-close-modal" disabled>Close</button>
       </div>
     </div>
@@ -164,7 +165,7 @@
       <div class="modal-log" id="rebuild-log" aria-live="polite"></div>
       <div class="modal-footer">
         <button class="btn btn-save" id="rebuild-save-report" style="display:none">Save Error Report</button>
-        <button class="btn btn-reboot" id="rebuild-reboot-btn" style="display:none">Reboot</button>
+        <button class="btn btn-reboot" id="rebuild-reboot-btn" style="display:none">Restart Entire System</button>
         <button class="btn btn-close-modal" id="rebuild-close-btn" disabled>Close</button>
       </div>
     </div>
@@ -240,26 +241,61 @@
         You will need it to log in to your computer<br />and the Sovran Hub at <em>sovransystemsos.local</em>.
       </p>
       <button class="security-reset-reboot-btn" id="security-reset-reboot-btn" disabled>
-        I have written down my new password — Reboot now
+        I have written down my new password — Restart Entire System
       </button>
     </div>
   </div>
 
   <!-- Reboot overlay -->
   <div class="reboot-overlay" id="reboot-overlay">
-    <div class="reboot-card">
-      <div class="reboot-icon">↻</div>
-      <h2 class="reboot-title">System Rebooting</h2>
+    <!-- Normal restarting card -->
+    <div class="reboot-card" id="reboot-main-card">
+      <div class="reboot-icon" aria-hidden="true">↻</div>
+      <h2 class="reboot-title">Restarting Entire System</h2>
       <p class="reboot-message">
-        Sovran_SystemsOS is now restarting.<br />
-        This page will automatically reconnect once the system is back online.
+        The entire computer is restarting, including the desktop and all hosted services.<br />
+        This page will reconnect automatically when Sovran_SystemsOS is back online.
       </p>
-      <div class="reboot-dots">
+      <div class="reboot-dots" aria-hidden="true">
         <span class="reboot-dot"></span>
         <span class="reboot-dot"></span>
         <span class="reboot-dot"></span>
       </div>
-      <p class="reboot-submessage">Stay tuned…</p>
+      <p class="reboot-submessage" id="reboot-submessage" aria-live="polite">Sending restart request…</p>
+    </div>
+    <!-- Error card (shown if restart request fails definitively) -->
+    <div class="reboot-card" id="reboot-error-card" style="display:none">
+      <div class="reboot-icon" aria-hidden="true">⚠</div>
+      <h2 class="reboot-title">Restart could not be started</h2>
+      <p class="reboot-message">
+        The computer did not begin restarting. No services were intentionally stopped. Please try again.
+      </p>
+      <div class="reboot-error-actions">
+        <button class="btn btn-close-modal" id="reboot-error-close-btn">Close</button>
+        <button class="btn btn-restart-amber" id="reboot-error-retry-btn">Try Again</button>
+      </div>
+    </div>
+  </div>
+
+  <!-- Restart Confirm Dialog -->
+  <div class="modal-overlay" id="restart-confirm-modal" role="dialog" aria-modal="true" aria-labelledby="restart-confirm-title">
+    <div class="creds-dialog domain-narrow-dialog">
+      <div class="creds-header">
+        <span class="creds-title" id="restart-confirm-title">Restart the entire computer?</span>
+      </div>
+      <div class="creds-body">
+        <div id="restart-conflict-box" class="restart-conflict-box" style="display:none">
+          <p class="restart-conflict-title">The system cannot restart right now.</p>
+          <p class="restart-conflict-desc">A system update, rebuild, backup, restore, or security operation is currently running. Wait for it to finish, then try again.</p>
+        </div>
+        <p class="support-desc"><strong>This will reboot the physical machine running Sovran_SystemsOS — not just the Hub.</strong></p>
+        <p class="support-desc">The desktop and all hosted services will stop temporarily and restart with the computer. Anyone currently using these services will be disconnected.</p>
+        <p class="support-desc">The system usually returns within 1–3 minutes. This page will reconnect automatically.</p>
+        <div class="domain-field-actions">
+          <button class="btn btn-close-modal" id="restart-confirm-cancel-btn">Cancel</button>
+          <button class="btn btn-restart-amber" id="restart-confirm-ok-btn">Restart Entire System</button>
+        </div>
+      </div>
     </div>
   </div>
 

app/sovran_systemsos_web/templates/onboarding.html

@@ -128,9 +128,8 @@
           <h2 class="onboarding-step-title">Domain Configuration</h2>
           <p class="onboarding-step-desc">
             Sovran_SystemsOS uses <strong><a href="https://njal.la" target="_blank" style="color: var(--accent-color);">Njal.la</a></strong> for domains and Dynamic DNS.
-            First, create an account at <strong>Njal.la</strong> and purchase a new domain, or create a subdomain from a domain you already own. Tip: Subdomains are free to create — you only need to purchase one domain, and you can add as many subdomains as you need at no extra cost.
-            Then, in the Njal.la web interface, create a <strong>Dynamic</strong> record pointing to this machine's external IP address (shown below).
-            Finally, paste the DDNS curl command from your Njal.la dashboard for each service below.
+            Create an account at Njal.la, then for each service below, add a <strong>Dynamic</strong> record — no IP needed, it auto-populates once the DDNS curl command runs.
+            Paste the curl command from your Njal.la dashboard for each service.
           </p>
         </div>
         <div class="onboarding-card" id="step-3-body">
@@ -149,14 +148,14 @@
       <div class="onboarding-panel" id="step-4" style="display:none">
         <div class="onboarding-step-header">
           <span class="onboarding-step-icon">🔌</span>
-          <h2 class="onboarding-step-title">Port Forwarding Check</h2>
+          <h2 class="onboarding-step-title">Router Setup</h2>
           <p class="onboarding-step-desc">
-            Forward these ports on your router to this machine. Each port only needs to be opened once — they are shared across all your services.
-            <strong>Ports 80 and 443 must be open for SSL certificates to work.</strong>
+            Forward these ports in your router to this Sovran_SystemsOS computer. These rules let people reach your services from outside your home network.
+            <strong>Ports 80 and 443 are required for HTTPS and SSL certificates.</strong>
           </p>
         </div>
         <div class="onboarding-card" id="step-4-body">
-          <p class="onboarding-loading">Checking ports…</p>
+          <p class="onboarding-loading">Loading router setup…</p>
         </div>
         <div class="onboarding-footer">
           <button class="btn btn-close-modal onboarding-btn-back" data-prev="3">← Back</button>

app/tests/test_hub_update_boot_staging.py

@@ -0,0 +1,44 @@
+import unittest
+from pathlib import Path
+
+
+HUB_NIX = Path(__file__).resolve().parents[2] / "modules" / "core" / "sovran-hub.nix"
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class HubUpdateBootStagingTests(unittest.TestCase):
+    def setUp(self):
+        self.source = HUB_NIX.read_text()
+        self.update_section = _section(
+            self.source,
+            'update-script = pkgs.writeShellScript "sovran-hub-update.sh" \'\'',
+            "# ── Rebuild wrapper script",
+        )
+        self.rebuild_section = _section(
+            self.source,
+            'rebuild-script = pkgs.writeShellScript "sovran-hub-rebuild.sh" \'\'',
+            "# ── Brave launcher wrapper",
+        )
+
+    def test_full_update_uses_boot_not_switch(self):
+        self.assertIn("nixos-rebuild boot --flake /etc/nixos", self.update_section)
+        self.assertNotIn("nixos-rebuild switch --flake /etc/nixos", self.update_section)
+
+    def test_full_update_marks_reboot_required(self):
+        self.assertIn('echo "REBOOT_REQUIRED" > "$STATUS"', self.update_section)
+
+    def test_rebuild_path_keeps_switch_semantics(self):
+        self.assertIn("nixos-rebuild switch --flake /etc/nixos", self.rebuild_section)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_local_domain_loopback_nix.py

@@ -0,0 +1,193 @@
+"""Structural regression tests for modules/core/local-domain-loopback.nix.
+
+Verifies that the sovran-hosts-update helper:
+  - is built as a pkgs.writeShellApplication with explicit runtimeInputs
+    (gawk, gnugrep, coreutils);
+  - uses ``lib.getExe hostsUpdateScript`` for both the systemd ExecStart and
+    the activation script so that both contexts share the same Nix-store
+    executable;
+  - does NOT point ExecStart at the raw /etc path;
+  - includes element-calling in the supported domain list;
+  - uses ``awk -v`` for safe marker-variable passing rather than interpolating
+    marker text directly into the awk program;
+  - retains the domain validation regex (idempotency / injection prevention);
+  - exposes /etc/sovran-hosts-update.sh as a symlink via ``source =`` (not a
+    second raw ``text =`` body);
+  - does NOT rely on environment.systemPackages for the helper's dependencies.
+"""
+
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+NIX_STRING_INDENT = 6
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FLAKE_SOURCE = (REPO_ROOT / "flake.nix").read_text()
+PRIMARY_NIXOS_CONFIGURATION_MATCH = re.search(
+    r"nixosConfigurations\.([A-Za-z0-9_-]+)\s*=",
+    FLAKE_SOURCE,
+)
+if PRIMARY_NIXOS_CONFIGURATION_MATCH is None:
+    raise RuntimeError("Could not determine the primary nixosConfigurations entry from flake.nix")
+PRIMARY_NIXOS_CONFIGURATION = PRIMARY_NIXOS_CONFIGURATION_MATCH.group(1)
+HELPER_BUILD_ATTR = (
+    f'.#nixosConfigurations.{PRIMARY_NIXOS_CONFIGURATION}.config.environment.etc.'
+    '"sovran-hosts-update.sh".source'
+)
+NIX_FILE = (
+    REPO_ROOT
+    / "modules"
+    / "core"
+    / "local-domain-loopback.nix"
+)
+
+
+class LocalDomainLoopbackNixStructureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = NIX_FILE.read_text()
+
+    def _helper_script(self) -> str:
+        start = self.source.index("text = ''") + len("text = ''")
+        end = self.source.index("    '';", start)
+        return "\n".join(
+            line[NIX_STRING_INDENT:]
+            if line.startswith(" " * NIX_STRING_INDENT)
+            else line
+            for line in self.source[start:end].splitlines()
+        ).lstrip("\n")
+
+    # ── writeShellApplication and explicit runtimeInputs ────────────────────
+
+    def test_uses_write_shell_application(self):
+        self.assertIn("pkgs.writeShellApplication", self.source)
+
+    def test_runtime_inputs_includes_coreutils(self):
+        self.assertIn("pkgs.coreutils", self.source)
+
+    def test_runtime_inputs_includes_gawk(self):
+        self.assertIn("pkgs.gawk", self.source)
+
+    def test_runtime_inputs_includes_gnugrep(self):
+        self.assertIn("pkgs.gnugrep", self.source)
+
+    def test_runtime_inputs_block_present(self):
+        self.assertIn("runtimeInputs", self.source)
+
+    # ── Both execution paths use lib.getExe ─────────────────────────────────
+
+    def test_exec_start_uses_lib_get_exe(self):
+        """systemd ExecStart must reference the Nix-store executable."""
+        self.assertIn("ExecStart = lib.getExe hostsUpdateScript", self.source)
+
+    def test_activation_script_uses_lib_get_exe(self):
+        """Activation text must call the same Nix-store executable."""
+        self.assertIn("${lib.getExe hostsUpdateScript}", self.source)
+
+    def test_exec_start_does_not_point_to_etc_path(self):
+        """ExecStart must NOT use the raw /etc path (which lacks a deterministic PATH)."""
+        self.assertNotIn('ExecStart = "/etc/sovran-hosts-update.sh"', self.source)
+
+    # ── /etc symlink uses source =, not a second text = body ────────────────
+
+    def test_etc_entry_uses_source_not_text(self):
+        """The /etc/sovran-hosts-update.sh entry must be a symlink (source =),
+        not a second raw script body (text =)."""
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source', self.source
+        )
+
+    def test_etc_source_points_to_get_exe(self):
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript',
+            self.source,
+        )
+
+    # ── element-calling domain is supported ─────────────────────────────────
+
+    def test_element_calling_domain_key_present(self):
+        self.assertIn("element-calling", self.source)
+
+    # ── Robust awk -v variable passing ──────────────────────────────────────
+
+    def test_awk_uses_dash_v_for_begin_marker(self):
+        """awk must receive the begin marker via -v, not by shell interpolation."""
+        self.assertIn('awk -v begin=', self.source)
+
+    def test_awk_uses_dash_v_for_end_marker(self):
+        self.assertIn('-v end=', self.source)
+
+    def test_awk_does_not_interpolate_marker_into_program(self):
+        """The old pattern interpolated $BEGIN_MARKER directly into the awk source."""
+        self.assertNotIn('/$BEGIN_MARKER', self.source)
+        self.assertNotIn('/$END_MARKER', self.source)
+
+    # ── Domain validation ────────────────────────────────────────────────────
+
+    def test_domain_validation_regex_present(self):
+        """The hostname validation regex must still be present for injection prevention."""
+        self.assertIn("grep -qE", self.source)
+        self.assertIn("[a-zA-Z0-9]", self.source)
+
+    def test_invalid_domain_warning_present(self):
+        self.assertIn("skipping invalid domain value", self.source)
+
+    # ── No environment.systemPackages reliance ───────────────────────────────
+
+    def test_no_environment_system_packages_for_helper(self):
+        """The helper's tools are declared via runtimeInputs; the module must
+        not add them to environment.systemPackages."""
+        self.assertNotIn("environment.systemPackages", self.source)
+
+    # ── Idempotency: existing Sovran block is removed before rewriting ───────
+
+    def test_existing_block_removal_logic_present(self):
+        """awk strip of the managed block must be present for idempotency."""
+        self.assertIn("skip=1", self.source)
+        self.assertIn("skip=0", self.source)
+
+    def test_managed_block_uses_grouped_append_redirect(self):
+        self.assertIn('} >> "$TMP"', self.source)
+        self.assertEqual(self.source.count('>> "$TMP"'), 1)
+
+    def test_helper_script_passes_shellcheck(self):
+        shellcheck = shutil.which("shellcheck")
+        if shellcheck is None:
+            self.skipTest("shellcheck is not installed")
+        proc = subprocess.run(
+            [shellcheck, "-s", "bash", "-"],
+            input=self._helper_script(),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        output = proc.stdout + proc.stderr
+        self.assertEqual(proc.returncode, 0, output)
+
+    def test_helper_derivation_builds_when_nix_available(self):
+        nix = shutil.which("nix")
+        if nix is None:
+            self.skipTest("nix is not installed")
+        proc = subprocess.run(
+            [
+                nix,
+                "build",
+                HELPER_BUILD_ATTR,
+                "--no-link",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+
+    # ── Activation script warns on failure rather than silently swallowing ───
+
+    def test_activation_script_emits_warning_on_failure(self):
+        self.assertIn("warning: sovran-hosts-update", self.source)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_loopback_diagnostics.py

@@ -0,0 +1,399 @@
+"""Tests for server-local loopback diagnostics and domain validation.
+
+Covers:
+- Domain value validation and injection prevention.
+- Loopback address detection (IPv4 and IPv6).
+- _resolve_all_addresses returning multiple addresses.
+- _check_domain_health_fast with loopback resolution.
+- _evaluate_domain_checklist with loopback override — no false dns_mismatch.
+- _evaluate_domain_checklist with genuine DNS mismatch — still reports error.
+- api_services health stays "healthy" when domain resolves to loopback.
+- api_services health stays "needs_attention" when DNS is genuinely wrong.
+- api_domains_check returns "local_override" for loopback-resolved domains.
+"""
+
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+# ---------------------------------------------------------------------------
+# Minimal stubs so server.py can be imported without the full FastAPI stack.
+# ---------------------------------------------------------------------------
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+                return _decorator
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server  # noqa: E402
+
+
+# ===========================================================================
+# Domain value validation
+# ===========================================================================
+
+class TestValidateDomainValue(unittest.TestCase):
+    """_validate_domain_value must reject anything that could corrupt /etc/hosts."""
+
+    def _v(self, value: str) -> bool:
+        return server._validate_domain_value(value)
+
+    # -- Valid values --------------------------------------------------------
+
+    def test_simple_domain_valid(self):
+        self.assertTrue(self._v("cloud.example.com"))
+
+    def test_subdomain_valid(self):
+        self.assertTrue(self._v("matrix.home.example.org"))
+
+    def test_single_label_with_tld_valid(self):
+        self.assertTrue(self._v("example.com"))
+
+    def test_hyphen_in_domain_valid(self):
+        self.assertTrue(self._v("my-nextcloud.example.com"))
+
+    # -- Injection / malformed values ----------------------------------------
+
+    def test_empty_string_invalid(self):
+        self.assertFalse(self._v(""))
+
+    def test_newline_injection_invalid(self):
+        self.assertFalse(self._v("evil.com\n127.0.0.1 other.host"))
+
+    def test_carriage_return_injection_invalid(self):
+        self.assertFalse(self._v("evil.com\r127.0.0.1 other.host"))
+
+    def test_space_injection_invalid(self):
+        self.assertFalse(self._v("evil.com 127.0.0.1"))
+
+    def test_hash_comment_injection_invalid(self):
+        self.assertFalse(self._v("evil.com# comment"))
+
+    def test_bare_hostname_no_dot_invalid(self):
+        self.assertFalse(self._v("localhost"))
+
+    def test_bare_ip_invalid(self):
+        self.assertFalse(self._v("192.168.1.1"))
+
+    def test_too_long_invalid(self):
+        self.assertFalse(self._v("a" * 254 + ".com"))
+
+    def test_leading_dot_invalid(self):
+        self.assertFalse(self._v(".example.com"))
+
+    def test_trailing_dot_invalid(self):
+        self.assertFalse(self._v("example.com."))
+
+
+# ===========================================================================
+# Loopback address detection
+# ===========================================================================
+
+class TestIsLoopbackAddress(unittest.TestCase):
+
+    def test_ipv4_loopback(self):
+        self.assertTrue(server._is_loopback_address("127.0.0.1"))
+
+    def test_ipv4_loopback_other(self):
+        self.assertTrue(server._is_loopback_address("127.0.0.2"))
+
+    def test_ipv4_loopback_high(self):
+        self.assertTrue(server._is_loopback_address("127.255.255.255"))
+
+    def test_ipv6_loopback(self):
+        self.assertTrue(server._is_loopback_address("::1"))
+
+    def test_public_ipv4_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("203.0.113.10"))
+
+    def test_private_ipv4_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("192.168.1.50"))
+
+    def test_ipv6_public_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("2001:db8::1"))
+
+    def test_invalid_string_not_loopback(self):
+        self.assertFalse(server._is_loopback_address("not-an-ip"))
+
+
+# ===========================================================================
+# _check_domain_health_fast
+# ===========================================================================
+
+class TestCheckDomainHealthFast(unittest.TestCase):
+    """_check_domain_health_fast returns True when there is an issue,
+    False when everything looks fine."""
+
+    def _fast(self, domain, external_ip, resolved_addrs):
+        with patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs):
+            return server._check_domain_health_fast(domain, external_ip)
+
+    def test_no_domain_no_issue(self):
+        # None/empty domain: the fast check reports True (handled by checklist).
+        result = server._check_domain_health_fast(None, "203.0.113.10")
+        self.assertTrue(result)
+
+    def test_empty_domain_no_issue(self):
+        result = server._check_domain_health_fast("", "203.0.113.10")
+        self.assertTrue(result)
+
+    def test_loopback_ipv4_no_issue(self):
+        """Loopback override must not be flagged as a DNS mismatch."""
+        result = self._fast("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        self.assertFalse(result)
+
+    def test_loopback_ipv6_no_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["::1"])
+        self.assertFalse(result)
+
+    def test_matches_external_ip_no_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["203.0.113.10"])
+        self.assertFalse(result)
+
+    def test_mismatch_is_an_issue(self):
+        result = self._fast("cloud.example.com", "203.0.113.10", ["198.51.100.1"])
+        self.assertTrue(result)
+
+    def test_unavailable_external_ip_no_issue(self):
+        result = self._fast("cloud.example.com", "unavailable", ["198.51.100.1"])
+        self.assertFalse(result)
+
+    def test_multiple_addresses_one_matches_no_issue(self):
+        """If any resolved address matches external_ip the check should pass."""
+        result = self._fast(
+            "cloud.example.com", "203.0.113.10",
+            ["198.51.100.1", "203.0.113.10"],
+        )
+        self.assertFalse(result)
+
+
+# ===========================================================================
+# _evaluate_domain_checklist — loopback override path
+# ===========================================================================
+
+class TestEvaluateDomainChecklistLoopback(unittest.TestCase):
+
+    def _eval(self, domain, external_ip, resolved_addrs, reachable_result=None):
+        with (
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_check_domain_reachable",
+                         return_value=reachable_result or {"reachable": True, "status_code": 200}),
+        ):
+            return server._evaluate_domain_checklist(domain, external_ip)
+
+    def test_loopback_dns_step_is_ok_not_error(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        dns_step = next(s for s in result["domain_check_steps"] if s["step"] == 2)
+        self.assertEqual(dns_step["status"], "ok")
+        self.assertNotIn("mismatch", dns_step.get("detail", "").lower())
+
+    def test_loopback_domain_status_is_local_override(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["127.0.0.1"])
+        self.assertEqual(result["domain_status"]["status"], "local_override")
+
+    def test_loopback_has_no_issues_when_reachable(self):
+        result = self._eval(
+            "cloud.example.com", "203.0.113.10", ["127.0.0.1"],
+            reachable_result={"reachable": True, "status_code": 200},
+        )
+        self.assertFalse(result["has_issues"])
+
+    def test_loopback_has_issues_when_caddy_unreachable(self):
+        """A loopback override with Caddy down should still report an issue."""
+        result = self._eval(
+            "cloud.example.com", "203.0.113.10", ["127.0.0.1"],
+            reachable_result={"reachable": False, "error": "connection refused"},
+        )
+        self.assertTrue(result["has_issues"])
+
+    def test_ipv6_loopback_no_issue(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["::1"])
+        self.assertEqual(result["domain_status"]["status"], "local_override")
+        self.assertFalse(result["has_issues"])
+
+    def test_genuine_mismatch_still_reports_error(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["198.51.100.1"])
+        self.assertEqual(result["domain_status"]["status"], "dns_mismatch")
+        self.assertTrue(result["has_issues"])
+
+    def test_correct_public_dns_still_reports_ok(self):
+        result = self._eval("cloud.example.com", "203.0.113.10", ["203.0.113.10"])
+        self.assertEqual(result["domain_status"]["status"], "connected")
+        self.assertFalse(result["has_issues"])
+
+    def test_no_domain_has_issues(self):
+        result = self._eval(None, "203.0.113.10", [])
+        self.assertTrue(result["has_issues"])
+
+
+# ===========================================================================
+# api_services — composite health with loopback
+# ===========================================================================
+
+class TestApiServicesLoopbackHealth(unittest.IsolatedAsyncioTestCase):
+
+    async def _get_health(self, resolved_addrs, cached_reachable):
+        """Return the health value for a single domain-requiring service."""
+        service_cfg = {
+            "services": [
+                {"unit": "caddy.service", "icon": "nextcloud", "enabled": True, "type": "system"}
+            ]
+        }
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"caddy.service": "nextcloud"}, clear=False),
+            patch("builtins.open", mock_open(read_data="cloud.example.com\n")),
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_is_domain_reachable_cached", return_value=cached_reachable),
+            patch.object(server, "_get_listening_ports",
+                         return_value={"tcp": {80, 443}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports",
+                         return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_cached_external_ip", "203.0.113.10"),
+        ):
+            results = await server.api_services()
+
+        return results[0]["health"]
+
+    async def test_loopback_and_reachable_is_healthy(self):
+        """Loopback override + Caddy reachable → healthy, not needs_attention."""
+        health = await self._get_health(["127.0.0.1"], cached_reachable=True)
+        self.assertEqual(health, "healthy")
+
+    async def test_loopback_and_caddy_down_is_needs_attention(self):
+        """Loopback override + Caddy unreachable → needs_attention (genuine issue)."""
+        health = await self._get_health(["127.0.0.1"], cached_reachable=False)
+        self.assertEqual(health, "needs_attention")
+
+    async def test_correct_dns_and_reachable_is_healthy(self):
+        health = await self._get_health(["203.0.113.10"], cached_reachable=True)
+        self.assertEqual(health, "healthy")
+
+    async def test_dns_mismatch_is_needs_attention(self):
+        health = await self._get_health(["198.51.100.1"], cached_reachable=True)
+        self.assertEqual(health, "needs_attention")
+
+
+# ===========================================================================
+# api_domains_check — loopback detection
+# ===========================================================================
+
+class TestApiDomainsCheckLoopback(unittest.IsolatedAsyncioTestCase):
+
+    async def _check(self, resolved_addrs, external_ip="203.0.113.10"):
+        with (
+            patch.object(server, "_resolve_all_addresses", return_value=resolved_addrs),
+            patch.object(server, "_cached_external_ip", external_ip),
+        ):
+            result = await server.api_domains_check(
+                MagicMock(domains=["cloud.example.com"])
+            )
+        return result["domains"][0]
+
+    async def test_loopback_ipv4_returns_local_override(self):
+        result = await self._check(["127.0.0.1"])
+        self.assertEqual(result["status"], "local_override")
+
+    async def test_loopback_ipv6_returns_local_override(self):
+        result = await self._check(["::1"])
+        self.assertEqual(result["status"], "local_override")
+
+    async def test_correct_dns_returns_connected(self):
+        result = await self._check(["203.0.113.10"])
+        self.assertEqual(result["status"], "connected")
+
+    async def test_mismatch_returns_dns_mismatch(self):
+        result = await self._check(["198.51.100.1"])
+        self.assertEqual(result["status"], "dns_mismatch")
+
+    async def test_no_resolution_returns_unresolvable(self):
+        result = await self._check([])
+        self.assertEqual(result["status"], "unresolvable")
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_manual_backup_workflow.py

@@ -0,0 +1,688 @@
+import asyncio
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BACKUP_SCRIPT = REPO_ROOT / "app" / "sovran_systemsos_web" / "scripts" / "sovran-hub-backup.sh"
+SERVER_FILE = REPO_ROOT / "app" / "sovran_systemsos_web" / "server.py"
+SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "support.js"
+NIX_HUB_FILE = REPO_ROOT / "modules" / "core" / "sovran-hub.nix"
+
+
+def _extract_bash_function(script_path: Path, func_name: str) -> str:
+    """Extract a bash function definition from a shell script using Python regex.
+
+    The pattern matches from 'funcname() {' to the first line whose only content
+    is '}' (the function closing brace, at column 0). This works correctly for
+    functions whose control structures (case/if/while/for) use indented braces,
+    because only the function's own closing brace sits at column 0. It would not
+    correctly extract a function that contains a nested function definition.
+
+    Returns the complete function definition, safe to inline into a test bash
+    script without eval or awk."""
+    source = script_path.read_text()
+    pattern = r"^" + re.escape(func_name) + r"\(\) \{.*?^}"
+    match = re.search(pattern, source, re.MULTILINE | re.DOTALL)
+    if match is None:
+        raise ValueError(f"Function '{func_name}' not found in {script_path}")
+    return match.group(0)
+
+
+class ManualBackupWorkflowTests(unittest.TestCase):
+    def test_backup_script_uses_tar_archives_with_checksums_and_exclusions(self):
+        source = BACKUP_SCRIPT.read_text()
+
+        self.assertIn("tar \\", source)
+        self.assertIn("--create", source)
+        self.assertIn("--one-file-system", source)
+        self.assertIn("sha256sum", source)
+        self.assertIn("export_postgresql_dumps", source)
+        self.assertIn("export_mariadb_dumps", source)
+        self.assertIn("export_lnd_scb_if_possible", source)
+        self.assertIn("--exclude='var/lib/bitcoind'", source)
+        self.assertIn("--exclude='var/lib/electrs'", source)
+        self.assertIn("--exclude='var/lib/lnd'", source)
+        self.assertIn("set_status \"RUNNING\"", source)
+        self.assertIn("set_status \"SUCCESS\"", source)
+        self.assertIn("set_status \"FAILED\"", source)
+        self.assertNotIn("rsync -a", source)
+
+    def test_backend_and_frontend_use_explicit_backup_terminal_states(self):
+        server_source = SERVER_FILE.read_text()
+        support_source = SUPPORT_JS.read_text()
+
+        self.assertIn("_write_backup_status, \"RUNNING\"", server_source)
+        self.assertIn("_monitor_backup_subprocess", server_source)
+        self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source)
+        self.assertIn("result === \"success\" || result === \"failed\"", support_source)
+
+    # ── Regression tests for exit-code-127 / missing-interpreter bug ──────────
+
+    def test_nix_service_path_includes_bash_and_gawk(self):
+        """Regression: pkgs.bash and pkgs.gawk must appear in the sovran-hub-web
+        service path so that the backup shell script and its awk calls can be
+        resolved at runtime without producing exit code 127."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertIn(
+            "pkgs.bash",
+            nix_source,
+            "pkgs.bash must be declared in the sovran-hub-web systemd service path",
+        )
+        self.assertIn(
+            "pkgs.gawk",
+            nix_source,
+            "pkgs.gawk must be declared in the sovran-hub-web systemd service path",
+        )
+
+    def test_server_resolves_bash_via_shutil_which(self):
+        """Regression: server must locate bash with shutil.which() rather than
+        relying on /usr/bin/env bash, so a missing interpreter is caught early
+        with a clear diagnostic instead of a cryptic exit-code-127 failure."""
+        source = SERVER_FILE.read_text()
+        self.assertIn(
+            'shutil.which("bash")',
+            source,
+            "server.py must resolve bash via shutil.which to catch missing interpreter",
+        )
+        self.assertNotIn(
+            '"/usr/bin/env", "bash"',
+            source,
+            "server.py must not use /usr/bin/env bash (relies on PATH, causes code 127)",
+        )
+
+    def test_server_logs_actionable_message_when_bash_missing(self):
+        """Regression: when bash is absent the server must write an actionable log
+        entry and set FAILED status before returning an HTTP error."""
+        source = SERVER_FILE.read_text()
+        self.assertIn(
+            "bash_path is None",
+            source,
+            "server.py must check that bash_path is not None before launching",
+        )
+        self.assertIn(
+            "interpreter",
+            source,
+            "server.py missing-bash error message must reference 'interpreter'",
+        )
+
+    def test_server_captures_stderr_from_backup_subprocess(self):
+        """Regression: the backup subprocess must use stderr=PIPE so that any
+        startup error (e.g. code 127 from a missing command) is captured and
+        surfaced in the backup log rather than being silently discarded."""
+        source = SERVER_FILE.read_text()
+        self.assertIn(
+            "stderr=asyncio.subprocess.PIPE",
+            source,
+            "backup subprocess must use stderr=PIPE to capture diagnostic output",
+        )
+        self.assertIn(
+            "stderr_text",
+            source,
+            "_monitor_backup_subprocess must capture and log stderr content",
+        )
+
+    def test_monitor_includes_stderr_detail_in_failed_message(self):
+        """Regression: _monitor_backup_subprocess must append captured stderr to
+        the FAILED log entry so the UI shows what went wrong (e.g. 'bash: not
+        found') rather than only the raw exit code."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("stderr_chunks", source)
+        self.assertIn("stderr_text", source)
+        # stderr detail is conditionally appended only when non-empty
+        self.assertIn('detail = f" — stderr: {stderr_text}"', source)
+
+    def test_exit_code_127_subprocess_stderr_drain(self):
+        """Behavioral regression: a subprocess that exits 127 must have its
+        stderr drained without deadlock by the async drain loop."""
+
+        async def _run():
+            proc = await asyncio.create_subprocess_exec(
+                "bash", "-c", "echo 'bash: command not found' >&2; exit 127",
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            chunks: list[bytes] = []
+
+            async def _drain():
+                async for line in proc.stderr:
+                    chunks.append(line)
+
+            drain_task = asyncio.create_task(_drain())
+            rc = await proc.wait()
+            await drain_task
+            return rc, b"".join(chunks).decode()
+
+        rc, stderr_out = asyncio.run(_run())
+        self.assertEqual(rc, 127)
+        self.assertIn("bash: command not found", stderr_out)
+
+    # ── Tests for home tar exit-1 tolerance (production fix) ──────────────────
+
+    def test_backup_script_has_home_tar_archive_function(self):
+        """The script must define create_home_tar_archive for /home only,
+        separate from the strict create_tar_archive used for other sources."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "create_home_tar_archive",
+            source,
+            "backup script must define create_home_tar_archive for /home",
+        )
+        self.assertIn(
+            "_is_home_warning_allowlisted",
+            source,
+            "backup script must define _is_home_warning_allowlisted helper",
+        )
+        self.assertIn(
+            "ARCHIVE_WARNINGS",
+            source,
+            "backup script must track nonfatal warnings in ARCHIVE_WARNINGS",
+        )
+
+    def test_backup_script_allowlist_covers_verified_gnu_tar_messages(self):
+        """The allowlist must contain the exact GNU tar (LC_ALL=C) strings for
+        the two permitted transient conditions."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "file changed as we read it",
+            source,
+            "allowlist must include exact GNU tar 'file changed as we read it' message",
+        )
+        self.assertIn(
+            "file removed before we read it",
+            source,
+            "allowlist must include exact GNU tar 'file removed before we read it' message",
+        )
+
+    def test_backup_script_stage3_uses_home_tar_function(self):
+        """Stage 3 must call create_home_tar_archive (not create_tar_archive)
+        so the /home archive tolerates live file changes on an active desktop."""
+        source = BACKUP_SCRIPT.read_text()
+        # The home archive call must use the tolerant function
+        self.assertIn(
+            'create_home_tar_archive "home.tar"',
+            source,
+            "Stage 3 must use create_home_tar_archive for home.tar",
+        )
+
+    def test_backup_script_strict_function_still_used_for_etc(self):
+        """create_tar_archive (strict) must still be used for /etc/nixos,
+        secrets, var-lib-lnd-clean, and var-lib — only /home is tolerant."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'create_tar_archive "etc-nixos.tar"',
+            source,
+            "etc-nixos must use strict create_tar_archive",
+        )
+        self.assertIn(
+            'create_tar_archive "var-lib.tar"',
+            source,
+            "var-lib must use strict create_tar_archive",
+        )
+
+    # ── Tests for partial-file atomic publish ─────────────────────────────────
+
+    def test_backup_script_uses_partial_path_and_atomic_rename(self):
+        """Archives must be written to a .partial path and atomically renamed
+        to the final name only after acceptance, so a failed run never leaves a
+        partial archive that appears complete."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            ".partial",
+            source,
+            "backup script must write to a .partial path before renaming",
+        )
+        self.assertIn(
+            'mv "$partial_path" "$archive_path"',
+            source,
+            "backup script must atomically rename partial to final archive path",
+        )
+        self.assertIn(
+            "PARTIAL_FILES",
+            source,
+            "backup script must track partial files for cleanup",
+        )
+
+    # ── Tests for INCOMPLETE / BACKUP_COMPLETE markers ────────────────────────
+
+    def test_backup_script_writes_incomplete_marker(self):
+        """A backup directory must be marked INCOMPLETE immediately after
+        creation, so interrupted or failed runs are identifiable."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'touch "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must create INCOMPLETE marker after mkdir",
+        )
+
+    def test_backup_script_writes_backup_complete_marker(self):
+        """A BACKUP_COMPLETE file must be written only after all work succeeds,
+        so restore tools can distinguish complete from incomplete backups."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'BACKUP_COMPLETE"',
+            source,
+            "backup script must write BACKUP_COMPLETE file on success",
+        )
+        self.assertIn(
+            'rm -f "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must remove INCOMPLETE marker on success",
+        )
+
+    # ── Tests for concurrency lock ─────────────────────────────────────────────
+
+    def test_backup_script_uses_flock_concurrency_lock(self):
+        """The script must acquire an exclusive flock before starting work to
+        prevent two simultaneous backup runs."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "flock",
+            source,
+            "backup script must use flock for concurrency protection",
+        )
+        self.assertIn(
+            "LOCK_FILE",
+            source,
+            "backup script must define a LOCK_FILE for the flock",
+        )
+
+    # ── Tests for expanded home exclusions ────────────────────────────────────
+
+    def test_backup_script_home_excludes_browser_caches(self):
+        """The home archive must exclude browser disk caches (volatile data)
+        while keeping profile data, bookmarks, and user documents."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "--exclude='home/*/.mozilla/firefox/*/cache2'",
+            source,
+            "backup script must exclude Firefox cache2 directory",
+        )
+        self.assertIn(
+            "--exclude='home/*/.config/google-chrome/*/Cache'",
+            source,
+            "backup script must exclude Chrome Cache directory",
+        )
+        self.assertIn(
+            "--exclude='home/*/.config/chromium/*/Cache'",
+            source,
+            "backup script must exclude Chromium Cache directory",
+        )
+        self.assertIn(
+            "--exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache'",
+            source,
+            "backup script must exclude Brave Cache directory",
+        )
+
+    def test_backup_script_home_excludes_other_volatile_caches(self):
+        """The home archive must exclude baloo, thumbnails, and X session
+        error logs — all volatile/reconstructable content."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "--exclude='home/*/.local/share/baloo'",
+            source,
+        )
+        self.assertIn(
+            "--exclude='home/*/.thumbnails'",
+            source,
+        )
+        self.assertIn(
+            "--exclude='home/*/.xsession-errors'",
+            source,
+        )
+
+    # ── Tests for require_cmd additions ───────────────────────────────────────
+
+    def test_backup_script_requires_mktemp_and_flock(self):
+        """The script must explicitly check for mktemp and flock via require_cmd
+        so that missing tools fail early with a clear error."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "require_cmd mktemp",
+            source,
+            "backup script must require mktemp",
+        )
+        self.assertIn(
+            "require_cmd flock",
+            source,
+            "backup script must require flock",
+        )
+
+    # ── Behavioral tests ──────────────────────────────────────────────────────
+
+    def test_allowlist_accepts_file_changed_message(self):
+        """Behavioral: _is_home_warning_allowlisted (from the actual backup script)
+        returns true for the exact GNU tar 'file changed as we read it' message (LC_ALL=C).
+        The function definition is extracted from the production script via Python regex
+        to ensure the test exercises the real allowlist without eval-of-awk risks."""
+        func_def = _extract_bash_function(BACKUP_SCRIPT, "_is_home_warning_allowlisted")
+        script = (
+            "#!/usr/bin/env bash\n"
+            + func_def + "\n"
+            + 'msgs=(\n'
+            + '  "tar: /home/user/firefox.db: file changed as we read it"\n'
+            + '  "tar: /home/user/.local/share/app: file removed before we read it"\n'
+            + ")\n"
+            + 'for msg in "${msgs[@]}"; do\n'
+            + '  _is_home_warning_allowlisted "$msg" || { echo "BLOCKED: $msg"; exit 1; }\n'
+            + '  echo "ALLOWED: $msg"\n'
+            + "done\n"
+        )
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as f:
+            f.write(script)
+            script_path = f.name
+        try:
+            result = subprocess.run(["bash", script_path], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("ALLOWED", result.stdout)
+            self.assertNotIn("BLOCKED", result.stdout)
+        finally:
+            os.unlink(script_path)
+
+    def test_allowlist_rejects_fatal_diagnostics(self):
+        """Behavioral: _is_home_warning_allowlisted (from the actual backup script)
+        returns false for permission errors, I/O errors, write errors, and other
+        fatal conditions."""
+        func_def = _extract_bash_function(BACKUP_SCRIPT, "_is_home_warning_allowlisted")
+        script = (
+            "#!/usr/bin/env bash\n"
+            + func_def + "\n"
+            + 'msgs=(\n'
+            + '  "tar: /home/user/file: Permission denied"\n'
+            + '  "tar: /dev/sdb: Cannot read: Input/output error"\n'
+            + '  "tar: Error is not recoverable: exiting now"\n'
+            + '  "Write error"\n'
+            + '  "tar: /home/user: Cannot stat: No such file or directory"\n'
+            + ")\n"
+            + 'for msg in "${msgs[@]}"; do\n'
+            + '  _is_home_warning_allowlisted "$msg" && { echo "ALLOWED_BUT_SHOULD_NOT: $msg"; exit 1; }\n'
+            + '  echo "CORRECTLY_BLOCKED: $msg"\n'
+            + "done\n"
+        )
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as f:
+            f.write(script)
+            script_path = f.name
+        try:
+            result = subprocess.run(["bash", script_path], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("CORRECTLY_BLOCKED", result.stdout)
+            self.assertNotIn("ALLOWED_BUT_SHOULD_NOT", result.stdout)
+        finally:
+            os.unlink(script_path)
+
+    def test_home_tar_exits_1_with_only_allowlisted_warnings_is_accepted(self):
+        """Behavioral: _create_archive_impl in HOME mode accepts tar exit 1 when
+        every diagnostic is allowlisted and publishes the final archive."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "home", "user1")
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(src, exist_ok=True)
+            os.makedirs(dest, exist_ok=True)
+            with open(os.path.join(src, "testfile.txt"), "w") as f:
+                f.write("test content\n")
+
+            # Emulate the allowlist + acceptance logic in HOME mode
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+BACKUP_DIR="{dest}"
+ARCHIVE_FILES=()
+ARCHIVE_WARNINGS=()
+PARTIAL_FILES=()
+
+log() {{ echo "$*"; }}
+fail() {{ echo "FAIL: $*" >&2; exit 1; }}
+
+_is_home_warning_allowlisted() {{
+  local msg="$1"
+  case "$msg" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}}
+
+_create_archive_impl() {{
+  local mode="$1"; shift
+  local archive_name="$1"; shift
+  local archive_path="$BACKUP_DIR/$archive_name"
+  local partial_path="${{archive_path}}.partial"
+  local diag_tmp
+  diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)"
+  PARTIAL_FILES+=("$partial_path" "$diag_tmp")
+
+  local tar_rc=0
+  LC_ALL=C tar --create --file "$partial_path" \
+    --numeric-owner --sparse --one-file-system \
+    "$@" 2>"$diag_tmp" || tar_rc=$?
+
+  # Inject a simulated allowlisted warning to match production scenario
+  echo "tar: {src}/testfile.txt: file changed as we read it" >> "$diag_tmp"
+  tar_rc=1  # simulate exit 1
+
+  local has_fatal_diag=0
+  if [[ -s "$diag_tmp" ]]; then
+    while IFS= read -r diag_line; do
+      [[ -n "$diag_line" ]] || continue
+      if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then
+        has_fatal_diag=1
+      fi
+    done < "$diag_tmp"
+  fi
+
+  local accept=0
+  if [[ "$tar_rc" -eq 0 ]]; then
+    accept=1
+  elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+    accept=1
+    ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings")
+  fi
+
+  [[ "$accept" -eq 1 ]] || {{ rm -f "$partial_path" "$diag_tmp"; fail "Rejected"; }}
+  [[ -s "$partial_path" ]] || {{ rm -f "$partial_path" "$diag_tmp"; fail "Empty"; }}
+
+  mv "$partial_path" "$archive_path"
+  rm -f "$diag_tmp"
+  ARCHIVE_FILES+=("$archive_name")
+  echo "CREATED: $archive_name"
+}}
+
+create_home_tar_archive() {{ _create_archive_impl HOME "$@"; }}
+
+create_home_tar_archive "home.tar" -C "{tmpdir}" home
+echo "WARNINGS: ${{#ARCHIVE_WARNINGS[@]}}"
+[[ -f "{dest}/home.tar" ]] && echo "FILE_EXISTS" || echo "FILE_MISSING"
+[[ ! -f "{dest}/home.tar.partial" ]] && echo "PARTIAL_CLEANED" || echo "PARTIAL_LEFT"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}\nstderr: {result.stderr}")
+            self.assertIn("CREATED: home.tar", result.stdout)
+            self.assertIn("FILE_EXISTS", result.stdout)
+            self.assertIn("PARTIAL_CLEANED", result.stdout)
+
+    def test_home_tar_exits_1_with_fatal_diagnostic_fails(self):
+        """Behavioral: _create_archive_impl in HOME mode must fail when tar exit 1
+        includes a non-allowlisted diagnostic (e.g. permission denied)."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+BACKUP_DIR="{dest}"
+ARCHIVE_FILES=()
+ARCHIVE_WARNINGS=()
+PARTIAL_FILES=()
+
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+log() {{ echo "$*"; }}
+
+_is_home_warning_allowlisted() {{
+  local msg="$1"
+  case "$msg" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}}
+
+# Simulate tar exit 1 with a fatal permission-denied diagnostic
+archive_name="home.tar"
+partial_path="$BACKUP_DIR/$archive_name.partial"
+diag_tmp=$(mktemp)
+echo "tar: /home/user/locked: Permission denied" > "$diag_tmp"
+
+tar_rc=1
+has_fatal_diag=0
+while IFS= read -r diag_line; do
+  [[ -n "$diag_line" ]] || continue
+  if ! _is_home_warning_allowlisted "$diag_line"; then
+    has_fatal_diag=1
+  fi
+done < "$diag_tmp"
+rm -f "$diag_tmp"
+
+if [[ "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+  echo "INCORRECTLY_ACCEPTED"
+  exit 0
+else
+  fail "tar exit 1 with Permission denied"
+fi
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}")
+            self.assertIn("CORRECTLY_FAILED", result.stdout)
+
+    def test_home_tar_exits_2_always_fails(self):
+        """Behavioral: tar exit code 2 (fatal) must always fail even in HOME mode."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+BACKUP_DIR="{dest}"
+ARCHIVE_WARNINGS=()
+
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+log() {{ echo "$*"; }}
+
+_is_home_warning_allowlisted() {{
+  case "$1" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}}
+
+tar_rc=2
+has_fatal_diag=0
+
+accept=0
+if [[ "$tar_rc" -eq 0 ]]; then
+  accept=1
+elif [[ "HOME" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+  accept=1
+fi
+
+if [[ "$accept" -eq 0 ]]; then
+  if [[ "$tar_rc" -gt 1 ]]; then
+    fail "tar exited with fatal code $tar_rc"
+  fi
+fi
+echo "SHOULD_NOT_REACH_HERE"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}")
+            self.assertIn("CORRECTLY_FAILED", result.stdout)
+            self.assertNotIn("SHOULD_NOT_REACH_HERE", result.stdout)
+
+    def test_partial_file_cleanup_on_failure(self):
+        """Behavioral: a .partial file must be removed when the archive creation
+        fails, so no incomplete archive is left on the destination."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            partial_path = os.path.join(dest, "home.tar.partial")
+
+            script = f"""#!/usr/bin/env bash
+BACKUP_DIR="{dest}"
+PARTIAL_FILES=()
+
+fail() {{
+  # Simulate cleanup removing partial files
+  for p in "${{PARTIAL_FILES[@]}}"; do
+    [[ -f "$p" ]] && rm -f "$p"
+  done
+  echo "FAILED: $*"
+  exit 1
+}}
+log() {{ :; }}
+
+partial_path="$BACKUP_DIR/home.tar.partial"
+diag_tmp=$(mktemp)
+PARTIAL_FILES+=("$partial_path" "$diag_tmp")
+
+# Simulate partial archive being written
+touch "$partial_path"
+
+# Simulate a fatal error
+fail "tar exited with code 2"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}")
+            self.assertFalse(
+                os.path.exists(partial_path),
+                "Partial file must be removed on failure",
+            )
+            self.assertIn("FAILED", result.stdout)
+
+    def test_archive_warnings_recorded_in_manifest(self):
+        """Source text: ARCHIVE_WARNINGS must be written into BACKUP_MANIFEST.txt
+        so the user can see which archives had nonfatal live-file warnings."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "ARCHIVE_WARNINGS[@]",
+            source,
+            "backup script must iterate ARCHIVE_WARNINGS in manifest generation",
+        )
+        self.assertIn(
+            "Nonfatal warnings:",
+            source,
+            "manifest must include a 'Nonfatal warnings:' section",
+        )
+
+    def test_backup_script_logs_warnings_before_success_message(self):
+        """Source text: when ARCHIVE_WARNINGS is non-empty, the script must log
+        the warnings before the 'All Finished!' success message."""
+        source = BACKUP_SCRIPT.read_text()
+        # Both constructs must be present
+        self.assertIn(
+            "${#ARCHIVE_WARNINGS[@]}",
+            source,
+            "backup script must check ARCHIVE_WARNINGS count before success message",
+        )
+        self.assertIn(
+            "All Finished!",
+            source,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_rdp_module_boot_setup.py

@@ -0,0 +1,122 @@
+import unittest
+from pathlib import Path
+
+
+RDP_NIX = Path(__file__).resolve().parents[2] / "modules" / "rdp.nix"
+USERNAME_READ = "USERNAME=\"$(tr -d '\\n' < \"$USERNAME_FILE\")\""
+USERNAME_LENGTH_GUARD = "if [ \"''${#USERNAME}\" -gt 32 ]; then"
+SHORT_PASSWORD_GUARD = 'if [ "\'\'${#PASSWORD}" -lt 8 ]; then'
+
+
+def _section(source: str, start: str, end: str) -> str:
+    start_idx = source.find(start)
+    if start_idx == -1:
+        raise AssertionError(f"Expected section start not found: {start!r}")
+    end_idx = source.find(end, start_idx)
+    if end_idx == -1:
+        raise AssertionError(f"Expected section end not found: {end!r}")
+    return source[start_idx:end_idx]
+
+
+class RdpModuleBootSetupTests(unittest.TestCase):
+    def setUp(self):
+        self.source = RDP_NIX.read_text()
+        self.gnome_service = _section(
+            self.source,
+            "systemd.services.gnome-remote-desktop = {",
+            "systemd.tmpfiles.rules = [",
+        )
+        self.setup_service = _section(
+            self.source,
+            "systemd.services.gnome-remote-desktop-setup = {",
+            "};\n}",
+        )
+
+    def test_does_not_redeclare_gnome_remote_desktop_user(self):
+        self.assertNotIn("users.users.gnome-remote-desktop", self.source)
+        self.assertNotIn("createHome = true;", self.source)
+
+    def test_main_service_requires_setup_before_starting(self):
+        self.assertIn('wantedBy = [ "graphical.target" ];', self.gnome_service)
+        self.assertIn('after = [ "gnome-remote-desktop-setup.service" ];', self.gnome_service)
+        self.assertIn('requires = [ "gnome-remote-desktop-setup.service" ];', self.gnome_service)
+
+    def test_setup_waits_for_configuration_service_and_bounded_timeout(self):
+        self.assertIn('wantedBy = [ "graphical.target" ];', self.setup_service)
+        self.assertIn('before = [ "gnome-remote-desktop.service" ];', self.setup_service)
+        self.assertIn('"dbus.service"', self.setup_service)
+        self.assertIn('"gnome-remote-desktop-configuration.service"', self.setup_service)
+        self.assertNotIn("RemainAfterExit", self.setup_service)
+        self.assertIn('TimeoutStartSec = "2min";', self.setup_service)
+        self.assertIn('timeout --kill-after=5s 10s', self.setup_service)
+        self.assertIn('echo "grdctl command timed out: $*" >&2', self.setup_service)
+        self.assertIn('echo "grdctl command failed (exit $rc): $*" >&2', self.setup_service)
+
+    def test_setup_runs_grdctl_directly_as_root(self):
+        # The oneshot service runs as root; grdctl --system is called directly.
+        # GRD 50.x invokes pkexec internally, but the call itself is plain
+        # grdctl --system, not a manual pkexec invocation.
+        self.assertIn('grdctl --system "$@"', self.setup_service)
+        self.assertNotIn("runuser", self.setup_service)
+        self.assertNotIn("sudo", self.setup_service)
+        # No direct Nix-store pkexec invocation (pkgs.polkit}/bin/pkexec).
+        self.assertNotIn("pkgs.polkit}/bin/pkexec", self.setup_service)
+
+    def test_privilege_escalation_packages_absent_from_setup_path(self):
+        self.assertNotIn("pkgs.polkit", self.setup_service)
+        self.assertNotIn("pkgs.util-linux", self.setup_service)
+
+    def test_run_wrappers_bin_prepended_to_path(self):
+        # /run/wrappers/bin must be prepended to PATH before any grdctl_system
+        # invocation so that grdctl --system resolves the NixOS setuid pkexec.
+        path_export = 'export PATH="/run/wrappers/bin:$PATH"'
+        grdctl_marker = "grdctl_system"
+        script = self.setup_service
+        path_idx = script.find(path_export)
+        grdctl_idx = script.find(grdctl_marker)
+        self.assertGreater(path_idx, -1, f"{path_export!r} not found in setup script")
+        self.assertGreater(
+            grdctl_idx, path_idx,
+            "PATH export must appear before the first grdctl_system usage",
+        )
+
+    def test_pkexec_preflight_check(self):
+        # A preflight must confirm /run/wrappers/bin/pkexec is executable
+        # with a clear error message before any GRD configuration changes.
+        self.assertIn("test -x /run/wrappers/bin/pkexec", self.setup_service)
+        self.assertIn(
+            "/run/wrappers/bin/pkexec is absent or not executable",
+            self.setup_service,
+        )
+
+    def test_hub_files_are_the_source_of_truth_for_username_and_password(self):
+        self.assertIn('DEFAULT_USERNAME="sovran"', self.setup_service)
+        self.assertIn('if [ ! -f "$USERNAME_FILE" ]; then', self.setup_service)
+        self.assertIn(USERNAME_READ, self.setup_service)
+        self.assertIn(USERNAME_LENGTH_GUARD, self.setup_service)
+        self.assertIn('case "$USERNAME" in', self.setup_service)
+        self.assertIn('[A-Za-z_][A-Za-z0-9_-]*)', self.setup_service)
+        self.assertIn("RDP username is too long (''${#USERNAME} characters, maximum 32)", self.setup_service)
+        self.assertIn("RDP username must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens", self.setup_service)
+        self.assertIn('if [ ! -f "$PASSWORD_FILE" ]; then', self.setup_service)
+        self.assertIn("tr -d '\\n'", self.setup_service)
+        self.assertIn('"$PASSWORD_FILE"', self.setup_service)
+        self.assertIn(SHORT_PASSWORD_GUARD, self.setup_service)
+        self.assertIn("RDP password is too short (''${#PASSWORD} characters, minimum 8)", self.setup_service)
+        self.assertIn('grdctl_system rdp set-credentials "$USERNAME" "$PASSWORD"', self.setup_service)
+        self.assertNotIn('grdctl --system rdp set-credentials sovran "$PASSWORD"', self.setup_service)
+
+    def test_secure_permissions_are_enforced_for_state_and_secret_files(self):
+        self.assertIn('"d /var/lib/gnome-remote-desktop/tls 0700', self.source)
+        self.assertIn("chmod 700", self.setup_service)
+        self.assertIn('chmod 600 "$USERNAME_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$PASSWORD_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$CRED_FILE"', self.setup_service)
+        self.assertIn('chmod 600 "$TLS_DIR/rdp-tls.key"', self.setup_service)
+        self.assertIn('chmod 644 "$TLS_DIR/rdp-tls.crt"', self.setup_service)
+        self.assertIn('LOCAL_IP="$(hostname -I | awk \'{print $1}\')"', self.setup_service)
+        self.assertIn('LOCAL_IP="127.0.0.1"', self.setup_service)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_service_detail_router_wording.py

@@ -0,0 +1,171 @@
+import unittest
+from pathlib import Path
+from unittest.mock import mock_open, patch
+import sys
+import types
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+def _install_web_stubs():
+    if "fastapi" in sys.modules:
+        return
+
+    class _HTTPException(Exception):
+        def __init__(self, status_code=None, detail=None):
+            super().__init__(detail)
+            self.status_code = status_code
+            self.detail = detail
+
+    class _FastAPI:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def mount(self, *args, **kwargs):
+            return None
+
+        def add_middleware(self, *args, **kwargs):
+            return None
+
+        def __getattr__(self, _name):
+            def _decorator_factory(*args, **kwargs):
+                def _decorator(func):
+                    return func
+
+                return _decorator
+
+            return _decorator_factory
+
+    class _BaseModel:
+        pass
+
+    class _StaticFiles:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _Jinja2Templates:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _BaseHTTPMiddleware:
+        pass
+
+    fastapi_module = types.ModuleType("fastapi")
+    fastapi_module.FastAPI = _FastAPI
+    fastapi_module.HTTPException = _HTTPException
+    sys.modules["fastapi"] = fastapi_module
+
+    responses_module = types.ModuleType("fastapi.responses")
+    responses_module.HTMLResponse = object
+    responses_module.JSONResponse = object
+    responses_module.RedirectResponse = object
+    sys.modules["fastapi.responses"] = responses_module
+
+    staticfiles_module = types.ModuleType("fastapi.staticfiles")
+    staticfiles_module.StaticFiles = _StaticFiles
+    sys.modules["fastapi.staticfiles"] = staticfiles_module
+
+    templating_module = types.ModuleType("fastapi.templating")
+    templating_module.Jinja2Templates = _Jinja2Templates
+    sys.modules["fastapi.templating"] = templating_module
+
+    requests_module = types.ModuleType("fastapi.requests")
+    requests_module.Request = object
+    sys.modules["fastapi.requests"] = requests_module
+
+    pydantic_module = types.ModuleType("pydantic")
+    pydantic_module.BaseModel = _BaseModel
+    sys.modules["pydantic"] = pydantic_module
+
+    starlette_base_module = types.ModuleType("starlette.middleware.base")
+    starlette_base_module.BaseHTTPMiddleware = _BaseHTTPMiddleware
+    sys.modules["starlette.middleware.base"] = starlette_base_module
+
+    starlette_middleware_module = types.ModuleType("starlette.middleware")
+    starlette_middleware_module.base = starlette_base_module
+    sys.modules["starlette.middleware"] = starlette_middleware_module
+
+    starlette_module = types.ModuleType("starlette")
+    starlette_module.middleware = starlette_middleware_module
+    sys.modules["starlette"] = starlette_module
+
+
+_install_web_stubs()
+from sovran_systemsos_web import server
+
+
+class ServiceDetailRouterWordingTests(unittest.IsolatedAsyncioTestCase):
+    async def test_livekit_service_detail_includes_internal_ip(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": {7881}, "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["internal_ip"], "192.168.1.44")
+        self.assertEqual(result["extra_ports"][0]["status"], "listening")
+        self.assertEqual(result["domain_check_steps"][-1]["label"], "Router Setup Needed")
+
+    async def test_livekit_router_step_uses_not_ready_yet_wording(self):
+        service_cfg = {
+            "services": [
+                {"unit": "livekit.service", "icon": "element-call", "enabled": True, "type": "system"}
+            ]
+        }
+        domain_eval = {
+            "domain_status": {"status": "ok"},
+            "domain_reachable": {"reachable": True},
+            "domain_check_steps": [],
+            "has_issues": False,
+        }
+
+        with (
+            patch.object(server, "load_config", return_value=service_cfg),
+            patch.object(server, "_read_hub_overrides", return_value=({}, None, None)),
+            patch.object(server.sysctl, "is_active", return_value="active"),
+            patch.dict(server.SERVICE_DOMAIN_MAP, {"livekit.service": "element-call"}, clear=False),
+            patch.dict(
+                server.SERVICE_PORT_REQUIREMENTS,
+                {"livekit.service": [{"port": "7881", "protocol": "TCP", "description": "LiveKit"}]},
+                clear=False,
+            ),
+            patch("builtins.open", mock_open(read_data="call.example.com\n")),
+            patch.object(server, "_evaluate_domain_checklist", return_value=domain_eval),
+            patch.object(server, "_get_internal_ip", return_value="192.168.1.44"),
+            patch.object(server, "_save_internal_ip"),
+            patch.object(server, "_get_listening_ports", return_value={"tcp": set(), "udp": set()}),
+            patch.object(server, "_get_firewall_allowed_ports", return_value={"tcp": set(), "udp": set()}),
+        ):
+            result = await server.api_service_detail("livekit.service")
+
+        self.assertEqual(result["extra_ports"][0]["status"], "closed")
+        self.assertIn("Not ready yet", result["domain_check_steps"][-1]["detail"])
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_template_response_signature.py

@@ -0,0 +1,99 @@
+"""Regression test for Starlette 1.1.0+ TemplateResponse keyword-argument style.
+
+Prior to this fix, the three HTML routes called:
+    templates.TemplateResponse("name.html", {"request": request, ...})
+which passes the context dict as the second positional argument.  With the
+updated Starlette/FastAPI versions shipped in NixOS unstable (Starlette 1.1.0,
+FastAPI 0.136.3) that positional argument is the template name, causing Jinja2
+to receive a dict as a cache key and raise:
+    TypeError: unhashable type: 'dict'
+
+The fix updates every call to use keyword arguments:
+    templates.TemplateResponse(request=request, name="name.html", context={...})
+"""
+
+import ast
+import unittest
+from pathlib import Path
+
+
+SERVER_PY = Path(__file__).resolve().parents[1] / "sovran_systemsos_web" / "server.py"
+
+
+def _template_response_calls(source: str):
+    """Return a list of ast.Call nodes that are TemplateResponse calls."""
+    tree = ast.parse(source)
+    calls = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func = node.func
+        if isinstance(func, ast.Attribute) and func.attr == "TemplateResponse":
+            calls.append(node)
+    return calls
+
+
+class TemplateResponseSignatureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = SERVER_PY.read_text()
+        self.calls = _template_response_calls(self.source)
+
+    def test_at_least_one_template_response_call_found(self):
+        self.assertGreater(len(self.calls), 0, "No TemplateResponse calls found in server.py")
+
+    def test_no_old_style_positional_dict_context(self):
+        """No TemplateResponse call should pass a dict literal as its second positional arg.
+
+        The old style was:
+            templates.TemplateResponse("name.html", {"request": request, ...})
+        where args[0] is a string and args[1] is a Dict node.  That pattern
+        triggers the Starlette 1.1.0 bug.
+        """
+        for call in self.calls:
+            positional = call.args
+            if len(positional) >= 2 and isinstance(positional[1], ast.Dict):
+                self.fail(
+                    f"Found old-style TemplateResponse call at line {call.lineno}: "
+                    "second positional argument is a dict literal.  "
+                    "Use keyword arguments (request=, name=, context=) instead."
+                )
+
+    def test_request_not_duplicated_in_context(self):
+        """The 'request' key must not appear inside the context= dict when
+        request= is already passed as a dedicated keyword argument."""
+        for call in self.calls:
+            kw_dict = {kw.arg: kw.value for kw in call.keywords if isinstance(kw, ast.keyword)}
+
+            if "request" not in kw_dict:
+                continue  # no request= kwarg, nothing to check
+
+            context_node = kw_dict.get("context")
+            if not isinstance(context_node, ast.Dict):
+                continue
+
+            for key_node in context_node.keys:
+                if isinstance(key_node, ast.Constant) and key_node.value == "request":
+                    self.fail(
+                        f"TemplateResponse at line {call.lineno} passes 'request' both as "
+                        "request= keyword argument and inside the context dict."
+                    )
+
+    def test_all_calls_use_keyword_arguments(self):
+        """Every TemplateResponse call should use keyword arguments for request, name,
+        and context rather than relying on positional ordering."""
+        for call in self.calls:
+            kw_args = {kw.arg for kw in call.keywords if isinstance(kw, ast.keyword)}
+            self.assertIn(
+                "request",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'request='.",
+            )
+            self.assertIn(
+                "name",
+                kw_args,
+                f"TemplateResponse at line {call.lineno} is missing keyword argument 'name='.",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

configuration.nix

@@ -149,8 +149,8 @@
     gparted pv unzip parted screen zenity
     libargon2 gnome-terminal libreoffice-fresh
     dig firefox wp-cli axel
-    lk-jwt-service livekit-libwebrtc livekit-cli livekit
-    matrix-synapse age
+    lk-jwt-service livekit-libwebrtc livekit
+    matrix-synapse age onlyoffice-desktopeditors
   ];
 
   # ── Shell ──────────────────────────────────────────────────

flake.lock

@@ -5,11 +5,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1780397635,
-        "narHash": "sha256-6WH7LKD6i91VLWoz4mEpoULtqVinCEZxG7ZjJPMSi3k=",
+        "lastModified": 1783519926,
+        "narHash": "sha256-2zwAN4lNitHFrHVnRZG3YcvpdtWOoF0cOBstxMeB1KI=",
         "owner": "emmanuelrosa",
         "repo": "btc-clients-nix",
-        "rev": "feacd7684dc6bfcd49c57764944a2049bbd71924",
+        "rev": "731a1e11c2fefb14f0aa4b1f03cfa85c19c28d71",
         "type": "github"
       },
       "original": {
@@ -52,11 +52,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1778716662,
-        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "lastModified": 1782949081,
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -108,11 +108,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1780218263,
-        "narHash": "sha256-T/f0pPDrH3Qc1VXyQXbK7yfHWRn90l3xwplc/nsxin4=",
+        "lastModified": 1782911660,
+        "narHash": "sha256-PbR+tJ5E/Ux+01UtdFKqblccVA4/FgWbkym4ev3VHHQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7fc393d1b46fa000d48ff14e8b6a3c9985f03af0",
+        "rev": "cf720c15e108d432d29041cc5a185630809acefb",
         "type": "github"
       },
       "original": {
@@ -139,11 +139,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1780453794,
-        "narHash": "sha256-bXMRa9VTsHSPXL4Cw8R6JJLQeY3Y/IP4+YJCYVmQ7FY=",
+        "lastModified": 1784432872,
+        "narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338",
+        "rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
         "type": "github"
       },
       "original": {
@@ -187,11 +187,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1780243769,
-        "narHash": "sha256-x5UQuRsH3MqI0U9afaXSNqzTPSeZlRLvFAav2Ux1pNw=",
+        "lastModified": 1784356753,
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "331800de5053fcebacf6813adb5db9c9dca22a0c",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
         "type": "github"
       },
       "original": {
@@ -203,11 +203,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1780336545,
-        "narHash": "sha256-vhVhuXzFrIOfcssC/9hDHx7MHzDKjF3keHuREOQqQiQ=",
+        "lastModified": 1783915482,
+        "narHash": "sha256-FmieJB8/OUvNxbkboi7+IGfIuSXY3nF/hZQm8kD0r50=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4df1b885d76a54e1aa1a318f8d16fd6005b6401f",
+        "rev": "6cdc7fc76e8bf7fde9fa43a849fcaaa70e230dee",
         "type": "github"
       },
       "original": {
@@ -224,11 +224,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1780646548,
-        "narHash": "sha256-Ckyl/l1XBmEwnaHcHD8PvBZk1uph0NqwbJ//CAvB7iE=",
+        "lastModified": 1784057377,
+        "narHash": "sha256-yycNej5//EsRbV10moBoh+/63vXEwZD1ZFEiRm6C9rQ=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "816a15282e58678dde831477964987d0262d4293",
+        "rev": "07180a087e4a00720dc0731cbcd8dec796974381",
         "type": "github"
       },
       "original": {

iso/common.nix

@@ -16,7 +16,6 @@ in
 {
   imports = [
     "${modulesPath}/installer/cd-dvd/installation-cd-graphical-gnome.nix"
-    ./branding.nix
   ];
 
   image.baseName = lib.mkForce "Sovran_SystemsOS";

iso/installer.py

@@ -1169,7 +1169,7 @@ class InstallerWindow(Adw.ApplicationWindow):
         btn_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=0)
         btn_box.set_halign(Gtk.Align.CENTER)
         btn_box.set_margin_bottom(32)
-        reboot_btn = Gtk.Button(label="I Have Written Down My Password — Reboot Now")
+        reboot_btn = Gtk.Button(label="I Have Written Down My Password — Restart Entire System")
         reboot_btn.add_css_class("suggested-action")
         reboot_btn.add_css_class("pill")
         reboot_btn.connect("clicked", lambda b: subprocess.run(["sudo", "reboot"]))

modules/core/caddy.nix

@@ -16,7 +16,10 @@ let
 in
 {
   services.caddy = {
-    enable = true;
+    # Only enable Caddy when at least one domain-based service needs it or
+    # the operator has defined custom vhosts.  This prevents Caddy from
+    # running on Desktop Only installs that have no web services configured.
+    enable = needsHttpsPorts || extraVhosts != "";
     user = "caddy";
     group = "root";
   };
@@ -94,10 +97,10 @@ EOF
 $MATRIX {
   reverse_proxy /_matrix/* http://localhost:8008
   reverse_proxy /_synapse/client/* http://localhost:8008
-}
-
-$MATRIX:8448 {
-  reverse_proxy http://localhost:8008
+  handle /.well-known/matrix/server {
+    header Content-Type application/json
+    respond \`{"m.server":"$MATRIX:443"}\` 200
+  }
 }
 EOF
         fi

modules/core/local-domain-loopback.nix

@@ -0,0 +1,171 @@
+{ config, pkgs, lib, ... }:
+
+# ── Server-local domain loopback overrides ────────────────────────────────────
+#
+# Some routers (especially newer ISP-provided devices) do not support NAT
+# loopback (hairpin NAT).  When a request originates on this computer and
+# targets a public domain name that resolves to the router's WAN address, the
+# router may refuse to loop the connection back in — causing Nextcloud, WordPress
+# background jobs, and other server-side callbacks to fail even when the service
+# is fully operational from the internet.
+#
+# This module installs a one-shot systemd service,
+# ``sovran-hosts-update.service``, that reads the configured service domains
+# from ``/var/lib/domains/`` at boot (and whenever triggered by the Hub after a
+# domain is saved) and writes ``127.0.0.1`` entries for them into a dedicated
+# Sovran-managed block in ``/etc/hosts``.
+#
+# With those entries in place:
+#   • Requests originating on this computer resolve the public domain name to
+#     127.0.0.1, reach Caddy directly, and never touch the router.
+#   • Caddy still receives the correct public hostname via TLS SNI so virtual-
+#     host routing and certificate validation continue to work.
+#   • The Sovran Hub can verify Caddy reachability locally without needing NAT
+#     loopback.
+#
+# Limitation: this does not help other devices on your home network (phones,
+# laptops).  Those devices resolve domains via the router's DNS and still depend
+# on NAT loopback (or require manual router DNS overrides).  For now, only
+# server-originated requests benefit from this override.
+#
+# On NixOS, /etc/hosts is normally a symlink into the Nix store and is
+# regenerated by the system activation script.  The ``system.activationScripts``
+# hook below converts it to a writable file each time the system is activated
+# (i.e. after every ``nixos-rebuild switch``) and then injects the Sovran block.
+# The same wrapped Nix-store executable is reused by both the activation hook
+# and the ``sovran-hosts-update.service`` unit, ensuring a deterministic runtime
+# PATH in every execution context.
+
+let
+  # ── Wrapped Nix-store executable ──────────────────────────────────────────
+  # Built with pkgs.writeShellApplication so that all required runtime tools
+  # (awk, grep, coreutils) are declared explicitly and injected into PATH by
+  # Nix.  Both the systemd service and the activation hook reference this same
+  # store-path executable — there is no second raw script body.
+  hostsUpdateScript = pkgs.writeShellApplication {
+    name = "sovran-hosts-update";
+
+    # Declare every external command the script calls.  These packages are
+    # added to the script's runtime PATH by writeShellApplication; nothing from
+    # the system PATH is relied upon.
+    runtimeInputs = [
+      pkgs.coreutils   # readlink, cp, mv, chmod, mktemp, rm, tr, head
+      pkgs.gawk        # awk
+      pkgs.gnugrep     # grep
+    ];
+
+    text = ''
+      # Regenerate the Sovran-managed loopback block in /etc/hosts.
+      # Safe to run multiple times — idempotent.
+
+      DOMAINS_DIR="/var/lib/domains"
+      HOSTS_FILE="/etc/hosts"
+      BEGIN_MARKER="# Sovran managed begin — server-local loopback overrides"
+      END_MARKER="# Sovran managed end"
+
+      # ── Step 1: ensure /etc/hosts is a regular writable file ──────────────
+      # On NixOS /etc/hosts starts as a symlink to the Nix store.  We replace
+      # it with a copy so we can append our block without touching the store.
+      if [ -L "$HOSTS_FILE" ]; then
+        TARGET=$(readlink -f "$HOSTS_FILE")
+        cp --no-preserve=all "$TARGET" "$HOSTS_FILE.sovran-tmp"
+        mv "$HOSTS_FILE.sovran-tmp" "$HOSTS_FILE"
+        chmod 644 "$HOSTS_FILE"
+      fi
+
+      # ── Step 2: remove any existing Sovran block ──────────────────────────
+      # Use a temp file so the operation is atomic.
+      # awk -v passes marker strings safely without shell interpolation.
+      TMP=$(mktemp "$HOSTS_FILE.XXXXXX")
+      trap 'rm -f "$TMP"' EXIT
+      awk -v begin="$BEGIN_MARKER" -v end="$END_MARKER" '
+        $0 == begin { skip=1; next }
+        $0 == end   { skip=0; next }
+        !skip
+      ' "$HOSTS_FILE" > "$TMP"
+
+      # ── Step 3: collect valid configured service domains ──────────────────
+      # NOTE: The hostname validation regex below must stay in sync with
+      # _SAFE_DOMAIN_RE in app/sovran_systemsos_web/server.py.
+      ENTRIES=""
+      for KEY in matrix wordpress nextcloud btcpayserver vaultwarden haven element-calling; do
+        FILE="$DOMAINS_DIR/$KEY"
+        [ -f "$FILE" ] || continue
+        # Read the domain value (strip all whitespace, limit to 253 chars)
+        DOMAIN=$(tr -d '[:space:]' < "$FILE" | head -c 253)
+        [ -z "$DOMAIN" ] && continue
+        # Validate: must match a reasonable hostname pattern (no injection)
+        if ! printf '%s' "$DOMAIN" | grep -qE \
+          '^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$'; then
+          echo "sovran-hosts-update: skipping invalid domain value for $KEY: $DOMAIN" >&2
+          continue
+        fi
+        ENTRIES="$ENTRIES
+127.0.0.1 $DOMAIN
+::1       $DOMAIN"
+      done
+
+      # ── Step 4: append the Sovran block if there are any entries ──────────
+      if [ -n "$ENTRIES" ]; then
+        {
+          printf '\n%s\n' "$BEGIN_MARKER"
+          printf '%s\n' "# These entries route configured service domains to local Caddy."
+          printf '%s\n' "# They are managed automatically — do not edit this block."
+          printf '%s\n' "$ENTRIES"
+          printf '%s\n' "$END_MARKER"
+        } >> "$TMP"
+      fi
+
+      # ── Step 5: atomically replace /etc/hosts ─────────────────────────────
+      mv "$TMP" "$HOSTS_FILE"
+      chmod 644 "$HOSTS_FILE"
+    '';
+  };
+
+in
+{
+  # ── /etc/sovran-hosts-update.sh — operator discoverability symlink ─────────
+  # Retain the familiar /etc path so administrators can inspect or manually
+  # invoke the helper.  The target is the wrapped Nix-store executable, so
+  # there is no second raw script body to keep in sync.
+  environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript;
+
+  # ── Systemd service ────────────────────────────────────────────────────────
+
+  systemd.services.sovran-hosts-update = {
+    description = "Update /etc/hosts with Sovran server-local loopback overrides";
+    documentation = [ "https://github.com/naturallaw777/sovran-systems" ];
+
+    # Run before Caddy so loopback entries are ready when it starts.
+    before = [
+      "caddy.service"
+      "network-online.target"
+    ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      # Point directly at the wrapped Nix-store executable, not the /etc path.
+      ExecStart = lib.getExe hostsUpdateScript;
+    };
+  };
+
+  # ── Activation script (runs after every nixos-rebuild switch) ─────────────
+  # This ensures the loopback block survives rebuilds that restore the /etc/hosts
+  # symlink.  The "users" and "etc" scripts must complete first.
+  # The same wrapped Nix-store executable used by the systemd service is
+  # referenced here, guaranteeing identical runtime dependencies in both
+  # execution contexts.
+
+  system.activationScripts.sovranDomainLoopback = {
+    text = ''
+      if [ -d /var/lib/domains ]; then
+        if ! ${lib.getExe hostsUpdateScript}; then
+          echo "warning: sovran-hosts-update: failed to update /etc/hosts loopback entries" >&2
+        fi
+      fi
+    '';
+    deps = [ "etc" "users" ];
+  };
+}

modules/core/role-logic.nix

@@ -3,6 +3,13 @@
 {
   config = lib.mkMerge [
 
+    # nix-bitcoin is globally imported by the flake (nixosModules.Sovran_SystemsOS).
+    # This default satisfies nix-bitcoin's generateSecrets assertion so that Desktop
+    # Only systems can evaluate without enabling any Bitcoin services.
+    {
+      nix-bitcoin.generateSecrets = lib.mkDefault true;
+    }
+
     # ── Server+Desktop Role (default) ─────────────────────────
     (lib.mkIf config.sovran_systemsOS.roles.server_plus_desktop {
       sovran_systemsOS.web.btcpayserver = lib.mkDefault true;
@@ -12,15 +19,24 @@
     (lib.mkIf config.sovran_systemsOS.roles.desktop {
       services.desktopManager.gnome.enable = true;
 
+      # Force all server/node services and features off so they cannot be
+      # accidentally enabled via custom.nix or option defaults on Desktop Only.
       sovran_systemsOS.services = {
-        synapse = lib.mkDefault false;
-        bitcoin = lib.mkDefault false;
-        vaultwarden = lib.mkDefault false;
-        wordpress = lib.mkDefault false;
-        nextcloud = lib.mkDefault false;
+        synapse = lib.mkForce false;
+        bitcoin = lib.mkForce false;
+        vaultwarden = lib.mkForce false;
+        wordpress = lib.mkForce false;
+        nextcloud = lib.mkForce false;
       };
 
-      sovran_systemsOS.web.btcpayserver = lib.mkDefault false;
+      sovran_systemsOS.features = {
+        haven = lib.mkForce false;
+        mempool = lib.mkForce false;
+        element-calling = lib.mkForce false;
+        bitcoin-core = lib.mkForce false;
+      };
+
+      sovran_systemsOS.web.btcpayserver = lib.mkForce false;
     })
 
     # ── Bitcoin Node Only Role ────────────────────────────────

modules/core/sovran-hub.nix

@@ -146,33 +146,16 @@ let
     echo ""
 
     if [ "$RC" -eq 0 ]; then
-      echo "── Step 2/3: nixos-rebuild ──────────────────────────"
-      SWITCH_OUT=$(nixos-rebuild switch --flake /etc/nixos --print-build-logs \
+      echo "── Step 2/3: nixos-rebuild boot (stage next reboot) ──"
+      BOOT_OUT=$(nixos-rebuild boot --flake /etc/nixos --print-build-logs \
         --option connect-timeout 10 \
         --option stalled-download-timeout 90 \
         --option download-attempts 7 \
         --option fallback true 2>&1)
-      SWITCH_RC=$?
-      echo "$SWITCH_OUT"
-      if [ "$SWITCH_RC" -eq 0 ]; then
-        echo "[OK] switch succeeded"
-      elif echo "$SWITCH_OUT" | grep -q "switchInhibitors\|Pre-switch checks failed"; then
-        echo ""
-        echo "  ✓ Build succeeded — a reboot is required to apply this update"
-        echo "  (Critical system components changed; running nixos-rebuild boot instead)"
-        if nixos-rebuild boot --flake /etc/nixos --print-build-logs \
-             --option connect-timeout 10 \
-             --option stalled-download-timeout 90 \
-             --option download-attempts 7 \
-             --option fallback true 2>&1; then
-          echo "REBOOT_REQUIRED" > "$STATUS"
-          exit 0
-        else
-          echo "[ERROR] nixos-rebuild boot also failed"
-          RC=1
-        fi
-      else
-        echo "[ERROR] nixos-rebuild switch failed"
+      BOOT_RC=$?
+      echo "$BOOT_OUT"
+      if [ "$BOOT_RC" -ne 0 ]; then
+        echo "[ERROR] nixos-rebuild boot failed"
         RC=1
       fi
       echo ""
@@ -188,9 +171,10 @@ let
 
     if [ "$RC" -eq 0 ]; then
       echo "══════════════════════════════════════════════════"
-      echo "  ✓ Update completed successfully"
+      echo "  ✓ Update staged successfully"
+      echo "  Reboot required to activate the new system"
       echo "══════════════════════════════════════════════════"
-      echo "SUCCESS" > "$STATUS"
+      echo "REBOOT_REQUIRED" > "$STATUS"
     else
       echo "══════════════════════════════════════════════════"
       echo "  ✗ Update failed — see errors above"
@@ -398,13 +382,24 @@ in
       };
 
       path = [
+        pkgs.bash
+        pkgs.gawk
         pkgs.qrencode
         pkgs.curl
         pkgs.iproute2
         pkgs.nftables
         pkgs.iptables
         pkgs.hostname
-      ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package;
+        pkgs.coreutils
+        pkgs.findutils
+        pkgs.gnugrep
+        pkgs.gnutar
+        pkgs.util-linux
+      ]
+      ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
+      ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
+      ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
+      ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
     };
 
     systemd.services.sovran-hub-update = {

modules/core/ssh-bootstrap.nix

@@ -31,7 +31,7 @@ lib.mkIf userExists {
   };
 
   systemd.services.factory-ssh-keygen = {
-    description = "Generate factory SSH key for ${userName} if missing";
+    description = "Generate or repair factory SSH key for ${userName}";
     wantedBy = [ "multi-user.target" ];
     after = [ "ssh-passphrase-setup.service" ];
     requires = [ "ssh-passphrase-setup.service" ];
@@ -39,14 +39,47 @@ lib.mkIf userExists {
       Type = "oneshot";
       RemainAfterExit = true;
     };
-    path = [ pkgs.openssh pkgs.coreutils ];
+    path = [ pkgs.openssh pkgs.coreutils pkgs.util-linux ];
     script = ''
-      if [ ! -f "${keyPath}" ]; then
-        PASSPHRASE=$(cat /var/lib/secrets/ssh-passphrase)
+      set -eu
+
+      PASSPHRASE=$(cat /var/lib/secrets/ssh-passphrase)
+      lock_file="${keyPath}.lock"
+
+      exec 9>"$lock_file"
+
+      if ! flock -n 9; then
+        echo "Factory SSH key setup is already running." >&2
+        exit 1
+      fi
+
+      generate_factory_key() {
         ssh-keygen -q -N "$PASSPHRASE" -t ed25519 -f "${keyPath}"
         chown ${userName}:users "${keyPath}" "${keyPath}.pub"
         chmod 600 "${keyPath}"
         chmod 644 "${keyPath}.pub"
+      }
+
+      if [ ! -f "${keyPath}" ]; then
+        generate_factory_key
+      elif ! ssh-keygen -y -P "$PASSPHRASE" -f "${keyPath}" >/dev/null 2>&1; then
+        backup_suffix="$(date -u +%Y%m%d_%H%M%S)-$$"
+        backup_path="${keyPath}.bak-$backup_suffix"
+        backup_index=0
+
+        while [ -e "$backup_path" ] || [ -e "$backup_path.pub" ]; do
+          backup_index=$((backup_index + 1))
+          backup_path="${keyPath}.bak-$backup_suffix-$backup_index"
+        done
+
+        echo "Existing factory SSH key does not match current passphrase; backing it up to $backup_path and generating a replacement."
+        mv "${keyPath}" "$backup_path"
+
+        if [ -f "${keyPath}.pub" ]; then
+          mv "${keyPath}.pub" "$backup_path.pub"
+        fi
+
+        generate_factory_key
       fi
     '';
   };

modules/element-calling.nix

@@ -34,8 +34,8 @@ lib.mkIf config.sovran_systemsOS.features.element-calling {
   };
 
   ####### ENSURE SERVICES START AFTER KEY EXISTS #######
-  systemd.services.livekit.after = [ "livekit-key-setup.service" ];
-  systemd.services.livekit.wants = [ "livekit-key-setup.service" ];
+  systemd.services.livekit.after = [ "livekit-key-setup.service" "livekit-turn-setup.service" ];
+  systemd.services.livekit.wants = [ "livekit-key-setup.service" "livekit-turn-setup.service" ];
   systemd.services.lk-jwt-service.after = [ "livekit-key-setup.service" ];
   systemd.services.lk-jwt-service.wants = [ "livekit-key-setup.service" ];
 
@@ -68,35 +68,54 @@ $MATRIX {
   header /.well-known/matrix/* Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
   header /.well-known/matrix/* Access-Control-Allow-Headers "X-Requested-With, Content-Type, Authorization"
   respond /.well-known/matrix/client \`{ "m.homeserver": {"base_url": "https://$MATRIX" }, "org.matrix.msc4143.rtc_foci": [{ "type":"livekit", "livekit_service_url":"https://$ELEMENT_CALLING/livekit/jwt" }] }\`
-}
-
-$MATRIX:8448 {
-  reverse_proxy http://localhost:8008
+  respond /.well-known/matrix/server \`{"m.server":"$MATRIX:443"}\`
 }
 
 $ELEMENT_CALLING {
-  handle /livekit/jwt/sfu/get {
+  # Route all current lk-jwt-service authorization endpoints to port 8073,
+  # stripping the /livekit/jwt prefix that Caddy adds on the public URL.
+  @lk_jwt path /livekit/jwt/sfu/get* /livekit/jwt/get_token* /livekit/jwt/healthz* /livekit/jwt/sfu_webhook* /livekit/jwt/delegate_delayed_leave*
+  handle @lk_jwt {
     uri strip_prefix /livekit/jwt
     reverse_proxy [::1]:8073 {
       header_up Host {host}
       header_up X-Forwarded-Server {host}
       header_up X-Real-IP {remote_host}
       header_up X-Forwarded-For {remote_host}
+      header_up X-Forwarded-Proto {scheme}
     }
   }
   handle {
-    reverse_proxy localhost:7880
+    reverse_proxy localhost:7880 {
+      header_up Host {host}
+      header_up X-Forwarded-Proto {scheme}
+      header_up X-Forwarded-For {remote_host}
+      header_up X-Real-IP {remote_host}
+      transport http {
+        read_timeout 300s
+        write_timeout 300s
+      }
+    }
   }
 }
 EOF
     '';
   };
 
-  ####### LIVEKIT RUNTIME CONFIG #######
-  systemd.services.livekit-runtime-config = {
-    description = "Generate LiveKit runtime config from domain files";
+  ####### LIVEKIT TURN SETUP (runtime cert + config) #######
+  # Replaces the old dead livekit-runtime-config.service. At runtime this:
+  #   * reads the matrix domain from /var/lib/domains/matrix (never hardcoded)
+  #   * copies Caddy's already-issued matrix cert/key into /var/lib/livekit
+  #     so LoadCredential can stage them for the (DynamicUser) livekit unit
+  #   * detects the primary network interface from the IPv4 default route so
+  #     LiveKit only advertises real ICE candidates — not VPN/container/private
+  #     addresses from interfaces like Tailscale or Docker bridges
+  #   * writes a complete LiveKit config (with turn.domain and interface
+  #     substituted) that the overridden ExecStart loads.
+  systemd.services.livekit-turn-setup = {
+    description = "Stage TURN cert and generate LiveKit runtime config from domain files";
+    after = [ "caddy.service" "livekit-key-setup.service" ];
     before = [ "livekit.service" ];
-    after = [ "livekit-key-setup.service" ];
     requiredBy = [ "livekit.service" ];
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
@@ -106,20 +125,63 @@ EOF
     unitConfig = {
       ConditionPathExists = "/var/lib/domains/element-calling";
     };
-    path = [ pkgs.coreutils ];
+    path = [ pkgs.coreutils pkgs.findutils pkgs.iproute2 pkgs.gawk ];
     script = ''
       MATRIX=$(cat /var/lib/domains/matrix)
 
       mkdir -p /run/livekit
 
-      cat > /run/livekit/runtime-config.yaml <<EOF
+      # Copy Caddy's already-issued matrix cert/key into LiveKit's state dir.
+      # The ACME CA hostname directory can vary, so glob for the domain dir.
+      CRT=$(find /var/lib/caddy -path "*/$MATRIX/$MATRIX.crt" | head -n1)
+      KEY=$(find /var/lib/caddy -path "*/$MATRIX/$MATRIX.key" | head -n1)
+      cp "$CRT" /var/lib/livekit/turn.crt
+      cp "$KEY" /var/lib/livekit/turn.key
+      chmod 640 /var/lib/livekit/turn.crt /var/lib/livekit/turn.key
+
+      # Detect the primary network interface from the IPv4 default route.
+      # Restricting LiveKit to this single interface prevents it from
+      # advertising VPN/container/private ICE candidates (e.g. Tailscale,
+      # Docker bridges) that remote peers cannot reach, which causes all
+      # ICE negotiation attempts to fail with responsesReceived: 0.
+      IFACE=$(ip -4 route show default | awk '/^default/ { for(i=1;i<=NF;i++) if($i=="dev" && (i+1)<=NF) { print $(i+1); exit } }')
+      if [ -z "$IFACE" ]; then
+        echo "ERROR: Could not detect a default-route network interface from 'ip -4 route show default'." >&2
+        echo "ERROR: Cannot generate a valid LiveKit config without a real interface to bind ICE candidates to." >&2
+        echo "ERROR: Ensure a default IPv4 route is configured, e.g.: ip route add default via <gateway> dev <interface>" >&2
+        echo "ERROR: Inspect the current routing table with: ip -4 route show" >&2
+        exit 1
+      fi
+      echo "Detected primary network interface: $IFACE"
+
+      # Generate the full LiveKit config the daemon will load. turn.domain and
+      # rtc.interfaces.includes are only known at runtime, so they are
+      # substituted here. The cert/key paths point at the LoadCredential-staged
+      # copies under /run/credentials.
+      cat > /run/livekit/livekit.yaml <<EOF
+port: 7880
+rtc:
+  use_external_ip: true
+  skip_external_ip_validation: true
+  tcp_port: 7881
+  udp_port: 7882
+  port_range_start: 30000
+  port_range_end: 40000
+  interfaces:
+    includes:
+      - $IFACE
+room:
+  auto_create: false
 turn:
+  enabled: true
   domain: $MATRIX
-  cert_file: /var/lib/livekit/$MATRIX.crt
-  key_file: /var/lib/livekit/$MATRIX.key
+  tls_port: 5349
+  udp_port: 3478
+  cert_file: /run/credentials/livekit.service/turn-cert
+  key_file: /run/credentials/livekit.service/turn-key
 EOF
 
-      chmod 640 /run/livekit/runtime-config.yaml
+      chmod 644 /run/livekit/livekit.yaml
     '';
   };
 
@@ -130,7 +192,11 @@ EOF
     keyFile = livekitKeyFile;
     settings = {
       rtc.use_external_ip = true;
+      rtc.skip_external_ip_validation = true;
+      rtc.tcp_port = 7881;
       rtc.udp_port = 7882;
+      rtc.port_range_start = 30000;
+      rtc.port_range_end = 40000;
       room.auto_create = false;
       turn = {
         enabled = true;
@@ -140,13 +206,27 @@ EOF
     };
   };
 
+  # Override ExecStart to load the runtime-generated config (which carries the
+  # runtime-only turn.domain), mirroring the Caddy ExecStart override pattern in
+  # modules/core/caddy.nix. Deliver the TURN cert/key via LoadCredential so they
+  # are readable under the upstream unit's DynamicUser=true sandbox without
+  # weakening it. Everything else about the standard unit is left intact.
+  systemd.services.livekit.serviceConfig.ExecStart = lib.mkForce [
+    ""
+    "${pkgs.livekit}/bin/livekit-server --config /run/credentials/livekit.service/livekit-config --key-file /run/credentials/livekit.service/livekit-secrets"
+  ];
+
+  systemd.services.livekit.serviceConfig.LoadCredential = [
+    "livekit-config:/run/livekit/livekit.yaml"
+    "livekit-secrets:${livekitKeyFile}"
+    "turn-cert:/var/lib/livekit/turn.crt"
+    "turn-key:/var/lib/livekit/turn.key"
+  ];
+
   networking.firewall.allowedTCPPorts = [ 5349 7881 ];
   networking.firewall.allowedUDPPorts = [ 3478 7882 ];
   networking.firewall.allowedUDPPortRanges = [
-    { from = 30000; to = 40000; }
-  ];
-  networking.firewall.allowedTCPPortRanges = [
-    { from = 30000; to = 40000; }
+    { from = 30000; to = 40000; } # LiveKit internal TURN relay range
   ];
 
   ####### JWT SERVICE RUNTIME CONFIG #######
@@ -166,11 +246,13 @@ EOF
     path = [ pkgs.coreutils ];
     script = ''
       ELEMENT_CALLING=$(cat /var/lib/domains/element-calling)
+      MATRIX=$(cat /var/lib/domains/matrix)
 
       mkdir -p /run/lk-jwt-service
 
       cat > /run/lk-jwt-service/env <<EOF
 LIVEKIT_URL=wss://$ELEMENT_CALLING
+LIVEKIT_FULL_ACCESS_HOMESERVERS=$MATRIX
 EOF
 
       chmod 640 /run/lk-jwt-service/env

modules/modules.nix

@@ -16,6 +16,7 @@
     ./core/remote-deploy.nix
     ./core/no-sleep.nix
     ./core/cpu-performance.nix
+    ./core/local-domain-loopback.nix
 
     # ── Always on (no flag) ───────────────────────────────────
     ./php.nix

modules/rdp.nix

@@ -2,70 +2,104 @@
 
 lib.mkIf config.sovran_systemsOS.features.rdp {
 
-  users.users.gnome-remote-desktop = {
-    isSystemUser = true;
-    group = "gnome-remote-desktop";
-    home = "/var/lib/gnome-remote-desktop";
-    createHome = true;
-  };
-  users.groups.gnome-remote-desktop = {};
-
   # Enable the GNOME Remote Desktop service at the system level
   services.gnome.gnome-remote-desktop.enable = true;
 
   # Open RDP port in the firewall
   networking.firewall.allowedTCPPorts = [ 3389 ];
 
-  # Ensure the service actually starts and waits for setup to complete
+  # Ensure the service only starts after setup succeeds
   systemd.services.gnome-remote-desktop = {
     wantedBy = [ "graphical.target" ];
     after = [ "gnome-remote-desktop-setup.service" ];
-    wants = [ "gnome-remote-desktop-setup.service" ];
+    requires = [ "gnome-remote-desktop-setup.service" ];
   };
 
   systemd.tmpfiles.rules = [
-    "d /var/lib/gnome-remote-desktop 0750 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local 0750 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local/share 0750 gnome-remote-desktop gnome-remote-desktop -"
-    "d /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop 0750 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local 0700 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local/share 0700 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop 0700 gnome-remote-desktop gnome-remote-desktop -"
+    "d /var/lib/gnome-remote-desktop/tls 0700 gnome-remote-desktop gnome-remote-desktop -"
   ];
 
   systemd.services.gnome-remote-desktop-setup = {
     description = "Configure GNOME Remote Desktop RDP";
-    wantedBy = [ "multi-user.target" ];
+    wantedBy = [ "graphical.target" ];
     before = [ "gnome-remote-desktop.service" ];
-    after = [ "systemd-tmpfiles-setup.service" "network-online.target" ];
-    wants = [ "network-online.target" ];
+    after = [
+      "dbus.service"
+      "systemd-tmpfiles-setup.service"
+      "network-online.target"
+      "gnome-remote-desktop-configuration.service"
+    ];
+    wants = [
+      "network-online.target"
+      "gnome-remote-desktop-configuration.service"
+    ];
     serviceConfig = {
       Type = "oneshot";
-      RemainAfterExit = true;
+      TimeoutStartSec = "2min";
     };
     path = [
-      pkgs.gnome-remote-desktop
-      pkgs.polkit
-      pkgs.openssl
-      pkgs.hostname
+      pkgs.coreutils
       pkgs.gawk
+      pkgs.gnome-remote-desktop
+      pkgs.hostname
+      pkgs.openssl
+      pkgs.systemd
     ];
     script = ''
-      # Ensure directory structure exists
-      mkdir -p /var/lib/gnome-remote-desktop/.local/share/gnome-remote-desktop
-      chown -R gnome-remote-desktop:gnome-remote-desktop /var/lib/gnome-remote-desktop
+      set -euo pipefail
 
-      TLS_DIR="/var/lib/gnome-remote-desktop/tls"
-      CRED_FILE="/var/lib/gnome-remote-desktop/rdp-credentials"
+      # GRD 50.x invokes pkexec internally for every grdctl --system call, even
+      # when the caller is root.  NixOS exposes the required setuid wrapper at
+      # /run/wrappers/bin/pkexec; the Nix-store polkit binary is not setuid and
+      # must not shadow it.  Prepend the wrapper directory so every subsequent
+      # grdctl --system resolves the correct binary.
+      export PATH="/run/wrappers/bin:$PATH"
+
+      STATE_DIR="/var/lib/gnome-remote-desktop"
+      TLS_DIR="$STATE_DIR/tls"
+      USERNAME_FILE="$STATE_DIR/rdp-username"
+      PASSWORD_FILE="$STATE_DIR/rdp-password"
+      CRED_FILE="$STATE_DIR/rdp-credentials"
+      DEFAULT_USERNAME="sovran"
+
+      grdctl_system() {
+        local rc=0
+
+        if timeout --kill-after=5s 10s \
+          grdctl --system "$@"; then
+          return 0
+        else
+          rc=$?
+        fi
+
+        if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
+          echo "grdctl command timed out: $*" >&2
+        fi
+        echo "grdctl command failed (exit $rc): $*" >&2
+
+        return "$rc"
+      }
+
+      mkdir -p "$STATE_DIR/.local/share/gnome-remote-desktop" "$TLS_DIR"
+      chown -R gnome-remote-desktop:gnome-remote-desktop "$STATE_DIR"
+      chmod 700 \
+        "$STATE_DIR" \
+        "$STATE_DIR/.local" \
+        "$STATE_DIR/.local/share" \
+        "$STATE_DIR/.local/share/gnome-remote-desktop" \
+        "$TLS_DIR"
 
-      # Regenerate TLS certificate if missing OR if ownership is wrong
-      # (disable/re-enable cycle can break ownership or grdctl state)
       NEED_REGEN=0
       if [ ! -f "$TLS_DIR/rdp-tls.crt" ] || [ ! -f "$TLS_DIR/rdp-tls.key" ]; then
         NEED_REGEN=1
-      elif [ "$(stat -c '%U' "$TLS_DIR/rdp-tls.key" 2>/dev/null)" != "gnome-remote-desktop" ]; then
+      elif [ "$(stat -c '%U:%G' "$TLS_DIR/rdp-tls.key" 2>/dev/null)" != "gnome-remote-desktop:gnome-remote-desktop" ]; then
         NEED_REGEN=1
       fi
 
       if [ "$NEED_REGEN" = "1" ]; then
-        mkdir -p "$TLS_DIR"
         rm -f "$TLS_DIR/rdp-tls.key" "$TLS_DIR/rdp-tls.crt"
         openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \
           -sha256 -nodes -days 3650 \
@@ -75,39 +109,59 @@ lib.mkIf config.sovran_systemsOS.features.rdp {
         echo "Generated new RDP TLS certificate"
       fi
 
-      # Always fix ownership and permissions (handles re-enable after disable)
-      chown -R gnome-remote-desktop:gnome-remote-desktop "$TLS_DIR"
+      chown gnome-remote-desktop:gnome-remote-desktop "$TLS_DIR/rdp-tls.key" "$TLS_DIR/rdp-tls.crt"
       chmod 600 "$TLS_DIR/rdp-tls.key"
       chmod 644 "$TLS_DIR/rdp-tls.crt"
 
-      # Configure TLS certificate
-      grdctl --system rdp set-tls-cert "$TLS_DIR/rdp-tls.crt"
-      grdctl --system rdp set-tls-key "$TLS_DIR/rdp-tls.key"
+      if [ ! -f "$USERNAME_FILE" ]; then
+        printf '%s\n' "$DEFAULT_USERNAME" > "$USERNAME_FILE"
+      fi
+      USERNAME="$(tr -d '\n' < "$USERNAME_FILE")"
+      if [ -z "$USERNAME" ]; then
+        USERNAME="$DEFAULT_USERNAME"
+        printf '%s\n' "$USERNAME" > "$USERNAME_FILE"
+      fi
+      if [ "''${#USERNAME}" -gt 32 ]; then
+        echo "RDP username is too long (''${#USERNAME} characters, maximum 32): $USERNAME from $USERNAME_FILE" >&2
+        exit 1
+      fi
+      case "$USERNAME" in
+        [A-Za-z_][A-Za-z0-9_-]*)
+          ;;
+        *)
+          echo "RDP username must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens: $USERNAME from $USERNAME_FILE" >&2
+          exit 1
+          ;;
+      esac
+      chown gnome-remote-desktop:gnome-remote-desktop "$USERNAME_FILE"
+      chmod 600 "$USERNAME_FILE"
 
-      # Generate password on first boot only
-      PASSWORD=""
-      if [ ! -f /var/lib/gnome-remote-desktop/rdp-password ]; then
-        PASSWORD=$(openssl rand -base64 16)
-        echo "$PASSWORD" > /var/lib/gnome-remote-desktop/rdp-password
-        chmod 600 /var/lib/gnome-remote-desktop/rdp-password
-      else
-        PASSWORD=$(cat /var/lib/gnome-remote-desktop/rdp-password)
+      if [ ! -f "$PASSWORD_FILE" ]; then
+        openssl rand -base64 16 > "$PASSWORD_FILE"
+      fi
+      PASSWORD="$(tr -d '\n' < "$PASSWORD_FILE")"
+      if [ -z "$PASSWORD" ]; then
+        echo "RDP password file is empty: $PASSWORD_FILE" >&2
+        exit 1
+      fi
+      if [ "''${#PASSWORD}" -lt 8 ]; then
+        echo "RDP password is too short (''${#PASSWORD} characters, minimum 8): $PASSWORD_FILE" >&2
+        exit 1
+      fi
+      chown gnome-remote-desktop:gnome-remote-desktop "$PASSWORD_FILE"
+      chmod 600 "$PASSWORD_FILE"
+
+      LOCAL_IP="$(hostname -I | awk '{print $1}')"
+      if [ -z "$LOCAL_IP" ]; then
+        LOCAL_IP="127.0.0.1"
       fi
 
-      # Write username to a separate file for the hub
-      echo "sovran" > /var/lib/gnome-remote-desktop/rdp-username
-      chmod 600 /var/lib/gnome-remote-desktop/rdp-username
-
-      # Get current IP address
-      LOCAL_IP=$(hostname -I | awk '{print $1}')
-
-      # Always rewrite the credentials file with the current IP
       cat > "$CRED_FILE" <<EOF
       ========================================
        GNOME Remote Desktop (RDP) Credentials
       ========================================
 
-       Username: sovran
+       Username: $USERNAME
        Password: $PASSWORD
 
        Connect from any RDP client to:
@@ -116,11 +170,22 @@ lib.mkIf config.sovran_systemsOS.features.rdp {
       ========================================
       EOF
 
+      chown gnome-remote-desktop:gnome-remote-desktop "$CRED_FILE"
       chmod 600 "$CRED_FILE"
 
-      # Enable RDP backend and set credentials
-      grdctl --system rdp enable
-      grdctl --system rdp set-credentials sovran "$PASSWORD"
+      # Preflight: the NixOS setuid pkexec wrapper must be present and executable
+      # before any grdctl --system call.  Absence means the system was booted
+      # without security.wrappers or the wrapper directory is not mounted yet.
+      if ! test -x /run/wrappers/bin/pkexec; then
+        echo "Preflight check failed: /run/wrappers/bin/pkexec is absent or not executable." >&2
+        echo "GNOME Remote Desktop system configuration requires the NixOS setuid pkexec wrapper at /run/wrappers/bin/pkexec." >&2
+        exit 1
+      fi
+
+      grdctl_system rdp enable
+      grdctl_system rdp set-tls-cert "$TLS_DIR/rdp-tls.crt"
+      grdctl_system rdp set-tls-key "$TLS_DIR/rdp-tls.key"
+      grdctl_system rdp set-credentials "$USERNAME" "$PASSWORD"
 
       echo "GNOME Remote Desktop RDP configured successfully"
     '';

modules/synapse.nix

@@ -250,9 +250,6 @@ CREDS
     '';
   };
 
-  networking.firewall.allowedTCPPorts = [ 8448 ];
-  networking.firewall.allowedUDPPorts = [ 8448 ];
-
   sovran_systemsOS.domainRequirements = [
     { name = "matrix"; label = "Matrix Synapse"; example = "matrix.yourdomain.com"; }
   ];