Merge pull request #332 from naturallaw777/copilot/fix-manual-backup-failure

Fix home.tar exit-1 failure on active desktop; harden full backup workflow

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -17,13 +17,26 @@ set -euo pipefail
 BACKUP_LOG="/var/log/sovran-hub-backup.log"
 BACKUP_STATUS="/var/log/sovran-hub-backup.status"
 MEDIA_ROOT="/run/media"
-MIN_FREE_GB=10
 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
 ROLE_STATE_NIX="/etc/nixos/role-state.nix"
+SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
+SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))
 
 # ── Internal drive labels/paths to NEVER use as backup targets ───
 INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
-INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/")
+INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
+
+FAILED_ALREADY=0
+BACKUP_COMPLETE=0
+LND_STOPPED=0
+LND_UNITS_TO_RESTART=()
+
+ARCHIVE_FILES=()
+ARCHIVE_WARNINGS=()
+PARTIAL_FILES=()
+DB_DUMP_FILES=()
+MANIFEST_EXCLUDES=()
+LND_BACKUP_NOTES=()
 
 # ── Logging helpers ──────────────────────────────────────────────
 
@@ -37,16 +50,74 @@ set_status() {
 }
 
 fail() {
+  FAILED_ALREADY=1
   log "ERROR: $*"
   set_status "FAILED"
   exit 1
 }
 
+cleanup() {
+  local rc=$?
+  local restart_failed=0
+
+  # Remove any partial archive files or temporary diagnostic files
+  if [[ "${#PARTIAL_FILES[@]}" -gt 0 ]]; then
+    local partial
+    for partial in "${PARTIAL_FILES[@]}"; do
+      [[ -f "$partial" ]] && rm -f "$partial" || true
+    done
+  fi
+
+  # Release the concurrency lock file descriptor if it was opened
+  [[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true
+
+  if [[ "$LND_STOPPED" -eq 1 ]]; then
+    log "Restarting previously active LND-related services…"
+    for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do
+      local unit="${LND_UNITS_TO_RESTART[$idx]}"
+      if systemctl start "$unit"; then
+        log "Started $unit"
+      else
+        log "ERROR: Failed to start $unit"
+        restart_failed=1
+      fi
+    done
+    LND_STOPPED=0
+  fi
+
+  if [[ "$restart_failed" -eq 1 ]]; then
+    rc=1
+    FAILED_ALREADY=1
+    set_status "FAILED"
+  fi
+
+  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
+    return
+  fi
+
+  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
+    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
+    set_status "FAILED"
+  fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
+}
+
+trap cleanup EXIT
+trap 'exit 1' INT TERM
+
+require_cmd() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
+}
+
 # ── Check whether a mount point is an internal drive ────────────
 
 is_internal() {
   local mnt="$1"
-  # Reject known internal mount points and their subdirectories
   for internal in "${INTERNAL_MOUNTS[@]}"; do
     if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
       return 0
@@ -59,22 +130,18 @@ is_internal() {
 
 find_external_drive() {
   local target=""
-  # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
-  if command -v lsblk &>/dev/null; then
+
   while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
-      # Must be a partition or disk, and be removable/hotplug
     [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
     [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
     [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal labels
     local skip=0
     for lbl in "${INTERNAL_LABELS[@]}"; do
       [[ "$label" == "$lbl" ]] && skip=1 && break
     done
     [[ "$skip" -eq 1 ]] && continue
 
-      # Filter out internal mount points
     is_internal "$mountpoint" && continue
 
     if mountpoint -q "$mountpoint" 2>/dev/null; then
@@ -84,14 +151,16 @@ find_external_drive() {
   done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
     python3 -c "
 import sys, json
-data = json.load(sys.stdin)
+
 def flatten(devs):
     for d in devs:
         yield d
         for c in d.get('children', []):
             yield from flatten([c])
+
+data = json.load(sys.stdin)
 for d in flatten(data.get('blockdevices', [])):
-    print('\t'.join([
+    print('\\t'.join([
         d.get('type') or '',
         str(d.get('hotplug') or '0'),
         str(d.get('rm') or '0'),
@@ -99,24 +168,10 @@ for d in flatten(data.get('blockdevices', [])):
         d.get('mountpoint') or '',
     ]))
 " 2>/dev/null || true)
-  fi
 
-  # Fallback: walk /run/media/ if lsblk produced nothing
   if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
     while IFS= read -r -d '' mnt; do
       is_internal "$mnt" && continue
-      # Check label via lsblk on the device backing this mount
-      local dev
-      dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
-      if [[ -n "$dev" ]]; then
-        local lbl
-        lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
-        local skip=0
-        for internal_lbl in "${INTERNAL_LABELS[@]}"; do
-          [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
-        done
-        [[ "$skip" -eq 1 ]] && continue
-      fi
       if mountpoint -q "$mnt" 2>/dev/null; then
         target="$mnt"
         break
@@ -128,16 +183,10 @@ for d in flatten(data.get('blockdevices', [])):
 }
 
 # ── Detect the configured system role ───────────────────────────
-#
-# Priority:
-#   1. Hub config JSON  (/var/lib/sovran-hub/config.json)  — "role" key
-#   2. role-state.nix   (/etc/nixos/role-state.nix)        — grep for true flag
-#   3. Default: server_plus_desktop
 
 detect_role() {
   local role="server_plus_desktop"
 
-  # 1. Try the Hub config JSON
   if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
     local r
     r=$(python3 -c \
@@ -149,7 +198,6 @@ detect_role() {
     fi
   fi
 
-  # 2. Fall back to parsing role-state.nix
   if [[ -f "$ROLE_STATE_NIX" ]]; then
     if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
       role="desktop"
@@ -161,6 +209,64 @@ detect_role() {
   echo "$role"
 }
 
+validate_target_mount() {
+  local target="$1"
+  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
+  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
+  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
+
+  local fstype=""
+  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
+  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
+
+  if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then
+    fail "Target '$target' must be exFAT (detected filesystem: $fstype)."
+  fi
+
+  if [[ "$fstype" == "fuseblk" ]]; then
+    local src_dev blk_type
+    src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true)
+    blk_type=""
+    if [[ -n "$src_dev" ]]; then
+      blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true)
+      [[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true)
+    fi
+    if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then
+      fail "Target '$target' is fuseblk but not identified as exFAT-compatible."
+    fi
+  fi
+
+  local write_test
+  write_test="$target/.sovran-write-test-$$"
+  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
+    fail "Target '$target' is not writable."
+  fi
+
+  log "Verified backup target filesystem: $fstype"
+}
+
+has_unit() {
+  systemctl cat "$1" >/dev/null 2>&1
+}
+
+is_unit_active() {
+  systemctl is-active --quiet "$1"
+}
+
+estimate_path_bytes() {
+  local path="$1"
+  shift || true
+  [[ -e "$path" ]] || {
+    echo 0
+    return
+  }
+
+  local size
+  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
+  [[ -n "$size" ]] || size=0
+  echo "$size"
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -169,6 +275,35 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"
 
+# ── Acquire exclusive run lock ────────────────────────────────────
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd tar
+require_cmd sha256sum
+require_cmd findmnt
+require_cmd lsblk
+require_cmd mountpoint
+require_cmd df
+require_cmd du
+require_cmd awk
+require_cmd sort
+require_cmd find
+require_cmd systemctl
+require_cmd hostname
+require_cmd date
+require_cmd python3
+require_cmd runuser
+require_cmd mktemp
+require_cmd flock
+
 # ── Detect system role ───────────────────────────────────────────
 
 ROLE="$(detect_role)"
@@ -184,7 +319,6 @@ log "Detected role: $ROLE_LABEL"
 
 if [[ -n "${BACKUP_TARGET:-}" ]]; then
   TARGET="$BACKUP_TARGET"
-  # Safety: never allow internal drives even if explicitly passed
   if is_internal "$TARGET"; then
     fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
   fi
@@ -193,106 +327,472 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. " \
-         "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
+    fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
-# ── Verify mount point ───────────────────────────────────────────
+validate_target_mount "$TARGET"
 
-[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist."
-mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
+# ── Plan role-aware source scope and exclusions ─────────────────
 
-# ── Check free disk space (require ≥ 10 GB) ──────────────────────
+LND_AVAILABLE=0
+if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then
+  LND_AVAILABLE=1
+fi
 
-FREE_KB=$(df -k --output=avail "$TARGET" | tail -1)
-FREE_GB=$(( FREE_KB / 1024 / 1024 ))
+if [[ "$ROLE" == "desktop" ]]; then
+  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)")
+else
+  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent")
+fi
+
+MANIFEST_EXCLUDES+=(
+  "/run/media/Second_Drive (never traversed)"
+  "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)"
+  "/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)"
+  "/var/lib/bitcoind (excluded from manual backup)"
+  "/var/lib/electrs (excluded from manual backup)"
+  "/var/lib/*/log and /var/lib/*/logs"
+  "/var/lib/*/cache and /var/lib/*/tmp"
+  "/home/*/.cache (system and application disk caches)"
+  "/home/*/.local/share/Trash and /home/*/Trash (trash directories)"
+  "/home/*/.mozilla/firefox/*/cache2 and */startupCache (Firefox volatile cache — profile data is kept)"
+  "/home/*/.config/google-chrome/*/Cache (Chrome disk cache — profile data is kept)"
+  "/home/*/.config/chromium/*/Cache (Chromium disk cache — profile data is kept)"
+  "/home/*/.config/BraveSoftware/Brave-Browser/*/Cache (Brave disk cache — profile data is kept)"
+  "/home/*/.local/share/baloo (KDE file indexer — rebuilt automatically)"
+  "/home/*/.thumbnails (thumbnail cache — rebuilt automatically)"
+  "/home/*/.xsession-errors and .xsession-errors.old (X session error logs)"
+)
+
+if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
+  MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive")
+fi
+
+# ── Estimate required free space ─────────────────────────────────
+
+ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
+HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
+SECRETS_BYTES=0
+if [[ "$ROLE" != "desktop" ]]; then
+  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
+fi
+
+VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='bitcoind' \
+  --exclude='electrs' \
+  --exclude='lnd' \
+  --exclude='*/log' \
+  --exclude='*/logs' \
+  --exclude='*/cache' \
+  --exclude='*/tmp')
+
+LND_BYTES=0
+if [[ "$LND_AVAILABLE" -eq 1 ]]; then
+  LND_BYTES=$(estimate_path_bytes /var/lib/lnd)
+fi
+
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES ))
+# Require 20% growth headroom plus an additional fixed 1 GiB safety margin.
+REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
+
+FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
+FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
+REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
+
+log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
+log "Required free space (with safety margin): ${REQUIRED_GB} GB"
 log "Free space on drive: ${FREE_GB} GB"
-(( FREE_GB >= MIN_FREE_GB )) || \
-  fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)."
+
+(( FREE_BYTES >= REQUIRED_BYTES )) || \
+  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
 
 # ── Create timestamped backup directory ─────────────────────────
 
 TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
 BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
-mkdir -p "$BACKUP_DIR"
+DB_DUMP_DIR="$BACKUP_DIR/database-dumps"
+mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR"
+# Write an INCOMPLETE marker immediately; it is removed only on successful completion.
+# Failed runs keep this marker so they are easily identifiable and not confused with
+# complete backups during restore selection.
+touch "$BACKUP_DIR/INCOMPLETE"
 log "Backup destination: $BACKUP_DIR"
 
-# ── Stage 1/4: NixOS configuration ──────────────────────────────
+create_tar_archive() {
+  _create_archive_impl STRICT "$@"
+}
+
+# ── Helper: classify a GNU tar (LC_ALL=C) diagnostic line ────────
+# Returns 0 (true) if the message is an allowlisted transient condition
+# that is safe to ignore for /home (live desktop file changes).
+# Only two verified GNU tar messages qualify; all others are fatal.
+_is_home_warning_allowlisted() {
+  local msg="$1"
+  case "$msg" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+# ── Internal archive builder ──────────────────────────────────────
+# mode: STRICT — any tar error fails.
+#       HOME   — tar exit 1 is accepted when every diagnostic is an
+#                allowlisted transient condition (live file changes).
+_create_archive_impl() {
+  local mode="$1"; shift
+  local archive_name="$1"; shift
+  local archive_path="$BACKUP_DIR/$archive_name"
+  local partial_path="${archive_path}.partial"
+  local diag_tmp
+  # mktemp creates files with mode 0600 (owner-only) by default, so tar
+  # diagnostics (which may include file paths) are not readable by other users.
+  diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)"
+  PARTIAL_FILES+=("$partial_path" "$diag_tmp")
+
+  log "Creating $archive_name …"
+
+  local tar_rc=0
+  LC_ALL=C tar \
+    --create \
+    --file "$partial_path" \
+    --numeric-owner \
+    --acls \
+    --xattrs \
+    --sparse \
+    --one-file-system \
+    "$@" 2>"$diag_tmp" || tar_rc=$?
+
+  # Log every tar diagnostic to the backup log so it appears in the Hub UI
+  local has_fatal_diag=0
+  if [[ -s "$diag_tmp" ]]; then
+    while IFS= read -r diag_line; do
+      [[ -n "$diag_line" ]] || continue
+      log "tar: $diag_line"
+      if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then
+        has_fatal_diag=1
+      fi
+    done < "$diag_tmp"
+  fi
+
+  local accept=0
+  if [[ "$tar_rc" -eq 0 ]]; then
+    accept=1
+  elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+    accept=1
+    log "NOTE: $archive_name completed with nonfatal warnings (live files changed during backup -- this is normal on an active desktop and does not affect the safety of your backup)."
+    ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings -- live files changed during backup (normal on an active desktop)")
+  fi
+
+  if [[ "$accept" -eq 0 ]]; then
+    rm -f "$partial_path" "$diag_tmp"
+    if [[ "$tar_rc" -gt 1 ]]; then
+      fail "tar exited with fatal code $tar_rc while creating $archive_name."
+    elif [[ "$mode" == "HOME" ]]; then
+      fail "tar exited with code $tar_rc and unrecognized diagnostics while creating $archive_name."
+    else
+      fail "tar exited with code $tar_rc while creating $archive_name."
+    fi
+  fi
+
+  # Verify the partial archive is non-empty and readable (spot-check first entry)
+  if [[ ! -s "$partial_path" ]]; then
+    rm -f "$partial_path" "$diag_tmp"
+    fail "Archive $archive_name is empty after creation."
+  fi
+
+  # Fast readability check: read only the first tar entry header (~512 bytes).
+  # head -1 closes the pipe after one line, sending SIGPIPE to tar which then
+  # exits early — so this is O(1) regardless of archive size.
+  local spot_entry
+  spot_entry="$(LC_ALL=C tar --list --file "$partial_path" 2>/dev/null | head -1 || true)"
+  if [[ -z "$spot_entry" ]]; then
+    rm -f "$partial_path" "$diag_tmp"
+    fail "Archive $archive_name failed readability check."
+  fi
+
+  # Atomic publish: rename partial to final path only after acceptance
+  mv "$partial_path" "$archive_path"
+  rm -f "$diag_tmp"
+  ARCHIVE_FILES+=("$archive_name")
+  log "Created archive: $archive_name"
+}
+
+# ── Home archive: tolerates allowlisted live-file warnings ───────
+create_home_tar_archive() {
+  _create_archive_impl HOME "$@"
+}
+
+export_postgresql_dumps() {
+  if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then
+    log "PostgreSQL tools/service not available — skipping PostgreSQL exports."
+    return
+  fi
+
+  if ! is_unit_active "postgresql.service"; then
+    log "PostgreSQL service is not active — skipping PostgreSQL exports."
+    return
+  fi
+
+  log "Exporting PostgreSQL globals and databases…"
+  local globals_file="$DB_DUMP_DIR/postgresql_globals.sql"
+  runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \
+    fail "Failed to export PostgreSQL globals."
+  DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql")
+
+  local dbs
+  dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true)
+
+  if [[ -z "$dbs" ]]; then
+    log "No non-template PostgreSQL application databases found."
+    return
+  fi
+
+  while IFS= read -r db; do
+    [[ -n "$db" ]] || continue
+    local safe_db
+    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
+    local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump"
+    runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \
+      fail "Failed to export PostgreSQL database '$db'."
+    DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump")
+  done <<< "$dbs"
+}
+
+export_mariadb_dumps() {
+  local dump_cmd=""
+  local query_cmd=""
+  local mariadb_unit=""
+
+  if command -v mariadb-dump >/dev/null 2>&1; then
+    dump_cmd="mariadb-dump"
+  elif command -v mysqldump >/dev/null 2>&1; then
+    dump_cmd="mysqldump"
+  fi
+
+  if command -v mariadb >/dev/null 2>&1; then
+    query_cmd="mariadb"
+  elif command -v mysql >/dev/null 2>&1; then
+    query_cmd="mysql"
+  fi
+
+  if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then
+    log "MariaDB dump/query tools not available — skipping MariaDB exports."
+    return
+  fi
+
+  if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then
+    mariadb_unit="mariadb.service"
+  elif has_unit "mysql.service" && is_unit_active "mysql.service"; then
+    mariadb_unit="mysql.service"
+  else
+    log "MariaDB service is not active — skipping MariaDB exports."
+    return
+  fi
+
+  log "Exporting MariaDB databases from ${mariadb_unit}…"
+
+  local dbs
+  dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true)
+  if [[ -z "$dbs" ]]; then
+    log "No MariaDB databases found."
+    return
+  fi
+
+  while IFS= read -r db; do
+    [[ -n "$db" ]] || continue
+    case "$db" in
+      information_schema|performance_schema|mysql|sys) continue ;;
+    esac
+
+    local safe_db out_file
+    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
+    out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql"
+
+    $dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \
+      fail "Failed to export MariaDB database '$db'."
+
+    DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql")
+  done <<< "$dbs"
+}
+
+export_lnd_scb_if_possible() {
+  [[ "$LND_AVAILABLE" -eq 1 ]] || return
+
+  local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb"
+  local attempts=(
+    "lncli exportchanbackup --all --output_file $scb_file"
+    "lncli -n mainnet exportchanbackup --all --output_file $scb_file"
+    "runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file"
+    "runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file"
+  )
+
+  if ! command -v lncli >/dev/null 2>&1; then
+    log "lncli not available — skipping Static Channel Backup export."
+    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)")
+    return
+  fi
+
+  if ! is_unit_active "lnd.service"; then
+    log "LND service is not active — skipping Static Channel Backup export."
+    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)")
+    return
+  fi
+
+  log "Exporting LND Static Channel Backup…"
+  local attempt
+  for attempt in "${attempts[@]}"; do
+    if eval "$attempt" >/dev/null 2>&1; then
+      DB_DUMP_FILES+=("lnd-static-channel-backup.scb")
+      LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli")
+      log "LND Static Channel Backup exported."
+      return
+    fi
+  done
+
+  log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations."
+  LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)")
+}
+
+capture_active_lnd_dependents() {
+  [[ "$LND_AVAILABLE" -eq 1 ]] || return
+
+  LND_UNITS_TO_RESTART=()
+  local raw_units=""
+  raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true)
+
+  while IFS= read -r unit; do
+    [[ -n "$unit" ]] || continue
+    if is_unit_active "$unit"; then
+      LND_UNITS_TO_RESTART+=("$unit")
+    fi
+  done <<< "$raw_units"
+
+  if is_unit_active "lnd.service"; then
+    LND_UNITS_TO_RESTART+=("lnd.service")
+  fi
+}
+
+stop_lnd_stack_if_needed() {
+  [[ "$LND_AVAILABLE" -eq 1 ]] || return
+
+  capture_active_lnd_dependents
+
+  if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then
+    log "No active LND-related services needed stopping."
+    return
+  fi
+
+  log "Stopping active services that depend on LND for clean /var/lib/lnd archive…"
+
+  local unit
+  for unit in "${LND_UNITS_TO_RESTART[@]}"; do
+    if [[ "$unit" == "lnd.service" ]]; then
+      continue
+    fi
+    systemctl stop "$unit" || fail "Failed to stop dependent service: $unit"
+    log "Stopped $unit"
+  done
+
+  if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then
+    systemctl stop lnd.service || fail "Failed to stop lnd.service"
+    log "Stopped lnd.service"
+  fi
+
+  LND_STOPPED=1
+}
+
+# ── Stage 1/5: NixOS configuration ──────────────────────────────
 
 log ""
-log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
+log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 1 failed while copying /etc/nixos"
+  create_tar_archive "etc-nixos.tar" -C / etc/nixos
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
 fi
 
-# ── Stage 2/4: Secrets ──────────────────────────────────────────
+# ── Stage 2/5: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/4: Secrets ───────────────────────────────────────"
-mkdir -p "$BACKUP_DIR/secrets"
-
+log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
-else
-  if [[ -e /etc/nix-bitcoin-secrets ]]; then
-    rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
+elif [[ -e /etc/nix-bitcoin-secrets ]]; then
+  create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets
 else
   log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
-fi
-
 log "Stage 2 complete."
 
-# ── Stage 3/4: Home directory ────────────────────────────────────
+# ── Stage 3/5: Home directory ───────────────────────────────────
 
 log ""
-log "── Stage 3/4: Home directory (/home) ───────────────────────"
+log "── Stage 3/5: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  rsync -a --info=progress2 \
-    --exclude='.cache/' \
-    --exclude='.local/share/Trash/' \
-    --exclude='*/Trash/' \
-    /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 3 failed while copying /home"
+  create_home_tar_archive "home.tar" \
+    -C / \
+    --exclude='home/*/.cache' \
+    --exclude='home/*/.local/share/Trash' \
+    --exclude='home/*/Trash' \
+    --exclude='home/*/.mozilla/firefox/*/cache2' \
+    --exclude='home/*/.mozilla/firefox/*/startupCache' \
+    --exclude='home/*/.mozilla/firefox/*/thumbnails' \
+    --exclude='home/*/.config/google-chrome/*/Cache' \
+    --exclude='home/*/.config/google-chrome/*/Code Cache' \
+    --exclude='home/*/.config/chromium/*/Cache' \
+    --exclude='home/*/.config/chromium/*/Code Cache' \
+    --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache' \
+    --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Code Cache' \
+    --exclude='home/*/.local/share/baloo' \
+    --exclude='home/*/.thumbnails' \
+    --exclude='home/*/.xsession-errors' \
+    --exclude='home/*/.xsession-errors.old' \
+    home
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/4: System data ───────────────────────────────────────
+# ── Stage 4/5: Database exports + LND artifacts ────────────────
 
 log ""
-log "── Stage 4/4: System data (/var/lib) ────────────────────────"
-if [[ "$ROLE" == "desktop" ]]; then
-  if [[ -d /var/lib ]]; then
-    rsync -a --info=progress2 \
-      --filter='- /lnd/***' \
-      --exclude='logs/' \
-      --exclude='log/' \
-      --exclude='*/logs/' \
-      --exclude='*/log/' \
-      /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      fail "Stage 4 failed while copying /var/lib for Desktop Only role"
-    log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)."
-  else
-    log "WARNING: /var/lib not found — skipping."
+log "── Stage 4/5: Database and LND consistency exports ─────────"
+export_postgresql_dumps
+export_mariadb_dumps
+export_lnd_scb_if_possible
+
+if [[ "$LND_AVAILABLE" -eq 1 ]]; then
+  stop_lnd_stack_if_needed
+  create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd
+  LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop")
 fi
-elif [[ -d /var/lib ]]; then
-  rsync -a --info=progress2 \
-    --exclude='logs/' \
-    --exclude='log/' \
-    --exclude='*/logs/' \
-    --exclude='*/log/' \
-    /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 4 failed while copying /var/lib"
+
 log "Stage 4 complete."
+
+# ── Stage 5/5: System data ──────────────────────────────────────
+
+log ""
+log "── Stage 5/5: System data (/var/lib) ───────────────────────"
+if [[ -d /var/lib ]]; then
+  VAR_LIB_EXCLUDES=(
+    --exclude='var/lib/bitcoind'
+    --exclude='var/lib/electrs'
+    --exclude='var/lib/*/log'
+    --exclude='var/lib/*/logs'
+    --exclude='var/lib/*/cache'
+    --exclude='var/lib/*/tmp'
+  )
+
+  if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
+    VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd')
+  fi
+
+  create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib
+  log "Stage 5 complete."
 else
   log "WARNING: /var/lib not found — skipping."
 fi
@@ -301,21 +801,110 @@ fi
 
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
+MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
+CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt"
+
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date)"
+  echo "Generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+  echo "Timestamp: $TIMESTAMP"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Contents:"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort
-} > "$BACKUP_DIR/BACKUP_MANIFEST.txt"
-log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo "Source paths included:"
+  echo "- /etc/nixos"
+  echo "- /home"
+  if [[ "$ROLE" != "desktop" ]]; then
+    echo "- /etc/nix-bitcoin-secrets (when present)"
+  fi
+  echo "- /var/lib"
+  echo ""
+  echo "Exclusions:"
+  for ex in "${MANIFEST_EXCLUDES[@]}"; do
+    echo "- $ex"
+  done
+  echo ""
+  echo "Archives:"
+  for archive in "${ARCHIVE_FILES[@]}"; do
+    echo "- $archive"
+  done
+  echo ""
+  echo "Database and LND exports:"
+  if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for dump in "${DB_DUMP_FILES[@]}"; do
+      echo "- $dump"
+    done
+    for note in "${LND_BACKUP_NOTES[@]}"; do
+      echo "- $note"
+    done
+  fi
+  echo ""
+  echo "Restore guidance:"
+  echo "- Verify artifacts: cd <backup_dir> && sha256sum -c SHA256SUMS.txt"
+  echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf <archive>.tar -C /"
+  echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql"
+  echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_<db>.dump"
+  echo "- MariaDB DB dump: mariadb <db_name> < database-dumps/mariadb_<db>.sql"
+  echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures"
+  echo "- Note: when restoring /var/lib, exclude raw DB directories (var/lib/postgresql, var/lib/mysql)"
+  echo "  and restore from native dumps instead for PostgreSQL and MariaDB"
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#ARCHIVE_WARNINGS[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for warning in "${ARCHIVE_WARNINGS[@]}"; do
+      echo "- $warning"
+    done
+  fi
+  echo ""
+  echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "from manual external backup because they already live on the internal second drive"
+  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
+  echo ""
+  echo "Artifact listing:"
+  # INCOMPLETE exists at this point (created at backup start); BACKUP_COMPLETE does not
+  # exist yet (written after checksums). Excluding both marker files here is intentional:
+  # INCOMPLETE is excluded so it doesn't appear as a data artifact, and BACKUP_COMPLETE
+  # is excluded defensively for consistency should the ordering ever change.
+  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f \
+    ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z | tr '\0' '\n'
+} > "$MANIFEST_FILE"
+
+# ── Generate checksums for all backup artifacts ─────────────────
+
+log "Generating SHA-256 checksums …"
+(
+  cd "$BACKUP_DIR"
+  while IFS= read -r -d '' file; do
+    sha256sum "$file"
+  done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z)
+) > "$CHECKSUM_FILE"
+
+log "Manifest written to $MANIFEST_FILE"
+log "Checksums written to $CHECKSUM_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
+if [[ "${#ARCHIVE_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${ARCHIVE_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that changed"
+  log "during backup, which is normal on an active desktop and does not affect your backup."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
 log "Please eject the drive safely before removing it from your Sovran Pro."
+
+# Remove incomplete marker and write completion marker only after all work succeeds
+rm -f "$BACKUP_DIR/INCOMPLETE"
+echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE"
+
+BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/server.py

@@ -1451,6 +1451,12 @@ def _read_backup_status() -> str:
         return "IDLE"
 
 
+def _write_backup_status(value: str) -> None:
+    """Write backup status file."""
+    with open(BACKUP_STATUS, "w") as f:
+        f.write(value)
+
+
 def _read_backup_log(offset: int = 0) -> tuple[str, int]:
     """Read the backup log file from the given byte offset.
     Returns (new_text, new_offset)."""
@@ -1467,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]:
         return "", 0
 
 
+def _append_backup_log(line: str) -> None:
+    """Append one line to backup log."""
+    with open(BACKUP_LOG, "a") as f:
+        f.write(line.rstrip("\n") + "\n")
+
+
 _INTERNAL_LABELS  = {"BTCEcoandBackup", "sovran_systemsos"}
 _INTERNAL_MOUNTS  = {"/", "/boot/efi"}
 _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive"
@@ -1481,6 +1493,41 @@ def _is_internal_mount(mnt: str) -> bool:
     return False
 
 
+def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
+    """Return whether the target filesystem type is supported for manual backup."""
+    fstype = (fstype or "").lower()
+    if fstype == "exfat":
+        return True
+    if fstype != "fuseblk":
+        return False
+
+    src_dev = ""
+    try:
+        result = subprocess.run(
+            ["findmnt", "-n", "-o", "SOURCE", "-T", path],
+            capture_output=True, text=True, timeout=5,
+        )
+        if result.returncode == 0:
+            src_dev = result.stdout.strip()
+    except Exception:
+        src_dev = ""
+
+    if not src_dev:
+        return False
+
+    for cmd in (
+        ["lsblk", "-no", "FSTYPE", src_dev],
+        ["blkid", "-o", "value", "-s", "TYPE", src_dev],
+    ):
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
+            if result.returncode == 0 and result.stdout.strip().lower() in {"exfat", "fuseblk"}:
+                return True
+        except Exception:
+            continue
+    return False
+
+
 def _detect_external_drives() -> list[dict]:
     """Scan for mounted external USB drives.
 
@@ -1490,7 +1537,8 @@ def _detect_external_drives() -> list[dict]:
     /run/media/ directly if lsblk is unavailable, applying the same
     label/path filters.
 
-    Returns a list of dicts with name, path, free_gb, total_gb.
+    Returns:
+        list[dict]: Each dict contains name, path, free_gb, total_gb, fstype.
     """
     import json as _json
     import subprocess as _subprocess
@@ -1501,7 +1549,7 @@ def _detect_external_drives() -> list[dict]:
     # ── Primary path: lsblk JSON ────────────────────────────────
     try:
         result = _subprocess.run(
-            ["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"],
+            ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"],
             capture_output=True, text=True, timeout=10
         )
         if result.returncode == 0:
@@ -1519,6 +1567,7 @@ def _detect_external_drives() -> list[dict]:
                 hotplug    = str(dev.get("hotplug", "0"))
                 rm         = str(dev.get("rm", "0"))
                 label      = dev.get("label") or ""
+                fstype     = (dev.get("fstype") or "").lower()
                 mountpoint = dev.get("mountpoint") or ""
 
                 if dev_type not in ("part", "disk"):
@@ -1544,6 +1593,7 @@ def _detect_external_drives() -> list[dict]:
                         "path":     mountpoint,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(mountpoint)
                 except OSError:
@@ -1577,11 +1627,20 @@ def _detect_external_drives() -> list[dict]:
                     st = os.statvfs(drive_path)
                     total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1)
                     free_gb  = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1)
+                    fstype = ""
+                    try:
+                        fstype = _subprocess.run(
+                            ["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path],
+                            capture_output=True, text=True, timeout=5
+                        ).stdout.strip().lower()
+                    except Exception:
+                        fstype = ""
                     drives.append({
                         "name":     drive_name,
                         "path":     drive_path,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(drive_path)
                 except OSError:
@@ -3682,6 +3741,37 @@ async def api_backup_drives():
     return {"drives": drives}
 
 
+async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
+    """Drain stderr, then mark status FAILED if backup subprocess exits unexpectedly."""
+    stderr_chunks: list[bytes] = []
+
+    async def _drain_stderr() -> None:
+        if proc.stderr is not None:
+            async for line in proc.stderr:
+                stderr_chunks.append(line)
+
+    drain_task = asyncio.create_task(_drain_stderr())
+    rc = await proc.wait()
+    await drain_task
+
+    if rc == 0:
+        return
+
+    loop = asyncio.get_event_loop()
+    status = await loop.run_in_executor(None, _read_backup_status)
+    if status in {"SUCCESS", "FAILED"}:
+        return
+
+    detail = ""
+    if stderr_chunks:
+        stderr_text = b"".join(stderr_chunks).decode("utf-8", errors="replace").strip()
+        if stderr_text:
+            detail = f" — stderr: {stderr_text}"
+    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc}).{detail}"
+    await loop.run_in_executor(None, _append_backup_log, msg)
+    await loop.run_in_executor(None, _write_backup_status, "FAILED")
+
+
 @app.post("/api/backup/run")
 async def api_backup_run(target: str = ""):
     """Start the backup script as a background subprocess.
@@ -3692,6 +3782,26 @@ async def api_backup_run(target: str = ""):
     if status == "RUNNING":
         return {"ok": True, "status": "already_running"}
 
+    drives = await loop.run_in_executor(None, _detect_external_drives)
+    if not drives:
+        raise HTTPException(status_code=400, detail="No external backup drive detected.")
+
+    drive_map = {d.get("path", ""): d for d in drives if d.get("path")}
+    if target:
+        if target not in drive_map:
+            raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.")
+        selected = drive_map[target]
+    else:
+        selected = drives[0]
+
+    selected_target = selected.get("path", "")
+    selected_fstype = (selected.get("fstype") or "").lower()
+    if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.",
+        )
+
     # Clear stale log before starting
     try:
         with open(BACKUP_LOG, "w") as f:
@@ -3699,21 +3809,48 @@ async def api_backup_run(target: str = ""):
     except OSError:
         pass
 
-    env = dict(os.environ)
-    if target:
-        env["BACKUP_TARGET"] = target
+    try:
+        await loop.run_in_executor(None, _write_backup_status, "RUNNING")
+    except OSError as exc:
+        raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}")
 
-    # Fire-and-forget: the script writes its own status/log files.
-    # Progress is read by the client via /api/backup/status (same pattern
-    # as /api/updates/run and the rebuild feature).
-    await asyncio.create_subprocess_exec(
-        "/usr/bin/env", "bash", BACKUP_SCRIPT,
-        stdout=asyncio.subprocess.DEVNULL,
-        stderr=asyncio.subprocess.DEVNULL,
-        env=env,
+    await loop.run_in_executor(
+        None,
+        _append_backup_log,
+        f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…",
     )
 
-    return {"ok": True, "status": "started"}
+    env = dict(os.environ)
+    env["BACKUP_TARGET"] = selected_target
+
+    bash_path = shutil.which("bash")
+    if bash_path is None:
+        no_bash_msg = (
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Cannot start backup:"
+            " interpreter 'bash' not found on PATH."
+            " Ensure pkgs.bash is in the sovran-hub-web service PATH."
+        )
+        await loop.run_in_executor(None, _append_backup_log, no_bash_msg)
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(
+            status_code=500,
+            detail="Backup interpreter (bash) not available. Check service PATH configuration.",
+        )
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bash_path, BACKUP_SCRIPT,
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except Exception as exc:
+        await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}")
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(status_code=500, detail="Failed to launch backup process.")
+
+    asyncio.create_task(_monitor_backup_subprocess(proc))
+    return {"ok": True, "status": "started", "target": selected_target}
 
 
 # ── Feature Manager endpoints ─────────────────────────────────────

app/sovran_systemsos_web/static/js/support.js

@@ -491,7 +491,7 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>At least 500 GB of free space on the drive</li>',
+        '<li>Enough free space for your selected backup data (the backup checks this before starting)</li>',
         '<li>Drive must be formatted as <strong>exFAT</strong></li>',
       '</ol>',
     '</div>',
@@ -584,9 +584,10 @@ async function pollBackupStatus() {
       logDiv.scrollTop = logDiv.scrollHeight;
     }
     _backupLogOffset = data.offset;
-    if (!data.running) {
+    const result = (data.result || "").toLowerCase();
+    if (result === "success" || result === "failed") {
       stopBackupPoll();
-      renderBackupDone(data.result === "success");
+      renderBackupDone(result === "success");
     }
   } catch (_) {}
 }

app/tests/test_local_domain_loopback_nix.py

@@ -0,0 +1,193 @@
+"""Structural regression tests for modules/core/local-domain-loopback.nix.
+
+Verifies that the sovran-hosts-update helper:
+  - is built as a pkgs.writeShellApplication with explicit runtimeInputs
+    (gawk, gnugrep, coreutils);
+  - uses ``lib.getExe hostsUpdateScript`` for both the systemd ExecStart and
+    the activation script so that both contexts share the same Nix-store
+    executable;
+  - does NOT point ExecStart at the raw /etc path;
+  - includes element-calling in the supported domain list;
+  - uses ``awk -v`` for safe marker-variable passing rather than interpolating
+    marker text directly into the awk program;
+  - retains the domain validation regex (idempotency / injection prevention);
+  - exposes /etc/sovran-hosts-update.sh as a symlink via ``source =`` (not a
+    second raw ``text =`` body);
+  - does NOT rely on environment.systemPackages for the helper's dependencies.
+"""
+
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+NIX_STRING_INDENT = 6
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FLAKE_SOURCE = (REPO_ROOT / "flake.nix").read_text()
+PRIMARY_NIXOS_CONFIGURATION_MATCH = re.search(
+    r"nixosConfigurations\.([A-Za-z0-9_-]+)\s*=",
+    FLAKE_SOURCE,
+)
+if PRIMARY_NIXOS_CONFIGURATION_MATCH is None:
+    raise RuntimeError("Could not determine the primary nixosConfigurations entry from flake.nix")
+PRIMARY_NIXOS_CONFIGURATION = PRIMARY_NIXOS_CONFIGURATION_MATCH.group(1)
+HELPER_BUILD_ATTR = (
+    f'.#nixosConfigurations.{PRIMARY_NIXOS_CONFIGURATION}.config.environment.etc.'
+    '"sovran-hosts-update.sh".source'
+)
+NIX_FILE = (
+    REPO_ROOT
+    / "modules"
+    / "core"
+    / "local-domain-loopback.nix"
+)
+
+
+class LocalDomainLoopbackNixStructureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = NIX_FILE.read_text()
+
+    def _helper_script(self) -> str:
+        start = self.source.index("text = ''") + len("text = ''")
+        end = self.source.index("    '';", start)
+        return "\n".join(
+            line[NIX_STRING_INDENT:]
+            if line.startswith(" " * NIX_STRING_INDENT)
+            else line
+            for line in self.source[start:end].splitlines()
+        ).lstrip("\n")
+
+    # ── writeShellApplication and explicit runtimeInputs ────────────────────
+
+    def test_uses_write_shell_application(self):
+        self.assertIn("pkgs.writeShellApplication", self.source)
+
+    def test_runtime_inputs_includes_coreutils(self):
+        self.assertIn("pkgs.coreutils", self.source)
+
+    def test_runtime_inputs_includes_gawk(self):
+        self.assertIn("pkgs.gawk", self.source)
+
+    def test_runtime_inputs_includes_gnugrep(self):
+        self.assertIn("pkgs.gnugrep", self.source)
+
+    def test_runtime_inputs_block_present(self):
+        self.assertIn("runtimeInputs", self.source)
+
+    # ── Both execution paths use lib.getExe ─────────────────────────────────
+
+    def test_exec_start_uses_lib_get_exe(self):
+        """systemd ExecStart must reference the Nix-store executable."""
+        self.assertIn("ExecStart = lib.getExe hostsUpdateScript", self.source)
+
+    def test_activation_script_uses_lib_get_exe(self):
+        """Activation text must call the same Nix-store executable."""
+        self.assertIn("${lib.getExe hostsUpdateScript}", self.source)
+
+    def test_exec_start_does_not_point_to_etc_path(self):
+        """ExecStart must NOT use the raw /etc path (which lacks a deterministic PATH)."""
+        self.assertNotIn('ExecStart = "/etc/sovran-hosts-update.sh"', self.source)
+
+    # ── /etc symlink uses source =, not a second text = body ────────────────
+
+    def test_etc_entry_uses_source_not_text(self):
+        """The /etc/sovran-hosts-update.sh entry must be a symlink (source =),
+        not a second raw script body (text =)."""
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source', self.source
+        )
+
+    def test_etc_source_points_to_get_exe(self):
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript',
+            self.source,
+        )
+
+    # ── element-calling domain is supported ─────────────────────────────────
+
+    def test_element_calling_domain_key_present(self):
+        self.assertIn("element-calling", self.source)
+
+    # ── Robust awk -v variable passing ──────────────────────────────────────
+
+    def test_awk_uses_dash_v_for_begin_marker(self):
+        """awk must receive the begin marker via -v, not by shell interpolation."""
+        self.assertIn('awk -v begin=', self.source)
+
+    def test_awk_uses_dash_v_for_end_marker(self):
+        self.assertIn('-v end=', self.source)
+
+    def test_awk_does_not_interpolate_marker_into_program(self):
+        """The old pattern interpolated $BEGIN_MARKER directly into the awk source."""
+        self.assertNotIn('/$BEGIN_MARKER', self.source)
+        self.assertNotIn('/$END_MARKER', self.source)
+
+    # ── Domain validation ────────────────────────────────────────────────────
+
+    def test_domain_validation_regex_present(self):
+        """The hostname validation regex must still be present for injection prevention."""
+        self.assertIn("grep -qE", self.source)
+        self.assertIn("[a-zA-Z0-9]", self.source)
+
+    def test_invalid_domain_warning_present(self):
+        self.assertIn("skipping invalid domain value", self.source)
+
+    # ── No environment.systemPackages reliance ───────────────────────────────
+
+    def test_no_environment_system_packages_for_helper(self):
+        """The helper's tools are declared via runtimeInputs; the module must
+        not add them to environment.systemPackages."""
+        self.assertNotIn("environment.systemPackages", self.source)
+
+    # ── Idempotency: existing Sovran block is removed before rewriting ───────
+
+    def test_existing_block_removal_logic_present(self):
+        """awk strip of the managed block must be present for idempotency."""
+        self.assertIn("skip=1", self.source)
+        self.assertIn("skip=0", self.source)
+
+    def test_managed_block_uses_grouped_append_redirect(self):
+        self.assertIn('} >> "$TMP"', self.source)
+        self.assertEqual(self.source.count('>> "$TMP"'), 1)
+
+    def test_helper_script_passes_shellcheck(self):
+        shellcheck = shutil.which("shellcheck")
+        if shellcheck is None:
+            self.skipTest("shellcheck is not installed")
+        proc = subprocess.run(
+            [shellcheck, "-s", "bash", "-"],
+            input=self._helper_script(),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        output = proc.stdout + proc.stderr
+        self.assertEqual(proc.returncode, 0, output)
+
+    def test_helper_derivation_builds_when_nix_available(self):
+        nix = shutil.which("nix")
+        if nix is None:
+            self.skipTest("nix is not installed")
+        proc = subprocess.run(
+            [
+                nix,
+                "build",
+                HELPER_BUILD_ATTR,
+                "--no-link",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+
+    # ── Activation script warns on failure rather than silently swallowing ───
+
+    def test_activation_script_emits_warning_on_failure(self):
+        self.assertIn("warning: sovran-hosts-update", self.source)
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_manual_backup_workflow.py

@@ -0,0 +1,688 @@
+import asyncio
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BACKUP_SCRIPT = REPO_ROOT / "app" / "sovran_systemsos_web" / "scripts" / "sovran-hub-backup.sh"
+SERVER_FILE = REPO_ROOT / "app" / "sovran_systemsos_web" / "server.py"
+SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "support.js"
+NIX_HUB_FILE = REPO_ROOT / "modules" / "core" / "sovran-hub.nix"
+
+
+def _extract_bash_function(script_path: Path, func_name: str) -> str:
+    """Extract a bash function definition from a shell script using Python regex.
+
+    The pattern matches from 'funcname() {' to the first line whose only content
+    is '}' (the function closing brace, at column 0). This works correctly for
+    functions whose control structures (case/if/while/for) use indented braces,
+    because only the function's own closing brace sits at column 0. It would not
+    correctly extract a function that contains a nested function definition.
+
+    Returns the complete function definition, safe to inline into a test bash
+    script without eval or awk."""
+    source = script_path.read_text()
+    pattern = r"^" + re.escape(func_name) + r"\(\) \{.*?^}"
+    match = re.search(pattern, source, re.MULTILINE | re.DOTALL)
+    if match is None:
+        raise ValueError(f"Function '{func_name}' not found in {script_path}")
+    return match.group(0)
+
+
+class ManualBackupWorkflowTests(unittest.TestCase):
+    def test_backup_script_uses_tar_archives_with_checksums_and_exclusions(self):
+        source = BACKUP_SCRIPT.read_text()
+
+        self.assertIn("tar \\", source)
+        self.assertIn("--create", source)
+        self.assertIn("--one-file-system", source)
+        self.assertIn("sha256sum", source)
+        self.assertIn("export_postgresql_dumps", source)
+        self.assertIn("export_mariadb_dumps", source)
+        self.assertIn("export_lnd_scb_if_possible", source)
+        self.assertIn("--exclude='var/lib/bitcoind'", source)
+        self.assertIn("--exclude='var/lib/electrs'", source)
+        self.assertIn("--exclude='var/lib/lnd'", source)
+        self.assertIn("set_status \"RUNNING\"", source)
+        self.assertIn("set_status \"SUCCESS\"", source)
+        self.assertIn("set_status \"FAILED\"", source)
+        self.assertNotIn("rsync -a", source)
+
+    def test_backend_and_frontend_use_explicit_backup_terminal_states(self):
+        server_source = SERVER_FILE.read_text()
+        support_source = SUPPORT_JS.read_text()
+
+        self.assertIn("_write_backup_status, \"RUNNING\"", server_source)
+        self.assertIn("_monitor_backup_subprocess", server_source)
+        self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source)
+        self.assertIn("result === \"success\" || result === \"failed\"", support_source)
+
+    # ── Regression tests for exit-code-127 / missing-interpreter bug ──────────
+
+    def test_nix_service_path_includes_bash_and_gawk(self):
+        """Regression: pkgs.bash and pkgs.gawk must appear in the sovran-hub-web
+        service path so that the backup shell script and its awk calls can be
+        resolved at runtime without producing exit code 127."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertIn(
+            "pkgs.bash",
+            nix_source,
+            "pkgs.bash must be declared in the sovran-hub-web systemd service path",
+        )
+        self.assertIn(
+            "pkgs.gawk",
+            nix_source,
+            "pkgs.gawk must be declared in the sovran-hub-web systemd service path",
+        )
+
+    def test_server_resolves_bash_via_shutil_which(self):
+        """Regression: server must locate bash with shutil.which() rather than
+        relying on /usr/bin/env bash, so a missing interpreter is caught early
+        with a clear diagnostic instead of a cryptic exit-code-127 failure."""
+        source = SERVER_FILE.read_text()
+        self.assertIn(
+            'shutil.which("bash")',
+            source,
+            "server.py must resolve bash via shutil.which to catch missing interpreter",
+        )
+        self.assertNotIn(
+            '"/usr/bin/env", "bash"',
+            source,
+            "server.py must not use /usr/bin/env bash (relies on PATH, causes code 127)",
+        )
+
+    def test_server_logs_actionable_message_when_bash_missing(self):
+        """Regression: when bash is absent the server must write an actionable log
+        entry and set FAILED status before returning an HTTP error."""
+        source = SERVER_FILE.read_text()
+        self.assertIn(
+            "bash_path is None",
+            source,
+            "server.py must check that bash_path is not None before launching",
+        )
+        self.assertIn(
+            "interpreter",
+            source,
+            "server.py missing-bash error message must reference 'interpreter'",
+        )
+
+    def test_server_captures_stderr_from_backup_subprocess(self):
+        """Regression: the backup subprocess must use stderr=PIPE so that any
+        startup error (e.g. code 127 from a missing command) is captured and
+        surfaced in the backup log rather than being silently discarded."""
+        source = SERVER_FILE.read_text()
+        self.assertIn(
+            "stderr=asyncio.subprocess.PIPE",
+            source,
+            "backup subprocess must use stderr=PIPE to capture diagnostic output",
+        )
+        self.assertIn(
+            "stderr_text",
+            source,
+            "_monitor_backup_subprocess must capture and log stderr content",
+        )
+
+    def test_monitor_includes_stderr_detail_in_failed_message(self):
+        """Regression: _monitor_backup_subprocess must append captured stderr to
+        the FAILED log entry so the UI shows what went wrong (e.g. 'bash: not
+        found') rather than only the raw exit code."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("stderr_chunks", source)
+        self.assertIn("stderr_text", source)
+        # stderr detail is conditionally appended only when non-empty
+        self.assertIn('detail = f" — stderr: {stderr_text}"', source)
+
+    def test_exit_code_127_subprocess_stderr_drain(self):
+        """Behavioral regression: a subprocess that exits 127 must have its
+        stderr drained without deadlock by the async drain loop."""
+
+        async def _run():
+            proc = await asyncio.create_subprocess_exec(
+                "bash", "-c", "echo 'bash: command not found' >&2; exit 127",
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            chunks: list[bytes] = []
+
+            async def _drain():
+                async for line in proc.stderr:
+                    chunks.append(line)
+
+            drain_task = asyncio.create_task(_drain())
+            rc = await proc.wait()
+            await drain_task
+            return rc, b"".join(chunks).decode()
+
+        rc, stderr_out = asyncio.run(_run())
+        self.assertEqual(rc, 127)
+        self.assertIn("bash: command not found", stderr_out)
+
+    # ── Tests for home tar exit-1 tolerance (production fix) ──────────────────
+
+    def test_backup_script_has_home_tar_archive_function(self):
+        """The script must define create_home_tar_archive for /home only,
+        separate from the strict create_tar_archive used for other sources."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "create_home_tar_archive",
+            source,
+            "backup script must define create_home_tar_archive for /home",
+        )
+        self.assertIn(
+            "_is_home_warning_allowlisted",
+            source,
+            "backup script must define _is_home_warning_allowlisted helper",
+        )
+        self.assertIn(
+            "ARCHIVE_WARNINGS",
+            source,
+            "backup script must track nonfatal warnings in ARCHIVE_WARNINGS",
+        )
+
+    def test_backup_script_allowlist_covers_verified_gnu_tar_messages(self):
+        """The allowlist must contain the exact GNU tar (LC_ALL=C) strings for
+        the two permitted transient conditions."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "file changed as we read it",
+            source,
+            "allowlist must include exact GNU tar 'file changed as we read it' message",
+        )
+        self.assertIn(
+            "file removed before we read it",
+            source,
+            "allowlist must include exact GNU tar 'file removed before we read it' message",
+        )
+
+    def test_backup_script_stage3_uses_home_tar_function(self):
+        """Stage 3 must call create_home_tar_archive (not create_tar_archive)
+        so the /home archive tolerates live file changes on an active desktop."""
+        source = BACKUP_SCRIPT.read_text()
+        # The home archive call must use the tolerant function
+        self.assertIn(
+            'create_home_tar_archive "home.tar"',
+            source,
+            "Stage 3 must use create_home_tar_archive for home.tar",
+        )
+
+    def test_backup_script_strict_function_still_used_for_etc(self):
+        """create_tar_archive (strict) must still be used for /etc/nixos,
+        secrets, var-lib-lnd-clean, and var-lib — only /home is tolerant."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'create_tar_archive "etc-nixos.tar"',
+            source,
+            "etc-nixos must use strict create_tar_archive",
+        )
+        self.assertIn(
+            'create_tar_archive "var-lib.tar"',
+            source,
+            "var-lib must use strict create_tar_archive",
+        )
+
+    # ── Tests for partial-file atomic publish ─────────────────────────────────
+
+    def test_backup_script_uses_partial_path_and_atomic_rename(self):
+        """Archives must be written to a .partial path and atomically renamed
+        to the final name only after acceptance, so a failed run never leaves a
+        partial archive that appears complete."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            ".partial",
+            source,
+            "backup script must write to a .partial path before renaming",
+        )
+        self.assertIn(
+            'mv "$partial_path" "$archive_path"',
+            source,
+            "backup script must atomically rename partial to final archive path",
+        )
+        self.assertIn(
+            "PARTIAL_FILES",
+            source,
+            "backup script must track partial files for cleanup",
+        )
+
+    # ── Tests for INCOMPLETE / BACKUP_COMPLETE markers ────────────────────────
+
+    def test_backup_script_writes_incomplete_marker(self):
+        """A backup directory must be marked INCOMPLETE immediately after
+        creation, so interrupted or failed runs are identifiable."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'touch "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must create INCOMPLETE marker after mkdir",
+        )
+
+    def test_backup_script_writes_backup_complete_marker(self):
+        """A BACKUP_COMPLETE file must be written only after all work succeeds,
+        so restore tools can distinguish complete from incomplete backups."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'BACKUP_COMPLETE"',
+            source,
+            "backup script must write BACKUP_COMPLETE file on success",
+        )
+        self.assertIn(
+            'rm -f "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must remove INCOMPLETE marker on success",
+        )
+
+    # ── Tests for concurrency lock ─────────────────────────────────────────────
+
+    def test_backup_script_uses_flock_concurrency_lock(self):
+        """The script must acquire an exclusive flock before starting work to
+        prevent two simultaneous backup runs."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "flock",
+            source,
+            "backup script must use flock for concurrency protection",
+        )
+        self.assertIn(
+            "LOCK_FILE",
+            source,
+            "backup script must define a LOCK_FILE for the flock",
+        )
+
+    # ── Tests for expanded home exclusions ────────────────────────────────────
+
+    def test_backup_script_home_excludes_browser_caches(self):
+        """The home archive must exclude browser disk caches (volatile data)
+        while keeping profile data, bookmarks, and user documents."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "--exclude='home/*/.mozilla/firefox/*/cache2'",
+            source,
+            "backup script must exclude Firefox cache2 directory",
+        )
+        self.assertIn(
+            "--exclude='home/*/.config/google-chrome/*/Cache'",
+            source,
+            "backup script must exclude Chrome Cache directory",
+        )
+        self.assertIn(
+            "--exclude='home/*/.config/chromium/*/Cache'",
+            source,
+            "backup script must exclude Chromium Cache directory",
+        )
+        self.assertIn(
+            "--exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache'",
+            source,
+            "backup script must exclude Brave Cache directory",
+        )
+
+    def test_backup_script_home_excludes_other_volatile_caches(self):
+        """The home archive must exclude baloo, thumbnails, and X session
+        error logs — all volatile/reconstructable content."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "--exclude='home/*/.local/share/baloo'",
+            source,
+        )
+        self.assertIn(
+            "--exclude='home/*/.thumbnails'",
+            source,
+        )
+        self.assertIn(
+            "--exclude='home/*/.xsession-errors'",
+            source,
+        )
+
+    # ── Tests for require_cmd additions ───────────────────────────────────────
+
+    def test_backup_script_requires_mktemp_and_flock(self):
+        """The script must explicitly check for mktemp and flock via require_cmd
+        so that missing tools fail early with a clear error."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "require_cmd mktemp",
+            source,
+            "backup script must require mktemp",
+        )
+        self.assertIn(
+            "require_cmd flock",
+            source,
+            "backup script must require flock",
+        )
+
+    # ── Behavioral tests ──────────────────────────────────────────────────────
+
+    def test_allowlist_accepts_file_changed_message(self):
+        """Behavioral: _is_home_warning_allowlisted (from the actual backup script)
+        returns true for the exact GNU tar 'file changed as we read it' message (LC_ALL=C).
+        The function definition is extracted from the production script via Python regex
+        to ensure the test exercises the real allowlist without eval-of-awk risks."""
+        func_def = _extract_bash_function(BACKUP_SCRIPT, "_is_home_warning_allowlisted")
+        script = (
+            "#!/usr/bin/env bash\n"
+            + func_def + "\n"
+            + 'msgs=(\n'
+            + '  "tar: /home/user/firefox.db: file changed as we read it"\n'
+            + '  "tar: /home/user/.local/share/app: file removed before we read it"\n'
+            + ")\n"
+            + 'for msg in "${msgs[@]}"; do\n'
+            + '  _is_home_warning_allowlisted "$msg" || { echo "BLOCKED: $msg"; exit 1; }\n'
+            + '  echo "ALLOWED: $msg"\n'
+            + "done\n"
+        )
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as f:
+            f.write(script)
+            script_path = f.name
+        try:
+            result = subprocess.run(["bash", script_path], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("ALLOWED", result.stdout)
+            self.assertNotIn("BLOCKED", result.stdout)
+        finally:
+            os.unlink(script_path)
+
+    def test_allowlist_rejects_fatal_diagnostics(self):
+        """Behavioral: _is_home_warning_allowlisted (from the actual backup script)
+        returns false for permission errors, I/O errors, write errors, and other
+        fatal conditions."""
+        func_def = _extract_bash_function(BACKUP_SCRIPT, "_is_home_warning_allowlisted")
+        script = (
+            "#!/usr/bin/env bash\n"
+            + func_def + "\n"
+            + 'msgs=(\n'
+            + '  "tar: /home/user/file: Permission denied"\n'
+            + '  "tar: /dev/sdb: Cannot read: Input/output error"\n'
+            + '  "tar: Error is not recoverable: exiting now"\n'
+            + '  "Write error"\n'
+            + '  "tar: /home/user: Cannot stat: No such file or directory"\n'
+            + ")\n"
+            + 'for msg in "${msgs[@]}"; do\n'
+            + '  _is_home_warning_allowlisted "$msg" && { echo "ALLOWED_BUT_SHOULD_NOT: $msg"; exit 1; }\n'
+            + '  echo "CORRECTLY_BLOCKED: $msg"\n'
+            + "done\n"
+        )
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as f:
+            f.write(script)
+            script_path = f.name
+        try:
+            result = subprocess.run(["bash", script_path], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("CORRECTLY_BLOCKED", result.stdout)
+            self.assertNotIn("ALLOWED_BUT_SHOULD_NOT", result.stdout)
+        finally:
+            os.unlink(script_path)
+
+    def test_home_tar_exits_1_with_only_allowlisted_warnings_is_accepted(self):
+        """Behavioral: _create_archive_impl in HOME mode accepts tar exit 1 when
+        every diagnostic is allowlisted and publishes the final archive."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "home", "user1")
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(src, exist_ok=True)
+            os.makedirs(dest, exist_ok=True)
+            with open(os.path.join(src, "testfile.txt"), "w") as f:
+                f.write("test content\n")
+
+            # Emulate the allowlist + acceptance logic in HOME mode
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+BACKUP_DIR="{dest}"
+ARCHIVE_FILES=()
+ARCHIVE_WARNINGS=()
+PARTIAL_FILES=()
+
+log() {{ echo "$*"; }}
+fail() {{ echo "FAIL: $*" >&2; exit 1; }}
+
+_is_home_warning_allowlisted() {{
+  local msg="$1"
+  case "$msg" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}}
+
+_create_archive_impl() {{
+  local mode="$1"; shift
+  local archive_name="$1"; shift
+  local archive_path="$BACKUP_DIR/$archive_name"
+  local partial_path="${{archive_path}}.partial"
+  local diag_tmp
+  diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)"
+  PARTIAL_FILES+=("$partial_path" "$diag_tmp")
+
+  local tar_rc=0
+  LC_ALL=C tar --create --file "$partial_path" \
+    --numeric-owner --sparse --one-file-system \
+    "$@" 2>"$diag_tmp" || tar_rc=$?
+
+  # Inject a simulated allowlisted warning to match production scenario
+  echo "tar: {src}/testfile.txt: file changed as we read it" >> "$diag_tmp"
+  tar_rc=1  # simulate exit 1
+
+  local has_fatal_diag=0
+  if [[ -s "$diag_tmp" ]]; then
+    while IFS= read -r diag_line; do
+      [[ -n "$diag_line" ]] || continue
+      if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then
+        has_fatal_diag=1
+      fi
+    done < "$diag_tmp"
+  fi
+
+  local accept=0
+  if [[ "$tar_rc" -eq 0 ]]; then
+    accept=1
+  elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+    accept=1
+    ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings")
+  fi
+
+  [[ "$accept" -eq 1 ]] || {{ rm -f "$partial_path" "$diag_tmp"; fail "Rejected"; }}
+  [[ -s "$partial_path" ]] || {{ rm -f "$partial_path" "$diag_tmp"; fail "Empty"; }}
+
+  mv "$partial_path" "$archive_path"
+  rm -f "$diag_tmp"
+  ARCHIVE_FILES+=("$archive_name")
+  echo "CREATED: $archive_name"
+}}
+
+create_home_tar_archive() {{ _create_archive_impl HOME "$@"; }}
+
+create_home_tar_archive "home.tar" -C "{tmpdir}" home
+echo "WARNINGS: ${{#ARCHIVE_WARNINGS[@]}}"
+[[ -f "{dest}/home.tar" ]] && echo "FILE_EXISTS" || echo "FILE_MISSING"
+[[ ! -f "{dest}/home.tar.partial" ]] && echo "PARTIAL_CLEANED" || echo "PARTIAL_LEFT"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}\nstderr: {result.stderr}")
+            self.assertIn("CREATED: home.tar", result.stdout)
+            self.assertIn("FILE_EXISTS", result.stdout)
+            self.assertIn("PARTIAL_CLEANED", result.stdout)
+
+    def test_home_tar_exits_1_with_fatal_diagnostic_fails(self):
+        """Behavioral: _create_archive_impl in HOME mode must fail when tar exit 1
+        includes a non-allowlisted diagnostic (e.g. permission denied)."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+BACKUP_DIR="{dest}"
+ARCHIVE_FILES=()
+ARCHIVE_WARNINGS=()
+PARTIAL_FILES=()
+
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+log() {{ echo "$*"; }}
+
+_is_home_warning_allowlisted() {{
+  local msg="$1"
+  case "$msg" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}}
+
+# Simulate tar exit 1 with a fatal permission-denied diagnostic
+archive_name="home.tar"
+partial_path="$BACKUP_DIR/$archive_name.partial"
+diag_tmp=$(mktemp)
+echo "tar: /home/user/locked: Permission denied" > "$diag_tmp"
+
+tar_rc=1
+has_fatal_diag=0
+while IFS= read -r diag_line; do
+  [[ -n "$diag_line" ]] || continue
+  if ! _is_home_warning_allowlisted "$diag_line"; then
+    has_fatal_diag=1
+  fi
+done < "$diag_tmp"
+rm -f "$diag_tmp"
+
+if [[ "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+  echo "INCORRECTLY_ACCEPTED"
+  exit 0
+else
+  fail "tar exit 1 with Permission denied"
+fi
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}")
+            self.assertIn("CORRECTLY_FAILED", result.stdout)
+
+    def test_home_tar_exits_2_always_fails(self):
+        """Behavioral: tar exit code 2 (fatal) must always fail even in HOME mode."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+BACKUP_DIR="{dest}"
+ARCHIVE_WARNINGS=()
+
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+log() {{ echo "$*"; }}
+
+_is_home_warning_allowlisted() {{
+  case "$1" in
+    *"file changed as we read it"*)   return 0 ;;
+    *"file removed before we read it"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}}
+
+tar_rc=2
+has_fatal_diag=0
+
+accept=0
+if [[ "$tar_rc" -eq 0 ]]; then
+  accept=1
+elif [[ "HOME" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
+  accept=1
+fi
+
+if [[ "$accept" -eq 0 ]]; then
+  if [[ "$tar_rc" -gt 1 ]]; then
+    fail "tar exited with fatal code $tar_rc"
+  fi
+fi
+echo "SHOULD_NOT_REACH_HERE"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}")
+            self.assertIn("CORRECTLY_FAILED", result.stdout)
+            self.assertNotIn("SHOULD_NOT_REACH_HERE", result.stdout)
+
+    def test_partial_file_cleanup_on_failure(self):
+        """Behavioral: a .partial file must be removed when the archive creation
+        fails, so no incomplete archive is left on the destination."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            partial_path = os.path.join(dest, "home.tar.partial")
+
+            script = f"""#!/usr/bin/env bash
+BACKUP_DIR="{dest}"
+PARTIAL_FILES=()
+
+fail() {{
+  # Simulate cleanup removing partial files
+  for p in "${{PARTIAL_FILES[@]}}"; do
+    [[ -f "$p" ]] && rm -f "$p"
+  done
+  echo "FAILED: $*"
+  exit 1
+}}
+log() {{ :; }}
+
+partial_path="$BACKUP_DIR/home.tar.partial"
+diag_tmp=$(mktemp)
+PARTIAL_FILES+=("$partial_path" "$diag_tmp")
+
+# Simulate partial archive being written
+touch "$partial_path"
+
+# Simulate a fatal error
+fail "tar exited with code 2"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}")
+            self.assertFalse(
+                os.path.exists(partial_path),
+                "Partial file must be removed on failure",
+            )
+            self.assertIn("FAILED", result.stdout)
+
+    def test_archive_warnings_recorded_in_manifest(self):
+        """Source text: ARCHIVE_WARNINGS must be written into BACKUP_MANIFEST.txt
+        so the user can see which archives had nonfatal live-file warnings."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "ARCHIVE_WARNINGS[@]",
+            source,
+            "backup script must iterate ARCHIVE_WARNINGS in manifest generation",
+        )
+        self.assertIn(
+            "Nonfatal warnings:",
+            source,
+            "manifest must include a 'Nonfatal warnings:' section",
+        )
+
+    def test_backup_script_logs_warnings_before_success_message(self):
+        """Source text: when ARCHIVE_WARNINGS is non-empty, the script must log
+        the warnings before the 'All Finished!' success message."""
+        source = BACKUP_SCRIPT.read_text()
+        # Both constructs must be present
+        self.assertIn(
+            "${#ARCHIVE_WARNINGS[@]}",
+            source,
+            "backup script must check ARCHIVE_WARNINGS count before success message",
+        )
+        self.assertIn(
+            "All Finished!",
+            source,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

modules/core/local-domain-loopback.nix

@@ -32,22 +32,31 @@
 # regenerated by the system activation script.  The ``system.activationScripts``
 # hook below converts it to a writable file each time the system is activated
 # (i.e. after every ``nixos-rebuild switch``) and then injects the Sovran block.
-# The same script is also run by the ``sovran-hosts-update.service`` unit so
-# that the Hub can trigger it immediately after saving a domain without
-# requiring a full rebuild.
+# The same wrapped Nix-store executable is reused by both the activation hook
+# and the ``sovran-hosts-update.service`` unit, ensuring a deterministic runtime
+# PATH in every execution context.
 
-{
-  # ── Helper script (stored in the Nix store, never reads /var/lib at eval) ──
+let
+  # ── Wrapped Nix-store executable ──────────────────────────────────────────
+  # Built with pkgs.writeShellApplication so that all required runtime tools
+  # (awk, grep, coreutils) are declared explicitly and injected into PATH by
+  # Nix.  Both the systemd service and the activation hook reference this same
+  # store-path executable — there is no second raw script body.
+  hostsUpdateScript = pkgs.writeShellApplication {
+    name = "sovran-hosts-update";
 
-  environment.systemPackages = [ pkgs.coreutils ];
+    # Declare every external command the script calls.  These packages are
+    # added to the script's runtime PATH by writeShellApplication; nothing from
+    # the system PATH is relied upon.
+    runtimeInputs = [
+      pkgs.coreutils   # readlink, cp, mv, chmod, mktemp, rm, tr, head
+      pkgs.gawk        # awk
+      pkgs.gnugrep     # grep
+    ];
 
-  environment.etc."sovran-hosts-update.sh" = {
-    mode = "0755";
     text = ''
-      #!/bin/sh
       # Regenerate the Sovran-managed loopback block in /etc/hosts.
       # Safe to run multiple times — idempotent.
-      set -eu
 
       DOMAINS_DIR="/var/lib/domains"
       HOSTS_FILE="/etc/hosts"
@@ -66,13 +75,14 @@
 
       # ── Step 2: remove any existing Sovran block ──────────────────────────
       # Use a temp file so the operation is atomic.
+      # awk -v passes marker strings safely without shell interpolation.
       TMP=$(mktemp "$HOSTS_FILE.XXXXXX")
       trap 'rm -f "$TMP"' EXIT
-      awk "
-        /^$BEGIN_MARKER\$/ { skip=1; next }
-        /^$END_MARKER\$/   { skip=0; next }
+      awk -v begin="$BEGIN_MARKER" -v end="$END_MARKER" '
+        $0 == begin { skip=1; next }
+        $0 == end   { skip=0; next }
         !skip
-      " "$HOSTS_FILE" > "$TMP"
+      ' "$HOSTS_FILE" > "$TMP"
 
       # ── Step 3: collect valid configured service domains ──────────────────
       # NOTE: The hostname validation regex below must stay in sync with
@@ -97,11 +107,13 @@
 
       # ── Step 4: append the Sovran block if there are any entries ──────────
       if [ -n "$ENTRIES" ]; then
-        printf '\n%s\n' "$BEGIN_MARKER" >> "$TMP"
-        printf '%s\n' "# These entries route configured service domains to local Caddy." >> "$TMP"
-        printf '%s\n' "# They are managed automatically — do not edit this block." >> "$TMP"
-        printf '%s\n' "$ENTRIES" >> "$TMP"
-        printf '%s\n' "$END_MARKER" >> "$TMP"
+        {
+          printf '\n%s\n' "$BEGIN_MARKER"
+          printf '%s\n' "# These entries route configured service domains to local Caddy."
+          printf '%s\n' "# They are managed automatically — do not edit this block."
+          printf '%s\n' "$ENTRIES"
+          printf '%s\n' "$END_MARKER"
+        } >> "$TMP"
       fi
 
       # ── Step 5: atomically replace /etc/hosts ─────────────────────────────
@@ -110,6 +122,14 @@
     '';
   };
 
+in
+{
+  # ── /etc/sovran-hosts-update.sh — operator discoverability symlink ─────────
+  # Retain the familiar /etc path so administrators can inspect or manually
+  # invoke the helper.  The target is the wrapped Nix-store executable, so
+  # there is no second raw script body to keep in sync.
+  environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript;
+
   # ── Systemd service ────────────────────────────────────────────────────────
 
   systemd.services.sovran-hosts-update = {
@@ -126,18 +146,24 @@
     serviceConfig = {
       Type = "oneshot";
       RemainAfterExit = true;
-      ExecStart = "/etc/sovran-hosts-update.sh";
+      # Point directly at the wrapped Nix-store executable, not the /etc path.
+      ExecStart = lib.getExe hostsUpdateScript;
     };
   };
 
   # ── Activation script (runs after every nixos-rebuild switch) ─────────────
   # This ensures the loopback block survives rebuilds that restore the /etc/hosts
   # symlink.  The "users" and "etc" scripts must complete first.
+  # The same wrapped Nix-store executable used by the systemd service is
+  # referenced here, guaranteeing identical runtime dependencies in both
+  # execution contexts.
 
   system.activationScripts.sovranDomainLoopback = {
     text = ''
-      if [ -x /etc/sovran-hosts-update.sh ] && [ -d /var/lib/domains ]; then
-        /etc/sovran-hosts-update.sh || true
+      if [ -d /var/lib/domains ]; then
+        if ! ${lib.getExe hostsUpdateScript}; then
+          echo "warning: sovran-hosts-update: failed to update /etc/hosts loopback entries" >&2
+        fi
       fi
     '';
     deps = [ "etc" "users" ];

modules/core/sovran-hub.nix

@@ -382,13 +382,24 @@ in
       };
 
       path = [
+        pkgs.bash
+        pkgs.gawk
         pkgs.qrencode
         pkgs.curl
         pkgs.iproute2
         pkgs.nftables
         pkgs.iptables
         pkgs.hostname
-      ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package;
+        pkgs.coreutils
+        pkgs.findutils
+        pkgs.gnugrep
+        pkgs.gnutar
+        pkgs.util-linux
+      ]
+      ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
+      ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
+      ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
+      ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
     };
 
     systemd.services.sovran-hub-update = {