Merge pull request #334 from naturallaw777/copilot/fix-manual-backup-rsync-failure

Fix rsync destination-directory failure: sync_tree with mkdir -p, mount re-verification, and path-safety guards

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ── Sovran Hub External Backup Script ────────────────────────────
-# Backs up Sovran_SystemsOS data to an external USB hard drive.
+# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
 # Designed for the Hub web UI (no GUI dependencies).
 #
 # Your Sovran Pro already backs up your data automatically to its
@@ -8,6 +8,15 @@
 # This script creates an additional copy on an external USB drive —
 # storing your data in a third location for maximum protection.
 #
+# The external drive must be formatted as ext4. Files are stored as
+# directly browsable files under Sovran_SystemsOS_Backup/current/.
+# Later runs update the same mirror and only transfer changed or new
+# files, making repeat backups fast.
+#
+# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
+# blockchain and Electrs index data are NOT included (they live on
+# the internal second drive).
+#
 # Usage:
 #   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
 #   (or run with no env var to auto-detect the first external USB drive)
@@ -28,20 +37,17 @@ INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
 
 FAILED_ALREADY=0
 BACKUP_COMPLETE=0
-LND_STOPPED=0
-LND_UNITS_TO_RESTART=()
+RSYNC_WARNINGS=()
 
-ARCHIVE_FILES=()
-ARCHIVE_WARNINGS=()
-PARTIAL_FILES=()
-DB_DUMP_FILES=()
-MANIFEST_EXCLUDES=()
-LND_BACKUP_NOTES=()
+# Stable rsync mirror sub-path under the target drive. Not timestamped
+# so later runs update the same destination and only transfer new or changed files.
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
 
 # ── Logging helpers ──────────────────────────────────────────────
 
 log() {
-  local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+  local msg
+  msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
   echo "$msg" | tee -a "$BACKUP_LOG"
 }
 
@@ -58,37 +64,10 @@ fail() {
 
 cleanup() {
   local rc=$?
-  local restart_failed=0
-
-  # Remove any partial archive files or temporary diagnostic files
-  if [[ "${#PARTIAL_FILES[@]}" -gt 0 ]]; then
-    local partial
-    for partial in "${PARTIAL_FILES[@]}"; do
-      [[ -f "$partial" ]] && rm -f "$partial" || true
-    done
-  fi
 
   # Release the concurrency lock file descriptor if it was opened
-  [[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true
-
-  if [[ "$LND_STOPPED" -eq 1 ]]; then
-    log "Restarting previously active LND-related services…"
-    for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do
-      local unit="${LND_UNITS_TO_RESTART[$idx]}"
-      if systemctl start "$unit"; then
-        log "Started $unit"
-      else
-        log "ERROR: Failed to start $unit"
-        restart_failed=1
-      fi
-    done
-    LND_STOPPED=0
-  fi
-
-  if [[ "$restart_failed" -eq 1 ]]; then
-    rc=1
-    FAILED_ALREADY=1
-    set_status "FAILED"
+  if [[ -n "${LOCK_FD:-}" ]]; then
+    exec {LOCK_FD}>&- 2>/dev/null || true
   fi
 
   if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
@@ -219,21 +198,8 @@ validate_target_mount() {
   fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
   [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
 
-  if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then
-    fail "Target '$target' must be exFAT (detected filesystem: $fstype)."
-  fi
-
-  if [[ "$fstype" == "fuseblk" ]]; then
-    local src_dev blk_type
-    src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true)
-    blk_type=""
-    if [[ -n "$src_dev" ]]; then
-      blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true)
-      [[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true)
-    fi
-    if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then
-      fail "Target '$target' is fuseblk but not identified as exFAT-compatible."
-    fi
+  if [[ "$fstype" != "ext4" ]]; then
+    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
   fi
 
   local write_test
@@ -245,14 +211,6 @@ validate_target_mount() {
   log "Verified backup target filesystem: $fstype"
 }
 
-has_unit() {
-  systemctl cat "$1" >/dev/null 2>&1
-}
-
-is_unit_active() {
-  systemctl is-active --quiet "$1"
-}
-
 estimate_path_bytes() {
   local path="$1"
   shift || true
@@ -267,6 +225,82 @@ estimate_path_bytes() {
   echo "$size"
 }
 
+# ── Sync one source tree to its backup destination ───────────────
+# Usage: sync_tree <label> <allow_vanished> <source> <destination> [rsync options...]
+#
+# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
+# Used for /home only — files may disappear while the desktop is active.
+# All other nonzero exit codes are always fatal.
+#
+# Before every rsync call this helper:
+#   1. Re-verifies $TARGET is still a mount point (fails if drive disconnected).
+#   2. Verifies the destination path remains beneath $BACKUP_DIR and $BACKUP_DIR
+#      remains beneath $TARGET (safe-path check).
+#   3. Creates the full destination directory hierarchy with mkdir -p so that
+#      rsync never fails trying to create a directory whose parent is absent.
+sync_tree() {
+  local label="$1"
+  local allow_vanished="$2"
+  local source="$3"
+  local destination="$4"
+  shift 4
+  # Remaining "$@" are rsync options (--exclude, etc.)
+
+  # ── Re-verify the external drive is still mounted ────────────────
+  mountpoint -q "$TARGET" 2>/dev/null || \
+    fail "Stage $label: external drive '$TARGET' is no longer mounted. Refusing to write."
+
+  # ── Verify path safety ────────────────────────────────────────────
+  # BACKUP_DIR must remain beneath TARGET.
+  case "$BACKUP_DIR" in
+    "$TARGET"/*) ;;
+    *) fail "Stage $label: BACKUP_DIR '$BACKUP_DIR' is outside TARGET '$TARGET'." ;;
+  esac
+  # Destination must remain beneath BACKUP_DIR.
+  case "$destination" in
+    "$BACKUP_DIR"/*|"$BACKUP_DIR") ;;
+    *) fail "Stage $label: destination '$destination' is outside BACKUP_DIR '$BACKUP_DIR'. Refusing to write." ;;
+  esac
+
+  # ── Create complete destination directory hierarchy ───────────────
+  # This is the fix for the production failure:
+  #   rsync: [Receiver] mkdir ".../current/etc/nixos" failed: No such file or directory
+  # mkdir -p creates all intermediate parents (e.g. current/etc/) before rsync runs.
+  mkdir -p -- "$destination" || \
+    fail "Stage $label: failed to create destination directory '$destination' (source: '$source')."
+
+  local rsync_err_tmp
+  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
+
+  local rc=0
+  rsync \
+    --archive \
+    --acls \
+    --xattrs \
+    --hard-links \
+    --numeric-ids \
+    --one-file-system \
+    --partial \
+    "$@" "$source" "$destination" 2>"$rsync_err_tmp" || rc=$?
+
+  if [[ -s "$rsync_err_tmp" ]]; then
+    while IFS= read -r rline; do
+      log "rsync: $rline"
+    done < "$rsync_err_tmp"
+  fi
+  rm -f "$rsync_err_tmp"
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
+  fi
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -286,22 +320,17 @@ exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
 flock --nonblock "$LOCK_FD" 2>/dev/null || \
   fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
 
-require_cmd tar
-require_cmd sha256sum
+require_cmd rsync
 require_cmd findmnt
 require_cmd lsblk
 require_cmd mountpoint
 require_cmd df
 require_cmd du
 require_cmd awk
-require_cmd sort
 require_cmd find
-require_cmd systemctl
 require_cmd hostname
 require_cmd date
 require_cmd python3
-require_cmd runuser
-require_cmd mktemp
 require_cmd flock
 
 # ── Detect system role ───────────────────────────────────────────
@@ -327,50 +356,32 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again."
+    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
 validate_target_mount "$TARGET"
 
-# ── Plan role-aware source scope and exclusions ─────────────────
+# ── Set up stable backup destination ────────────────────────────
+# Subsequent runs update the same mirror, transferring only new or changed files.
 
-LND_AVAILABLE=0
-if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then
-  LND_AVAILABLE=1
-fi
+BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
+mkdir -p -- "$BACKUP_DIR"
 
-if [[ "$ROLE" == "desktop" ]]; then
-  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)")
-else
-  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent")
-fi
+# Remove any stale BACKUP_COMPLETE left by a previous successful run.
+# The new run will re-earn it only after all stages succeed.
+rm -f "$BACKUP_DIR/BACKUP_COMPLETE"
 
-MANIFEST_EXCLUDES+=(
-  "/run/media/Second_Drive (never traversed)"
-  "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)"
-  "/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)"
-  "/var/lib/bitcoind (excluded from manual backup)"
-  "/var/lib/electrs (excluded from manual backup)"
-  "/var/lib/*/log and /var/lib/*/logs"
-  "/var/lib/*/cache and /var/lib/*/tmp"
-  "/home/*/.cache (system and application disk caches)"
-  "/home/*/.local/share/Trash and /home/*/Trash (trash directories)"
-  "/home/*/.mozilla/firefox/*/cache2 and */startupCache (Firefox volatile cache — profile data is kept)"
-  "/home/*/.config/google-chrome/*/Cache (Chrome disk cache — profile data is kept)"
-  "/home/*/.config/chromium/*/Cache (Chromium disk cache — profile data is kept)"
-  "/home/*/.config/BraveSoftware/Brave-Browser/*/Cache (Brave disk cache — profile data is kept)"
-  "/home/*/.local/share/baloo (KDE file indexer — rebuilt automatically)"
-  "/home/*/.thumbnails (thumbnail cache — rebuilt automatically)"
-  "/home/*/.xsession-errors and .xsession-errors.old (X session error logs)"
-)
-
-if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
-  MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive")
-fi
+# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
+# after all rsync stages and manifest write succeed. Failed or interrupted
+# runs keep this marker so they are clearly identifiable.
+touch "$BACKUP_DIR/INCOMPLETE"
+log "Backup destination: $BACKUP_DIR"
 
 # ── Estimate required free space ─────────────────────────────────
+# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
+# from the estimate to avoid inflating the required size.
 
 ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
 HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
@@ -380,21 +391,20 @@ if [[ "$ROLE" != "desktop" ]]; then
 fi
 
 VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='postgresql' \
+  --exclude='mysql' \
+  --exclude='mariadb' \
   --exclude='bitcoind' \
   --exclude='electrs' \
-  --exclude='lnd' \
   --exclude='*/log' \
   --exclude='*/logs' \
   --exclude='*/cache' \
   --exclude='*/tmp')
 
-LND_BYTES=0
-if [[ "$LND_AVAILABLE" -eq 1 ]]; then
-  LND_BYTES=$(estimate_path_bytes /var/lib/lnd)
-fi
-
-ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES ))
-# Require 20% growth headroom plus an additional fixed 1 GiB safety margin.
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
+# Require 20% growth headroom plus a fixed 1 GiB safety margin.
+# Later incremental runs need far less space, but a conservative first-run
+# check protects against running out of space mid-backup.
 REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
 
 FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
@@ -408,391 +418,79 @@ log "Free space on drive: ${FREE_GB} GB"
 (( FREE_BYTES >= REQUIRED_BYTES )) || \
   fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
 
-# ── Create timestamped backup directory ─────────────────────────
-
-TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
-BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
-DB_DUMP_DIR="$BACKUP_DIR/database-dumps"
-mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR"
-# Write an INCOMPLETE marker immediately; it is removed only on successful completion.
-# Failed runs keep this marker so they are easily identifiable and not confused with
-# complete backups during restore selection.
-touch "$BACKUP_DIR/INCOMPLETE"
-log "Backup destination: $BACKUP_DIR"
-
-create_tar_archive() {
-  _create_archive_impl STRICT "$@"
-}
-
-# ── Helper: classify a GNU tar (LC_ALL=C) diagnostic line ────────
-# Returns 0 (true) if the message is an allowlisted transient condition
-# that is safe to ignore for /home (live desktop file changes).
-# Only two verified GNU tar messages qualify; all others are fatal.
-_is_home_warning_allowlisted() {
-  local msg="$1"
-  case "$msg" in
-    *"file changed as we read it"*)   return 0 ;;
-    *"file removed before we read it"*) return 0 ;;
-    *) return 1 ;;
-  esac
-}
-
-# ── Internal archive builder ──────────────────────────────────────
-# mode: STRICT — any tar error fails.
-#       HOME   — tar exit 1 is accepted when every diagnostic is an
-#                allowlisted transient condition (live file changes).
-_create_archive_impl() {
-  local mode="$1"; shift
-  local archive_name="$1"; shift
-  local archive_path="$BACKUP_DIR/$archive_name"
-  local partial_path="${archive_path}.partial"
-  local diag_tmp
-  # mktemp creates files with mode 0600 (owner-only) by default, so tar
-  # diagnostics (which may include file paths) are not readable by other users.
-  diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)"
-  PARTIAL_FILES+=("$partial_path" "$diag_tmp")
-
-  log "Creating $archive_name …"
-
-  local tar_rc=0
-  LC_ALL=C tar \
-    --create \
-    --file "$partial_path" \
-    --numeric-owner \
-    --acls \
-    --xattrs \
-    --sparse \
-    --one-file-system \
-    "$@" 2>"$diag_tmp" || tar_rc=$?
-
-  # Log every tar diagnostic to the backup log so it appears in the Hub UI
-  local has_fatal_diag=0
-  if [[ -s "$diag_tmp" ]]; then
-    while IFS= read -r diag_line; do
-      [[ -n "$diag_line" ]] || continue
-      log "tar: $diag_line"
-      if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then
-        has_fatal_diag=1
-      fi
-    done < "$diag_tmp"
-  fi
-
-  local accept=0
-  if [[ "$tar_rc" -eq 0 ]]; then
-    accept=1
-  elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
-    accept=1
-    log "NOTE: $archive_name completed with nonfatal warnings (live files changed during backup -- this is normal on an active desktop and does not affect the safety of your backup)."
-    ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings -- live files changed during backup (normal on an active desktop)")
-  fi
-
-  if [[ "$accept" -eq 0 ]]; then
-    rm -f "$partial_path" "$diag_tmp"
-    if [[ "$tar_rc" -gt 1 ]]; then
-      fail "tar exited with fatal code $tar_rc while creating $archive_name."
-    elif [[ "$mode" == "HOME" ]]; then
-      fail "tar exited with code $tar_rc and unrecognized diagnostics while creating $archive_name."
-    else
-      fail "tar exited with code $tar_rc while creating $archive_name."
-    fi
-  fi
-
-  # Verify the partial archive is non-empty and readable (spot-check first entry)
-  if [[ ! -s "$partial_path" ]]; then
-    rm -f "$partial_path" "$diag_tmp"
-    fail "Archive $archive_name is empty after creation."
-  fi
-
-  # Fast readability check: read only the first tar entry header (~512 bytes).
-  # head -1 closes the pipe after one line, sending SIGPIPE to tar which then
-  # exits early — so this is O(1) regardless of archive size.
-  local spot_entry
-  spot_entry="$(LC_ALL=C tar --list --file "$partial_path" 2>/dev/null | head -1 || true)"
-  if [[ -z "$spot_entry" ]]; then
-    rm -f "$partial_path" "$diag_tmp"
-    fail "Archive $archive_name failed readability check."
-  fi
-
-  # Atomic publish: rename partial to final path only after acceptance
-  mv "$partial_path" "$archive_path"
-  rm -f "$diag_tmp"
-  ARCHIVE_FILES+=("$archive_name")
-  log "Created archive: $archive_name"
-}
-
-# ── Home archive: tolerates allowlisted live-file warnings ───────
-create_home_tar_archive() {
-  _create_archive_impl HOME "$@"
-}
-
-export_postgresql_dumps() {
-  if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then
-    log "PostgreSQL tools/service not available — skipping PostgreSQL exports."
-    return
-  fi
-
-  if ! is_unit_active "postgresql.service"; then
-    log "PostgreSQL service is not active — skipping PostgreSQL exports."
-    return
-  fi
-
-  log "Exporting PostgreSQL globals and databases…"
-  local globals_file="$DB_DUMP_DIR/postgresql_globals.sql"
-  runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \
-    fail "Failed to export PostgreSQL globals."
-  DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql")
-
-  local dbs
-  dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true)
-
-  if [[ -z "$dbs" ]]; then
-    log "No non-template PostgreSQL application databases found."
-    return
-  fi
-
-  while IFS= read -r db; do
-    [[ -n "$db" ]] || continue
-    local safe_db
-    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
-    local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump"
-    runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \
-      fail "Failed to export PostgreSQL database '$db'."
-    DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump")
-  done <<< "$dbs"
-}
-
-export_mariadb_dumps() {
-  local dump_cmd=""
-  local query_cmd=""
-  local mariadb_unit=""
-
-  if command -v mariadb-dump >/dev/null 2>&1; then
-    dump_cmd="mariadb-dump"
-  elif command -v mysqldump >/dev/null 2>&1; then
-    dump_cmd="mysqldump"
-  fi
-
-  if command -v mariadb >/dev/null 2>&1; then
-    query_cmd="mariadb"
-  elif command -v mysql >/dev/null 2>&1; then
-    query_cmd="mysql"
-  fi
-
-  if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then
-    log "MariaDB dump/query tools not available — skipping MariaDB exports."
-    return
-  fi
-
-  if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then
-    mariadb_unit="mariadb.service"
-  elif has_unit "mysql.service" && is_unit_active "mysql.service"; then
-    mariadb_unit="mysql.service"
-  else
-    log "MariaDB service is not active — skipping MariaDB exports."
-    return
-  fi
-
-  log "Exporting MariaDB databases from ${mariadb_unit}…"
-
-  local dbs
-  dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true)
-  if [[ -z "$dbs" ]]; then
-    log "No MariaDB databases found."
-    return
-  fi
-
-  while IFS= read -r db; do
-    [[ -n "$db" ]] || continue
-    case "$db" in
-      information_schema|performance_schema|mysql|sys) continue ;;
-    esac
-
-    local safe_db out_file
-    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
-    out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql"
-
-    $dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \
-      fail "Failed to export MariaDB database '$db'."
-
-    DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql")
-  done <<< "$dbs"
-}
-
-export_lnd_scb_if_possible() {
-  [[ "$LND_AVAILABLE" -eq 1 ]] || return
-
-  local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb"
-  local attempts=(
-    "lncli exportchanbackup --all --output_file $scb_file"
-    "lncli -n mainnet exportchanbackup --all --output_file $scb_file"
-    "runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file"
-    "runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file"
-  )
-
-  if ! command -v lncli >/dev/null 2>&1; then
-    log "lncli not available — skipping Static Channel Backup export."
-    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)")
-    return
-  fi
-
-  if ! is_unit_active "lnd.service"; then
-    log "LND service is not active — skipping Static Channel Backup export."
-    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)")
-    return
-  fi
-
-  log "Exporting LND Static Channel Backup…"
-  local attempt
-  for attempt in "${attempts[@]}"; do
-    if eval "$attempt" >/dev/null 2>&1; then
-      DB_DUMP_FILES+=("lnd-static-channel-backup.scb")
-      LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli")
-      log "LND Static Channel Backup exported."
-      return
-    fi
-  done
-
-  log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations."
-  LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)")
-}
-
-capture_active_lnd_dependents() {
-  [[ "$LND_AVAILABLE" -eq 1 ]] || return
-
-  LND_UNITS_TO_RESTART=()
-  local raw_units=""
-  raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true)
-
-  while IFS= read -r unit; do
-    [[ -n "$unit" ]] || continue
-    if is_unit_active "$unit"; then
-      LND_UNITS_TO_RESTART+=("$unit")
-    fi
-  done <<< "$raw_units"
-
-  if is_unit_active "lnd.service"; then
-    LND_UNITS_TO_RESTART+=("lnd.service")
-  fi
-}
-
-stop_lnd_stack_if_needed() {
-  [[ "$LND_AVAILABLE" -eq 1 ]] || return
-
-  capture_active_lnd_dependents
-
-  if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then
-    log "No active LND-related services needed stopping."
-    return
-  fi
-
-  log "Stopping active services that depend on LND for clean /var/lib/lnd archive…"
-
-  local unit
-  for unit in "${LND_UNITS_TO_RESTART[@]}"; do
-    if [[ "$unit" == "lnd.service" ]]; then
-      continue
-    fi
-    systemctl stop "$unit" || fail "Failed to stop dependent service: $unit"
-    log "Stopped $unit"
-  done
-
-  if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then
-    systemctl stop lnd.service || fail "Failed to stop lnd.service"
-    log "Stopped lnd.service"
-  fi
-
-  LND_STOPPED=1
-}
-
-# ── Stage 1/5: NixOS configuration ──────────────────────────────
+# ── Stage 1/4: NixOS configuration ──────────────────────────────
 
 log ""
-log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────"
+log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  create_tar_archive "etc-nixos.tar" -C / etc/nixos
+  sync_tree "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
 fi
 
-# ── Stage 2/5: Secrets ──────────────────────────────────────────
+# ── Stage 2/4: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────"
+log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
 elif [[ -e /etc/nix-bitcoin-secrets ]]; then
-  create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets
+  sync_tree "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
 else
   log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
 log "Stage 2 complete."
 
-# ── Stage 3/5: Home directory ───────────────────────────────────
+# ── Stage 3/4: Home directory ───────────────────────────────────
+# Rsync exit code 24 (vanished source files) is treated as nonfatal here
+# because the desktop may be active and files can disappear between the
+# directory scan and the copy. All other nonzero exit codes remain fatal.
 
 log ""
-log "── Stage 3/5: Home directory (/home) ───────────────────────"
+log "── Stage 3/4: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  create_home_tar_archive "home.tar" \
-    -C / \
-    --exclude='home/*/.cache' \
-    --exclude='home/*/.local/share/Trash' \
-    --exclude='home/*/Trash' \
-    --exclude='home/*/.mozilla/firefox/*/cache2' \
-    --exclude='home/*/.mozilla/firefox/*/startupCache' \
-    --exclude='home/*/.mozilla/firefox/*/thumbnails' \
-    --exclude='home/*/.config/google-chrome/*/Cache' \
-    --exclude='home/*/.config/google-chrome/*/Code Cache' \
-    --exclude='home/*/.config/chromium/*/Cache' \
-    --exclude='home/*/.config/chromium/*/Code Cache' \
-    --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache' \
-    --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Code Cache' \
-    --exclude='home/*/.local/share/baloo' \
-    --exclude='home/*/.thumbnails' \
-    --exclude='home/*/.xsession-errors' \
-    --exclude='home/*/.xsession-errors.old' \
-    home
+  sync_tree "/home" yes /home/ "$BACKUP_DIR/home/" \
+    --exclude='.cache/' \
+    --exclude='.local/share/Trash/' \
+    --exclude='Trash/' \
+    --exclude='.mozilla/firefox/*/cache2/' \
+    --exclude='.mozilla/firefox/*/startupCache/' \
+    --exclude='.mozilla/firefox/*/thumbnails/' \
+    --exclude='.config/google-chrome/*/Cache/' \
+    --exclude='.config/google-chrome/*/Code Cache/' \
+    --exclude='.config/chromium/*/Cache/' \
+    --exclude='.config/chromium/*/Code Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
+    --exclude='.local/share/baloo/' \
+    --exclude='.thumbnails/' \
+    --exclude='.xsession-errors' \
+    --exclude='.xsession-errors.old'
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/5: Database exports + LND artifacts ────────────────
+# ── Stage 4/4: System data ──────────────────────────────────────
+# PostgreSQL/MariaDB raw database directories are excluded. Application
+# databases must be backed up separately with native database tools.
+# Bitcoin/Electrs data are excluded; they live on the internal second drive.
 
 log ""
-log "── Stage 4/5: Database and LND consistency exports ─────────"
-export_postgresql_dumps
-export_mariadb_dumps
-export_lnd_scb_if_possible
-
-if [[ "$LND_AVAILABLE" -eq 1 ]]; then
-  stop_lnd_stack_if_needed
-  create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd
-  LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop")
-fi
-
-log "Stage 4 complete."
-
-# ── Stage 5/5: System data ──────────────────────────────────────
-
-log ""
-log "── Stage 5/5: System data (/var/lib) ───────────────────────"
+log "── Stage 4/4: System data (/var/lib) ───────────────────────"
 if [[ -d /var/lib ]]; then
-  VAR_LIB_EXCLUDES=(
-    --exclude='var/lib/bitcoind'
-    --exclude='var/lib/electrs'
-    --exclude='var/lib/*/log'
-    --exclude='var/lib/*/logs'
-    --exclude='var/lib/*/cache'
-    --exclude='var/lib/*/tmp'
-  )
-
-  if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
-    VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd')
-  fi
-
-  create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib
-  log "Stage 5 complete."
+  sync_tree "/var/lib" no /var/lib/ "$BACKUP_DIR/var/lib/" \
+    --exclude='postgresql/' \
+    --exclude='mysql/' \
+    --exclude='mariadb/' \
+    --exclude='bitcoind/' \
+    --exclude='electrs/' \
+    --exclude='*/log/' \
+    --exclude='*/logs/' \
+    --exclude='*/cache/' \
+    --exclude='*/tmp/'
+  log "Stage 4 complete."
 else
   log "WARNING: /var/lib not found — skipping."
 fi
@@ -802,109 +500,90 @@ fi
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
 MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
-CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt"
 
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
-  echo "Timestamp: $TIMESTAMP"
+  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Source paths included:"
-  echo "- /etc/nixos"
-  echo "- /home"
+  echo "Backup type: Live rsync mirror (directly browsable files)"
+  echo "Location:    ${BACKUP_DIR}"
+  echo ""
+  echo "Source paths mirrored:"
+  echo "- /etc/nixos  →  current/etc/nixos/"
   if [[ "$ROLE" != "desktop" ]]; then
-    echo "- /etc/nix-bitcoin-secrets (when present)"
+    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
   fi
-  echo "- /var/lib"
+  echo "- /home  →  current/home/"
+  echo "- /var/lib  →  current/var/lib/"
   echo ""
   echo "Exclusions:"
-  for ex in "${MANIFEST_EXCLUDES[@]}"; do
-    echo "- $ex"
-  done
+  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
+  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
+  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
+  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
+  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
+  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
+  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
   echo ""
-  echo "Archives:"
-  for archive in "${ARCHIVE_FILES[@]}"; do
-    echo "- $archive"
-  done
-  echo ""
-  echo "Database and LND exports:"
-  if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then
-    echo "- none"
-  else
-    for dump in "${DB_DUMP_FILES[@]}"; do
-      echo "- $dump"
-    done
-    for note in "${LND_BACKUP_NOTES[@]}"; do
-      echo "- $note"
-    done
-  fi
+  echo "Important limitations:"
+  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
+  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
+  echo "  applications, their data must be backed up separately using native tools."
+  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
+  echo "  reconstructable or stored on the internal second drive."
+  echo "- This is a live file-level mirror, not a transactional database backup."
+  echo "  Files being written during the backup may be in an inconsistent state."
   echo ""
   echo "Restore guidance:"
-  echo "- Verify artifacts: cd <backup_dir> && sha256sum -c SHA256SUMS.txt"
-  echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf <archive>.tar -C /"
-  echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql"
-  echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_<db>.dump"
-  echo "- MariaDB DB dump: mariadb <db_name> < database-dumps/mariadb_<db>.sql"
-  echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures"
-  echo "- Note: when restoring /var/lib, exclude raw DB directories (var/lib/postgresql, var/lib/mysql)"
-  echo "  and restore from native dumps instead for PostgreSQL and MariaDB"
+  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
+  echo "- To restore a directory:"
+  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
+  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
+  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
+  echo "- To copy individual files:"
+  echo "    sudo cp -a current/home/username/ /home/username/"
+  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
+  echo "  hardware-configuration.nix for the new hardware before rebuilding."
   echo ""
   echo "Nonfatal warnings:"
-  if [[ "${#ARCHIVE_WARNINGS[@]}" -eq 0 ]]; then
+  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
     echo "- none"
   else
-    for warning in "${ARCHIVE_WARNINGS[@]}"; do
+    for warning in "${RSYNC_WARNINGS[@]}"; do
       echo "- $warning"
     done
   fi
   echo ""
-  echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
   echo "from manual external backup because they already live on the internal second drive"
   echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
-  echo ""
-  echo "Artifact listing:"
-  # INCOMPLETE exists at this point (created at backup start); BACKUP_COMPLETE does not
-  # exist yet (written after checksums). Excluding both marker files here is intentional:
-  # INCOMPLETE is excluded so it doesn't appear as a data artifact, and BACKUP_COMPLETE
-  # is excluded defensively for consistency should the ordering ever change.
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f \
-    ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z | tr '\0' '\n'
 } > "$MANIFEST_FILE"
 
-# ── Generate checksums for all backup artifacts ─────────────────
-
-log "Generating SHA-256 checksums …"
-(
-  cd "$BACKUP_DIR"
-  while IFS= read -r -d '' file; do
-    sha256sum "$file"
-  done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z)
-) > "$CHECKSUM_FILE"
-
 log "Manifest written to $MANIFEST_FILE"
-log "Checksums written to $CHECKSUM_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
-if [[ "${#ARCHIVE_WARNINGS[@]}" -gt 0 ]]; then
+if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
   log "Backup completed with nonfatal warnings:"
-  for warning in "${ARCHIVE_WARNINGS[@]}"; do
+  for warning in "${RSYNC_WARNINGS[@]}"; do
     log "  WARNING: $warning"
   done
-  log "Your important data is backed up. The warnings above indicate files that changed"
-  log "during backup, which is normal on an active desktop and does not affect your backup."
+  log "Your important data is backed up. The warnings above indicate files that"
+  log "vanished during backup, which is normal on an active desktop."
   log ""
 fi
 log "All Finished! Your data is now backed up to a third location."
+log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
 log "Please eject the drive safely before removing it from your Sovran Pro."
 
-# Remove incomplete marker and write completion marker only after all work succeeds
+# Remove incomplete marker and write completion marker only after all work succeeds.
+# A later successful run will update the same mirror and replace any INCOMPLETE state.
 rm -f "$BACKUP_DIR/INCOMPLETE"
-echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE"
+date -u '+%Y-%m-%dT%H:%M:%SZ' > "$BACKUP_DIR/BACKUP_COMPLETE"
 
 BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/server.py

@@ -1494,38 +1494,13 @@ def _is_internal_mount(mnt: str) -> bool:
 
 
 def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
-    """Return whether the target filesystem type is supported for manual backup."""
+    """Return whether the target filesystem type is supported for manual backup.
+
+    Manual Backup requires ext4 for Linux metadata preservation (ACLs, xattrs,
+    hard links). exFAT, FAT32, NTFS, and other filesystems are not supported.
+    """
     fstype = (fstype or "").lower()
-    if fstype == "exfat":
-        return True
-    if fstype != "fuseblk":
-        return False
-
-    src_dev = ""
-    try:
-        result = subprocess.run(
-            ["findmnt", "-n", "-o", "SOURCE", "-T", path],
-            capture_output=True, text=True, timeout=5,
-        )
-        if result.returncode == 0:
-            src_dev = result.stdout.strip()
-    except Exception:
-        src_dev = ""
-
-    if not src_dev:
-        return False
-
-    for cmd in (
-        ["lsblk", "-no", "FSTYPE", src_dev],
-        ["blkid", "-o", "value", "-s", "TYPE", src_dev],
-    ):
-        try:
-            result = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
-            if result.returncode == 0 and result.stdout.strip().lower() in {"exfat", "fuseblk"}:
-                return True
-        except Exception:
-            continue
-    return False
+    return fstype == "ext4"
 
 
 def _detect_external_drives() -> list[dict]:
@@ -3799,7 +3774,7 @@ async def api_backup_run(target: str = ""):
     if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
         raise HTTPException(
             status_code=400,
-            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.",
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup. Manual Backup requires an ext4-formatted drive.",
         )
 
     # Clear stale log before starting

app/sovran_systemsos_web/static/js/support.js

@@ -491,8 +491,9 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>Enough free space for your selected backup data (the backup checks this before starting)</li>',
-        '<li>Drive must be formatted as <strong>exFAT</strong></li>',
+        '<li>Enough free space for your data (the backup checks this before starting)</li>',
+        '<li>Drive must be formatted as <strong>ext4</strong> (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or <code>mkfs.ext4</code> — note that formatting erases all data on the drive.</li>',
+        '<li>The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.</li>',
       '</ol>',
     '</div>',
 
@@ -501,17 +502,25 @@ function renderBackupReady(drives) {
       '<ol class="support-backup-steps">',
         '<li>NixOS configuration (<code>/etc/nixos</code>)</li>',
         '<li>nix-bitcoin secrets (<code>/etc/nix-bitcoin-secrets</code>)</li>',
-        '<li>System service data (<code>/var/lib</code>) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets</li>',
+        '<li>System service data (<code>/var/lib</code>) — excluding databases and blockchain data (see note below)</li>',
         '<li>Home directory (<code>/home</code>)</li>',
       '</ol>',
     '</div>',
 
     '<div class="support-wallet-box support-wallet-warning">',
+      '<div class="support-wallet-header">',
+        '<span class="support-wallet-icon">\u2139\ufe0f</span>',
+        '<span class="support-wallet-title">Database and Blockchain Data</span>',
+      '</div>',
+      '<p class="support-wallet-desc">Application databases stored in PostgreSQL or MariaDB/MySQL are <strong>not included</strong> in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.</p>',
+    '</div>',
+
+    '<div class="support-wallet-box support-wallet-protected">',
       '<div class="support-wallet-header">',
         '<span class="support-wallet-icon">\u23f1\ufe0f</span>',
         '<span class="support-wallet-title">Time Estimate</span>',
       '</div>',
-      '<p class="support-wallet-desc">This backup can take <strong>up to 4 hours</strong> depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026</p>',
+      '<p class="support-wallet-desc">The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.</p>',
     '</div>',
 
     driveSelector,
@@ -619,7 +628,7 @@ function renderBackupDone(success) {
       '<div class="support-section">',
       '<div class="support-icon-big">\u26a0\ufe0f</div>',
       '<h3 class="support-heading">Backup Failed</h3>',
-      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.</p>',
+      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.</p>',
       '<div class="modal-log" id="backup-log-fail" style="text-align:left;"></div>',
       '<button class="btn support-btn-done" id="btn-backup-close">Close</button>',
       '</div>',

modules/core/sovran-hub.nix

@@ -393,7 +393,8 @@ in
         pkgs.coreutils
         pkgs.findutils
         pkgs.gnugrep
-        pkgs.gnutar
+        pkgs.rsync
+        pkgs.acl
         pkgs.util-linux
       ]
       ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package