updated nixpkgs and btc clients

Fix rsync destination-directory failure: sync_tree with mkdir -p, mount re-verification, and path-safety guards

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ── Sovran Hub External Backup Script ────────────────────────────
-# Backs up Sovran_SystemsOS data to an external USB hard drive.
+# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
 # Designed for the Hub web UI (no GUI dependencies).
 #
 # Your Sovran Pro already backs up your data automatically to its
@@ -8,6 +8,15 @@
 # This script creates an additional copy on an external USB drive —
 # storing your data in a third location for maximum protection.
 #
+# The external drive must be formatted as ext4. Files are stored as
+# directly browsable files under Sovran_SystemsOS_Backup/current/.
+# Later runs update the same mirror and only transfer changed or new
+# files, making repeat backups fast.
+#
+# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
+# blockchain and Electrs index data are NOT included (they live on
+# the internal second drive).
+#
 # Usage:
 #   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
 #   (or run with no env var to auto-detect the first external USB drive)
@@ -28,18 +37,17 @@ INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
 
 FAILED_ALREADY=0
 BACKUP_COMPLETE=0
-LND_STOPPED=0
+RSYNC_WARNINGS=()
-LND_UNITS_TO_RESTART=()
 
-ARCHIVE_FILES=()
+# Stable rsync mirror sub-path under the target drive. Not timestamped
-DB_DUMP_FILES=()
+# so later runs update the same destination and only transfer new or changed files.
-MANIFEST_EXCLUDES=()
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
-LND_BACKUP_NOTES=()
 
 # ── Logging helpers ──────────────────────────────────────────────
 
 log() {
-  local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+  local msg
+  msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
   echo "$msg" | tee -a "$BACKUP_LOG"
 }
 
@@ -56,26 +64,10 @@ fail() {
 
 cleanup() {
   local rc=$?
-  local restart_failed=0
 
-  if [[ "$LND_STOPPED" -eq 1 ]]; then
+  # Release the concurrency lock file descriptor if it was opened
-    log "Restarting previously active LND-related services…"
+  if [[ -n "${LOCK_FD:-}" ]]; then
-    for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do
+    exec {LOCK_FD}>&- 2>/dev/null || true
-      local unit="${LND_UNITS_TO_RESTART[$idx]}"
-      if systemctl start "$unit"; then
-        log "Started $unit"
-      else
-        log "ERROR: Failed to start $unit"
-        restart_failed=1
-      fi
-    done
-    LND_STOPPED=0
-  fi
-
-  if [[ "$restart_failed" -eq 1 ]]; then
-    rc=1
-    FAILED_ALREADY=1
-    set_status "FAILED"
   fi
 
   if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
@@ -86,6 +78,11 @@ cleanup() {
     log "ERROR: Backup terminated unexpectedly (exit code $rc)."
     set_status "FAILED"
   fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
 }
 
 trap cleanup EXIT
@@ -201,21 +198,8 @@ validate_target_mount() {
   fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
   [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
 
-  if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then
+  if [[ "$fstype" != "ext4" ]]; then
-    fail "Target '$target' must be exFAT (detected filesystem: $fstype)."
+    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
-  fi
-
-  if [[ "$fstype" == "fuseblk" ]]; then
-    local src_dev blk_type
-    src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true)
-    blk_type=""
-    if [[ -n "$src_dev" ]]; then
-      blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true)
-      [[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true)
-    fi
-    if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then
-      fail "Target '$target' is fuseblk but not identified as exFAT-compatible."
-    fi
   fi
 
   local write_test
@@ -227,14 +211,6 @@ validate_target_mount() {
   log "Verified backup target filesystem: $fstype"
 }
 
-has_unit() {
-  systemctl cat "$1" >/dev/null 2>&1
-}
-
-is_unit_active() {
-  systemctl is-active --quiet "$1"
-}
-
 estimate_path_bytes() {
   local path="$1"
   shift || true
@@ -249,6 +225,82 @@ estimate_path_bytes() {
   echo "$size"
 }
 
+# ── Sync one source tree to its backup destination ───────────────
+# Usage: sync_tree <label> <allow_vanished> <source> <destination> [rsync options...]
+#
+# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
+# Used for /home only — files may disappear while the desktop is active.
+# All other nonzero exit codes are always fatal.
+#
+# Before every rsync call this helper:
+#   1. Re-verifies $TARGET is still a mount point (fails if drive disconnected).
+#   2. Verifies the destination path remains beneath $BACKUP_DIR and $BACKUP_DIR
+#      remains beneath $TARGET (safe-path check).
+#   3. Creates the full destination directory hierarchy with mkdir -p so that
+#      rsync never fails trying to create a directory whose parent is absent.
+sync_tree() {
+  local label="$1"
+  local allow_vanished="$2"
+  local source="$3"
+  local destination="$4"
+  shift 4
+  # Remaining "$@" are rsync options (--exclude, etc.)
+
+  # ── Re-verify the external drive is still mounted ────────────────
+  mountpoint -q "$TARGET" 2>/dev/null || \
+    fail "Stage $label: external drive '$TARGET' is no longer mounted. Refusing to write."
+
+  # ── Verify path safety ────────────────────────────────────────────
+  # BACKUP_DIR must remain beneath TARGET.
+  case "$BACKUP_DIR" in
+    "$TARGET"/*) ;;
+    *) fail "Stage $label: BACKUP_DIR '$BACKUP_DIR' is outside TARGET '$TARGET'." ;;
+  esac
+  # Destination must remain beneath BACKUP_DIR.
+  case "$destination" in
+    "$BACKUP_DIR"/*|"$BACKUP_DIR") ;;
+    *) fail "Stage $label: destination '$destination' is outside BACKUP_DIR '$BACKUP_DIR'. Refusing to write." ;;
+  esac
+
+  # ── Create complete destination directory hierarchy ───────────────
+  # This is the fix for the production failure:
+  #   rsync: [Receiver] mkdir ".../current/etc/nixos" failed: No such file or directory
+  # mkdir -p creates all intermediate parents (e.g. current/etc/) before rsync runs.
+  mkdir -p -- "$destination" || \
+    fail "Stage $label: failed to create destination directory '$destination' (source: '$source')."
+
+  local rsync_err_tmp
+  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
+
+  local rc=0
+  rsync \
+    --archive \
+    --acls \
+    --xattrs \
+    --hard-links \
+    --numeric-ids \
+    --one-file-system \
+    --partial \
+    "$@" "$source" "$destination" 2>"$rsync_err_tmp" || rc=$?
+
+  if [[ -s "$rsync_err_tmp" ]]; then
+    while IFS= read -r rline; do
+      log "rsync: $rline"
+    done < "$rsync_err_tmp"
+  fi
+  rm -f "$rsync_err_tmp"
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
+  fi
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -257,21 +309,29 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"
 
-require_cmd tar
+# ── Acquire exclusive run lock ────────────────────────────────────
-require_cmd sha256sum
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd rsync
 require_cmd findmnt
 require_cmd lsblk
 require_cmd mountpoint
 require_cmd df
 require_cmd du
 require_cmd awk
-require_cmd sort
 require_cmd find
-require_cmd systemctl
 require_cmd hostname
 require_cmd date
 require_cmd python3
-require_cmd runuser
+require_cmd flock
 
 # ── Detect system role ───────────────────────────────────────────
 
@@ -296,42 +356,32 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again."
+    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
 validate_target_mount "$TARGET"
 
-# ── Plan role-aware source scope and exclusions ─────────────────
+# ── Set up stable backup destination ────────────────────────────
+# Subsequent runs update the same mirror, transferring only new or changed files.
 
-LND_AVAILABLE=0
+BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
-if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then
+mkdir -p -- "$BACKUP_DIR"
-  LND_AVAILABLE=1
-fi
 
-if [[ "$ROLE" == "desktop" ]]; then
+# Remove any stale BACKUP_COMPLETE left by a previous successful run.
-  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)")
+# The new run will re-earn it only after all stages succeed.
-else
+rm -f "$BACKUP_DIR/BACKUP_COMPLETE"
-  MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent")
-fi
 
-MANIFEST_EXCLUDES+=(
+# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
-  "/run/media/Second_Drive (never traversed)"
+# after all rsync stages and manifest write succeed. Failed or interrupted
-  "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)"
+# runs keep this marker so they are clearly identifiable.
-  "/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)"
+touch "$BACKUP_DIR/INCOMPLETE"
-  "/var/lib/bitcoind (excluded from manual backup)"
+log "Backup destination: $BACKUP_DIR"
-  "/var/lib/electrs (excluded from manual backup)"
-  "/var/lib/*/log and /var/lib/*/logs"
-  "/var/lib/*/cache and /var/lib/*/tmp"
-  "/home/*/.cache and /home/*/.local/share/Trash"
-)
-
-if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
-  MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive")
-fi
 
 # ── Estimate required free space ─────────────────────────────────
+# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
+# from the estimate to avoid inflating the required size.
 
 ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
 HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
@@ -341,21 +391,20 @@ if [[ "$ROLE" != "desktop" ]]; then
 fi
 
 VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='postgresql' \
+  --exclude='mysql' \
+  --exclude='mariadb' \
   --exclude='bitcoind' \
   --exclude='electrs' \
-  --exclude='lnd' \
   --exclude='*/log' \
   --exclude='*/logs' \
   --exclude='*/cache' \
   --exclude='*/tmp')
 
-LND_BYTES=0
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
-if [[ "$LND_AVAILABLE" -eq 1 ]]; then
+# Require 20% growth headroom plus a fixed 1 GiB safety margin.
-  LND_BYTES=$(estimate_path_bytes /var/lib/lnd)
+# Later incremental runs need far less space, but a conservative first-run
-fi
+# check protects against running out of space mid-backup.
-
-ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES ))
-# Require 20% growth headroom plus an additional fixed 1 GiB safety margin.
 REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
 
 FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
@@ -369,289 +418,79 @@ log "Free space on drive: ${FREE_GB} GB"
 (( FREE_BYTES >= REQUIRED_BYTES )) || \
   fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
 
-# ── Create timestamped backup directory ─────────────────────────
+# ── Stage 1/4: NixOS configuration ──────────────────────────────
-
-TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
-BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
-DB_DUMP_DIR="$BACKUP_DIR/database-dumps"
-mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR"
-log "Backup destination: $BACKUP_DIR"
-
-create_tar_archive() {
-  local archive_name="$1"
-  shift
-  local archive_path="$BACKUP_DIR/$archive_name"
-
-  log "Creating $archive_name …"
-  tar \
-    --create \
-    --file "$archive_path" \
-    --numeric-owner \
-    --acls \
-    --xattrs \
-    --sparse \
-    --one-file-system \
-    "$@"
-
-  ARCHIVE_FILES+=("$archive_name")
-  log "Created archive: $archive_name"
-}
-
-export_postgresql_dumps() {
-  if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then
-    log "PostgreSQL tools/service not available — skipping PostgreSQL exports."
-    return
-  fi
-
-  if ! is_unit_active "postgresql.service"; then
-    log "PostgreSQL service is not active — skipping PostgreSQL exports."
-    return
-  fi
-
-  log "Exporting PostgreSQL globals and databases…"
-  local globals_file="$DB_DUMP_DIR/postgresql_globals.sql"
-  runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \
-    fail "Failed to export PostgreSQL globals."
-  DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql")
-
-  local dbs
-  dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true)
-
-  if [[ -z "$dbs" ]]; then
-    log "No non-template PostgreSQL application databases found."
-    return
-  fi
-
-  while IFS= read -r db; do
-    [[ -n "$db" ]] || continue
-    local safe_db
-    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
-    local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump"
-    runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \
-      fail "Failed to export PostgreSQL database '$db'."
-    DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump")
-  done <<< "$dbs"
-}
-
-export_mariadb_dumps() {
-  local dump_cmd=""
-  local query_cmd=""
-  local mariadb_unit=""
-
-  if command -v mariadb-dump >/dev/null 2>&1; then
-    dump_cmd="mariadb-dump"
-  elif command -v mysqldump >/dev/null 2>&1; then
-    dump_cmd="mysqldump"
-  fi
-
-  if command -v mariadb >/dev/null 2>&1; then
-    query_cmd="mariadb"
-  elif command -v mysql >/dev/null 2>&1; then
-    query_cmd="mysql"
-  fi
-
-  if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then
-    log "MariaDB dump/query tools not available — skipping MariaDB exports."
-    return
-  fi
-
-  if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then
-    mariadb_unit="mariadb.service"
-  elif has_unit "mysql.service" && is_unit_active "mysql.service"; then
-    mariadb_unit="mysql.service"
-  else
-    log "MariaDB service is not active — skipping MariaDB exports."
-    return
-  fi
-
-  log "Exporting MariaDB databases from ${mariadb_unit}…"
-
-  local dbs
-  dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true)
-  if [[ -z "$dbs" ]]; then
-    log "No MariaDB databases found."
-    return
-  fi
-
-  while IFS= read -r db; do
-    [[ -n "$db" ]] || continue
-    case "$db" in
-      information_schema|performance_schema|mysql|sys) continue ;;
-    esac
-
-    local safe_db out_file
-    safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
-    out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql"
-
-    $dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \
-      fail "Failed to export MariaDB database '$db'."
-
-    DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql")
-  done <<< "$dbs"
-}
-
-export_lnd_scb_if_possible() {
-  [[ "$LND_AVAILABLE" -eq 1 ]] || return
-
-  local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb"
-  local attempts=(
-    "lncli exportchanbackup --all --output_file $scb_file"
-    "lncli -n mainnet exportchanbackup --all --output_file $scb_file"
-    "runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file"
-    "runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file"
-  )
-
-  if ! command -v lncli >/dev/null 2>&1; then
-    log "lncli not available — skipping Static Channel Backup export."
-    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)")
-    return
-  fi
-
-  if ! is_unit_active "lnd.service"; then
-    log "LND service is not active — skipping Static Channel Backup export."
-    LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)")
-    return
-  fi
-
-  log "Exporting LND Static Channel Backup…"
-  local attempt
-  for attempt in "${attempts[@]}"; do
-    if eval "$attempt" >/dev/null 2>&1; then
-      DB_DUMP_FILES+=("lnd-static-channel-backup.scb")
-      LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli")
-      log "LND Static Channel Backup exported."
-      return
-    fi
-  done
-
-  log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations."
-  LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)")
-}
-
-capture_active_lnd_dependents() {
-  [[ "$LND_AVAILABLE" -eq 1 ]] || return
-
-  LND_UNITS_TO_RESTART=()
-  local raw_units=""
-  raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true)
-
-  while IFS= read -r unit; do
-    [[ -n "$unit" ]] || continue
-    if is_unit_active "$unit"; then
-      LND_UNITS_TO_RESTART+=("$unit")
-    fi
-  done <<< "$raw_units"
-
-  if is_unit_active "lnd.service"; then
-    LND_UNITS_TO_RESTART+=("lnd.service")
-  fi
-}
-
-stop_lnd_stack_if_needed() {
-  [[ "$LND_AVAILABLE" -eq 1 ]] || return
-
-  capture_active_lnd_dependents
-
-  if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then
-    log "No active LND-related services needed stopping."
-    return
-  fi
-
-  log "Stopping active services that depend on LND for clean /var/lib/lnd archive…"
-
-  local unit
-  for unit in "${LND_UNITS_TO_RESTART[@]}"; do
-    if [[ "$unit" == "lnd.service" ]]; then
-      continue
-    fi
-    systemctl stop "$unit" || fail "Failed to stop dependent service: $unit"
-    log "Stopped $unit"
-  done
-
-  if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then
-    systemctl stop lnd.service || fail "Failed to stop lnd.service"
-    log "Stopped lnd.service"
-  fi
-
-  LND_STOPPED=1
-}
-
-# ── Stage 1/5: NixOS configuration ──────────────────────────────
 
 log ""
-log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────"
+log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  create_tar_archive "etc-nixos.tar" -C / etc/nixos
+  sync_tree "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
 fi
 
-# ── Stage 2/5: Secrets ──────────────────────────────────────────
+# ── Stage 2/4: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────"
+log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
 elif [[ -e /etc/nix-bitcoin-secrets ]]; then
-  create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets
+  sync_tree "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
 else
   log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
 log "Stage 2 complete."
 
-# ── Stage 3/5: Home directory ───────────────────────────────────
+# ── Stage 3/4: Home directory ───────────────────────────────────
+# Rsync exit code 24 (vanished source files) is treated as nonfatal here
+# because the desktop may be active and files can disappear between the
+# directory scan and the copy. All other nonzero exit codes remain fatal.
 
 log ""
-log "── Stage 3/5: Home directory (/home) ───────────────────────"
+log "── Stage 3/4: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  create_tar_archive "home.tar" \
+  sync_tree "/home" yes /home/ "$BACKUP_DIR/home/" \
-    -C / \
+    --exclude='.cache/' \
-    --exclude='home/*/.cache' \
+    --exclude='.local/share/Trash/' \
-    --exclude='home/*/.local/share/Trash' \
+    --exclude='Trash/' \
-    --exclude='home/*/Trash' \
+    --exclude='.mozilla/firefox/*/cache2/' \
-    home
+    --exclude='.mozilla/firefox/*/startupCache/' \
+    --exclude='.mozilla/firefox/*/thumbnails/' \
+    --exclude='.config/google-chrome/*/Cache/' \
+    --exclude='.config/google-chrome/*/Code Cache/' \
+    --exclude='.config/chromium/*/Cache/' \
+    --exclude='.config/chromium/*/Code Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
+    --exclude='.local/share/baloo/' \
+    --exclude='.thumbnails/' \
+    --exclude='.xsession-errors' \
+    --exclude='.xsession-errors.old'
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/5: Database exports + LND artifacts ────────────────
+# ── Stage 4/4: System data ──────────────────────────────────────
+# PostgreSQL/MariaDB raw database directories are excluded. Application
+# databases must be backed up separately with native database tools.
+# Bitcoin/Electrs data are excluded; they live on the internal second drive.
 
 log ""
-log "── Stage 4/5: Database and LND consistency exports ─────────"
+log "── Stage 4/4: System data (/var/lib) ───────────────────────"
-export_postgresql_dumps
-export_mariadb_dumps
-export_lnd_scb_if_possible
-
-if [[ "$LND_AVAILABLE" -eq 1 ]]; then
-  stop_lnd_stack_if_needed
-  create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd
-  LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop")
-fi
-
-log "Stage 4 complete."
-
-# ── Stage 5/5: System data ──────────────────────────────────────
-
-log ""
-log "── Stage 5/5: System data (/var/lib) ───────────────────────"
 if [[ -d /var/lib ]]; then
-  VAR_LIB_EXCLUDES=(
+  sync_tree "/var/lib" no /var/lib/ "$BACKUP_DIR/var/lib/" \
-    --exclude='var/lib/bitcoind'
+    --exclude='postgresql/' \
-    --exclude='var/lib/electrs'
+    --exclude='mysql/' \
-    --exclude='var/lib/*/log'
+    --exclude='mariadb/' \
-    --exclude='var/lib/*/logs'
+    --exclude='bitcoind/' \
-    --exclude='var/lib/*/cache'
+    --exclude='electrs/' \
-    --exclude='var/lib/*/tmp'
+    --exclude='*/log/' \
-  )
+    --exclude='*/logs/' \
-
+    --exclude='*/cache/' \
-  if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
+    --exclude='*/tmp/'
-    VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd')
+  log "Stage 4 complete."
-  fi
-
-  create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib
-  log "Stage 5 complete."
 else
   log "WARNING: /var/lib not found — skipping."
 fi
@@ -661,80 +500,90 @@ fi
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
 MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
-CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt"
 
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
-  echo "Timestamp: $TIMESTAMP"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Source paths included:"
+  echo "Backup type: Live rsync mirror (directly browsable files)"
-  echo "- /etc/nixos"
+  echo "Location:    ${BACKUP_DIR}"
-  echo "- /home"
+  echo ""
+  echo "Source paths mirrored:"
+  echo "- /etc/nixos  →  current/etc/nixos/"
   if [[ "$ROLE" != "desktop" ]]; then
-    echo "- /etc/nix-bitcoin-secrets (when present)"
+    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
   fi
-  echo "- /var/lib"
+  echo "- /home  →  current/home/"
+  echo "- /var/lib  →  current/var/lib/"
   echo ""
   echo "Exclusions:"
-  for ex in "${MANIFEST_EXCLUDES[@]}"; do
+  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
-    echo "- $ex"
+  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
-  done
+  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
+  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
+  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
+  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
+  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
   echo ""
-  echo "Archives:"
+  echo "Important limitations:"
-  for archive in "${ARCHIVE_FILES[@]}"; do
+  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
-    echo "- $archive"
+  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
-  done
+  echo "  applications, their data must be backed up separately using native tools."
+  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
+  echo "  reconstructable or stored on the internal second drive."
+  echo "- This is a live file-level mirror, not a transactional database backup."
+  echo "  Files being written during the backup may be in an inconsistent state."
   echo ""
-  echo "Database and LND exports:"
+  echo "Restore guidance:"
-  if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then
+  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
+  echo "- To restore a directory:"
+  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
+  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
+  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
+  echo "- To copy individual files:"
+  echo "    sudo cp -a current/home/username/ /home/username/"
+  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
+  echo "  hardware-configuration.nix for the new hardware before rebuilding."
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
     echo "- none"
   else
-    for dump in "${DB_DUMP_FILES[@]}"; do
+    for warning in "${RSYNC_WARNINGS[@]}"; do
-      echo "- $dump"
+      echo "- $warning"
-    done
-    for note in "${LND_BACKUP_NOTES[@]}"; do
-      echo "- $note"
     done
   fi
   echo ""
-  echo "Restore guidance:"
+  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
-  echo "- Verify artifacts: cd <backup_dir> && sha256sum -c SHA256SUMS.txt"
-  echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf <archive>.tar -C /"
-  echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql"
-  echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_<db>.dump"
-  echo "- MariaDB DB dump: mariadb <db_name> < database-dumps/mariadb_<db>.sql"
-  echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures"
-  echo ""
-  echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded"
   echo "from manual external backup because they already live on the internal second drive"
   echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
-  echo ""
-  echo "Artifact listing:"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f | sort
 } > "$MANIFEST_FILE"
 
-# ── Generate checksums for all backup artifacts ─────────────────
-
-log "Generating SHA-256 checksums …"
-(
-  cd "$BACKUP_DIR"
-  while IFS= read -r -d '' file; do
-    sha256sum "$file"
-  done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' -print0 | sort -z)
-) > "$CHECKSUM_FILE"
-
 log "Manifest written to $MANIFEST_FILE"
-log "Checksums written to $CHECKSUM_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
+if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${RSYNC_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that"
+  log "vanished during backup, which is normal on an active desktop."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
+log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
 log "Please eject the drive safely before removing it from your Sovran Pro."
 
+# Remove incomplete marker and write completion marker only after all work succeeds.
+# A later successful run will update the same mirror and replace any INCOMPLETE state.
+rm -f "$BACKUP_DIR/INCOMPLETE"
+date -u '+%Y-%m-%dT%H:%M:%SZ' > "$BACKUP_DIR/BACKUP_COMPLETE"
+
 BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/server.py

@@ -1494,38 +1494,13 @@ def _is_internal_mount(mnt: str) -> bool:
 
 
 def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
-    """Return whether the target filesystem type is supported for manual backup."""
+    """Return whether the target filesystem type is supported for manual backup.
+
+    Manual Backup requires ext4 for Linux metadata preservation (ACLs, xattrs,
+    hard links). exFAT, FAT32, NTFS, and other filesystems are not supported.
+    """
     fstype = (fstype or "").lower()
-    if fstype == "exfat":
+    return fstype == "ext4"
-        return True
-    if fstype != "fuseblk":
-        return False
-
-    src_dev = ""
-    try:
-        result = subprocess.run(
-            ["findmnt", "-n", "-o", "SOURCE", "-T", path],
-            capture_output=True, text=True, timeout=5,
-        )
-        if result.returncode == 0:
-            src_dev = result.stdout.strip()
-    except Exception:
-        src_dev = ""
-
-    if not src_dev:
-        return False
-
-    for cmd in (
-        ["lsblk", "-no", "FSTYPE", src_dev],
-        ["blkid", "-o", "value", "-s", "TYPE", src_dev],
-    ):
-        try:
-            result = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
-            if result.returncode == 0 and result.stdout.strip().lower() in {"exfat", "fuseblk"}:
-                return True
-        except Exception:
-            continue
-    return False
 
 
 def _detect_external_drives() -> list[dict]:
@@ -3742,8 +3717,18 @@ async def api_backup_drives():
 
 
 async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
-    """Mark status FAILED if backup subprocess exits unexpectedly."""
+    """Drain stderr, then mark status FAILED if backup subprocess exits unexpectedly."""
+    stderr_chunks: list[bytes] = []
+
+    async def _drain_stderr() -> None:
+        if proc.stderr is not None:
+            async for line in proc.stderr:
+                stderr_chunks.append(line)
+
+    drain_task = asyncio.create_task(_drain_stderr())
     rc = await proc.wait()
+    await drain_task
+
     if rc == 0:
         return
 
@@ -3752,7 +3737,12 @@ async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
     if status in {"SUCCESS", "FAILED"}:
         return
 
-    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc})."
+    detail = ""
+    if stderr_chunks:
+        stderr_text = b"".join(stderr_chunks).decode("utf-8", errors="replace").strip()
+        if stderr_text:
+            detail = f" — stderr: {stderr_text}"
+    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc}).{detail}"
     await loop.run_in_executor(None, _append_backup_log, msg)
     await loop.run_in_executor(None, _write_backup_status, "FAILED")
 
@@ -3784,7 +3774,7 @@ async def api_backup_run(target: str = ""):
     if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
         raise HTTPException(
             status_code=400,
-            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.",
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup. Manual Backup requires an ext4-formatted drive.",
         )
 
     # Clear stale log before starting
@@ -3808,11 +3798,25 @@ async def api_backup_run(target: str = ""):
     env = dict(os.environ)
     env["BACKUP_TARGET"] = selected_target
 
+    bash_path = shutil.which("bash")
+    if bash_path is None:
+        no_bash_msg = (
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Cannot start backup:"
+            " interpreter 'bash' not found on PATH."
+            " Ensure pkgs.bash is in the sovran-hub-web service PATH."
+        )
+        await loop.run_in_executor(None, _append_backup_log, no_bash_msg)
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(
+            status_code=500,
+            detail="Backup interpreter (bash) not available. Check service PATH configuration.",
+        )
+
     try:
         proc = await asyncio.create_subprocess_exec(
-            "/usr/bin/env", "bash", BACKUP_SCRIPT,
+            bash_path, BACKUP_SCRIPT,
             stdout=asyncio.subprocess.DEVNULL,
-            stderr=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.PIPE,
             env=env,
         )
     except Exception as exc:

app/sovran_systemsos_web/static/js/support.js

@@ -491,8 +491,9 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>Enough free space for your selected backup data (the backup checks this before starting)</li>',
+        '<li>Enough free space for your data (the backup checks this before starting)</li>',
-        '<li>Drive must be formatted as <strong>exFAT</strong></li>',
+        '<li>Drive must be formatted as <strong>ext4</strong> (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or <code>mkfs.ext4</code> — note that formatting erases all data on the drive.</li>',
+        '<li>The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.</li>',
       '</ol>',
     '</div>',
 
@@ -501,17 +502,25 @@ function renderBackupReady(drives) {
       '<ol class="support-backup-steps">',
         '<li>NixOS configuration (<code>/etc/nixos</code>)</li>',
         '<li>nix-bitcoin secrets (<code>/etc/nix-bitcoin-secrets</code>)</li>',
-        '<li>System service data (<code>/var/lib</code>) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets</li>',
+        '<li>System service data (<code>/var/lib</code>) — excluding databases and blockchain data (see note below)</li>',
         '<li>Home directory (<code>/home</code>)</li>',
       '</ol>',
     '</div>',
 
     '<div class="support-wallet-box support-wallet-warning">',
+      '<div class="support-wallet-header">',
+        '<span class="support-wallet-icon">\u2139\ufe0f</span>',
+        '<span class="support-wallet-title">Database and Blockchain Data</span>',
+      '</div>',
+      '<p class="support-wallet-desc">Application databases stored in PostgreSQL or MariaDB/MySQL are <strong>not included</strong> in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.</p>',
+    '</div>',
+
+    '<div class="support-wallet-box support-wallet-protected">',
       '<div class="support-wallet-header">',
         '<span class="support-wallet-icon">\u23f1\ufe0f</span>',
         '<span class="support-wallet-title">Time Estimate</span>',
       '</div>',
-      '<p class="support-wallet-desc">This backup can take <strong>up to 4 hours</strong> depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026</p>',
+      '<p class="support-wallet-desc">The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.</p>',
     '</div>',
 
     driveSelector,
@@ -619,7 +628,7 @@ function renderBackupDone(success) {
       '<div class="support-section">',
       '<div class="support-icon-big">\u26a0\ufe0f</div>',
       '<h3 class="support-heading">Backup Failed</h3>',
-      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.</p>',
+      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.</p>',
       '<div class="modal-log" id="backup-log-fail" style="text-align:left;"></div>',
       '<button class="btn support-btn-done" id="btn-backup-close">Close</button>',
       '</div>',

flake.lock

@@ -5,11 +5,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1783519926,
+        "lastModified": 1784932759,
-        "narHash": "sha256-2zwAN4lNitHFrHVnRZG3YcvpdtWOoF0cOBstxMeB1KI=",
+        "narHash": "sha256-44/iCx+wiYukHGhvPm65ppJZ3FZZbp6f9JE5isz8TsA=",
         "owner": "emmanuelrosa",
         "repo": "btc-clients-nix",
-        "rev": "731a1e11c2fefb14f0aa4b1f03cfa85c19c28d71",
+        "rev": "8aab86c245ab9a2bea0d72175d6fd663a892af9f",
         "type": "github"
       },
       "original": {
@@ -139,11 +139,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1783856661,
+        "lastModified": 1784856561,
-        "narHash": "sha256-ZGP04e+Q6WyQJGA9ZvI5CL6+heGQldbAG9U1T9NGvmU=",
+        "narHash": "sha256-J+Bx1Z6Oeoj2FgnBhRMKyUhhtDoOpTgXYaVLZpDjW4A=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "569d578509928497eddc3fdbf94a799027050be4",
+        "rev": "597283ad8aa0b331c788e97c4c262d58877074ef",
         "type": "github"
       },
       "original": {
@@ -187,11 +187,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1783776592,
+        "lastModified": 1784796856,
-        "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=",
+        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3",
+        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
         "type": "github"
       },
       "original": {
@@ -203,11 +203,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1783791668,
+        "lastModified": 1784555310,
-        "narHash": "sha256-zbcZ1dmBTPfJ7Mlqh/yLEPGpgJnwuv4Xr1xucy2WqMA=",
+        "narHash": "sha256-/FCliTPgiuV1owejZFNx3Ch9irdvkOfOFl+HHZ+DrtM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "716c7a2664ca8325617b8a7fbb609273f2c4cae7",
+        "rev": "421eebfd0ec7bccd4abe826ce62d7e6e83129493",
         "type": "github"
       },
       "original": {
@@ -224,11 +224,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1783941741,
+        "lastModified": 1784814601,
-        "narHash": "sha256-F+3M1IZrJa920cx2/k2AMKqedEodxLF7COJVkLJwUBo=",
+        "narHash": "sha256-T32JXjZ7kIbhBn8/Har171yGg6IBdl97cxAWameqZDE=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "e6715f01d9f56f07a27a01386b85ae22b06f0705",
+        "rev": "f316e949e0ed9df0e1e0bf645c6dce721d4e230e",
         "type": "github"
       },
       "original": {

modules/core/sovran-hub.nix

@@ -382,6 +382,8 @@ in
       };
 
       path = [
+        pkgs.bash
+        pkgs.gawk
         pkgs.qrencode
         pkgs.curl
         pkgs.iproute2
@@ -391,7 +393,8 @@ in
         pkgs.coreutils
         pkgs.findutils
         pkgs.gnugrep
-        pkgs.gnutar
+        pkgs.rsync
+        pkgs.acl
         pkgs.util-linux
       ]
       ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package