Merge pull request #333 from naturallaw777/copilot/replace-manual-backup-implementation

Replace Manual Backup: tar+DB+LND → ext4+rsync mirror
refactor: rename CURRENT_DIR_NAME to BACKUP_SUBPATH for clarity
2026-07-20 22:51:38 +00:00 · 2026-07-20 22:47:53 +00:00 · 2026-07-20 22:45:04 +00:00 · 2026-07-20 22:34:57 +00:00 · 2026-07-19 16:59:19 -05:00 · 2026-07-18 18:21:07 +00:00
8 changed files with 1501 additions and 171 deletions
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ── Sovran Hub External Backup Script ────────────────────────────
-# Backs up Sovran_SystemsOS data to an external USB hard drive.
+# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
 # Designed for the Hub web UI (no GUI dependencies).
 #
 # Your Sovran Pro already backs up your data automatically to its
@@ -8,6 +8,15 @@
 # This script creates an additional copy on an external USB drive —
 # storing your data in a third location for maximum protection.
 #
+# The external drive must be formatted as ext4. Files are stored as
+# directly browsable files under Sovran_SystemsOS_Backup/current/.
+# Later runs update the same mirror and only transfer changed or new
+# files, making repeat backups fast.
+#
+# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
+# blockchain and Electrs index data are NOT included (they live on
+# the internal second drive).
+#
 # Usage:
 #   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
 #   (or run with no env var to auto-detect the first external USB drive)
@@ -17,13 +26,22 @@ set -euo pipefail
 BACKUP_LOG="/var/log/sovran-hub-backup.log"
 BACKUP_STATUS="/var/log/sovran-hub-backup.status"
 MEDIA_ROOT="/run/media"
-MIN_FREE_GB=10
 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
 ROLE_STATE_NIX="/etc/nixos/role-state.nix"
+SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
+SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))

 # ── Internal drive labels/paths to NEVER use as backup targets ───
 INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
-INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/")
+INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
+
+FAILED_ALREADY=0
+BACKUP_COMPLETE=0
+RSYNC_WARNINGS=()
+
+# Stable rsync mirror sub-path under the target drive. Not timestamped
+# so later runs update the same destination and only transfer new or changed files.
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"

 # ── Logging helpers ──────────────────────────────────────────────

@@ -37,16 +55,45 @@ set_status() {
 }

 fail() {
+  FAILED_ALREADY=1
  log "ERROR: $*"
  set_status "FAILED"
  exit 1
 }

+cleanup() {
+  local rc=$?
+
+  # Release the concurrency lock file descriptor if it was opened
+  [[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true
+
+  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
+    return
+  fi
+
+  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
+    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
+    set_status "FAILED"
+  fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
+}
+
+trap cleanup EXIT
+trap 'exit 1' INT TERM
+
+require_cmd() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
+}
+
 # ── Check whether a mount point is an internal drive ────────────

 is_internal() {
  local mnt="$1"
-  # Reject known internal mount points and their subdirectories
  for internal in "${INTERNAL_MOUNTS[@]}"; do
    if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
      return 0
@@ -59,39 +106,37 @@ is_internal() {

 find_external_drive() {
  local target=""
-  # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
-  if command -v lsblk &>/dev/null; then
-    while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
-      # Must be a partition or disk, and be removable/hotplug
-      [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
-      [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
-      [[ -n "$mountpoint" ]] || continue

-      # Filter out internal labels
-      local skip=0
-      for lbl in "${INTERNAL_LABELS[@]}"; do
-        [[ "$label" == "$lbl" ]] && skip=1 && break
-      done
-      [[ "$skip" -eq 1 ]] && continue
+  while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
+    [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
+    [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
+    [[ -n "$mountpoint" ]] || continue

-      # Filter out internal mount points
-      is_internal "$mountpoint" && continue
+    local skip=0
+    for lbl in "${INTERNAL_LABELS[@]}"; do
+      [[ "$label" == "$lbl" ]] && skip=1 && break
+    done
+    [[ "$skip" -eq 1 ]] && continue

-      if mountpoint -q "$mountpoint" 2>/dev/null; then
-        target="$mountpoint"
-        break
-      fi
-    done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
-      python3 -c "
+    is_internal "$mountpoint" && continue
+
+    if mountpoint -q "$mountpoint" 2>/dev/null; then
+      target="$mountpoint"
+      break
+    fi
+  done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
+    python3 -c "
 import sys, json
-data = json.load(sys.stdin)
+
 def flatten(devs):
    for d in devs:
        yield d
        for c in d.get('children', []):
            yield from flatten([c])
+
+data = json.load(sys.stdin)
 for d in flatten(data.get('blockdevices', [])):
-    print('\t'.join([
+    print('\\t'.join([
        d.get('type') or '',
        str(d.get('hotplug') or '0'),
        str(d.get('rm') or '0'),
@@ -99,24 +144,10 @@ for d in flatten(data.get('blockdevices', [])):
        d.get('mountpoint') or '',
    ]))
 " 2>/dev/null || true)
-  fi

-  # Fallback: walk /run/media/ if lsblk produced nothing
  if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
    while IFS= read -r -d '' mnt; do
      is_internal "$mnt" && continue
-      # Check label via lsblk on the device backing this mount
-      local dev
-      dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
-      if [[ -n "$dev" ]]; then
-        local lbl
-        lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
-        local skip=0
-        for internal_lbl in "${INTERNAL_LABELS[@]}"; do
-          [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
-        done
-        [[ "$skip" -eq 1 ]] && continue
-      fi
      if mountpoint -q "$mnt" 2>/dev/null; then
        target="$mnt"
        break
@@ -128,16 +159,10 @@ for d in flatten(data.get('blockdevices', [])):
 }

 # ── Detect the configured system role ───────────────────────────
-#
-# Priority:
-#   1. Hub config JSON  (/var/lib/sovran-hub/config.json)  — "role" key
-#   2. role-state.nix   (/etc/nixos/role-state.nix)        — grep for true flag
-#   3. Default: server_plus_desktop

 detect_role() {
  local role="server_plus_desktop"

-  # 1. Try the Hub config JSON
  if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
    local r
    r=$(python3 -c \
@@ -149,7 +174,6 @@ detect_role() {
    fi
  fi

-  # 2. Fall back to parsing role-state.nix
  if [[ -f "$ROLE_STATE_NIX" ]]; then
    if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
      role="desktop"
@@ -161,6 +185,85 @@ detect_role() {
  echo "$role"
 }

+validate_target_mount() {
+  local target="$1"
+  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
+  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
+  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
+
+  local fstype=""
+  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
+  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
+
+  if [[ "$fstype" != "ext4" ]]; then
+    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
+  fi
+
+  local write_test
+  write_test="$target/.sovran-write-test-$$"
+  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
+    fail "Target '$target' is not writable."
+  fi
+
+  log "Verified backup target filesystem: $fstype"
+}
+
+estimate_path_bytes() {
+  local path="$1"
+  shift || true
+  [[ -e "$path" ]] || {
+    echo 0
+    return
+  }
+
+  local size
+  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
+  [[ -n "$size" ]] || size=0
+  echo "$size"
+}
+
+# ── Run rsync for one source tree ────────────────────────────────
+# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
+# For /home only — files may disappear while the desktop is active.
+# All other nonzero exit codes are always fatal.
+run_rsync() {
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+  # Remaining args are passed directly to rsync (filters + source + dest).
+
+  local rsync_err_tmp
+  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
+
+  local rc=0
+  rsync \
+    --archive \
+    --acls \
+    --xattrs \
+    --hard-links \
+    --numeric-ids \
+    --one-file-system \
+    --partial \
+    "$@" 2>"$rsync_err_tmp" || rc=$?
+
+  if [[ -s "$rsync_err_tmp" ]]; then
+    while IFS= read -r rline; do
+      log "rsync: $rline"
+    done < "$rsync_err_tmp"
+  fi
+  rm -f "$rsync_err_tmp"
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
+  fi
+}
+
 # ── Initialise log file ──────────────────────────────────────────

 : > "$BACKUP_LOG"
@@ -169,14 +272,38 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"

+# ── Acquire exclusive run lock ────────────────────────────────────
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd rsync
+require_cmd findmnt
+require_cmd lsblk
+require_cmd mountpoint
+require_cmd df
+require_cmd du
+require_cmd awk
+require_cmd find
+require_cmd hostname
+require_cmd date
+require_cmd python3
+require_cmd flock
+
 # ── Detect system role ───────────────────────────────────────────

 ROLE="$(detect_role)"
 case "$ROLE" in
-  desktop)            ROLE_LABEL="Desktop Only" ;;
-  node)               ROLE_LABEL="Node (Bitcoin-only)" ;;
-  server_plus_desktop) ROLE_LABEL="Server + Desktop" ;;
-  *)                  ROLE_LABEL="$ROLE" ;;
+  desktop)              ROLE_LABEL="Desktop Only" ;;
+  node)                 ROLE_LABEL="Node (Bitcoin-only)" ;;
+  server_plus_desktop)  ROLE_LABEL="Server + Desktop" ;;
+  *)                    ROLE_LABEL="$ROLE" ;;
 esac
 log "Detected role: $ROLE_LABEL"

@@ -184,7 +311,6 @@ log "Detected role: $ROLE_LABEL"

 if [[ -n "${BACKUP_TARGET:-}" ]]; then
  TARGET="$BACKUP_TARGET"
-  # Safety: never allow internal drives even if explicitly passed
  if is_internal "$TARGET"; then
    fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
  fi
@@ -193,39 +319,70 @@ else
  log "Auto-detecting external USB drives…"
  TARGET="$(find_external_drive)"
  if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. " \
-         "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
+    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
  fi
  log "Detected external drive: $TARGET"
 fi

-# ── Verify mount point ───────────────────────────────────────────
+validate_target_mount "$TARGET"

-[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist."
-mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
+# ── Set up stable backup destination ────────────────────────────
+# Subsequent runs update the same mirror, transferring only new or changed files.

-# ── Check free disk space (require ≥ 10 GB) ──────────────────────
-
-FREE_KB=$(df -k --output=avail "$TARGET" | tail -1)
-FREE_GB=$(( FREE_KB / 1024 / 1024 ))
-log "Free space on drive: ${FREE_GB} GB"
-(( FREE_GB >= MIN_FREE_GB )) || \
-  fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)."
-
-# ── Create timestamped backup directory ─────────────────────────
-
-TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
-BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
+BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
 mkdir -p "$BACKUP_DIR"
+
+# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
+# after all rsync stages and manifest write succeed. Failed or interrupted
+# runs keep this marker so they are clearly identifiable.
+touch "$BACKUP_DIR/INCOMPLETE"
 log "Backup destination: $BACKUP_DIR"

+# ── Estimate required free space ─────────────────────────────────
+# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
+# from the estimate to avoid inflating the required size.
+
+ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
+HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
+SECRETS_BYTES=0
+if [[ "$ROLE" != "desktop" ]]; then
+  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
+fi
+
+VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='postgresql' \
+  --exclude='mysql' \
+  --exclude='mariadb' \
+  --exclude='bitcoind' \
+  --exclude='electrs' \
+  --exclude='*/log' \
+  --exclude='*/logs' \
+  --exclude='*/cache' \
+  --exclude='*/tmp')
+
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
+# Require 20% growth headroom plus a fixed 1 GiB safety margin.
+# Later incremental runs need far less space, but a conservative first-run
+# check protects against running out of space mid-backup.
+REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
+
+FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
+FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
+REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
+
+log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
+log "Required free space (with safety margin): ${REQUIRED_GB} GB"
+log "Free space on drive: ${FREE_GB} GB"
+
+(( FREE_BYTES >= REQUIRED_BYTES )) || \
+  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
+
 # ── Stage 1/4: NixOS configuration ──────────────────────────────

 log ""
 log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 1 failed while copying /etc/nixos"
+  run_rsync "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
  log "Stage 1 complete."
 else
  log "WARNING: /etc/nixos not found — skipping."
@@ -234,64 +391,66 @@ fi
 # ── Stage 2/4: Secrets ──────────────────────────────────────────

 log ""
-log "── Stage 2/4: Secrets ───────────────────────────────────────"
-mkdir -p "$BACKUP_DIR/secrets"
-
+log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
  log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
+elif [[ -e /etc/nix-bitcoin-secrets ]]; then
+  run_rsync "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
 else
-  if [[ -e /etc/nix-bitcoin-secrets ]]; then
-    rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
-  else
-    log "  (not found: /etc/nix-bitcoin-secrets — skipping)"
-  fi
+  log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
-
 log "Stage 2 complete."

-# ── Stage 3/4: Home directory ────────────────────────────────────
+# ── Stage 3/4: Home directory ───────────────────────────────────
+# Rsync exit code 24 (vanished source files) is treated as nonfatal here
+# because the desktop may be active and files can disappear between the
+# directory scan and the copy. All other nonzero exit codes remain fatal.

 log ""
 log "── Stage 3/4: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  rsync -a --info=progress2 \
+  run_rsync "/home" yes \
    --exclude='.cache/' \
    --exclude='.local/share/Trash/' \
-    --exclude='*/Trash/' \
-    /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 3 failed while copying /home"
+    --exclude='Trash/' \
+    --exclude='.mozilla/firefox/*/cache2/' \
+    --exclude='.mozilla/firefox/*/startupCache/' \
+    --exclude='.mozilla/firefox/*/thumbnails/' \
+    --exclude='.config/google-chrome/*/Cache/' \
+    --exclude='.config/google-chrome/*/Code Cache/' \
+    --exclude='.config/chromium/*/Cache/' \
+    --exclude='.config/chromium/*/Code Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
+    --exclude='.local/share/baloo/' \
+    --exclude='.thumbnails/' \
+    --exclude='.xsession-errors' \
+    --exclude='.xsession-errors.old' \
+    /home/ "$BACKUP_DIR/home/"
  log "Stage 3 complete."
 else
  log "WARNING: /home not found — skipping."
 fi

-# ── Stage 4/4: System data ───────────────────────────────────────
+# ── Stage 4/4: System data ──────────────────────────────────────
+# PostgreSQL/MariaDB raw database directories are excluded. Application
+# databases must be backed up separately with native database tools.
+# Bitcoin/Electrs data are excluded; they live on the internal second drive.

 log ""
-log "── Stage 4/4: System data (/var/lib) ────────────────────────"
-if [[ "$ROLE" == "desktop" ]]; then
-  if [[ -d /var/lib ]]; then
-    rsync -a --info=progress2 \
-      --filter='- /lnd/***' \
-      --exclude='logs/' \
-      --exclude='log/' \
-      --exclude='*/logs/' \
-      --exclude='*/log/' \
-      /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      fail "Stage 4 failed while copying /var/lib for Desktop Only role"
-    log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)."
-  else
-    log "WARNING: /var/lib not found — skipping."
-  fi
-elif [[ -d /var/lib ]]; then
-  rsync -a --info=progress2 \
-    --exclude='logs/' \
-    --exclude='log/' \
-    --exclude='*/logs/' \
+log "── Stage 4/4: System data (/var/lib) ───────────────────────"
+if [[ -d /var/lib ]]; then
+  run_rsync "/var/lib" no \
+    --exclude='postgresql/' \
+    --exclude='mysql/' \
+    --exclude='mariadb/' \
+    --exclude='bitcoind/' \
+    --exclude='electrs/' \
    --exclude='*/log/' \
-    /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 4 failed while copying /var/lib"
+    --exclude='*/logs/' \
+    --exclude='*/cache/' \
+    --exclude='*/tmp/' \
+    /var/lib/ "$BACKUP_DIR/var/lib/"
  log "Stage 4 complete."
 else
  log "WARNING: /var/lib not found — skipping."
@@ -301,21 +460,91 @@ fi

 log ""
 log "Generating BACKUP_MANIFEST.txt …"
+MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
+
 {
  echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date)"
+  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
  echo "Hostname:  $(hostname)"
  echo "Role:      $ROLE_LABEL"
  echo "Target:    $TARGET"
  echo ""
-  echo "Contents:"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort
-} > "$BACKUP_DIR/BACKUP_MANIFEST.txt"
-log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo "Backup type: Live rsync mirror (directly browsable files)"
+  echo "Location:    ${BACKUP_DIR}"
+  echo ""
+  echo "Source paths mirrored:"
+  echo "- /etc/nixos  →  current/etc/nixos/"
+  if [[ "$ROLE" != "desktop" ]]; then
+    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
+  fi
+  echo "- /home  →  current/home/"
+  echo "- /var/lib  →  current/var/lib/"
+  echo ""
+  echo "Exclusions:"
+  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
+  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
+  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
+  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
+  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
+  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
+  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
+  echo ""
+  echo "Important limitations:"
+  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
+  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
+  echo "  applications, their data must be backed up separately using native tools."
+  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
+  echo "  reconstructable or stored on the internal second drive."
+  echo "- This is a live file-level mirror, not a transactional database backup."
+  echo "  Files being written during the backup may be in an inconsistent state."
+  echo ""
+  echo "Restore guidance:"
+  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
+  echo "- To restore a directory:"
+  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
+  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
+  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
+  echo "- To copy individual files:"
+  echo "    sudo cp -a current/home/username/ /home/username/"
+  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
+  echo "  hardware-configuration.nix for the new hardware before rebuilding."
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for warning in "${RSYNC_WARNINGS[@]}"; do
+      echo "- $warning"
+    done
+  fi
+  echo ""
+  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "from manual external backup because they already live on the internal second drive"
+  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
+} > "$MANIFEST_FILE"
+
+log "Manifest written to $MANIFEST_FILE"

 # ── Done ─────────────────────────────────────────────────────────

 log ""
+if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${RSYNC_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that"
+  log "vanished during backup, which is normal on an active desktop."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
+log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
 log "Please eject the drive safely before removing it from your Sovran Pro."
+
+# Remove incomplete marker and write completion marker only after all work succeeds.
+# A later successful run will update the same mirror and replace any INCOMPLETE state.
+rm -f "$BACKUP_DIR/INCOMPLETE"
+echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE"
+
+BACKUP_COMPLETE=1
 set_status "SUCCESS"
@@ -1451,6 +1451,12 @@ def _read_backup_status() -> str:
        return "IDLE"


+def _write_backup_status(value: str) -> None:
+    """Write backup status file."""
+    with open(BACKUP_STATUS, "w") as f:
+        f.write(value)
+
+
 def _read_backup_log(offset: int = 0) -> tuple[str, int]:
    """Read the backup log file from the given byte offset.
    Returns (new_text, new_offset)."""
@@ -1467,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]:
        return "", 0


+def _append_backup_log(line: str) -> None:
+    """Append one line to backup log."""
+    with open(BACKUP_LOG, "a") as f:
+        f.write(line.rstrip("\n") + "\n")
+
+
 _INTERNAL_LABELS  = {"BTCEcoandBackup", "sovran_systemsos"}
 _INTERNAL_MOUNTS  = {"/", "/boot/efi"}
 _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive"
@@ -1481,6 +1493,16 @@ def _is_internal_mount(mnt: str) -> bool:
    return False


+def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
+    """Return whether the target filesystem type is supported for manual backup.
+
+    Manual Backup requires ext4 for Linux metadata preservation (ACLs, xattrs,
+    hard links). exFAT, FAT32, NTFS, and other filesystems are not supported.
+    """
+    fstype = (fstype or "").lower()
+    return fstype == "ext4"
+
+
 def _detect_external_drives() -> list[dict]:
    """Scan for mounted external USB drives.

@@ -1490,7 +1512,8 @@ def _detect_external_drives() -> list[dict]:
    /run/media/ directly if lsblk is unavailable, applying the same
    label/path filters.

-    Returns a list of dicts with name, path, free_gb, total_gb.
+    Returns:
+        list[dict]: Each dict contains name, path, free_gb, total_gb, fstype.
    """
    import json as _json
    import subprocess as _subprocess
@@ -1501,7 +1524,7 @@ def _detect_external_drives() -> list[dict]:
    # ── Primary path: lsblk JSON ────────────────────────────────
    try:
        result = _subprocess.run(
-            ["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"],
+            ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"],
            capture_output=True, text=True, timeout=10
        )
        if result.returncode == 0:
@@ -1519,6 +1542,7 @@ def _detect_external_drives() -> list[dict]:
                hotplug    = str(dev.get("hotplug", "0"))
                rm         = str(dev.get("rm", "0"))
                label      = dev.get("label") or ""
+                fstype     = (dev.get("fstype") or "").lower()
                mountpoint = dev.get("mountpoint") or ""

                if dev_type not in ("part", "disk"):
@@ -1544,6 +1568,7 @@ def _detect_external_drives() -> list[dict]:
                        "path":     mountpoint,
                        "free_gb":  free_gb,
                        "total_gb": total_gb,
+                        "fstype":   fstype,
                    })
                    seen_paths.add(mountpoint)
                except OSError:
@@ -1577,11 +1602,20 @@ def _detect_external_drives() -> list[dict]:
                    st = os.statvfs(drive_path)
                    total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1)
                    free_gb  = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1)
+                    fstype = ""
+                    try:
+                        fstype = _subprocess.run(
+                            ["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path],
+                            capture_output=True, text=True, timeout=5
+                        ).stdout.strip().lower()
+                    except Exception:
+                        fstype = ""
                    drives.append({
                        "name":     drive_name,
                        "path":     drive_path,
                        "free_gb":  free_gb,
                        "total_gb": total_gb,
+                        "fstype":   fstype,
                    })
                    seen_paths.add(drive_path)
                except OSError:
@@ -3682,6 +3716,37 @@ async def api_backup_drives():
    return {"drives": drives}


+async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
+    """Drain stderr, then mark status FAILED if backup subprocess exits unexpectedly."""
+    stderr_chunks: list[bytes] = []
+
+    async def _drain_stderr() -> None:
+        if proc.stderr is not None:
+            async for line in proc.stderr:
+                stderr_chunks.append(line)
+
+    drain_task = asyncio.create_task(_drain_stderr())
+    rc = await proc.wait()
+    await drain_task
+
+    if rc == 0:
+        return
+
+    loop = asyncio.get_event_loop()
+    status = await loop.run_in_executor(None, _read_backup_status)
+    if status in {"SUCCESS", "FAILED"}:
+        return
+
+    detail = ""
+    if stderr_chunks:
+        stderr_text = b"".join(stderr_chunks).decode("utf-8", errors="replace").strip()
+        if stderr_text:
+            detail = f" — stderr: {stderr_text}"
+    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc}).{detail}"
+    await loop.run_in_executor(None, _append_backup_log, msg)
+    await loop.run_in_executor(None, _write_backup_status, "FAILED")
+
+
@app.post("/api/backup/run")
 async def api_backup_run(target: str = ""):
    """Start the backup script as a background subprocess.
@@ -3692,6 +3757,26 @@ async def api_backup_run(target: str = ""):
    if status == "RUNNING":
        return {"ok": True, "status": "already_running"}

+    drives = await loop.run_in_executor(None, _detect_external_drives)
+    if not drives:
+        raise HTTPException(status_code=400, detail="No external backup drive detected.")
+
+    drive_map = {d.get("path", ""): d for d in drives if d.get("path")}
+    if target:
+        if target not in drive_map:
+            raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.")
+        selected = drive_map[target]
+    else:
+        selected = drives[0]
+
+    selected_target = selected.get("path", "")
+    selected_fstype = (selected.get("fstype") or "").lower()
+    if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup. Manual Backup requires an ext4-formatted drive.",
+        )
+
    # Clear stale log before starting
    try:
        with open(BACKUP_LOG, "w") as f:
@@ -3699,21 +3784,48 @@ async def api_backup_run(target: str = ""):
    except OSError:
        pass

-    env = dict(os.environ)
-    if target:
-        env["BACKUP_TARGET"] = target
+    try:
+        await loop.run_in_executor(None, _write_backup_status, "RUNNING")
+    except OSError as exc:
+        raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}")

-    # Fire-and-forget: the script writes its own status/log files.
-    # Progress is read by the client via /api/backup/status (same pattern
-    # as /api/updates/run and the rebuild feature).
-    await asyncio.create_subprocess_exec(
-        "/usr/bin/env", "bash", BACKUP_SCRIPT,
-        stdout=asyncio.subprocess.DEVNULL,
-        stderr=asyncio.subprocess.DEVNULL,
-        env=env,
+    await loop.run_in_executor(
+        None,
+        _append_backup_log,
+        f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…",
    )

-    return {"ok": True, "status": "started"}
+    env = dict(os.environ)
+    env["BACKUP_TARGET"] = selected_target
+
+    bash_path = shutil.which("bash")
+    if bash_path is None:
+        no_bash_msg = (
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Cannot start backup:"
+            " interpreter 'bash' not found on PATH."
+            " Ensure pkgs.bash is in the sovran-hub-web service PATH."
+        )
+        await loop.run_in_executor(None, _append_backup_log, no_bash_msg)
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(
+            status_code=500,
+            detail="Backup interpreter (bash) not available. Check service PATH configuration.",
+        )
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bash_path, BACKUP_SCRIPT,
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except Exception as exc:
+        await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}")
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(status_code=500, detail="Failed to launch backup process.")
+
+    asyncio.create_task(_monitor_backup_subprocess(proc))
+    return {"ok": True, "status": "started", "target": selected_target}


 # ── Feature Manager endpoints ─────────────────────────────────────
@@ -491,8 +491,9 @@ function renderBackupReady(drives) {
      '<div class="support-steps-title">Requirements</div>',
      '<ol class="support-backup-steps">',
        '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>At least 500 GB of free space on the drive</li>',
-        '<li>Drive must be formatted as <strong>exFAT</strong></li>',
+        '<li>Enough free space for your data (the backup checks this before starting)</li>',
+        '<li>Drive must be formatted as <strong>ext4</strong> (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or <code>mkfs.ext4</code> — note that formatting erases all data on the drive.</li>',
+        '<li>The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.</li>',
      '</ol>',
    '</div>',

@@ -501,17 +502,25 @@ function renderBackupReady(drives) {
      '<ol class="support-backup-steps">',
        '<li>NixOS configuration (<code>/etc/nixos</code>)</li>',
        '<li>nix-bitcoin secrets (<code>/etc/nix-bitcoin-secrets</code>)</li>',
-        '<li>System service data (<code>/var/lib</code>) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets</li>',
+        '<li>System service data (<code>/var/lib</code>) — excluding databases and blockchain data (see note below)</li>',
        '<li>Home directory (<code>/home</code>)</li>',
      '</ol>',
    '</div>',

    '<div class="support-wallet-box support-wallet-warning">',
+      '<div class="support-wallet-header">',
+        '<span class="support-wallet-icon">\u2139\ufe0f</span>',
+        '<span class="support-wallet-title">Database and Blockchain Data</span>',
+      '</div>',
+      '<p class="support-wallet-desc">Application databases stored in PostgreSQL or MariaDB/MySQL are <strong>not included</strong> in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.</p>',
+    '</div>',
+
+    '<div class="support-wallet-box support-wallet-protected">',
      '<div class="support-wallet-header">',
        '<span class="support-wallet-icon">\u23f1\ufe0f</span>',
        '<span class="support-wallet-title">Time Estimate</span>',
      '</div>',
-      '<p class="support-wallet-desc">This backup can take <strong>up to 4 hours</strong> depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026</p>',
+      '<p class="support-wallet-desc">The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.</p>',
    '</div>',

    driveSelector,
@@ -584,9 +593,10 @@ async function pollBackupStatus() {
      logDiv.scrollTop = logDiv.scrollHeight;
    }
    _backupLogOffset = data.offset;
-    if (!data.running) {
+    const result = (data.result || "").toLowerCase();
+    if (result === "success" || result === "failed") {
      stopBackupPoll();
-      renderBackupDone(data.result === "success");
+      renderBackupDone(result === "success");
    }
  } catch (_) {}
 }
@@ -618,7 +628,7 @@ function renderBackupDone(success) {
      '<div class="support-section">',
      '<div class="support-icon-big">\u26a0\ufe0f</div>',
      '<h3 class="support-heading">Backup Failed</h3>',
-      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.</p>',
+      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.</p>',
      '<div class="modal-log" id="backup-log-fail" style="text-align:left;"></div>',
      '<button class="btn support-btn-done" id="btn-backup-close">Close</button>',
      '</div>',
@@ -0,0 +1,193 @@
+"""Structural regression tests for modules/core/local-domain-loopback.nix.
+
+Verifies that the sovran-hosts-update helper:
+  - is built as a pkgs.writeShellApplication with explicit runtimeInputs
+    (gawk, gnugrep, coreutils);
+  - uses ``lib.getExe hostsUpdateScript`` for both the systemd ExecStart and
+    the activation script so that both contexts share the same Nix-store
+    executable;
+  - does NOT point ExecStart at the raw /etc path;
+  - includes element-calling in the supported domain list;
+  - uses ``awk -v`` for safe marker-variable passing rather than interpolating
+    marker text directly into the awk program;
+  - retains the domain validation regex (idempotency / injection prevention);
+  - exposes /etc/sovran-hosts-update.sh as a symlink via ``source =`` (not a
+    second raw ``text =`` body);
+  - does NOT rely on environment.systemPackages for the helper's dependencies.
+"""
+
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+NIX_STRING_INDENT = 6
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FLAKE_SOURCE = (REPO_ROOT / "flake.nix").read_text()
+PRIMARY_NIXOS_CONFIGURATION_MATCH = re.search(
+    r"nixosConfigurations\.([A-Za-z0-9_-]+)\s*=",
+    FLAKE_SOURCE,
+)
+if PRIMARY_NIXOS_CONFIGURATION_MATCH is None:
+    raise RuntimeError("Could not determine the primary nixosConfigurations entry from flake.nix")
+PRIMARY_NIXOS_CONFIGURATION = PRIMARY_NIXOS_CONFIGURATION_MATCH.group(1)
+HELPER_BUILD_ATTR = (
+    f'.#nixosConfigurations.{PRIMARY_NIXOS_CONFIGURATION}.config.environment.etc.'
+    '"sovran-hosts-update.sh".source'
+)
+NIX_FILE = (
+    REPO_ROOT
+    / "modules"
+    / "core"
+    / "local-domain-loopback.nix"
+)
+
+
+class LocalDomainLoopbackNixStructureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = NIX_FILE.read_text()
+
+    def _helper_script(self) -> str:
+        start = self.source.index("text = ''") + len("text = ''")
+        end = self.source.index("    '';", start)
+        return "\n".join(
+            line[NIX_STRING_INDENT:]
+            if line.startswith(" " * NIX_STRING_INDENT)
+            else line
+            for line in self.source[start:end].splitlines()
+        ).lstrip("\n")
+
+    # ── writeShellApplication and explicit runtimeInputs ────────────────────
+
+    def test_uses_write_shell_application(self):
+        self.assertIn("pkgs.writeShellApplication", self.source)
+
+    def test_runtime_inputs_includes_coreutils(self):
+        self.assertIn("pkgs.coreutils", self.source)
+
+    def test_runtime_inputs_includes_gawk(self):
+        self.assertIn("pkgs.gawk", self.source)
+
+    def test_runtime_inputs_includes_gnugrep(self):
+        self.assertIn("pkgs.gnugrep", self.source)
+
+    def test_runtime_inputs_block_present(self):
+        self.assertIn("runtimeInputs", self.source)
+
+    # ── Both execution paths use lib.getExe ─────────────────────────────────
+
+    def test_exec_start_uses_lib_get_exe(self):
+        """systemd ExecStart must reference the Nix-store executable."""
+        self.assertIn("ExecStart = lib.getExe hostsUpdateScript", self.source)
+
+    def test_activation_script_uses_lib_get_exe(self):
+        """Activation text must call the same Nix-store executable."""
+        self.assertIn("${lib.getExe hostsUpdateScript}", self.source)
+
+    def test_exec_start_does_not_point_to_etc_path(self):
+        """ExecStart must NOT use the raw /etc path (which lacks a deterministic PATH)."""
+        self.assertNotIn('ExecStart = "/etc/sovran-hosts-update.sh"', self.source)
+
+    # ── /etc symlink uses source =, not a second text = body ────────────────
+
+    def test_etc_entry_uses_source_not_text(self):
+        """The /etc/sovran-hosts-update.sh entry must be a symlink (source =),
+        not a second raw script body (text =)."""
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source', self.source
+        )
+
+    def test_etc_source_points_to_get_exe(self):
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript',
+            self.source,
+        )
+
+    # ── element-calling domain is supported ─────────────────────────────────
+
+    def test_element_calling_domain_key_present(self):
+        self.assertIn("element-calling", self.source)
+
+    # ── Robust awk -v variable passing ──────────────────────────────────────
+
+    def test_awk_uses_dash_v_for_begin_marker(self):
+        """awk must receive the begin marker via -v, not by shell interpolation."""
+        self.assertIn('awk -v begin=', self.source)
+
+    def test_awk_uses_dash_v_for_end_marker(self):
+        self.assertIn('-v end=', self.source)
+
+    def test_awk_does_not_interpolate_marker_into_program(self):
+        """The old pattern interpolated $BEGIN_MARKER directly into the awk source."""
+        self.assertNotIn('/$BEGIN_MARKER', self.source)
+        self.assertNotIn('/$END_MARKER', self.source)
+
+    # ── Domain validation ────────────────────────────────────────────────────
+
+    def test_domain_validation_regex_present(self):
+        """The hostname validation regex must still be present for injection prevention."""
+        self.assertIn("grep -qE", self.source)
+        self.assertIn("[a-zA-Z0-9]", self.source)
+
+    def test_invalid_domain_warning_present(self):
+        self.assertIn("skipping invalid domain value", self.source)
+
+    # ── No environment.systemPackages reliance ───────────────────────────────
+
+    def test_no_environment_system_packages_for_helper(self):
+        """The helper's tools are declared via runtimeInputs; the module must
+        not add them to environment.systemPackages."""
+        self.assertNotIn("environment.systemPackages", self.source)
+
+    # ── Idempotency: existing Sovran block is removed before rewriting ───────
+
+    def test_existing_block_removal_logic_present(self):
+        """awk strip of the managed block must be present for idempotency."""
+        self.assertIn("skip=1", self.source)
+        self.assertIn("skip=0", self.source)
+
+    def test_managed_block_uses_grouped_append_redirect(self):
+        self.assertIn('} >> "$TMP"', self.source)
+        self.assertEqual(self.source.count('>> "$TMP"'), 1)
+
+    def test_helper_script_passes_shellcheck(self):
+        shellcheck = shutil.which("shellcheck")
+        if shellcheck is None:
+            self.skipTest("shellcheck is not installed")
+        proc = subprocess.run(
+            [shellcheck, "-s", "bash", "-"],
+            input=self._helper_script(),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        output = proc.stdout + proc.stderr
+        self.assertEqual(proc.returncode, 0, output)
+
+    def test_helper_derivation_builds_when_nix_available(self):
+        nix = shutil.which("nix")
+        if nix is None:
+            self.skipTest("nix is not installed")
+        proc = subprocess.run(
+            [
+                nix,
+                "build",
+                HELPER_BUILD_ATTR,
+                "--no-link",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+
+    # ── Activation script warns on failure rather than silently swallowing ───
+
+    def test_activation_script_emits_warning_on_failure(self):
+        self.assertIn("warning: sovran-hosts-update", self.source)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,748 @@
+import asyncio
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BACKUP_SCRIPT = REPO_ROOT / "app" / "sovran_systemsos_web" / "scripts" / "sovran-hub-backup.sh"
+SERVER_FILE = REPO_ROOT / "app" / "sovran_systemsos_web" / "server.py"
+SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "support.js"
+NIX_HUB_FILE = REPO_ROOT / "modules" / "core" / "sovran-hub.nix"
+
+
+class ManualBackupWorkflowTests(unittest.TestCase):
+    # ── Core design: rsync, no tar, no DB, no LND ─────────────────────────────
+
+    def test_backup_script_uses_rsync_not_tar(self):
+        """New design: backup must use rsync, not tar archives."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("rsync", source, "backup script must use rsync")
+        self.assertIn("--archive", source, "rsync must be called with --archive flag")
+        self.assertNotIn("tar --create", source, "backup script must not create tar archives")
+        self.assertNotIn("--file ", source, "backup script must not write tar --file output")
+
+    def test_backup_script_has_no_database_dump_functions(self):
+        """Removed: PostgreSQL and MariaDB dump functions must not exist."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("pg_dump", source, "backup script must not contain pg_dump")
+        self.assertNotIn("pg_dumpall", source, "backup script must not contain pg_dumpall")
+        self.assertNotIn("mariadb-dump", source, "backup script must not contain mariadb-dump")
+        self.assertNotIn("mysqldump", source, "backup script must not contain mysqldump")
+        self.assertNotIn("export_postgresql_dumps", source)
+        self.assertNotIn("export_mariadb_dumps", source)
+
+    def test_backup_script_has_no_lnd_service_orchestration(self):
+        """Removed: LND stop/restart and SCB export must not exist."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("export_lnd_scb_if_possible", source)
+        self.assertNotIn("stop_lnd_stack_if_needed", source)
+        self.assertNotIn("lncli", source, "backup script must not call lncli")
+        self.assertNotIn("LND_STOPPED", source, "backup script must not track LND_STOPPED state")
+        self.assertNotIn("LND_UNITS_TO_RESTART", source)
+
+    def test_backup_script_has_no_tar_dependencies(self):
+        """Removed: sha256sum checksums for tar artifacts must not exist."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("sha256sum", source, "backup script must not generate tar checksums")
+        self.assertNotIn("SHA256SUMS", source, "backup script must not write SHA256SUMS.txt")
+
+    # ── Rsync options ──────────────────────────────────────────────────────────
+
+    def test_backup_script_rsync_uses_metadata_preserving_options(self):
+        """Rsync must be called with Linux metadata-preserving options."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--archive", source)
+        self.assertIn("--acls", source)
+        self.assertIn("--xattrs", source)
+        self.assertIn("--hard-links", source)
+        self.assertIn("--numeric-ids", source)
+        self.assertIn("--one-file-system", source)
+
+    def test_backup_script_rsync_no_delete(self):
+        """Rsync must NOT use --delete or --delete-delay.
+        Accidental source deletion must not silently wipe the backup copy."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("--delete", source,
+            "rsync must not use --delete; accidental source deletion must not erase backup")
+
+    def test_backup_script_uses_stable_current_mirror_path(self):
+        """Backup must write to a stable 'current/' path, not timestamped directories.
+        Later runs should update the same mirror."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("Sovran_SystemsOS_Backup/current", source,
+            "backup must use a stable 'current' mirror path")
+        self.assertIn("BACKUP_SUBPATH", source,
+            "stable sub-path must be defined in BACKUP_SUBPATH variable")
+        # Must not create new timestamped directories per run
+        self.assertNotIn(
+            "date '+%Y%m%d_%H%M%S'",
+            source,
+            "backup must not create timestamped per-run directories",
+        )
+
+    def test_backup_script_source_destination_mapping(self):
+        """Each source tree must be synced to a matching destination path."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("/etc/nixos/", source)
+        self.assertIn('"$BACKUP_DIR/etc/nixos/"', source)
+        self.assertIn("/home/", source)
+        self.assertIn('"$BACKUP_DIR/home/"', source)
+        self.assertIn("/var/lib/", source)
+        self.assertIn('"$BACKUP_DIR/var/lib/"', source)
+
+    # ── ext4 filesystem validation ─────────────────────────────────────────────
+
+    def test_backup_script_requires_ext4_filesystem(self):
+        """Backup script must reject non-ext4 filesystems and require ext4."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'fstype" != "ext4"',
+            source,
+            "backup script must check for ext4 filesystem",
+        )
+        self.assertIn("ext4", source, "backup script must reference ext4")
+        self.assertNotIn(
+            '"exfat"',
+            source,
+            "backup script must not accept exFAT (old requirement)",
+        )
+        self.assertNotIn(
+            '"fuseblk"',
+            source,
+            "backup script must not accept fuseblk/NTFS",
+        )
+
+    def test_backup_script_rejects_unsupported_filesystems_in_error_message(self):
+        """The ext4 rejection message must mention the unsupported filesystem types."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "exFAT, FAT32, and NTFS are not supported",
+            source,
+            "error message must clearly list unsupported filesystem types",
+        )
+
+    def test_backend_accepts_ext4_rejects_exfat(self):
+        """Backend _is_supported_backup_fstype must accept ext4 and reject exFAT."""
+        server_source = SERVER_FILE.read_text()
+        # Must accept ext4
+        self.assertIn(
+            'return fstype == "ext4"',
+            server_source,
+            "backend must accept only ext4",
+        )
+        # Must not accept exFAT
+        self.assertNotIn(
+            'fstype == "exfat"',
+            server_source,
+            "backend must not accept exFAT",
+        )
+        self.assertNotIn(
+            'fuseblk',
+            server_source.split("_is_supported_backup_fstype")[1][:500],
+            "backend must not accept fuseblk",
+        )
+
+    def test_frontend_requires_ext4_not_exfat(self):
+        """Frontend UI copy must require ext4 and not mention exFAT as the requirement."""
+        support_source = SUPPORT_JS.read_text()
+        self.assertIn(
+            "ext4",
+            support_source,
+            "support.js must mention ext4 as the required filesystem",
+        )
+        # The word exFAT must not appear as a requirement (it may appear in a
+        # note about unsupported formats, but the requirement itself must say ext4)
+        requirements_section = re.search(
+            r"Requirements.*?What gets backed up",
+            support_source,
+            re.DOTALL,
+        )
+        if requirements_section:
+            req_text = requirements_section.group(0)
+            self.assertNotIn(
+                "Drive must be formatted as <strong>exFAT</strong>",
+                req_text,
+                "Requirements section must not say 'formatted as exFAT'",
+            )
+            self.assertIn(
+                "ext4",
+                req_text,
+                "Requirements section must say ext4",
+            )
+
+    def test_frontend_failure_message_says_ext4(self):
+        """Failure message in renderBackupDone must say ext4, not exFAT."""
+        support_source = SUPPORT_JS.read_text()
+        self.assertNotIn(
+            "formatted as exFAT",
+            support_source,
+            "failure message must not say 'formatted as exFAT'",
+        )
+        self.assertIn(
+            "formatted as ext4",
+            support_source,
+            "failure message must say 'formatted as ext4'",
+        )
+
+    # ── Database exclusions ────────────────────────────────────────────────────
+
+    def test_backup_script_excludes_postgresql_and_mariadb(self):
+        """PostgreSQL and MariaDB raw database directories must be excluded."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='postgresql/'", source,
+            "rsync must exclude postgresql directory")
+        self.assertIn("--exclude='mysql/'", source,
+            "rsync must exclude mysql directory")
+        self.assertIn("--exclude='mariadb/'", source,
+            "rsync must exclude mariadb directory")
+
+    def test_backup_script_excludes_bitcoin_and_electrs(self):
+        """Bitcoin blockchain and Electrs index data must be excluded."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='bitcoind/'", source,
+            "rsync must exclude bitcoind directory")
+        self.assertIn("--exclude='electrs/'", source,
+            "rsync must exclude electrs directory")
+
+    def test_manifest_states_database_exclusion(self):
+        """The manifest must explicitly state that PostgreSQL and MariaDB are excluded."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "PostgreSQL and MariaDB/MySQL application databases are NOT included",
+            source,
+            "manifest must state that databases are not included",
+        )
+        self.assertIn(
+            "Bitcoin blockchain data and Electrs indexes are NOT included",
+            source,
+            "manifest must state that blockchain data is not included",
+        )
+
+    def test_manifest_has_no_tar_restore_instructions(self):
+        """Manifest must not mention tar extraction, pg_restore, or LND SCB."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn(
+            "tar --acls --xattrs",
+            source,
+            "manifest must not give tar restore instructions",
+        )
+        self.assertNotIn(
+            "pg_restore",
+            source,
+            "manifest must not reference pg_restore",
+        )
+        self.assertNotIn(
+            "lnd-static-channel-backup.scb",
+            source,
+            "manifest must not reference LND SCB restore procedures",
+        )
+
+    def test_manifest_has_rsync_restore_guidance(self):
+        """Manifest must provide rsync-based restore guidance."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            "rsync -aAXH --numeric-ids",
+            source,
+            "manifest must show rsync restore command",
+        )
+
+    # ── Exit code 24 (vanished files) for /home only ──────────────────────────
+
+    def test_backup_script_exit_code_24_nonfatal_for_home(self):
+        """Rsync exit code 24 (vanished files) must be nonfatal for /home only."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            '"$rc" -eq 24',
+            source,
+            "backup script must handle rsync exit code 24",
+        )
+        self.assertIn(
+            'allow_vanished" == "yes"',
+            source,
+            "exit 24 acceptance must be gated on allow_vanished flag",
+        )
+
+    def test_backup_script_home_stage_allows_vanished(self):
+        """Stage 3 (/home) must call run_rsync with allow_vanished=yes."""
+        source = BACKUP_SCRIPT.read_text()
+        # The /home call must pass "yes" as the allow_vanished argument
+        self.assertIn(
+            'run_rsync "/home" yes',
+            source,
+            "/home stage must pass allow_vanished=yes to run_rsync",
+        )
+
+    def test_backup_script_other_stages_disallow_vanished(self):
+        """Stages other than /home must call run_rsync with allow_vanished=no."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'run_rsync "/etc/nixos" no',
+            source,
+            "/etc/nixos stage must use allow_vanished=no",
+        )
+        self.assertIn(
+            'run_rsync "/var/lib" no',
+            source,
+            "/var/lib stage must use allow_vanished=no",
+        )
+
+    # ── INCOMPLETE / BACKUP_COMPLETE markers ──────────────────────────────────
+
+    def test_backup_script_writes_incomplete_marker(self):
+        """A backup directory must be marked INCOMPLETE immediately after
+        creation, so interrupted or failed runs are identifiable."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'touch "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must create INCOMPLETE marker after mkdir",
+        )
+
+    def test_backup_script_writes_backup_complete_marker(self):
+        """A BACKUP_COMPLETE file must be written only after all work succeeds."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn(
+            'BACKUP_COMPLETE"',
+            source,
+            "backup script must write BACKUP_COMPLETE file on success",
+        )
+        self.assertIn(
+            'rm -f "$BACKUP_DIR/INCOMPLETE"',
+            source,
+            "backup script must remove INCOMPLETE marker on success",
+        )
+
+    # ── Concurrency lock ──────────────────────────────────────────────────────
+
+    def test_backup_script_uses_flock_concurrency_lock(self):
+        """The script must acquire an exclusive flock before starting work."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("flock", source, "backup script must use flock")
+        self.assertIn("LOCK_FILE", source, "backup script must define LOCK_FILE")
+
+    # ── Status states ─────────────────────────────────────────────────────────
+
+    def test_backup_script_uses_running_success_failed_states(self):
+        """Status values must be RUNNING, SUCCESS, and FAILED."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn('set_status "RUNNING"', source)
+        self.assertIn('set_status "SUCCESS"', source)
+        self.assertIn('set_status "FAILED"', source)
+
+    def test_backend_and_frontend_use_explicit_backup_terminal_states(self):
+        """Backend and frontend must use consistent terminal state handling."""
+        server_source = SERVER_FILE.read_text()
+        support_source = SUPPORT_JS.read_text()
+
+        self.assertIn("_write_backup_status, \"RUNNING\"", server_source)
+        self.assertIn("_monitor_backup_subprocess", server_source)
+        self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source)
+        self.assertIn("result === \"success\" || result === \"failed\"", support_source)
+
+    # ── Nix service PATH ──────────────────────────────────────────────────────
+
+    def test_nix_service_path_includes_rsync_and_acl(self):
+        """pkgs.rsync and pkgs.acl must appear in the sovran-hub-web service path
+        so that rsync with ACL/xattr support is available at runtime."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertIn(
+            "pkgs.rsync",
+            nix_source,
+            "pkgs.rsync must be declared in the sovran-hub-web service path",
+        )
+        self.assertIn(
+            "pkgs.acl",
+            nix_source,
+            "pkgs.acl must be declared in the sovran-hub-web service path",
+        )
+
+    def test_nix_service_path_includes_bash_and_gawk(self):
+        """Regression: pkgs.bash and pkgs.gawk must appear in the service path."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertIn("pkgs.bash", nix_source,
+            "pkgs.bash must be declared in the sovran-hub-web service path")
+        self.assertIn("pkgs.gawk", nix_source,
+            "pkgs.gawk must be declared in the sovran-hub-web service path")
+
+    def test_nix_service_path_has_no_gnutar(self):
+        """pkgs.gnutar must be removed from the service path (no longer needed)."""
+        nix_source = NIX_HUB_FILE.read_text()
+        self.assertNotIn(
+            "pkgs.gnutar",
+            nix_source,
+            "pkgs.gnutar must be removed from the service path (tar is no longer used)",
+        )
+
+    # ── Regression tests for exit-code-127 / missing-interpreter bug ──────────
+
+    def test_server_resolves_bash_via_shutil_which(self):
+        """Regression: server must locate bash with shutil.which()."""
+        source = SERVER_FILE.read_text()
+        self.assertIn('shutil.which("bash")', source)
+        self.assertNotIn('"/usr/bin/env", "bash"', source)
+
+    def test_server_logs_actionable_message_when_bash_missing(self):
+        """Regression: when bash is absent the server must write an actionable log."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("bash_path is None", source)
+        self.assertIn("interpreter", source)
+
+    def test_server_captures_stderr_from_backup_subprocess(self):
+        """Regression: backup subprocess must use stderr=PIPE."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("stderr=asyncio.subprocess.PIPE", source)
+        self.assertIn("stderr_text", source)
+
+    def test_monitor_includes_stderr_detail_in_failed_message(self):
+        """Regression: _monitor_backup_subprocess must append captured stderr."""
+        source = SERVER_FILE.read_text()
+        self.assertIn("stderr_chunks", source)
+        self.assertIn("stderr_text", source)
+        self.assertIn('detail = f" — stderr: {stderr_text}"', source)
+
+    def test_exit_code_127_subprocess_stderr_drain(self):
+        """Behavioral regression: a subprocess that exits 127 drains stderr without deadlock."""
+        async def _run():
+            proc = await asyncio.create_subprocess_exec(
+                "bash", "-c", "echo 'bash: command not found' >&2; exit 127",
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            chunks: list[bytes] = []
+
+            async def _drain():
+                async for line in proc.stderr:
+                    chunks.append(line)
+
+            drain_task = asyncio.create_task(_drain())
+            rc = await proc.wait()
+            await drain_task
+            return rc, b"".join(chunks).decode()
+
+        rc, stderr_out = asyncio.run(_run())
+        self.assertEqual(rc, 127)
+        self.assertIn("bash: command not found", stderr_out)
+
+    # ── Rsync exit code 24 behavioral tests ──────────────────────────────────
+
+    def test_run_rsync_exit_24_nonfatal_for_home(self):
+        """Behavioral: run_rsync with allow_vanished=yes treats exit 24 as a
+        nonfatal warning (recorded in RSYNC_WARNINGS) and does not fail."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+RSYNC_WARNINGS=()
+
+log() {{ echo "$*"; }}
+fail() {{ echo "FAILED: $*" >&2; exit 1; }}
+
+run_rsync() {{
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+
+  local rc=24  # simulate rsync exit 24
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop)."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc)"
+  fi
+}}
+
+run_rsync "/home" yes /home/ "{dest}/home/"
+echo "WARNINGS: ${{#RSYNC_WARNINGS[@]}}"
+echo "SUCCESS"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("SUCCESS", result.stdout)
+            self.assertIn("WARNINGS: 1", result.stdout)
+
+    def test_run_rsync_exit_24_fatal_for_non_home(self):
+        """Behavioral: run_rsync with allow_vanished=no treats exit 24 as fatal."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(dest, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+RSYNC_WARNINGS=()
+
+log() {{ echo "$*"; }}
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+
+run_rsync() {{
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+
+  local rc=24  # simulate rsync exit 24
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    RSYNC_WARNINGS+=("$label: vanished")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc)"
+  fi
+}}
+
+run_rsync "/etc/nixos" no /etc/nixos/ "{dest}/etc/nixos/"
+echo "SHOULD_NOT_REACH_HERE"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 1)
+            self.assertIn("CORRECTLY_FAILED", result.stdout)
+            self.assertNotIn("SHOULD_NOT_REACH_HERE", result.stdout)
+
+    def test_run_rsync_other_nonzero_codes_always_fatal(self):
+        """Behavioral: rsync exit codes other than 0 and 24 (for home) are fatal."""
+        for rc in [1, 10, 11, 12, 23, 25]:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                dest = os.path.join(tmpdir, "backup")
+                os.makedirs(dest, exist_ok=True)
+
+                script = f"""#!/usr/bin/env bash
+RSYNC_WARNINGS=()
+
+fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }}
+log() {{ echo "$*"; }}
+
+run_rsync() {{
+  local label="$1"
+  local allow_vanished="$2"
+  shift 2
+
+  local rc={rc}  # simulated exit code
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    RSYNC_WARNINGS+=("$label: vanished")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc)"
+  fi
+}}
+
+run_rsync "/home" yes /home/ "{dest}/home/"
+echo "SHOULD_NOT_REACH_HERE"
+"""
+                script_file = os.path.join(tmpdir, "test.sh")
+                with open(script_file, "w") as sf:
+                    sf.write(script)
+                result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+                self.assertEqual(result.returncode, 1,
+                    f"Exit code {rc} should fail: {result.stdout}")
+                self.assertIn("CORRECTLY_FAILED", result.stdout)
+
+    # ── Behavioral: repeated runs target same 'current' mirror ──────────────
+
+    def test_repeated_runs_use_same_current_directory(self):
+        """Behavioral: a second call to the backup logic must sync to the same
+        BACKUP_DIR (not create a new timestamped directory) so only changed
+        files are transferred."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            target = os.path.join(tmpdir, "drive")
+            os.makedirs(target, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+TARGET="{target}"
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
+BACKUP_DIR="${{TARGET}}/${{BACKUP_SUBPATH}}"
+mkdir -p "$BACKUP_DIR"
+echo "$BACKUP_DIR"
+"""
+            for _ in range(2):
+                script_file = os.path.join(tmpdir, "test.sh")
+                with open(script_file, "w") as sf:
+                    sf.write(script)
+                result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+                self.assertEqual(result.returncode, 0)
+                backup_dir = result.stdout.strip()
+                self.assertEqual(
+                    backup_dir,
+                    os.path.join(target, "Sovran_SystemsOS_Backup", "current"),
+                )
+
+    # ── Behavioral: INCOMPLETE/BACKUP_COMPLETE marker lifecycle ─────────────
+
+    def test_incomplete_marker_written_before_work_starts(self):
+        """Behavioral: INCOMPLETE marker must exist during backup and be removed on success."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            backup_dir = os.path.join(tmpdir, "current")
+            os.makedirs(backup_dir, exist_ok=True)
+
+            script = f"""#!/usr/bin/env bash
+set -euo pipefail
+BACKUP_DIR="{backup_dir}"
+BACKUP_COMPLETE=0
+
+touch "$BACKUP_DIR/INCOMPLETE"
+[[ -f "$BACKUP_DIR/INCOMPLETE" ]] && echo "INCOMPLETE_EXISTS"
+
+# Simulate successful completion
+rm -f "$BACKUP_DIR/INCOMPLETE"
+echo "done" > "$BACKUP_DIR/BACKUP_COMPLETE"
+BACKUP_COMPLETE=1
+
+[[ ! -f "$BACKUP_DIR/INCOMPLETE" ]] && echo "INCOMPLETE_REMOVED"
+[[ -f "$BACKUP_DIR/BACKUP_COMPLETE" ]] && echo "COMPLETE_EXISTS"
+"""
+            script_file = os.path.join(tmpdir, "test.sh")
+            with open(script_file, "w") as sf:
+                sf.write(script)
+            result = subprocess.run(["bash", script_file], capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
+            self.assertIn("INCOMPLETE_EXISTS", result.stdout)
+            self.assertIn("INCOMPLETE_REMOVED", result.stdout)
+            self.assertIn("COMPLETE_EXISTS", result.stdout)
+
+    # ── Behavioral: actual rsync with real source/dest trees ─────────────────
+
+    def test_rsync_mirrors_source_to_dest(self):
+        """Behavioral: rsync correctly mirrors a source tree to a destination."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "etc", "nixos")
+            dest = os.path.join(tmpdir, "backup", "etc", "nixos")
+            os.makedirs(src, exist_ok=True)
+            os.makedirs(os.path.join(tmpdir, "backup", "etc"), exist_ok=True)
+
+            # Write test files
+            with open(os.path.join(src, "configuration.nix"), "w") as f:
+                f.write("{ ... }: {}\n")
+            with open(os.path.join(src, "custom.nix"), "w") as f:
+                f.write("{ ... }: {}\n")
+
+            result = subprocess.run(
+                ["rsync", "--archive", "--one-file-system",
+                 src + "/", dest + "/"],
+                capture_output=True, text=True,
+            )
+            self.assertEqual(result.returncode, 0, f"rsync failed: {result.stderr}")
+            self.assertTrue(os.path.exists(os.path.join(dest, "configuration.nix")))
+            self.assertTrue(os.path.exists(os.path.join(dest, "custom.nix")))
+
+    def test_rsync_second_run_only_transfers_changes(self):
+        """Behavioral: a second rsync run to the same dest only copies changed files."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = os.path.join(tmpdir, "source")
+            dest = os.path.join(tmpdir, "backup")
+            os.makedirs(src)
+            os.makedirs(dest)
+
+            with open(os.path.join(src, "file.txt"), "w") as f:
+                f.write("initial content\n")
+
+            # First run
+            r1 = subprocess.run(
+                ["rsync", "--archive", "--one-file-system", src + "/", dest + "/"],
+                capture_output=True, text=True,
+            )
+            self.assertEqual(r1.returncode, 0)
+
+            # Modify one file and add another; advance mtime so rsync detects the change
+            with open(os.path.join(src, "file.txt"), "w") as f:
+                f.write("updated content\n")
+            import time as _time
+            future_mtime = _time.time() + 2
+            os.utime(os.path.join(src, "file.txt"), (future_mtime, future_mtime))
+
+            with open(os.path.join(src, "new.txt"), "w") as f:
+                f.write("new file\n")
+
+            # Second run
+            r2 = subprocess.run(
+                ["rsync", "--archive", "--one-file-system", src + "/", dest + "/"],
+                capture_output=True, text=True,
+            )
+            self.assertEqual(r2.returncode, 0)
+
+            # Both files exist in dest
+            with open(os.path.join(dest, "file.txt")) as fh:
+                self.assertEqual(fh.read(), "updated content\n")
+            self.assertTrue(os.path.exists(os.path.join(dest, "new.txt")))
+
+    # ── Script syntax check ──────────────────────────────────────────────────
+
+    def test_backup_script_passes_bash_syntax_check(self):
+        """bash -n must report no syntax errors in the backup script."""
+        result = subprocess.run(
+            ["bash", "-n", str(BACKUP_SCRIPT)],
+            capture_output=True, text=True,
+        )
+        self.assertEqual(
+            result.returncode, 0,
+            f"bash -n failed:\n{result.stderr}",
+        )
+
+    # ── Home exclusions ──────────────────────────────────────────────────────
+
+    def test_backup_script_home_excludes_browser_caches(self):
+        """Home sync must exclude browser disk caches."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='.mozilla/firefox/*/cache2/'", source)
+        self.assertIn("--exclude='.config/google-chrome/*/Cache/'", source)
+        self.assertIn("--exclude='.config/chromium/*/Cache/'", source)
+        self.assertIn("--exclude='.config/BraveSoftware/Brave-Browser/*/Cache/'", source)
+
+    def test_backup_script_home_excludes_other_volatile_caches(self):
+        """Home sync must exclude baloo, thumbnails, and X session error logs."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("--exclude='.local/share/baloo/'", source)
+        self.assertIn("--exclude='.thumbnails/'", source)
+        self.assertIn("--exclude='.xsession-errors'", source)
+
+    # ── require_cmd ──────────────────────────────────────────────────────────
+
+    def test_backup_script_requires_rsync_and_flock(self):
+        """Script must require rsync and flock via require_cmd."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertIn("require_cmd rsync", source)
+        self.assertIn("require_cmd flock", source)
+
+    def test_backup_script_does_not_require_tar_or_sha256sum(self):
+        """Script must not require tar or sha256sum (no longer used)."""
+        source = BACKUP_SCRIPT.read_text()
+        self.assertNotIn("require_cmd tar", source)
+        self.assertNotIn("require_cmd sha256sum", source)
+
+    # ── Frontend database limitation notice ──────────────────────────────────
+
+    def test_frontend_mentions_database_limitation(self):
+        """Frontend must explain that PostgreSQL/MariaDB databases are excluded."""
+        support_source = SUPPORT_JS.read_text()
+        self.assertIn(
+            "PostgreSQL",
+            support_source,
+            "UI must mention PostgreSQL exclusion",
+        )
+        self.assertIn(
+            "not included",
+            support_source,
+            "UI must explain that databases are not included",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -139,11 +139,11 @@
    },
    "nixpkgs-stable": {
      "locked": {
-        "lastModified": 1783856661,
-        "narHash": "sha256-ZGP04e+Q6WyQJGA9ZvI5CL6+heGQldbAG9U1T9NGvmU=",
+        "lastModified": 1784432872,
+        "narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "569d578509928497eddc3fdbf94a799027050be4",
+        "rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
        "type": "github"
      },
      "original": {
@@ -187,11 +187,11 @@
    },
    "nixpkgs_3": {
      "locked": {
-        "lastModified": 1783776592,
-        "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=",
+        "lastModified": 1784356753,
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
        "type": "github"
      },
      "original": {
@@ -203,11 +203,11 @@
    },
    "nixpkgs_4": {
      "locked": {
-        "lastModified": 1783791668,
-        "narHash": "sha256-zbcZ1dmBTPfJ7Mlqh/yLEPGpgJnwuv4Xr1xucy2WqMA=",
+        "lastModified": 1783915482,
+        "narHash": "sha256-FmieJB8/OUvNxbkboi7+IGfIuSXY3nF/hZQm8kD0r50=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "716c7a2664ca8325617b8a7fbb609273f2c4cae7",
+        "rev": "6cdc7fc76e8bf7fde9fa43a849fcaaa70e230dee",
        "type": "github"
      },
      "original": {
@@ -224,11 +224,11 @@
        "systems": "systems_2"
      },
      "locked": {
-        "lastModified": 1783941741,
-        "narHash": "sha256-F+3M1IZrJa920cx2/k2AMKqedEodxLF7COJVkLJwUBo=",
+        "lastModified": 1784057377,
+        "narHash": "sha256-yycNej5//EsRbV10moBoh+/63vXEwZD1ZFEiRm6C9rQ=",
        "owner": "nix-community",
        "repo": "nixvim",
-        "rev": "e6715f01d9f56f07a27a01386b85ae22b06f0705",
+        "rev": "07180a087e4a00720dc0731cbcd8dec796974381",
        "type": "github"
      },
      "original": {
@@ -32,22 +32,31 @@
 # regenerated by the system activation script.  The ``system.activationScripts``
 # hook below converts it to a writable file each time the system is activated
 # (i.e. after every ``nixos-rebuild switch``) and then injects the Sovran block.
-# The same script is also run by the ``sovran-hosts-update.service`` unit so
-# that the Hub can trigger it immediately after saving a domain without
-# requiring a full rebuild.
+# The same wrapped Nix-store executable is reused by both the activation hook
+# and the ``sovran-hosts-update.service`` unit, ensuring a deterministic runtime
+# PATH in every execution context.

-{
-  # ── Helper script (stored in the Nix store, never reads /var/lib at eval) ──
+let
+  # ── Wrapped Nix-store executable ──────────────────────────────────────────
+  # Built with pkgs.writeShellApplication so that all required runtime tools
+  # (awk, grep, coreutils) are declared explicitly and injected into PATH by
+  # Nix.  Both the systemd service and the activation hook reference this same
+  # store-path executable — there is no second raw script body.
+  hostsUpdateScript = pkgs.writeShellApplication {
+    name = "sovran-hosts-update";

-  environment.systemPackages = [ pkgs.coreutils ];
+    # Declare every external command the script calls.  These packages are
+    # added to the script's runtime PATH by writeShellApplication; nothing from
+    # the system PATH is relied upon.
+    runtimeInputs = [
+      pkgs.coreutils   # readlink, cp, mv, chmod, mktemp, rm, tr, head
+      pkgs.gawk        # awk
+      pkgs.gnugrep     # grep
+    ];

-  environment.etc."sovran-hosts-update.sh" = {
-    mode = "0755";
    text = ''
-      #!/bin/sh
      # Regenerate the Sovran-managed loopback block in /etc/hosts.
      # Safe to run multiple times — idempotent.
-      set -eu

      DOMAINS_DIR="/var/lib/domains"
      HOSTS_FILE="/etc/hosts"
@@ -66,13 +75,14 @@

      # ── Step 2: remove any existing Sovran block ──────────────────────────
      # Use a temp file so the operation is atomic.
+      # awk -v passes marker strings safely without shell interpolation.
      TMP=$(mktemp "$HOSTS_FILE.XXXXXX")
      trap 'rm -f "$TMP"' EXIT
-      awk "
-        /^$BEGIN_MARKER\$/ { skip=1; next }
-        /^$END_MARKER\$/   { skip=0; next }
+      awk -v begin="$BEGIN_MARKER" -v end="$END_MARKER" '
+        $0 == begin { skip=1; next }
+        $0 == end   { skip=0; next }
        !skip
-      " "$HOSTS_FILE" > "$TMP"
+      ' "$HOSTS_FILE" > "$TMP"

      # ── Step 3: collect valid configured service domains ──────────────────
      # NOTE: The hostname validation regex below must stay in sync with
@@ -97,11 +107,13 @@

      # ── Step 4: append the Sovran block if there are any entries ──────────
      if [ -n "$ENTRIES" ]; then
-        printf '\n%s\n' "$BEGIN_MARKER" >> "$TMP"
-        printf '%s\n' "# These entries route configured service domains to local Caddy." >> "$TMP"
-        printf '%s\n' "# They are managed automatically — do not edit this block." >> "$TMP"
-        printf '%s\n' "$ENTRIES" >> "$TMP"
-        printf '%s\n' "$END_MARKER" >> "$TMP"
+        {
+          printf '\n%s\n' "$BEGIN_MARKER"
+          printf '%s\n' "# These entries route configured service domains to local Caddy."
+          printf '%s\n' "# They are managed automatically — do not edit this block."
+          printf '%s\n' "$ENTRIES"
+          printf '%s\n' "$END_MARKER"
+        } >> "$TMP"
      fi

      # ── Step 5: atomically replace /etc/hosts ─────────────────────────────
@@ -110,6 +122,14 @@
    '';
  };

+in
+{
+  # ── /etc/sovran-hosts-update.sh — operator discoverability symlink ─────────
+  # Retain the familiar /etc path so administrators can inspect or manually
+  # invoke the helper.  The target is the wrapped Nix-store executable, so
+  # there is no second raw script body to keep in sync.
+  environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript;
+
  # ── Systemd service ────────────────────────────────────────────────────────

  systemd.services.sovran-hosts-update = {
@@ -126,18 +146,24 @@
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
-      ExecStart = "/etc/sovran-hosts-update.sh";
+      # Point directly at the wrapped Nix-store executable, not the /etc path.
+      ExecStart = lib.getExe hostsUpdateScript;
    };
  };

  # ── Activation script (runs after every nixos-rebuild switch) ─────────────
  # This ensures the loopback block survives rebuilds that restore the /etc/hosts
  # symlink.  The "users" and "etc" scripts must complete first.
+  # The same wrapped Nix-store executable used by the systemd service is
+  # referenced here, guaranteeing identical runtime dependencies in both
+  # execution contexts.

  system.activationScripts.sovranDomainLoopback = {
    text = ''
-      if [ -x /etc/sovran-hosts-update.sh ] && [ -d /var/lib/domains ]; then
-        /etc/sovran-hosts-update.sh || true
+      if [ -d /var/lib/domains ]; then
+        if ! ${lib.getExe hostsUpdateScript}; then
+          echo "warning: sovran-hosts-update: failed to update /etc/hosts loopback entries" >&2
+        fi
      fi
    '';
    deps = [ "etc" "users" ];
@@ -382,13 +382,25 @@ in
      };

      path = [
+        pkgs.bash
+        pkgs.gawk
        pkgs.qrencode
        pkgs.curl
        pkgs.iproute2
        pkgs.nftables
        pkgs.iptables
        pkgs.hostname
-      ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package;
+        pkgs.coreutils
+        pkgs.findutils
+        pkgs.gnugrep
+        pkgs.rsync
+        pkgs.acl
+        pkgs.util-linux
+      ]
+      ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
+      ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
+      ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
+      ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
    };

    systemd.services.sovran-hub-update = {
Author	SHA1	Message	Date
Sovran SystemsandGitHub	b98c82886f	Merge pull request #333 from naturallaw777/copilot/replace-manual-backup-implementation Replace Manual Backup: tar+DB+LND → ext4+rsync mirror	2026-07-20 22:51:38 +00:00
copilot-swe-agent[bot]andGitHub	e16eaabdde	refactor: rename CURRENT_DIR_NAME to BACKUP_SUBPATH for clarity	2026-07-20 22:47:53 +00:00
copilot-swe-agent[bot]andGitHub	0fa804a430	feat: replace tar+DB+LND backup with ext4+rsync workflow - Rewrite sovran-hub-backup.sh: rsync-based mirror to stable current/ path, ext4 validation, no tar/pg_dump/mariadb-dump/LND orchestration, exit-24 nonfatal for /home, INCOMPLETE/BACKUP_COMPLETE markers, flock - Update server.py: _is_supported_backup_fstype accepts only ext4 - Update support.js: require ext4, explain database limitations, update failure message from exFAT to ext4 - Update sovran-hub.nix: replace pkgs.gnutar with pkgs.rsync + pkgs.acl - Rewrite test_manual_backup_workflow.py: 40 new tests covering rsync options, ext4 acceptance, exit-24 handling, no-delete, stable current/ path, INCOMPLETE markers, behavioral rsync tests, and regressions	2026-07-20 22:45:04 +00:00
copilot-swe-agent[bot]andGitHub	01db44f0d2	Initial plan	2026-07-20 22:34:57 +00:00
naturallaw777	978a82ade3	nixpkgs update	2026-07-19 16:59:19 -05:00
Sovran SystemsandGitHub	c202c69898	Merge pull request #332 from naturallaw777/copilot/fix-manual-backup-failure Fix home.tar exit-1 failure on active desktop; harden full backup workflow	2026-07-18 18:21:07 +00:00
copilot-swe-agent[bot]andGitHub	d1251bf238	Final polish: explicit LOCK_FD close, tar --list SIGPIPE comment, marker timing comment, _extract_bash_function docstring	2026-07-18 18:11:19 +00:00
copilot-swe-agent[bot]andGitHub	de25110493	Address code review round 2: Python regex for function extraction, bash 4.1+ comment, find -print0 for manifest, mktemp 0600 comment	2026-07-18 18:08:33 +00:00
copilot-swe-agent[bot]andGitHub	ab31f4f60b	Address code review: source allowlist function from script in tests, fix find exclusions for BACKUP_COMPLETE, simplify log message	2026-07-18 18:05:46 +00:00
copilot-swe-agent[bot]andGitHub	f5a5e99de4	Fix test assertion pattern and replace em dashes in new log messages	2026-07-18 18:02:41 +00:00
copilot-swe-agent[bot]andGitHub	a30d4d5e74	Fix home.tar exit-1 failure: add tar tolerance, partial atomics, INCOMPLETE marker, flock, browser cache exclusions	2026-07-18 18:00:29 +00:00
copilot-swe-agent[bot]andGitHub	12b8872847	Initial plan	2026-07-18 17:44:54 +00:00
Sovran SystemsandGitHub	56e0c4dcd2	Merge pull request #331 from naturallaw777/copilot/fix-manual-backup-failure Fix Manual Backup exit-code-127: add bash+gawk to service PATH, harden launcher	2026-07-18 16:40:29 +00:00
copilot-swe-agent[bot]andGitHub	22aa251d64	Address code review: compute stderr_text only when chunks are present	2026-07-18 16:30:38 +00:00
copilot-swe-agent[bot]andGitHub	35c4de412f	Fix Manual Backup exit-code-127: add bash+gawk to service PATH, harden launcher	2026-07-18 16:29:10 +00:00
copilot-swe-agent[bot]andGitHub	16067be909	Initial plan	2026-07-18 16:23:49 +00:00
Sovran SystemsandGitHub	36abece7f9	Merge pull request #330 from naturallaw777/copilot/implement-manual-backup-workflow Implement reliable exFAT Manual Backup with tar artifacts, DB exports, and lifecycle hardening	2026-07-18 15:31:38 +00:00
copilot-swe-agent[bot]andGitHub	36187c0504	Refine backup validation and manifest details	2026-07-17 17:00:32 +00:00
copilot-swe-agent[bot]andGitHub	992c806ed7	Address validation feedback for backup workflow	2026-07-17 16:58:34 +00:00
copilot-swe-agent[bot]andGitHub	9f3d3e7670	Implement reliable exFAT manual backup workflow	2026-07-17 16:56:08 +00:00
copilot-swe-agent[bot]andGitHub	0ec8203557	Initial plan	2026-07-17 16:49:35 +00:00
Sovran SystemsandGitHub	ab4de8da7d	Merge pull request #329 from naturallaw777/copilot/fix-nix-build-regression Fix `sovran-hosts-update` ShellCheck build regression from `writeShellApplication`	2026-07-16 20:40:41 +00:00
copilot-swe-agent[bot]andGitHub	92cf417760	test: harden flake configuration detection	2026-07-16 20:38:48 +00:00
copilot-swe-agent[bot]andGitHub	ec4c1c851b	test: derive nix helper build attr from flake	2026-07-16 20:37:48 +00:00
copilot-swe-agent[bot]andGitHub	38f49e9161	fix: group sovran hosts append redirection	2026-07-16 20:36:33 +00:00
copilot-swe-agent[bot]andGitHub	c853853616	Initial plan	2026-07-16 20:33:53 +00:00
Sovran SystemsandGitHub	3e2bc106b1	Merge pull request #328 from naturallaw777/copilot/fix-sovran-hosts-update-runtime-dependency fix(local-domain-loopback): replace raw /etc script with writeShellApplication, declare explicit runtimeInputs	2026-07-16 20:28:39 +00:00
copilot-swe-agent[bot]andGitHub	d4f8c7b431	fix: convert sovran-hosts-update to writeShellApplication with explicit runtimeInputs - Replace environment.etc raw script with pkgs.writeShellApplication - Declare runtimeInputs: pkgs.coreutils, pkgs.gawk, pkgs.gnugrep - Use awk -v for safe marker variable passing (no shell interpolation) - Point systemd ExecStart and activation script at lib.getExe hostsUpdateScript - Keep /etc/sovran-hosts-update.sh as a source symlink for operator discoverability - Remove environment.systemPackages reliance - Emit warning (not silently swallow) on activation failure - Add structural regression tests (19 new tests, all passing)	2026-07-16 20:26:59 +00:00
copilot-swe-agent[bot]andGitHub	dcbac4760f	Initial plan	2026-07-16 20:23:44 +00:00