Merge pull request #334 from naturallaw777/copilot/fix-manual-backup-rsync-failure

Fix rsync destination-directory failure: sync_tree with mkdir -p, mount re-verification, and path-safety guards

app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # ── Sovran Hub External Backup Script ────────────────────────────
-# Backs up Sovran_SystemsOS data to an external USB hard drive.
+# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
 # Designed for the Hub web UI (no GUI dependencies).
 #
 # Your Sovran Pro already backs up your data automatically to its
@@ -8,6 +8,15 @@
 # This script creates an additional copy on an external USB drive —
 # storing your data in a third location for maximum protection.
 #
+# The external drive must be formatted as ext4. Files are stored as
+# directly browsable files under Sovran_SystemsOS_Backup/current/.
+# Later runs update the same mirror and only transfer changed or new
+# files, making repeat backups fast.
+#
+# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
+# blockchain and Electrs index data are NOT included (they live on
+# the internal second drive).
+#
 # Usage:
 #   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
 #   (or run with no env var to auto-detect the first external USB drive)
@@ -17,18 +26,28 @@ set -euo pipefail
 BACKUP_LOG="/var/log/sovran-hub-backup.log"
 BACKUP_STATUS="/var/log/sovran-hub-backup.status"
 MEDIA_ROOT="/run/media"
-MIN_FREE_GB=10
 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
 ROLE_STATE_NIX="/etc/nixos/role-state.nix"
+SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
+SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))
 
 # ── Internal drive labels/paths to NEVER use as backup targets ───
 INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
-INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/")
+INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
+
+FAILED_ALREADY=0
+BACKUP_COMPLETE=0
+RSYNC_WARNINGS=()
+
+# Stable rsync mirror sub-path under the target drive. Not timestamped
+# so later runs update the same destination and only transfer new or changed files.
+BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"
 
 # ── Logging helpers ──────────────────────────────────────────────
 
 log() {
-  local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+  local msg
+  msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
   echo "$msg" | tee -a "$BACKUP_LOG"
 }
 
@@ -37,16 +56,47 @@ set_status() {
 }
 
 fail() {
+  FAILED_ALREADY=1
   log "ERROR: $*"
   set_status "FAILED"
   exit 1
 }
 
+cleanup() {
+  local rc=$?
+
+  # Release the concurrency lock file descriptor if it was opened
+  if [[ -n "${LOCK_FD:-}" ]]; then
+    exec {LOCK_FD}>&- 2>/dev/null || true
+  fi
+
+  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
+    return
+  fi
+
+  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
+    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
+    set_status "FAILED"
+  fi
+
+  # Mark the backup directory as incomplete so failed runs are identifiable
+  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
+    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
+  fi
+}
+
+trap cleanup EXIT
+trap 'exit 1' INT TERM
+
+require_cmd() {
+  local cmd="$1"
+  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
+}
+
 # ── Check whether a mount point is an internal drive ────────────
 
 is_internal() {
   local mnt="$1"
-  # Reject known internal mount points and their subdirectories
   for internal in "${INTERNAL_MOUNTS[@]}"; do
     if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
       return 0
@@ -59,39 +109,37 @@ is_internal() {
 
 find_external_drive() {
   local target=""
-  # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
-  if command -v lsblk &>/dev/null; then
-    while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
-      # Must be a partition or disk, and be removable/hotplug
-      [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
-      [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
-      [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal labels
-      local skip=0
-      for lbl in "${INTERNAL_LABELS[@]}"; do
-        [[ "$label" == "$lbl" ]] && skip=1 && break
-      done
-      [[ "$skip" -eq 1 ]] && continue
+  while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
+    [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
+    [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
+    [[ -n "$mountpoint" ]] || continue
 
-      # Filter out internal mount points
-      is_internal "$mountpoint" && continue
+    local skip=0
+    for lbl in "${INTERNAL_LABELS[@]}"; do
+      [[ "$label" == "$lbl" ]] && skip=1 && break
+    done
+    [[ "$skip" -eq 1 ]] && continue
 
-      if mountpoint -q "$mountpoint" 2>/dev/null; then
-        target="$mountpoint"
-        break
-      fi
-    done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
-      python3 -c "
+    is_internal "$mountpoint" && continue
+
+    if mountpoint -q "$mountpoint" 2>/dev/null; then
+      target="$mountpoint"
+      break
+    fi
+  done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
+    python3 -c "
 import sys, json
-data = json.load(sys.stdin)
+
 def flatten(devs):
     for d in devs:
         yield d
         for c in d.get('children', []):
             yield from flatten([c])
+
+data = json.load(sys.stdin)
 for d in flatten(data.get('blockdevices', [])):
-    print('\t'.join([
+    print('\\t'.join([
         d.get('type') or '',
         str(d.get('hotplug') or '0'),
         str(d.get('rm') or '0'),
@@ -99,24 +147,10 @@ for d in flatten(data.get('blockdevices', [])):
         d.get('mountpoint') or '',
     ]))
 " 2>/dev/null || true)
-  fi
 
-  # Fallback: walk /run/media/ if lsblk produced nothing
   if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
     while IFS= read -r -d '' mnt; do
       is_internal "$mnt" && continue
-      # Check label via lsblk on the device backing this mount
-      local dev
-      dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
-      if [[ -n "$dev" ]]; then
-        local lbl
-        lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
-        local skip=0
-        for internal_lbl in "${INTERNAL_LABELS[@]}"; do
-          [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
-        done
-        [[ "$skip" -eq 1 ]] && continue
-      fi
       if mountpoint -q "$mnt" 2>/dev/null; then
         target="$mnt"
         break
@@ -128,16 +162,10 @@ for d in flatten(data.get('blockdevices', [])):
 }
 
 # ── Detect the configured system role ───────────────────────────
-#
-# Priority:
-#   1. Hub config JSON  (/var/lib/sovran-hub/config.json)  — "role" key
-#   2. role-state.nix   (/etc/nixos/role-state.nix)        — grep for true flag
-#   3. Default: server_plus_desktop
 
 detect_role() {
   local role="server_plus_desktop"
 
-  # 1. Try the Hub config JSON
   if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
     local r
     r=$(python3 -c \
@@ -149,7 +177,6 @@ detect_role() {
     fi
   fi
 
-  # 2. Fall back to parsing role-state.nix
   if [[ -f "$ROLE_STATE_NIX" ]]; then
     if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
       role="desktop"
@@ -161,6 +188,119 @@ detect_role() {
   echo "$role"
 }
 
+validate_target_mount() {
+  local target="$1"
+  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
+  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
+  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
+
+  local fstype=""
+  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
+  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
+
+  if [[ "$fstype" != "ext4" ]]; then
+    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
+  fi
+
+  local write_test
+  write_test="$target/.sovran-write-test-$$"
+  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
+    fail "Target '$target' is not writable."
+  fi
+
+  log "Verified backup target filesystem: $fstype"
+}
+
+estimate_path_bytes() {
+  local path="$1"
+  shift || true
+  [[ -e "$path" ]] || {
+    echo 0
+    return
+  }
+
+  local size
+  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
+  [[ -n "$size" ]] || size=0
+  echo "$size"
+}
+
+# ── Sync one source tree to its backup destination ───────────────
+# Usage: sync_tree <label> <allow_vanished> <source> <destination> [rsync options...]
+#
+# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
+# Used for /home only — files may disappear while the desktop is active.
+# All other nonzero exit codes are always fatal.
+#
+# Before every rsync call this helper:
+#   1. Re-verifies $TARGET is still a mount point (fails if drive disconnected).
+#   2. Verifies the destination path remains beneath $BACKUP_DIR and $BACKUP_DIR
+#      remains beneath $TARGET (safe-path check).
+#   3. Creates the full destination directory hierarchy with mkdir -p so that
+#      rsync never fails trying to create a directory whose parent is absent.
+sync_tree() {
+  local label="$1"
+  local allow_vanished="$2"
+  local source="$3"
+  local destination="$4"
+  shift 4
+  # Remaining "$@" are rsync options (--exclude, etc.)
+
+  # ── Re-verify the external drive is still mounted ────────────────
+  mountpoint -q "$TARGET" 2>/dev/null || \
+    fail "Stage $label: external drive '$TARGET' is no longer mounted. Refusing to write."
+
+  # ── Verify path safety ────────────────────────────────────────────
+  # BACKUP_DIR must remain beneath TARGET.
+  case "$BACKUP_DIR" in
+    "$TARGET"/*) ;;
+    *) fail "Stage $label: BACKUP_DIR '$BACKUP_DIR' is outside TARGET '$TARGET'." ;;
+  esac
+  # Destination must remain beneath BACKUP_DIR.
+  case "$destination" in
+    "$BACKUP_DIR"/*|"$BACKUP_DIR") ;;
+    *) fail "Stage $label: destination '$destination' is outside BACKUP_DIR '$BACKUP_DIR'. Refusing to write." ;;
+  esac
+
+  # ── Create complete destination directory hierarchy ───────────────
+  # This is the fix for the production failure:
+  #   rsync: [Receiver] mkdir ".../current/etc/nixos" failed: No such file or directory
+  # mkdir -p creates all intermediate parents (e.g. current/etc/) before rsync runs.
+  mkdir -p -- "$destination" || \
+    fail "Stage $label: failed to create destination directory '$destination' (source: '$source')."
+
+  local rsync_err_tmp
+  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
+
+  local rc=0
+  rsync \
+    --archive \
+    --acls \
+    --xattrs \
+    --hard-links \
+    --numeric-ids \
+    --one-file-system \
+    --partial \
+    "$@" "$source" "$destination" 2>"$rsync_err_tmp" || rc=$?
+
+  if [[ -s "$rsync_err_tmp" ]]; then
+    while IFS= read -r rline; do
+      log "rsync: $rline"
+    done < "$rsync_err_tmp"
+  fi
+  rm -f "$rsync_err_tmp"
+
+  if [[ "$rc" -eq 0 ]]; then
+    return 0
+  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
+    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
+    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
+    return 0
+  else
+    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
+  fi
+}
+
 # ── Initialise log file ──────────────────────────────────────────
 
 : > "$BACKUP_LOG"
@@ -169,14 +309,38 @@ set_status "RUNNING"
 log "=== Sovran_SystemsOS External Hub Backup ==="
 log "Starting backup process…"
 
+# ── Acquire exclusive run lock ────────────────────────────────────
+# Prevents two simultaneous backup runs (e.g. from double-click or
+# stale RUNNING status after a Hub restart).
+
+LOCK_FILE="/var/lock/sovran-hub-backup.lock"
+# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
+exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
+  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
+flock --nonblock "$LOCK_FD" 2>/dev/null || \
+  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
+
+require_cmd rsync
+require_cmd findmnt
+require_cmd lsblk
+require_cmd mountpoint
+require_cmd df
+require_cmd du
+require_cmd awk
+require_cmd find
+require_cmd hostname
+require_cmd date
+require_cmd python3
+require_cmd flock
+
 # ── Detect system role ───────────────────────────────────────────
 
 ROLE="$(detect_role)"
 case "$ROLE" in
-  desktop)            ROLE_LABEL="Desktop Only" ;;
-  node)               ROLE_LABEL="Node (Bitcoin-only)" ;;
-  server_plus_desktop) ROLE_LABEL="Server + Desktop" ;;
-  *)                  ROLE_LABEL="$ROLE" ;;
+  desktop)              ROLE_LABEL="Desktop Only" ;;
+  node)                 ROLE_LABEL="Node (Bitcoin-only)" ;;
+  server_plus_desktop)  ROLE_LABEL="Server + Desktop" ;;
+  *)                    ROLE_LABEL="$ROLE" ;;
 esac
 log "Detected role: $ROLE_LABEL"
 
@@ -184,7 +348,6 @@ log "Detected role: $ROLE_LABEL"
 
 if [[ -n "${BACKUP_TARGET:-}" ]]; then
   TARGET="$BACKUP_TARGET"
-  # Safety: never allow internal drives even if explicitly passed
   if is_internal "$TARGET"; then
     fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
   fi
@@ -193,39 +356,74 @@ else
   log "Auto-detecting external USB drives…"
   TARGET="$(find_external_drive)"
   if [[ -z "$TARGET" ]]; then
-    fail "No external USB drive detected. " \
-         "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
+    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
   fi
   log "Detected external drive: $TARGET"
 fi
 
-# ── Verify mount point ───────────────────────────────────────────
+validate_target_mount "$TARGET"
 
-[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist."
-mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
+# ── Set up stable backup destination ────────────────────────────
+# Subsequent runs update the same mirror, transferring only new or changed files.
 
-# ── Check free disk space (require ≥ 10 GB) ──────────────────────
+BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
+mkdir -p -- "$BACKUP_DIR"
 
-FREE_KB=$(df -k --output=avail "$TARGET" | tail -1)
-FREE_GB=$(( FREE_KB / 1024 / 1024 ))
-log "Free space on drive: ${FREE_GB} GB"
-(( FREE_GB >= MIN_FREE_GB )) || \
-  fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)."
+# Remove any stale BACKUP_COMPLETE left by a previous successful run.
+# The new run will re-earn it only after all stages succeed.
+rm -f "$BACKUP_DIR/BACKUP_COMPLETE"
 
-# ── Create timestamped backup directory ─────────────────────────
-
-TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
-BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
-mkdir -p "$BACKUP_DIR"
+# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
+# after all rsync stages and manifest write succeed. Failed or interrupted
+# runs keep this marker so they are clearly identifiable.
+touch "$BACKUP_DIR/INCOMPLETE"
 log "Backup destination: $BACKUP_DIR"
 
+# ── Estimate required free space ─────────────────────────────────
+# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
+# from the estimate to avoid inflating the required size.
+
+ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
+HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
+SECRETS_BYTES=0
+if [[ "$ROLE" != "desktop" ]]; then
+  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
+fi
+
+VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
+  --exclude='postgresql' \
+  --exclude='mysql' \
+  --exclude='mariadb' \
+  --exclude='bitcoind' \
+  --exclude='electrs' \
+  --exclude='*/log' \
+  --exclude='*/logs' \
+  --exclude='*/cache' \
+  --exclude='*/tmp')
+
+ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
+# Require 20% growth headroom plus a fixed 1 GiB safety margin.
+# Later incremental runs need far less space, but a conservative first-run
+# check protects against running out of space mid-backup.
+REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
+
+FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
+FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
+REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
+
+log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
+log "Required free space (with safety margin): ${REQUIRED_GB} GB"
+log "Free space on drive: ${FREE_GB} GB"
+
+(( FREE_BYTES >= REQUIRED_BYTES )) || \
+  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
+
 # ── Stage 1/4: NixOS configuration ──────────────────────────────
 
 log ""
 log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
 if [[ -d /etc/nixos ]]; then
-  rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 1 failed while copying /etc/nixos"
+  sync_tree "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
   log "Stage 1 complete."
 else
   log "WARNING: /etc/nixos not found — skipping."
@@ -234,64 +432,64 @@ fi
 # ── Stage 2/4: Secrets ──────────────────────────────────────────
 
 log ""
-log "── Stage 2/4: Secrets ───────────────────────────────────────"
-mkdir -p "$BACKUP_DIR/secrets"
-
+log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
 if [[ "$ROLE" == "desktop" ]]; then
   log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
+elif [[ -e /etc/nix-bitcoin-secrets ]]; then
+  sync_tree "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
 else
-  if [[ -e /etc/nix-bitcoin-secrets ]]; then
-    rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
-  else
-    log "  (not found: /etc/nix-bitcoin-secrets — skipping)"
-  fi
+  log "(not found: /etc/nix-bitcoin-secrets — skipping)"
 fi
-
 log "Stage 2 complete."
 
-# ── Stage 3/4: Home directory ────────────────────────────────────
+# ── Stage 3/4: Home directory ───────────────────────────────────
+# Rsync exit code 24 (vanished source files) is treated as nonfatal here
+# because the desktop may be active and files can disappear between the
+# directory scan and the copy. All other nonzero exit codes remain fatal.
 
 log ""
 log "── Stage 3/4: Home directory (/home) ───────────────────────"
 if [[ -d /home ]]; then
-  rsync -a --info=progress2 \
+  sync_tree "/home" yes /home/ "$BACKUP_DIR/home/" \
     --exclude='.cache/' \
     --exclude='.local/share/Trash/' \
-    --exclude='*/Trash/' \
-    /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 3 failed while copying /home"
+    --exclude='Trash/' \
+    --exclude='.mozilla/firefox/*/cache2/' \
+    --exclude='.mozilla/firefox/*/startupCache/' \
+    --exclude='.mozilla/firefox/*/thumbnails/' \
+    --exclude='.config/google-chrome/*/Cache/' \
+    --exclude='.config/google-chrome/*/Code Cache/' \
+    --exclude='.config/chromium/*/Cache/' \
+    --exclude='.config/chromium/*/Code Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
+    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
+    --exclude='.local/share/baloo/' \
+    --exclude='.thumbnails/' \
+    --exclude='.xsession-errors' \
+    --exclude='.xsession-errors.old'
   log "Stage 3 complete."
 else
   log "WARNING: /home not found — skipping."
 fi
 
-# ── Stage 4/4: System data ───────────────────────────────────────
+# ── Stage 4/4: System data ──────────────────────────────────────
+# PostgreSQL/MariaDB raw database directories are excluded. Application
+# databases must be backed up separately with native database tools.
+# Bitcoin/Electrs data are excluded; they live on the internal second drive.
 
 log ""
-log "── Stage 4/4: System data (/var/lib) ────────────────────────"
-if [[ "$ROLE" == "desktop" ]]; then
-  if [[ -d /var/lib ]]; then
-    rsync -a --info=progress2 \
-      --filter='- /lnd/***' \
-      --exclude='logs/' \
-      --exclude='log/' \
-      --exclude='*/logs/' \
-      --exclude='*/log/' \
-      /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-      fail "Stage 4 failed while copying /var/lib for Desktop Only role"
-    log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)."
-  else
-    log "WARNING: /var/lib not found — skipping."
-  fi
-elif [[ -d /var/lib ]]; then
-  rsync -a --info=progress2 \
-    --exclude='logs/' \
-    --exclude='log/' \
-    --exclude='*/logs/' \
+log "── Stage 4/4: System data (/var/lib) ───────────────────────"
+if [[ -d /var/lib ]]; then
+  sync_tree "/var/lib" no /var/lib/ "$BACKUP_DIR/var/lib/" \
+    --exclude='postgresql/' \
+    --exclude='mysql/' \
+    --exclude='mariadb/' \
+    --exclude='bitcoind/' \
+    --exclude='electrs/' \
     --exclude='*/log/' \
-    /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
-    fail "Stage 4 failed while copying /var/lib"
+    --exclude='*/logs/' \
+    --exclude='*/cache/' \
+    --exclude='*/tmp/'
   log "Stage 4 complete."
 else
   log "WARNING: /var/lib not found — skipping."
@@ -301,21 +499,91 @@ fi
 
 log ""
 log "Generating BACKUP_MANIFEST.txt …"
+MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
+
 {
   echo "Sovran_SystemsOS Backup Manifest"
-  echo "Generated: $(date)"
+  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
   echo "Hostname:  $(hostname)"
   echo "Role:      $ROLE_LABEL"
   echo "Target:    $TARGET"
   echo ""
-  echo "Contents:"
-  find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort
-} > "$BACKUP_DIR/BACKUP_MANIFEST.txt"
-log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt"
+  echo "Backup type: Live rsync mirror (directly browsable files)"
+  echo "Location:    ${BACKUP_DIR}"
+  echo ""
+  echo "Source paths mirrored:"
+  echo "- /etc/nixos  →  current/etc/nixos/"
+  if [[ "$ROLE" != "desktop" ]]; then
+    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
+  fi
+  echo "- /home  →  current/home/"
+  echo "- /var/lib  →  current/var/lib/"
+  echo ""
+  echo "Exclusions:"
+  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
+  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
+  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
+  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
+  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
+  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
+  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
+  echo ""
+  echo "Important limitations:"
+  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
+  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
+  echo "  applications, their data must be backed up separately using native tools."
+  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
+  echo "  reconstructable or stored on the internal second drive."
+  echo "- This is a live file-level mirror, not a transactional database backup."
+  echo "  Files being written during the backup may be in an inconsistent state."
+  echo ""
+  echo "Restore guidance:"
+  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
+  echo "- To restore a directory:"
+  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
+  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
+  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
+  echo "- To copy individual files:"
+  echo "    sudo cp -a current/home/username/ /home/username/"
+  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
+  echo "  hardware-configuration.nix for the new hardware before rebuilding."
+  echo ""
+  echo "Nonfatal warnings:"
+  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
+    echo "- none"
+  else
+    for warning in "${RSYNC_WARNINGS[@]}"; do
+      echo "- $warning"
+    done
+  fi
+  echo ""
+  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
+  echo "from manual external backup because they already live on the internal second drive"
+  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
+} > "$MANIFEST_FILE"
+
+log "Manifest written to $MANIFEST_FILE"
 
 # ── Done ─────────────────────────────────────────────────────────
 
 log ""
+if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
+  log "Backup completed with nonfatal warnings:"
+  for warning in "${RSYNC_WARNINGS[@]}"; do
+    log "  WARNING: $warning"
+  done
+  log "Your important data is backed up. The warnings above indicate files that"
+  log "vanished during backup, which is normal on an active desktop."
+  log ""
+fi
 log "All Finished! Your data is now backed up to a third location."
+log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
 log "Please eject the drive safely before removing it from your Sovran Pro."
+
+# Remove incomplete marker and write completion marker only after all work succeeds.
+# A later successful run will update the same mirror and replace any INCOMPLETE state.
+rm -f "$BACKUP_DIR/INCOMPLETE"
+date -u '+%Y-%m-%dT%H:%M:%SZ' > "$BACKUP_DIR/BACKUP_COMPLETE"
+
+BACKUP_COMPLETE=1
 set_status "SUCCESS"

app/sovran_systemsos_web/server.py

@@ -1451,6 +1451,12 @@ def _read_backup_status() -> str:
         return "IDLE"
 
 
+def _write_backup_status(value: str) -> None:
+    """Write backup status file."""
+    with open(BACKUP_STATUS, "w") as f:
+        f.write(value)
+
+
 def _read_backup_log(offset: int = 0) -> tuple[str, int]:
     """Read the backup log file from the given byte offset.
     Returns (new_text, new_offset)."""
@@ -1467,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]:
         return "", 0
 
 
+def _append_backup_log(line: str) -> None:
+    """Append one line to backup log."""
+    with open(BACKUP_LOG, "a") as f:
+        f.write(line.rstrip("\n") + "\n")
+
+
 _INTERNAL_LABELS  = {"BTCEcoandBackup", "sovran_systemsos"}
 _INTERNAL_MOUNTS  = {"/", "/boot/efi"}
 _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive"
@@ -1481,6 +1493,16 @@ def _is_internal_mount(mnt: str) -> bool:
     return False
 
 
+def _is_supported_backup_fstype(path: str, fstype: str) -> bool:
+    """Return whether the target filesystem type is supported for manual backup.
+
+    Manual Backup requires ext4 for Linux metadata preservation (ACLs, xattrs,
+    hard links). exFAT, FAT32, NTFS, and other filesystems are not supported.
+    """
+    fstype = (fstype or "").lower()
+    return fstype == "ext4"
+
+
 def _detect_external_drives() -> list[dict]:
     """Scan for mounted external USB drives.
 
@@ -1490,7 +1512,8 @@ def _detect_external_drives() -> list[dict]:
     /run/media/ directly if lsblk is unavailable, applying the same
     label/path filters.
 
-    Returns a list of dicts with name, path, free_gb, total_gb.
+    Returns:
+        list[dict]: Each dict contains name, path, free_gb, total_gb, fstype.
     """
     import json as _json
     import subprocess as _subprocess
@@ -1501,7 +1524,7 @@ def _detect_external_drives() -> list[dict]:
     # ── Primary path: lsblk JSON ────────────────────────────────
     try:
         result = _subprocess.run(
-            ["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"],
+            ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"],
             capture_output=True, text=True, timeout=10
         )
         if result.returncode == 0:
@@ -1519,6 +1542,7 @@ def _detect_external_drives() -> list[dict]:
                 hotplug    = str(dev.get("hotplug", "0"))
                 rm         = str(dev.get("rm", "0"))
                 label      = dev.get("label") or ""
+                fstype     = (dev.get("fstype") or "").lower()
                 mountpoint = dev.get("mountpoint") or ""
 
                 if dev_type not in ("part", "disk"):
@@ -1544,6 +1568,7 @@ def _detect_external_drives() -> list[dict]:
                         "path":     mountpoint,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(mountpoint)
                 except OSError:
@@ -1577,11 +1602,20 @@ def _detect_external_drives() -> list[dict]:
                     st = os.statvfs(drive_path)
                     total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1)
                     free_gb  = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1)
+                    fstype = ""
+                    try:
+                        fstype = _subprocess.run(
+                            ["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path],
+                            capture_output=True, text=True, timeout=5
+                        ).stdout.strip().lower()
+                    except Exception:
+                        fstype = ""
                     drives.append({
                         "name":     drive_name,
                         "path":     drive_path,
                         "free_gb":  free_gb,
                         "total_gb": total_gb,
+                        "fstype":   fstype,
                     })
                     seen_paths.add(drive_path)
                 except OSError:
@@ -3682,6 +3716,37 @@ async def api_backup_drives():
     return {"drives": drives}
 
 
+async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
+    """Drain stderr, then mark status FAILED if backup subprocess exits unexpectedly."""
+    stderr_chunks: list[bytes] = []
+
+    async def _drain_stderr() -> None:
+        if proc.stderr is not None:
+            async for line in proc.stderr:
+                stderr_chunks.append(line)
+
+    drain_task = asyncio.create_task(_drain_stderr())
+    rc = await proc.wait()
+    await drain_task
+
+    if rc == 0:
+        return
+
+    loop = asyncio.get_event_loop()
+    status = await loop.run_in_executor(None, _read_backup_status)
+    if status in {"SUCCESS", "FAILED"}:
+        return
+
+    detail = ""
+    if stderr_chunks:
+        stderr_text = b"".join(stderr_chunks).decode("utf-8", errors="replace").strip()
+        if stderr_text:
+            detail = f" — stderr: {stderr_text}"
+    msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc}).{detail}"
+    await loop.run_in_executor(None, _append_backup_log, msg)
+    await loop.run_in_executor(None, _write_backup_status, "FAILED")
+
+
 @app.post("/api/backup/run")
 async def api_backup_run(target: str = ""):
     """Start the backup script as a background subprocess.
@@ -3692,6 +3757,26 @@ async def api_backup_run(target: str = ""):
     if status == "RUNNING":
         return {"ok": True, "status": "already_running"}
 
+    drives = await loop.run_in_executor(None, _detect_external_drives)
+    if not drives:
+        raise HTTPException(status_code=400, detail="No external backup drive detected.")
+
+    drive_map = {d.get("path", ""): d for d in drives if d.get("path")}
+    if target:
+        if target not in drive_map:
+            raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.")
+        selected = drive_map[target]
+    else:
+        selected = drives[0]
+
+    selected_target = selected.get("path", "")
+    selected_fstype = (selected.get("fstype") or "").lower()
+    if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup. Manual Backup requires an ext4-formatted drive.",
+        )
+
     # Clear stale log before starting
     try:
         with open(BACKUP_LOG, "w") as f:
@@ -3699,21 +3784,48 @@ async def api_backup_run(target: str = ""):
     except OSError:
         pass
 
-    env = dict(os.environ)
-    if target:
-        env["BACKUP_TARGET"] = target
+    try:
+        await loop.run_in_executor(None, _write_backup_status, "RUNNING")
+    except OSError as exc:
+        raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}")
 
-    # Fire-and-forget: the script writes its own status/log files.
-    # Progress is read by the client via /api/backup/status (same pattern
-    # as /api/updates/run and the rebuild feature).
-    await asyncio.create_subprocess_exec(
-        "/usr/bin/env", "bash", BACKUP_SCRIPT,
-        stdout=asyncio.subprocess.DEVNULL,
-        stderr=asyncio.subprocess.DEVNULL,
-        env=env,
+    await loop.run_in_executor(
+        None,
+        _append_backup_log,
+        f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…",
     )
 
-    return {"ok": True, "status": "started"}
+    env = dict(os.environ)
+    env["BACKUP_TARGET"] = selected_target
+
+    bash_path = shutil.which("bash")
+    if bash_path is None:
+        no_bash_msg = (
+            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Cannot start backup:"
+            " interpreter 'bash' not found on PATH."
+            " Ensure pkgs.bash is in the sovran-hub-web service PATH."
+        )
+        await loop.run_in_executor(None, _append_backup_log, no_bash_msg)
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(
+            status_code=500,
+            detail="Backup interpreter (bash) not available. Check service PATH configuration.",
+        )
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bash_path, BACKUP_SCRIPT,
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except Exception as exc:
+        await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}")
+        await loop.run_in_executor(None, _write_backup_status, "FAILED")
+        raise HTTPException(status_code=500, detail="Failed to launch backup process.")
+
+    asyncio.create_task(_monitor_backup_subprocess(proc))
+    return {"ok": True, "status": "started", "target": selected_target}
 
 
 # ── Feature Manager endpoints ─────────────────────────────────────

app/sovran_systemsos_web/static/js/support.js

@@ -491,8 +491,9 @@ function renderBackupReady(drives) {
       '<div class="support-steps-title">Requirements</div>',
       '<ol class="support-backup-steps">',
         '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
-        '<li>At least 500 GB of free space on the drive</li>',
-        '<li>Drive must be formatted as <strong>exFAT</strong></li>',
+        '<li>Enough free space for your data (the backup checks this before starting)</li>',
+        '<li>Drive must be formatted as <strong>ext4</strong> (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or <code>mkfs.ext4</code> — note that formatting erases all data on the drive.</li>',
+        '<li>The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.</li>',
       '</ol>',
     '</div>',
 
@@ -501,17 +502,25 @@ function renderBackupReady(drives) {
       '<ol class="support-backup-steps">',
         '<li>NixOS configuration (<code>/etc/nixos</code>)</li>',
         '<li>nix-bitcoin secrets (<code>/etc/nix-bitcoin-secrets</code>)</li>',
-        '<li>System service data (<code>/var/lib</code>) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets</li>',
+        '<li>System service data (<code>/var/lib</code>) — excluding databases and blockchain data (see note below)</li>',
         '<li>Home directory (<code>/home</code>)</li>',
       '</ol>',
     '</div>',
 
     '<div class="support-wallet-box support-wallet-warning">',
+      '<div class="support-wallet-header">',
+        '<span class="support-wallet-icon">\u2139\ufe0f</span>',
+        '<span class="support-wallet-title">Database and Blockchain Data</span>',
+      '</div>',
+      '<p class="support-wallet-desc">Application databases stored in PostgreSQL or MariaDB/MySQL are <strong>not included</strong> in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.</p>',
+    '</div>',
+
+    '<div class="support-wallet-box support-wallet-protected">',
       '<div class="support-wallet-header">',
         '<span class="support-wallet-icon">\u23f1\ufe0f</span>',
         '<span class="support-wallet-title">Time Estimate</span>',
       '</div>',
-      '<p class="support-wallet-desc">This backup can take <strong>up to 4 hours</strong> depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026</p>',
+      '<p class="support-wallet-desc">The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.</p>',
     '</div>',
 
     driveSelector,
@@ -584,9 +593,10 @@ async function pollBackupStatus() {
       logDiv.scrollTop = logDiv.scrollHeight;
     }
     _backupLogOffset = data.offset;
-    if (!data.running) {
+    const result = (data.result || "").toLowerCase();
+    if (result === "success" || result === "failed") {
       stopBackupPoll();
-      renderBackupDone(data.result === "success");
+      renderBackupDone(result === "success");
     }
   } catch (_) {}
 }
@@ -618,7 +628,7 @@ function renderBackupDone(success) {
       '<div class="support-section">',
       '<div class="support-icon-big">\u26a0\ufe0f</div>',
       '<h3 class="support-heading">Backup Failed</h3>',
-      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.</p>',
+      '<p class="support-desc">The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.</p>',
       '<div class="modal-log" id="backup-log-fail" style="text-align:left;"></div>',
       '<button class="btn support-btn-done" id="btn-backup-close">Close</button>',
       '</div>',

app/tests/test_local_domain_loopback_nix.py

@@ -0,0 +1,193 @@
+"""Structural regression tests for modules/core/local-domain-loopback.nix.
+
+Verifies that the sovran-hosts-update helper:
+  - is built as a pkgs.writeShellApplication with explicit runtimeInputs
+    (gawk, gnugrep, coreutils);
+  - uses ``lib.getExe hostsUpdateScript`` for both the systemd ExecStart and
+    the activation script so that both contexts share the same Nix-store
+    executable;
+  - does NOT point ExecStart at the raw /etc path;
+  - includes element-calling in the supported domain list;
+  - uses ``awk -v`` for safe marker-variable passing rather than interpolating
+    marker text directly into the awk program;
+  - retains the domain validation regex (idempotency / injection prevention);
+  - exposes /etc/sovran-hosts-update.sh as a symlink via ``source =`` (not a
+    second raw ``text =`` body);
+  - does NOT rely on environment.systemPackages for the helper's dependencies.
+"""
+
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+NIX_STRING_INDENT = 6
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FLAKE_SOURCE = (REPO_ROOT / "flake.nix").read_text()
+PRIMARY_NIXOS_CONFIGURATION_MATCH = re.search(
+    r"nixosConfigurations\.([A-Za-z0-9_-]+)\s*=",
+    FLAKE_SOURCE,
+)
+if PRIMARY_NIXOS_CONFIGURATION_MATCH is None:
+    raise RuntimeError("Could not determine the primary nixosConfigurations entry from flake.nix")
+PRIMARY_NIXOS_CONFIGURATION = PRIMARY_NIXOS_CONFIGURATION_MATCH.group(1)
+HELPER_BUILD_ATTR = (
+    f'.#nixosConfigurations.{PRIMARY_NIXOS_CONFIGURATION}.config.environment.etc.'
+    '"sovran-hosts-update.sh".source'
+)
+NIX_FILE = (
+    REPO_ROOT
+    / "modules"
+    / "core"
+    / "local-domain-loopback.nix"
+)
+
+
+class LocalDomainLoopbackNixStructureTests(unittest.TestCase):
+    def setUp(self):
+        self.source = NIX_FILE.read_text()
+
+    def _helper_script(self) -> str:
+        start = self.source.index("text = ''") + len("text = ''")
+        end = self.source.index("    '';", start)
+        return "\n".join(
+            line[NIX_STRING_INDENT:]
+            if line.startswith(" " * NIX_STRING_INDENT)
+            else line
+            for line in self.source[start:end].splitlines()
+        ).lstrip("\n")
+
+    # ── writeShellApplication and explicit runtimeInputs ────────────────────
+
+    def test_uses_write_shell_application(self):
+        self.assertIn("pkgs.writeShellApplication", self.source)
+
+    def test_runtime_inputs_includes_coreutils(self):
+        self.assertIn("pkgs.coreutils", self.source)
+
+    def test_runtime_inputs_includes_gawk(self):
+        self.assertIn("pkgs.gawk", self.source)
+
+    def test_runtime_inputs_includes_gnugrep(self):
+        self.assertIn("pkgs.gnugrep", self.source)
+
+    def test_runtime_inputs_block_present(self):
+        self.assertIn("runtimeInputs", self.source)
+
+    # ── Both execution paths use lib.getExe ─────────────────────────────────
+
+    def test_exec_start_uses_lib_get_exe(self):
+        """systemd ExecStart must reference the Nix-store executable."""
+        self.assertIn("ExecStart = lib.getExe hostsUpdateScript", self.source)
+
+    def test_activation_script_uses_lib_get_exe(self):
+        """Activation text must call the same Nix-store executable."""
+        self.assertIn("${lib.getExe hostsUpdateScript}", self.source)
+
+    def test_exec_start_does_not_point_to_etc_path(self):
+        """ExecStart must NOT use the raw /etc path (which lacks a deterministic PATH)."""
+        self.assertNotIn('ExecStart = "/etc/sovran-hosts-update.sh"', self.source)
+
+    # ── /etc symlink uses source =, not a second text = body ────────────────
+
+    def test_etc_entry_uses_source_not_text(self):
+        """The /etc/sovran-hosts-update.sh entry must be a symlink (source =),
+        not a second raw script body (text =)."""
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source', self.source
+        )
+
+    def test_etc_source_points_to_get_exe(self):
+        self.assertIn(
+            'environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript',
+            self.source,
+        )
+
+    # ── element-calling domain is supported ─────────────────────────────────
+
+    def test_element_calling_domain_key_present(self):
+        self.assertIn("element-calling", self.source)
+
+    # ── Robust awk -v variable passing ──────────────────────────────────────
+
+    def test_awk_uses_dash_v_for_begin_marker(self):
+        """awk must receive the begin marker via -v, not by shell interpolation."""
+        self.assertIn('awk -v begin=', self.source)
+
+    def test_awk_uses_dash_v_for_end_marker(self):
+        self.assertIn('-v end=', self.source)
+
+    def test_awk_does_not_interpolate_marker_into_program(self):
+        """The old pattern interpolated $BEGIN_MARKER directly into the awk source."""
+        self.assertNotIn('/$BEGIN_MARKER', self.source)
+        self.assertNotIn('/$END_MARKER', self.source)
+
+    # ── Domain validation ────────────────────────────────────────────────────
+
+    def test_domain_validation_regex_present(self):
+        """The hostname validation regex must still be present for injection prevention."""
+        self.assertIn("grep -qE", self.source)
+        self.assertIn("[a-zA-Z0-9]", self.source)
+
+    def test_invalid_domain_warning_present(self):
+        self.assertIn("skipping invalid domain value", self.source)
+
+    # ── No environment.systemPackages reliance ───────────────────────────────
+
+    def test_no_environment_system_packages_for_helper(self):
+        """The helper's tools are declared via runtimeInputs; the module must
+        not add them to environment.systemPackages."""
+        self.assertNotIn("environment.systemPackages", self.source)
+
+    # ── Idempotency: existing Sovran block is removed before rewriting ───────
+
+    def test_existing_block_removal_logic_present(self):
+        """awk strip of the managed block must be present for idempotency."""
+        self.assertIn("skip=1", self.source)
+        self.assertIn("skip=0", self.source)
+
+    def test_managed_block_uses_grouped_append_redirect(self):
+        self.assertIn('} >> "$TMP"', self.source)
+        self.assertEqual(self.source.count('>> "$TMP"'), 1)
+
+    def test_helper_script_passes_shellcheck(self):
+        shellcheck = shutil.which("shellcheck")
+        if shellcheck is None:
+            self.skipTest("shellcheck is not installed")
+        proc = subprocess.run(
+            [shellcheck, "-s", "bash", "-"],
+            input=self._helper_script(),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        output = proc.stdout + proc.stderr
+        self.assertEqual(proc.returncode, 0, output)
+
+    def test_helper_derivation_builds_when_nix_available(self):
+        nix = shutil.which("nix")
+        if nix is None:
+            self.skipTest("nix is not installed")
+        proc = subprocess.run(
+            [
+                nix,
+                "build",
+                HELPER_BUILD_ATTR,
+                "--no-link",
+            ],
+            cwd=REPO_ROOT,
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+
+    # ── Activation script warns on failure rather than silently swallowing ───
+
+    def test_activation_script_emits_warning_on_failure(self):
+        self.assertIn("warning: sovran-hosts-update", self.source)
+
+
+if __name__ == "__main__":
+    unittest.main()

flake.lock

@@ -139,11 +139,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1783856661,
-        "narHash": "sha256-ZGP04e+Q6WyQJGA9ZvI5CL6+heGQldbAG9U1T9NGvmU=",
+        "lastModified": 1784432872,
+        "narHash": "sha256-n3gKTBIV4ZA5VQpUakffBe3KGu4+mhPoA34rrqS0GkA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "569d578509928497eddc3fdbf94a799027050be4",
+        "rev": "fd1462031fdee08f65fd0b4c6b64e22239a77870",
         "type": "github"
       },
       "original": {
@@ -187,11 +187,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1783776592,
-        "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=",
+        "lastModified": 1784356753,
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
         "type": "github"
       },
       "original": {
@@ -203,11 +203,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1783791668,
-        "narHash": "sha256-zbcZ1dmBTPfJ7Mlqh/yLEPGpgJnwuv4Xr1xucy2WqMA=",
+        "lastModified": 1783915482,
+        "narHash": "sha256-FmieJB8/OUvNxbkboi7+IGfIuSXY3nF/hZQm8kD0r50=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "716c7a2664ca8325617b8a7fbb609273f2c4cae7",
+        "rev": "6cdc7fc76e8bf7fde9fa43a849fcaaa70e230dee",
         "type": "github"
       },
       "original": {
@@ -224,11 +224,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1783941741,
-        "narHash": "sha256-F+3M1IZrJa920cx2/k2AMKqedEodxLF7COJVkLJwUBo=",
+        "lastModified": 1784057377,
+        "narHash": "sha256-yycNej5//EsRbV10moBoh+/63vXEwZD1ZFEiRm6C9rQ=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "e6715f01d9f56f07a27a01386b85ae22b06f0705",
+        "rev": "07180a087e4a00720dc0731cbcd8dec796974381",
         "type": "github"
       },
       "original": {

modules/core/local-domain-loopback.nix

@@ -32,22 +32,31 @@
 # regenerated by the system activation script.  The ``system.activationScripts``
 # hook below converts it to a writable file each time the system is activated
 # (i.e. after every ``nixos-rebuild switch``) and then injects the Sovran block.
-# The same script is also run by the ``sovran-hosts-update.service`` unit so
-# that the Hub can trigger it immediately after saving a domain without
-# requiring a full rebuild.
+# The same wrapped Nix-store executable is reused by both the activation hook
+# and the ``sovran-hosts-update.service`` unit, ensuring a deterministic runtime
+# PATH in every execution context.
 
-{
-  # ── Helper script (stored in the Nix store, never reads /var/lib at eval) ──
+let
+  # ── Wrapped Nix-store executable ──────────────────────────────────────────
+  # Built with pkgs.writeShellApplication so that all required runtime tools
+  # (awk, grep, coreutils) are declared explicitly and injected into PATH by
+  # Nix.  Both the systemd service and the activation hook reference this same
+  # store-path executable — there is no second raw script body.
+  hostsUpdateScript = pkgs.writeShellApplication {
+    name = "sovran-hosts-update";
 
-  environment.systemPackages = [ pkgs.coreutils ];
+    # Declare every external command the script calls.  These packages are
+    # added to the script's runtime PATH by writeShellApplication; nothing from
+    # the system PATH is relied upon.
+    runtimeInputs = [
+      pkgs.coreutils   # readlink, cp, mv, chmod, mktemp, rm, tr, head
+      pkgs.gawk        # awk
+      pkgs.gnugrep     # grep
+    ];
 
-  environment.etc."sovran-hosts-update.sh" = {
-    mode = "0755";
     text = ''
-      #!/bin/sh
       # Regenerate the Sovran-managed loopback block in /etc/hosts.
       # Safe to run multiple times — idempotent.
-      set -eu
 
       DOMAINS_DIR="/var/lib/domains"
       HOSTS_FILE="/etc/hosts"
@@ -66,13 +75,14 @@
 
       # ── Step 2: remove any existing Sovran block ──────────────────────────
       # Use a temp file so the operation is atomic.
+      # awk -v passes marker strings safely without shell interpolation.
       TMP=$(mktemp "$HOSTS_FILE.XXXXXX")
       trap 'rm -f "$TMP"' EXIT
-      awk "
-        /^$BEGIN_MARKER\$/ { skip=1; next }
-        /^$END_MARKER\$/   { skip=0; next }
+      awk -v begin="$BEGIN_MARKER" -v end="$END_MARKER" '
+        $0 == begin { skip=1; next }
+        $0 == end   { skip=0; next }
         !skip
-      " "$HOSTS_FILE" > "$TMP"
+      ' "$HOSTS_FILE" > "$TMP"
 
       # ── Step 3: collect valid configured service domains ──────────────────
       # NOTE: The hostname validation regex below must stay in sync with
@@ -97,11 +107,13 @@
 
       # ── Step 4: append the Sovran block if there are any entries ──────────
       if [ -n "$ENTRIES" ]; then
-        printf '\n%s\n' "$BEGIN_MARKER" >> "$TMP"
-        printf '%s\n' "# These entries route configured service domains to local Caddy." >> "$TMP"
-        printf '%s\n' "# They are managed automatically — do not edit this block." >> "$TMP"
-        printf '%s\n' "$ENTRIES" >> "$TMP"
-        printf '%s\n' "$END_MARKER" >> "$TMP"
+        {
+          printf '\n%s\n' "$BEGIN_MARKER"
+          printf '%s\n' "# These entries route configured service domains to local Caddy."
+          printf '%s\n' "# They are managed automatically — do not edit this block."
+          printf '%s\n' "$ENTRIES"
+          printf '%s\n' "$END_MARKER"
+        } >> "$TMP"
       fi
 
       # ── Step 5: atomically replace /etc/hosts ─────────────────────────────
@@ -110,6 +122,14 @@
     '';
   };
 
+in
+{
+  # ── /etc/sovran-hosts-update.sh — operator discoverability symlink ─────────
+  # Retain the familiar /etc path so administrators can inspect or manually
+  # invoke the helper.  The target is the wrapped Nix-store executable, so
+  # there is no second raw script body to keep in sync.
+  environment.etc."sovran-hosts-update.sh".source = lib.getExe hostsUpdateScript;
+
   # ── Systemd service ────────────────────────────────────────────────────────
 
   systemd.services.sovran-hosts-update = {
@@ -126,18 +146,24 @@
     serviceConfig = {
       Type = "oneshot";
       RemainAfterExit = true;
-      ExecStart = "/etc/sovran-hosts-update.sh";
+      # Point directly at the wrapped Nix-store executable, not the /etc path.
+      ExecStart = lib.getExe hostsUpdateScript;
     };
   };
 
   # ── Activation script (runs after every nixos-rebuild switch) ─────────────
   # This ensures the loopback block survives rebuilds that restore the /etc/hosts
   # symlink.  The "users" and "etc" scripts must complete first.
+  # The same wrapped Nix-store executable used by the systemd service is
+  # referenced here, guaranteeing identical runtime dependencies in both
+  # execution contexts.
 
   system.activationScripts.sovranDomainLoopback = {
     text = ''
-      if [ -x /etc/sovran-hosts-update.sh ] && [ -d /var/lib/domains ]; then
-        /etc/sovran-hosts-update.sh || true
+      if [ -d /var/lib/domains ]; then
+        if ! ${lib.getExe hostsUpdateScript}; then
+          echo "warning: sovran-hosts-update: failed to update /etc/hosts loopback entries" >&2
+        fi
       fi
     '';
     deps = [ "etc" "users" ];

modules/core/sovran-hub.nix

@@ -382,13 +382,25 @@ in
       };
 
       path = [
+        pkgs.bash
+        pkgs.gawk
         pkgs.qrencode
         pkgs.curl
         pkgs.iproute2
         pkgs.nftables
         pkgs.iptables
         pkgs.hostname
-      ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package;
+        pkgs.coreutils
+        pkgs.findutils
+        pkgs.gnugrep
+        pkgs.rsync
+        pkgs.acl
+        pkgs.util-linux
+      ]
+      ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
+      ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
+      ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
+      ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
     };
 
     systemd.services.sovran-hub-update = {