Sovran_SystemsOS/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh

#!/usr/bin/env bash
# ── Sovran Hub External Backup Script ────────────────────────────
# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
# Designed for the Hub web UI (no GUI dependencies).
#
# Your Sovran Pro already backs up your data automatically to its
# internal second drive (BTCEcoandBackup at /run/media/Second_Drive).
# This script creates an additional copy on an external USB drive —
# storing your data in a third location for maximum protection.
#
# The external drive must be formatted as ext4. Files are stored as
# directly browsable files under Sovran_SystemsOS_Backup/current/.
# Later runs update the same mirror and only transfer changed or new
# files, making repeat backups fast.
#
# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
# blockchain and Electrs index data are NOT included (they live on
# the internal second drive).
#
# Usage:
#   BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
#   (or run with no env var to auto-detect the first external USB drive)

set -euo pipefail

BACKUP_LOG="/var/log/sovran-hub-backup.log"
BACKUP_STATUS="/var/log/sovran-hub-backup.status"
MEDIA_ROOT="/run/media"
HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
ROLE_STATE_NIX="/etc/nixos/role-state.nix"
SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))

# ── Internal drive labels/paths to NEVER use as backup targets ───
INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")

FAILED_ALREADY=0
BACKUP_COMPLETE=0
RSYNC_WARNINGS=()

# Stable rsync mirror sub-path under the target drive. Not timestamped
# so later runs update the same destination and only transfer new or changed files.
BACKUP_SUBPATH="Sovran_SystemsOS_Backup/current"

# ── Logging helpers ──────────────────────────────────────────────

log() {
  local msg
  msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
  echo "$msg" | tee -a "$BACKUP_LOG"
}

set_status() {
  echo "$1" > "$BACKUP_STATUS"
}

fail() {
  FAILED_ALREADY=1
  log "ERROR: $*"
  set_status "FAILED"
  exit 1
}

cleanup() {
  local rc=$?

  # Release the concurrency lock file descriptor if it was opened
  if [[ -n "${LOCK_FD:-}" ]]; then
    exec {LOCK_FD}>&- 2>/dev/null || true
  fi

  if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
    return
  fi

  if [[ "$FAILED_ALREADY" -eq 0 ]]; then
    log "ERROR: Backup terminated unexpectedly (exit code $rc)."
    set_status "FAILED"
  fi

  # Mark the backup directory as incomplete so failed runs are identifiable
  if [[ -n "${BACKUP_DIR:-}" && -d "${BACKUP_DIR:-}" && ! -f "${BACKUP_DIR:-}/BACKUP_COMPLETE" ]]; then
    touch "${BACKUP_DIR}/INCOMPLETE" 2>/dev/null || true
  fi
}

trap cleanup EXIT
trap 'exit 1' INT TERM

require_cmd() {
  local cmd="$1"
  command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
}

# ── Check whether a mount point is an internal drive ────────────

is_internal() {
  local mnt="$1"
  for internal in "${INTERNAL_MOUNTS[@]}"; do
    if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
      return 0
    fi
  done
  return 1
}

# ── Use lsblk to find the first genuine external USB drive ───────

find_external_drive() {
  local target=""

  while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
    [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
    [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
    [[ -n "$mountpoint" ]] || continue

    local skip=0
    for lbl in "${INTERNAL_LABELS[@]}"; do
      [[ "$label" == "$lbl" ]] && skip=1 && break
    done
    [[ "$skip" -eq 1 ]] && continue

    is_internal "$mountpoint" && continue

    if mountpoint -q "$mountpoint" 2>/dev/null; then
      target="$mountpoint"
      break
    fi
  done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
    python3 -c "
import sys, json

def flatten(devs):
    for d in devs:
        yield d
        for c in d.get('children', []):
            yield from flatten([c])

data = json.load(sys.stdin)
for d in flatten(data.get('blockdevices', [])):
    print('\\t'.join([
        d.get('type') or '',
        str(d.get('hotplug') or '0'),
        str(d.get('rm') or '0'),
        d.get('label') or '',
        d.get('mountpoint') or '',
    ]))
" 2>/dev/null || true)

  if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
    while IFS= read -r -d '' mnt; do
      is_internal "$mnt" && continue
      if mountpoint -q "$mnt" 2>/dev/null; then
        target="$mnt"
        break
      fi
    done < <(find "$MEDIA_ROOT" -mindepth 2 -maxdepth 2 -type d -print0 2>/dev/null)
  fi

  echo "$target"
}

# ── Detect the configured system role ───────────────────────────

detect_role() {
  local role="server_plus_desktop"

  if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
    local r
    r=$(python3 -c \
      "import json,sys; d=json.load(open(sys.argv[1])); print(d.get('role',''))" \
      "$HUB_CONFIG_JSON" 2>/dev/null || true)
    if [[ -n "$r" ]]; then
      echo "$r"
      return
    fi
  fi

  if [[ -f "$ROLE_STATE_NIX" ]]; then
    if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
      role="desktop"
    elif grep -q 'roles\.node = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
      role="node"
    fi
  fi

  echo "$role"
}

validate_target_mount() {
  local target="$1"
  [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
  [[ -d "$target" ]] || fail "Target path '$target' does not exist."
  mountpoint -q "$target" || fail "Target path '$target' is not a mount point."

  local fstype=""
  fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
  [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."

  if [[ "$fstype" != "ext4" ]]; then
    fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
  fi

  local write_test
  write_test="$target/.sovran-write-test-$$"
  if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
    fail "Target '$target' is not writable."
  fi

  log "Verified backup target filesystem: $fstype"
}

estimate_path_bytes() {
  local path="$1"
  shift || true
  [[ -e "$path" ]] || {
    echo 0
    return
  }

  local size
  size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
  [[ -n "$size" ]] || size=0
  echo "$size"
}

# ── Sync one source tree to its backup destination ───────────────
# Usage: sync_tree <label> <allow_vanished> <source> <destination> [rsync options...]
#
# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
# Used for /home only — files may disappear while the desktop is active.
# All other nonzero exit codes are always fatal.
#
# Before every rsync call this helper:
#   1. Re-verifies $TARGET is still a mount point (fails if drive disconnected).
#   2. Verifies the destination path remains beneath $BACKUP_DIR and $BACKUP_DIR
#      remains beneath $TARGET (safe-path check).
#   3. Creates the full destination directory hierarchy with mkdir -p so that
#      rsync never fails trying to create a directory whose parent is absent.
sync_tree() {
  local label="$1"
  local allow_vanished="$2"
  local source="$3"
  local destination="$4"
  shift 4
  # Remaining "$@" are rsync options (--exclude, etc.)

  # ── Re-verify the external drive is still mounted ────────────────
  mountpoint -q "$TARGET" 2>/dev/null || \
    fail "Stage $label: external drive '$TARGET' is no longer mounted. Refusing to write."

  # ── Verify path safety ────────────────────────────────────────────
  # BACKUP_DIR must remain beneath TARGET.
  case "$BACKUP_DIR" in
    "$TARGET"/*) ;;
    *) fail "Stage $label: BACKUP_DIR '$BACKUP_DIR' is outside TARGET '$TARGET'." ;;
  esac
  # Destination must remain beneath BACKUP_DIR.
  case "$destination" in
    "$BACKUP_DIR"/*|"$BACKUP_DIR") ;;
    *) fail "Stage $label: destination '$destination' is outside BACKUP_DIR '$BACKUP_DIR'. Refusing to write." ;;
  esac

  # ── Create complete destination directory hierarchy ───────────────
  # This is the fix for the production failure:
  #   rsync: [Receiver] mkdir ".../current/etc/nixos" failed: No such file or directory
  # mkdir -p creates all intermediate parents (e.g. current/etc/) before rsync runs.
  mkdir -p -- "$destination" || \
    fail "Stage $label: failed to create destination directory '$destination' (source: '$source')."

  local rsync_err_tmp
  rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"

  local rc=0
  rsync \
    --archive \
    --acls \
    --xattrs \
    --hard-links \
    --numeric-ids \
    --one-file-system \
    --partial \
    "$@" "$source" "$destination" 2>"$rsync_err_tmp" || rc=$?

  if [[ -s "$rsync_err_tmp" ]]; then
    while IFS= read -r rline; do
      log "rsync: $rline"
    done < "$rsync_err_tmp"
  fi
  rm -f "$rsync_err_tmp"

  if [[ "$rc" -eq 0 ]]; then
    return 0
  elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
    log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
    RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
    return 0
  else
    fail "rsync failed for $label (exit code $rc). See the rsync errors above."
  fi
}

# ── Initialise log file ──────────────────────────────────────────

: > "$BACKUP_LOG"
set_status "RUNNING"

log "=== Sovran_SystemsOS External Hub Backup ==="
log "Starting backup process…"

# ── Acquire exclusive run lock ────────────────────────────────────
# Prevents two simultaneous backup runs (e.g. from double-click or
# stale RUNNING status after a Hub restart).

LOCK_FILE="/var/lock/sovran-hub-backup.lock"
# Note: exec {LOCK_FD}>>file requires bash 4.1+ (NixOS provides bash 5.x).
exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
  fail "Cannot open lock file: $LOCK_FILE. Ensure /var/lock is writable."
flock --nonblock "$LOCK_FD" 2>/dev/null || \
  fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."

require_cmd rsync
require_cmd findmnt
require_cmd lsblk
require_cmd mountpoint
require_cmd df
require_cmd du
require_cmd awk
require_cmd find
require_cmd hostname
require_cmd date
require_cmd python3
require_cmd flock

# ── Detect system role ───────────────────────────────────────────

ROLE="$(detect_role)"
case "$ROLE" in
  desktop)              ROLE_LABEL="Desktop Only" ;;
  node)                 ROLE_LABEL="Node (Bitcoin-only)" ;;
  server_plus_desktop)  ROLE_LABEL="Server + Desktop" ;;
  *)                    ROLE_LABEL="$ROLE" ;;
esac
log "Detected role: $ROLE_LABEL"

# ── Detect target drive ──────────────────────────────────────────

if [[ -n "${BACKUP_TARGET:-}" ]]; then
  TARGET="$BACKUP_TARGET"
  if is_internal "$TARGET"; then
    fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
  fi
  log "Using specified backup target: $TARGET"
else
  log "Auto-detecting external USB drives…"
  TARGET="$(find_external_drive)"
  if [[ -z "$TARGET" ]]; then
    fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
  fi
  log "Detected external drive: $TARGET"
fi

validate_target_mount "$TARGET"

# ── Set up stable backup destination ────────────────────────────
# Subsequent runs update the same mirror, transferring only new or changed files.

BACKUP_DIR="${TARGET}/${BACKUP_SUBPATH}"
mkdir -p -- "$BACKUP_DIR"

# Remove any stale BACKUP_COMPLETE left by a previous successful run.
# The new run will re-earn it only after all stages succeed.
rm -f "$BACKUP_DIR/BACKUP_COMPLETE"

# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
# after all rsync stages and manifest write succeed. Failed or interrupted
# runs keep this marker so they are clearly identifiable.
touch "$BACKUP_DIR/INCOMPLETE"
log "Backup destination: $BACKUP_DIR"

# ── Estimate required free space ─────────────────────────────────
# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
# from the estimate to avoid inflating the required size.

ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
SECRETS_BYTES=0
if [[ "$ROLE" != "desktop" ]]; then
  SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
fi

VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
  --exclude='postgresql' \
  --exclude='mysql' \
  --exclude='mariadb' \
  --exclude='bitcoind' \
  --exclude='electrs' \
  --exclude='*/log' \
  --exclude='*/logs' \
  --exclude='*/cache' \
  --exclude='*/tmp')

ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
# Require 20% growth headroom plus a fixed 1 GiB safety margin.
# Later incremental runs need far less space, but a conservative first-run
# check protects against running out of space mid-backup.
REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))

FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))

log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
log "Required free space (with safety margin): ${REQUIRED_GB} GB"
log "Free space on drive: ${FREE_GB} GB"

(( FREE_BYTES >= REQUIRED_BYTES )) || \
  fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."

# ── Stage 1/4: NixOS configuration ──────────────────────────────

log ""
log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
if [[ -d /etc/nixos ]]; then
  sync_tree "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
  log "Stage 1 complete."
else
  log "WARNING: /etc/nixos not found — skipping."
fi

# ── Stage 2/4: Secrets ──────────────────────────────────────────

log ""
log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
if [[ "$ROLE" == "desktop" ]]; then
  log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
elif [[ -e /etc/nix-bitcoin-secrets ]]; then
  sync_tree "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
else
  log "(not found: /etc/nix-bitcoin-secrets — skipping)"
fi
log "Stage 2 complete."

# ── Stage 3/4: Home directory ───────────────────────────────────
# Rsync exit code 24 (vanished source files) is treated as nonfatal here
# because the desktop may be active and files can disappear between the
# directory scan and the copy. All other nonzero exit codes remain fatal.

log ""
log "── Stage 3/4: Home directory (/home) ───────────────────────"
if [[ -d /home ]]; then
  sync_tree "/home" yes /home/ "$BACKUP_DIR/home/" \
    --exclude='.cache/' \
    --exclude='.local/share/Trash/' \
    --exclude='Trash/' \
    --exclude='.mozilla/firefox/*/cache2/' \
    --exclude='.mozilla/firefox/*/startupCache/' \
    --exclude='.mozilla/firefox/*/thumbnails/' \
    --exclude='.config/google-chrome/*/Cache/' \
    --exclude='.config/google-chrome/*/Code Cache/' \
    --exclude='.config/chromium/*/Cache/' \
    --exclude='.config/chromium/*/Code Cache/' \
    --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
    --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
    --exclude='.local/share/baloo/' \
    --exclude='.thumbnails/' \
    --exclude='.xsession-errors' \
    --exclude='.xsession-errors.old'
  log "Stage 3 complete."
else
  log "WARNING: /home not found — skipping."
fi

# ── Stage 4/4: System data ──────────────────────────────────────
# PostgreSQL/MariaDB raw database directories are excluded. Application
# databases must be backed up separately with native database tools.
# Bitcoin/Electrs data are excluded; they live on the internal second drive.

log ""
log "── Stage 4/4: System data (/var/lib) ───────────────────────"
if [[ -d /var/lib ]]; then
  sync_tree "/var/lib" no /var/lib/ "$BACKUP_DIR/var/lib/" \
    --exclude='postgresql/' \
    --exclude='mysql/' \
    --exclude='mariadb/' \
    --exclude='bitcoind/' \
    --exclude='electrs/' \
    --exclude='*/log/' \
    --exclude='*/logs/' \
    --exclude='*/cache/' \
    --exclude='*/tmp/'
  log "Stage 4 complete."
else
  log "WARNING: /var/lib not found — skipping."
fi

# ── Generate manifest ────────────────────────────────────────────

log ""
log "Generating BACKUP_MANIFEST.txt …"
MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"

{
  echo "Sovran_SystemsOS Backup Manifest"
  echo "Updated:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
  echo "Hostname:  $(hostname)"
  echo "Role:      $ROLE_LABEL"
  echo "Target:    $TARGET"
  echo ""
  echo "Backup type: Live rsync mirror (directly browsable files)"
  echo "Location:    ${BACKUP_DIR}"
  echo ""
  echo "Source paths mirrored:"
  echo "- /etc/nixos  →  current/etc/nixos/"
  if [[ "$ROLE" != "desktop" ]]; then
    echo "- /etc/nix-bitcoin-secrets (when present)  →  current/etc/nix-bitcoin-secrets/"
  fi
  echo "- /home  →  current/home/"
  echo "- /var/lib  →  current/var/lib/"
  echo ""
  echo "Exclusions:"
  echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
  echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
  echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
  echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
  echo "- /run/media/Second_Drive (internal second drive — never traversed)"
  echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
  echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
  echo ""
  echo "Important limitations:"
  echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
  echo "  backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
  echo "  applications, their data must be backed up separately using native tools."
  echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
  echo "  reconstructable or stored on the internal second drive."
  echo "- This is a live file-level mirror, not a transactional database backup."
  echo "  Files being written during the backup may be in an inconsistent state."
  echo ""
  echo "Restore guidance:"
  echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
  echo "- To restore a directory:"
  echo "    sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
  echo "    sudo rsync -aAXH --numeric-ids current/home/ /home/"
  echo "    sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
  echo "- To copy individual files:"
  echo "    sudo cp -a current/home/username/ /home/username/"
  echo "- When restoring /etc/nixos to replacement hardware, regenerate"
  echo "  hardware-configuration.nix for the new hardware before rebuilding."
  echo ""
  echo "Nonfatal warnings:"
  if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
    echo "- none"
  else
    for warning in "${RSYNC_WARNINGS[@]}"; do
      echo "- $warning"
    done
  fi
  echo ""
  echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
  echo "from manual external backup because they already live on the internal second drive"
  echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
} > "$MANIFEST_FILE"

log "Manifest written to $MANIFEST_FILE"

# ── Done ─────────────────────────────────────────────────────────

log ""
if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
  log "Backup completed with nonfatal warnings:"
  for warning in "${RSYNC_WARNINGS[@]}"; do
    log "  WARNING: $warning"
  done
  log "Your important data is backed up. The warnings above indicate files that"
  log "vanished during backup, which is normal on an active desktop."
  log ""
fi
log "All Finished! Your data is now backed up to a third location."
log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
log "Please eject the drive safely before removing it from your Sovran Pro."

# Remove incomplete marker and write completion marker only after all work succeeds.
# A later successful run will update the same mirror and replace any INCOMPLETE state.
rm -f "$BACKUP_DIR/INCOMPLETE"
date -u '+%Y-%m-%dT%H:%M:%SZ' > "$BACKUP_DIR/BACKUP_COMPLETE"

BACKUP_COMPLETE=1
set_status "SUCCESS"