diff --git a/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh b/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh index d6a285f..d402255 100755 --- a/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh +++ b/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh @@ -17,13 +17,24 @@ set -euo pipefail BACKUP_LOG="/var/log/sovran-hub-backup.log" BACKUP_STATUS="/var/log/sovran-hub-backup.status" MEDIA_ROOT="/run/media" -MIN_FREE_GB=10 HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json" ROLE_STATE_NIX="/etc/nixos/role-state.nix" +SECOND_DRIVE_MOUNT="/run/media/Second_Drive" +SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024)) # ── Internal drive labels/paths to NEVER use as backup targets ─── INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos") -INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/") +INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/") + +FAILED_ALREADY=0 +BACKUP_COMPLETE=0 +LND_STOPPED=0 +LND_UNITS_TO_RESTART=() + +ARCHIVE_FILES=() +DB_DUMP_FILES=() +MANIFEST_EXCLUDES=() +LND_BACKUP_NOTES=() # ── Logging helpers ────────────────────────────────────────────── @@ -37,16 +48,58 @@ set_status() { } fail() { + FAILED_ALREADY=1 log "ERROR: $*" set_status "FAILED" exit 1 } +cleanup() { + local rc=$? + local restart_failed=0 + + if [[ "$LND_STOPPED" -eq 1 ]]; then + log "Restarting previously active LND-related services…" + for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do + local unit="${LND_UNITS_TO_RESTART[$idx]}" + if systemctl start "$unit"; then + log "Started $unit" + else + log "ERROR: Failed to start $unit" + restart_failed=1 + fi + done + LND_STOPPED=0 + fi + + if [[ "$restart_failed" -eq 1 ]]; then + rc=1 + FAILED_ALREADY=1 + set_status "FAILED" + fi + + if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then + return + fi + + if [[ "$FAILED_ALREADY" -eq 0 ]]; then + log "ERROR: Backup terminated unexpectedly (exit code $rc)." + set_status "FAILED" + fi +} + +trap cleanup EXIT +trap 'exit 1' INT TERM + +require_cmd() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd" +} + # ── Check whether a mount point is an internal drive ──────────── is_internal() { local mnt="$1" - # Reject known internal mount points and their subdirectories for internal in "${INTERNAL_MOUNTS[@]}"; do if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then return 0 @@ -59,39 +112,37 @@ is_internal() { find_external_drive() { local target="" - # lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE - if command -v lsblk &>/dev/null; then - while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do - # Must be a partition or disk, and be removable/hotplug - [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue - [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue - [[ -n "$mountpoint" ]] || continue - # Filter out internal labels - local skip=0 - for lbl in "${INTERNAL_LABELS[@]}"; do - [[ "$label" == "$lbl" ]] && skip=1 && break - done - [[ "$skip" -eq 1 ]] && continue + while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do + [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue + [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue + [[ -n "$mountpoint" ]] || continue - # Filter out internal mount points - is_internal "$mountpoint" && continue + local skip=0 + for lbl in "${INTERNAL_LABELS[@]}"; do + [[ "$label" == "$lbl" ]] && skip=1 && break + done + [[ "$skip" -eq 1 ]] && continue - if mountpoint -q "$mountpoint" 2>/dev/null; then - target="$mountpoint" - break - fi - done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \ - python3 -c " + is_internal "$mountpoint" && continue + + if mountpoint -q "$mountpoint" 2>/dev/null; then + target="$mountpoint" + break + fi + done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \ + python3 -c " import sys, json -data = json.load(sys.stdin) + def flatten(devs): for d in devs: yield d for c in d.get('children', []): yield from flatten([c]) + +data = json.load(sys.stdin) for d in flatten(data.get('blockdevices', [])): - print('\t'.join([ + print('\\t'.join([ d.get('type') or '', str(d.get('hotplug') or '0'), str(d.get('rm') or '0'), @@ -99,24 +150,10 @@ for d in flatten(data.get('blockdevices', [])): d.get('mountpoint') or '', ])) " 2>/dev/null || true) - fi - # Fallback: walk /run/media/ if lsblk produced nothing if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then while IFS= read -r -d '' mnt; do is_internal "$mnt" && continue - # Check label via lsblk on the device backing this mount - local dev - dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true) - if [[ -n "$dev" ]]; then - local lbl - lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true) - local skip=0 - for internal_lbl in "${INTERNAL_LABELS[@]}"; do - [[ "$lbl" == "$internal_lbl" ]] && skip=1 && break - done - [[ "$skip" -eq 1 ]] && continue - fi if mountpoint -q "$mnt" 2>/dev/null; then target="$mnt" break @@ -128,16 +165,10 @@ for d in flatten(data.get('blockdevices', [])): } # ── Detect the configured system role ─────────────────────────── -# -# Priority: -# 1. Hub config JSON (/var/lib/sovran-hub/config.json) — "role" key -# 2. role-state.nix (/etc/nixos/role-state.nix) — grep for true flag -# 3. Default: server_plus_desktop detect_role() { local role="server_plus_desktop" - # 1. Try the Hub config JSON if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then local r r=$(python3 -c \ @@ -149,7 +180,6 @@ detect_role() { fi fi - # 2. Fall back to parsing role-state.nix if [[ -f "$ROLE_STATE_NIX" ]]; then if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then role="desktop" @@ -161,6 +191,64 @@ detect_role() { echo "$role" } +validate_target_mount() { + local target="$1" + [[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT." + [[ -d "$target" ]] || fail "Target path '$target' does not exist." + mountpoint -q "$target" || fail "Target path '$target' is not a mount point." + + local fstype="" + fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true) + [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'." + + if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then + fail "Target '$target' must be exFAT (detected filesystem: $fstype)." + fi + + if [[ "$fstype" == "fuseblk" ]]; then + local src_dev blk_type + src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true) + blk_type="" + if [[ -n "$src_dev" ]]; then + blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true) + [[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true) + fi + if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then + fail "Target '$target' is fuseblk but not identified as exFAT-compatible." + fi + fi + + local write_test + write_test="$target/.sovran-write-test-$$" + if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then + fail "Target '$target' is not writable." + fi + + log "Verified backup target filesystem: $fstype" +} + +has_unit() { + systemctl cat "$1" >/dev/null 2>&1 +} + +is_unit_active() { + systemctl is-active --quiet "$1" +} + +estimate_path_bytes() { + local path="$1" + shift || true + [[ -e "$path" ]] || { + echo 0 + return + } + + local size + size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true) + [[ -n "$size" ]] || size=0 + echo "$size" +} + # ── Initialise log file ────────────────────────────────────────── : > "$BACKUP_LOG" @@ -169,14 +257,30 @@ set_status "RUNNING" log "=== Sovran_SystemsOS External Hub Backup ===" log "Starting backup process…" +require_cmd tar +require_cmd sha256sum +require_cmd findmnt +require_cmd lsblk +require_cmd mountpoint +require_cmd df +require_cmd du +require_cmd awk +require_cmd sort +require_cmd find +require_cmd systemctl +require_cmd hostname +require_cmd date +require_cmd python3 +require_cmd runuser + # ── Detect system role ─────────────────────────────────────────── ROLE="$(detect_role)" case "$ROLE" in - desktop) ROLE_LABEL="Desktop Only" ;; - node) ROLE_LABEL="Node (Bitcoin-only)" ;; - server_plus_desktop) ROLE_LABEL="Server + Desktop" ;; - *) ROLE_LABEL="$ROLE" ;; + desktop) ROLE_LABEL="Desktop Only" ;; + node) ROLE_LABEL="Node (Bitcoin-only)" ;; + server_plus_desktop) ROLE_LABEL="Server + Desktop" ;; + *) ROLE_LABEL="$ROLE" ;; esac log "Detected role: $ROLE_LABEL" @@ -184,7 +288,6 @@ log "Detected role: $ROLE_LABEL" if [[ -n "${BACKUP_TARGET:-}" ]]; then TARGET="$BACKUP_TARGET" - # Safety: never allow internal drives even if explicitly passed if is_internal "$TARGET"; then fail "Target '$TARGET' is an internal system drive and cannot be used for external backup." fi @@ -193,106 +296,361 @@ else log "Auto-detecting external USB drives…" TARGET="$(find_external_drive)" if [[ -z "$TARGET" ]]; then - fail "No external USB drive detected. " \ - "Please plug in an exFAT-formatted USB drive (≥500 GB) and try again." + fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again." fi log "Detected external drive: $TARGET" fi -# ── Verify mount point ─────────────────────────────────────────── +validate_target_mount "$TARGET" -[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist." -mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point." +# ── Plan role-aware source scope and exclusions ───────────────── -# ── Check free disk space (require ≥ 10 GB) ────────────────────── +LND_AVAILABLE=0 +if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then + LND_AVAILABLE=1 +fi -FREE_KB=$(df -k --output=avail "$TARGET" | tail -1) -FREE_GB=$(( FREE_KB / 1024 / 1024 )) +if [[ "$ROLE" == "desktop" ]]; then + MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)") +else + MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent") +fi + +MANIFEST_EXCLUDES+=( + "/run/media/Second_Drive (never traversed)" + "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)" + "/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)" + "/var/lib/bitcoind (excluded from manual backup)" + "/var/lib/electrs (excluded from manual backup)" + "/var/lib/*/log and /var/lib/*/logs" + "/var/lib/*/cache and /var/lib/*/tmp" + "/home/*/.cache and /home/*/.local/share/Trash" +) + +if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then + MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive") +fi + +# ── Estimate required free space ───────────────────────────────── + +ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos) +HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash') +SECRETS_BYTES=0 +if [[ "$ROLE" != "desktop" ]]; then + SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets) +fi + +VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \ + --exclude='bitcoind' \ + --exclude='electrs' \ + --exclude='lnd' \ + --exclude='*/log' \ + --exclude='*/logs' \ + --exclude='*/cache' \ + --exclude='*/tmp') + +LND_BYTES=0 +if [[ "$LND_AVAILABLE" -eq 1 ]]; then + LND_BYTES=$(estimate_path_bytes /var/lib/lnd) +fi + +ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES )) +REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES )) + +FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ') +FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 )) +REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 )) + +log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB" +log "Required free space (with safety margin): ${REQUIRED_GB} GB" log "Free space on drive: ${FREE_GB} GB" -(( FREE_GB >= MIN_FREE_GB )) || \ - fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)." + +(( FREE_BYTES >= REQUIRED_BYTES )) || \ + fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)." # ── Create timestamped backup directory ───────────────────────── TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}" -mkdir -p "$BACKUP_DIR" +DB_DUMP_DIR="$BACKUP_DIR/database-dumps" +mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR" log "Backup destination: $BACKUP_DIR" -# ── Stage 1/4: NixOS configuration ────────────────────────────── +create_tar_archive() { + local archive_name="$1" + shift + local archive_path="$BACKUP_DIR/$archive_name" + + log "Creating $archive_name …" + tar \ + --create \ + --file "$archive_path" \ + --numeric-owner \ + --acls \ + --xattrs \ + --sparse \ + --one-file-system \ + "$@" + + ARCHIVE_FILES+=("$archive_name") + log "Created archive: $archive_name" +} + +export_postgresql_dumps() { + if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then + log "PostgreSQL tools/service not available — skipping PostgreSQL exports." + return + fi + + if ! is_unit_active "postgresql.service"; then + log "PostgreSQL service is not active — skipping PostgreSQL exports." + return + fi + + log "Exporting PostgreSQL globals and databases…" + local globals_file="$DB_DUMP_DIR/postgresql_globals.sql" + runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \ + fail "Failed to export PostgreSQL globals." + DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql") + + local dbs + dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true) + + if [[ -z "$dbs" ]]; then + log "No non-template PostgreSQL application databases found." + return + fi + + while IFS= read -r db; do + [[ -n "$db" ]] || continue + local safe_db + safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')" + local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump" + runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \ + fail "Failed to export PostgreSQL database '$db'." + DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump") + done <<< "$dbs" +} + +export_mariadb_dumps() { + local dump_cmd="" + local query_cmd="" + local mariadb_unit="" + + if command -v mariadb-dump >/dev/null 2>&1; then + dump_cmd="mariadb-dump" + elif command -v mysqldump >/dev/null 2>&1; then + dump_cmd="mysqldump" + fi + + if command -v mariadb >/dev/null 2>&1; then + query_cmd="mariadb" + elif command -v mysql >/dev/null 2>&1; then + query_cmd="mysql" + fi + + if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then + log "MariaDB dump/query tools not available — skipping MariaDB exports." + return + fi + + if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then + mariadb_unit="mariadb.service" + elif has_unit "mysql.service" && is_unit_active "mysql.service"; then + mariadb_unit="mysql.service" + else + log "MariaDB service is not active — skipping MariaDB exports." + return + fi + + log "Exporting MariaDB databases from ${mariadb_unit}…" + + local dbs + dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true) + if [[ -z "$dbs" ]]; then + log "No MariaDB databases found." + return + fi + + while IFS= read -r db; do + [[ -n "$db" ]] || continue + case "$db" in + information_schema|performance_schema|mysql|sys) continue ;; + esac + + local safe_db out_file + safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')" + out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql" + + $dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \ + fail "Failed to export MariaDB database '$db'." + + DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql") + done <<< "$dbs" +} + +export_lnd_scb_if_possible() { + [[ "$LND_AVAILABLE" -eq 1 ]] || return + + local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb" + local attempts=( + "lncli exportchanbackup --all --output_file $scb_file" + "lncli -n mainnet exportchanbackup --all --output_file $scb_file" + "runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file" + "runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file" + ) + + if ! command -v lncli >/dev/null 2>&1; then + log "lncli not available — skipping Static Channel Backup export." + LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)") + return + fi + + if ! is_unit_active "lnd.service"; then + log "LND service is not active — skipping Static Channel Backup export." + LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)") + return + fi + + log "Exporting LND Static Channel Backup…" + local attempt + for attempt in "${attempts[@]}"; do + if eval "$attempt" >/dev/null 2>&1; then + DB_DUMP_FILES+=("lnd-static-channel-backup.scb") + LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli") + log "LND Static Channel Backup exported." + return + fi + done + + log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations." + LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)") +} + +capture_active_lnd_dependents() { + [[ "$LND_AVAILABLE" -eq 1 ]] || return + + LND_UNITS_TO_RESTART=() + local raw_units="" + raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true) + + while IFS= read -r unit; do + [[ -n "$unit" ]] || continue + if is_unit_active "$unit"; then + LND_UNITS_TO_RESTART+=("$unit") + fi + done <<< "$raw_units" + + if is_unit_active "lnd.service"; then + LND_UNITS_TO_RESTART+=("lnd.service") + fi +} + +stop_lnd_stack_if_needed() { + [[ "$LND_AVAILABLE" -eq 1 ]] || return + + capture_active_lnd_dependents + + if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then + log "No active LND-related services needed stopping." + return + fi + + log "Stopping active services that depend on LND for clean /var/lib/lnd archive…" + + local unit + for unit in "${LND_UNITS_TO_RESTART[@]}"; do + if [[ "$unit" == "lnd.service" ]]; then + continue + fi + systemctl stop "$unit" || fail "Failed to stop dependent service: $unit" + log "Stopped $unit" + done + + if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then + systemctl stop lnd.service || fail "Failed to stop lnd.service" + log "Stopped lnd.service" + fi + + LND_STOPPED=1 +} + +# ── Stage 1/5: NixOS configuration ────────────────────────────── log "" -log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────" +log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────" if [[ -d /etc/nixos ]]; then - rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \ - fail "Stage 1 failed while copying /etc/nixos" + create_tar_archive "etc-nixos.tar" -C / etc/nixos log "Stage 1 complete." else log "WARNING: /etc/nixos not found — skipping." fi -# ── Stage 2/4: Secrets ────────────────────────────────────────── +# ── Stage 2/5: Secrets ────────────────────────────────────────── log "" -log "── Stage 2/4: Secrets ───────────────────────────────────────" -mkdir -p "$BACKUP_DIR/secrets" - +log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────" if [[ "$ROLE" == "desktop" ]]; then log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role." +elif [[ -e /etc/nix-bitcoin-secrets ]]; then + create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets else - if [[ -e /etc/nix-bitcoin-secrets ]]; then - rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \ - log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing." - else - log " (not found: /etc/nix-bitcoin-secrets — skipping)" - fi + log "(not found: /etc/nix-bitcoin-secrets — skipping)" fi - log "Stage 2 complete." -# ── Stage 3/4: Home directory ──────────────────────────────────── +# ── Stage 3/5: Home directory ─────────────────────────────────── log "" -log "── Stage 3/4: Home directory (/home) ───────────────────────" +log "── Stage 3/5: Home directory (/home) ───────────────────────" if [[ -d /home ]]; then - rsync -a --info=progress2 \ - --exclude='.cache/' \ - --exclude='.local/share/Trash/' \ - --exclude='*/Trash/' \ - /home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \ - fail "Stage 3 failed while copying /home" + create_tar_archive "home.tar" \ + -C / \ + --exclude='home/*/.cache' \ + --exclude='home/*/.local/share/Trash' \ + --exclude='home/*/Trash' \ + home log "Stage 3 complete." else log "WARNING: /home not found — skipping." fi -# ── Stage 4/4: System data ─────────────────────────────────────── +# ── Stage 4/5: Database exports + LND artifacts ──────────────── log "" -log "── Stage 4/4: System data (/var/lib) ────────────────────────" -if [[ "$ROLE" == "desktop" ]]; then - if [[ -d /var/lib ]]; then - rsync -a --info=progress2 \ - --filter='- /lnd/***' \ - --exclude='logs/' \ - --exclude='log/' \ - --exclude='*/logs/' \ - --exclude='*/log/' \ - /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \ - fail "Stage 4 failed while copying /var/lib for Desktop Only role" - log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)." - else - log "WARNING: /var/lib not found — skipping." +log "── Stage 4/5: Database and LND consistency exports ─────────" +export_postgresql_dumps +export_mariadb_dumps +export_lnd_scb_if_possible + +if [[ "$LND_AVAILABLE" -eq 1 ]]; then + stop_lnd_stack_if_needed + create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd + LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop") +fi + +log "Stage 4 complete." + +# ── Stage 5/5: System data ────────────────────────────────────── + +log "" +log "── Stage 5/5: System data (/var/lib) ───────────────────────" +if [[ -d /var/lib ]]; then + VAR_LIB_EXCLUDES=( + --exclude='var/lib/bitcoind' + --exclude='var/lib/electrs' + --exclude='var/lib/*/log' + --exclude='var/lib/*/logs' + --exclude='var/lib/*/cache' + --exclude='var/lib/*/tmp' + ) + + if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then + VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd') fi -elif [[ -d /var/lib ]]; then - rsync -a --info=progress2 \ - --exclude='logs/' \ - --exclude='log/' \ - --exclude='*/logs/' \ - --exclude='*/log/' \ - /var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \ - fail "Stage 4 failed while copying /var/lib" - log "Stage 4 complete." + + create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib + log "Stage 5 complete." else log "WARNING: /var/lib not found — skipping." fi @@ -301,21 +659,83 @@ fi log "" log "Generating BACKUP_MANIFEST.txt …" +MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt" +CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt" + { echo "Sovran_SystemsOS Backup Manifest" - echo "Generated: $(date)" + echo "Generated: $(date -Is)" + echo "Timestamp: $TIMESTAMP" echo "Hostname: $(hostname)" echo "Role: $ROLE_LABEL" echo "Target: $TARGET" echo "" - echo "Contents:" - find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort -} > "$BACKUP_DIR/BACKUP_MANIFEST.txt" -log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt" + echo "Source paths included:" + echo "- /etc/nixos" + echo "- /home" + if [[ "$ROLE" != "desktop" ]]; then + echo "- /etc/nix-bitcoin-secrets (when present)" + fi + echo "- /var/lib" + echo "" + echo "Exclusions:" + for ex in "${MANIFEST_EXCLUDES[@]}"; do + echo "- $ex" + done + echo "" + echo "Archives:" + for archive in "${ARCHIVE_FILES[@]}"; do + echo "- $archive" + done + echo "" + echo "Database and LND exports:" + if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then + echo "- none" + else + for dump in "${DB_DUMP_FILES[@]}"; do + echo "- $dump" + done + for note in "${LND_BACKUP_NOTES[@]}"; do + echo "- $note" + done + fi + echo "" + echo "Restore guidance:" + echo "- Verify artifacts: cd '$BACKUP_DIR' && sha256sum -c SHA256SUMS.txt" + echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf .tar -C /" + echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql" + echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_.dump" + echo "- MariaDB DB dump: mariadb < database-dumps/mariadb_.sql" + echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures" + echo "" + echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded" + echo "from manual external backup because they already live on the internal second drive" + echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data." + echo "" + echo "Artifact listing:" + find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f | sort +} > "$MANIFEST_FILE" + +ARCHIVE_FILES+=("BACKUP_MANIFEST.txt") + +# ── Generate checksums for all backup artifacts ───────────────── + +log "Generating SHA-256 checksums …" +( + cd "$BACKUP_DIR" + while IFS= read -r -d '' file; do + sha256sum "$file" + done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' -print0 | sort -z) +) > "$CHECKSUM_FILE" + +log "Manifest written to $MANIFEST_FILE" +log "Checksums written to $CHECKSUM_FILE" # ── Done ───────────────────────────────────────────────────────── log "" log "All Finished! Your data is now backed up to a third location." log "Please eject the drive safely before removing it from your Sovran Pro." + +BACKUP_COMPLETE=1 set_status "SUCCESS" diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index 76674f3..5b8f999 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -1451,6 +1451,12 @@ def _read_backup_status() -> str: return "IDLE" +def _write_backup_status(value: str) -> None: + """Write backup status file.""" + with open(BACKUP_STATUS, "w") as f: + f.write(value) + + def _read_backup_log(offset: int = 0) -> tuple[str, int]: """Read the backup log file from the given byte offset. Returns (new_text, new_offset).""" @@ -1467,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]: return "", 0 +def _append_backup_log(line: str) -> None: + """Append one line to backup log.""" + with open(BACKUP_LOG, "a") as f: + f.write(line.rstrip("\n") + "\n") + + _INTERNAL_LABELS = {"BTCEcoandBackup", "sovran_systemsos"} _INTERNAL_MOUNTS = {"/", "/boot/efi"} _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive" @@ -1490,7 +1502,7 @@ def _detect_external_drives() -> list[dict]: /run/media/ directly if lsblk is unavailable, applying the same label/path filters. - Returns a list of dicts with name, path, free_gb, total_gb. + Returns a list of dicts with name, path, free_gb, total_gb, fstype. """ import json as _json import subprocess as _subprocess @@ -1501,7 +1513,7 @@ def _detect_external_drives() -> list[dict]: # ── Primary path: lsblk JSON ──────────────────────────────── try: result = _subprocess.run( - ["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"], + ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"], capture_output=True, text=True, timeout=10 ) if result.returncode == 0: @@ -1519,6 +1531,7 @@ def _detect_external_drives() -> list[dict]: hotplug = str(dev.get("hotplug", "0")) rm = str(dev.get("rm", "0")) label = dev.get("label") or "" + fstype = (dev.get("fstype") or "").lower() mountpoint = dev.get("mountpoint") or "" if dev_type not in ("part", "disk"): @@ -1544,6 +1557,7 @@ def _detect_external_drives() -> list[dict]: "path": mountpoint, "free_gb": free_gb, "total_gb": total_gb, + "fstype": fstype, }) seen_paths.add(mountpoint) except OSError: @@ -1577,11 +1591,20 @@ def _detect_external_drives() -> list[dict]: st = os.statvfs(drive_path) total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1) free_gb = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1) + fstype = "" + try: + fstype = _subprocess.run( + ["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path], + capture_output=True, text=True, timeout=5 + ).stdout.strip().lower() + except Exception: + fstype = "" drives.append({ "name": drive_name, "path": drive_path, "free_gb": free_gb, "total_gb": total_gb, + "fstype": fstype, }) seen_paths.add(drive_path) except OSError: @@ -3682,6 +3705,22 @@ async def api_backup_drives(): return {"drives": drives} +async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None: + """Mark status FAILED if backup subprocess exits unexpectedly.""" + rc = await proc.wait() + if rc == 0: + return + + loop = asyncio.get_event_loop() + status = await loop.run_in_executor(None, _read_backup_status) + if status in {"SUCCESS", "FAILED"}: + return + + msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc})." + await loop.run_in_executor(None, _append_backup_log, msg) + await loop.run_in_executor(None, _write_backup_status, "FAILED") + + @app.post("/api/backup/run") async def api_backup_run(target: str = ""): """Start the backup script as a background subprocess. @@ -3692,6 +3731,26 @@ async def api_backup_run(target: str = ""): if status == "RUNNING": return {"ok": True, "status": "already_running"} + drives = await loop.run_in_executor(None, _detect_external_drives) + if not drives: + raise HTTPException(status_code=400, detail="No external backup drive detected.") + + drive_map = {d.get("path", ""): d for d in drives if d.get("path")} + if target: + if target not in drive_map: + raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.") + selected = drive_map[target] + else: + selected = drives[0] + + selected_target = selected.get("path", "") + selected_fstype = (selected.get("fstype") or "").lower() + if selected_fstype and selected_fstype not in {"exfat", "fuseblk"}: + raise HTTPException( + status_code=400, + detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.", + ) + # Clear stale log before starting try: with open(BACKUP_LOG, "w") as f: @@ -3699,21 +3758,34 @@ async def api_backup_run(target: str = ""): except OSError: pass - env = dict(os.environ) - if target: - env["BACKUP_TARGET"] = target + try: + await loop.run_in_executor(None, _write_backup_status, "RUNNING") + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}") - # Fire-and-forget: the script writes its own status/log files. - # Progress is read by the client via /api/backup/status (same pattern - # as /api/updates/run and the rebuild feature). - await asyncio.create_subprocess_exec( - "/usr/bin/env", "bash", BACKUP_SCRIPT, - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL, - env=env, + await loop.run_in_executor( + None, + _append_backup_log, + f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…", ) - return {"ok": True, "status": "started"} + env = dict(os.environ) + env["BACKUP_TARGET"] = selected_target + + try: + proc = await asyncio.create_subprocess_exec( + "/usr/bin/env", "bash", BACKUP_SCRIPT, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + env=env, + ) + except Exception as exc: + await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}") + await loop.run_in_executor(None, _write_backup_status, "FAILED") + raise HTTPException(status_code=500, detail="Failed to launch backup process.") + + asyncio.create_task(_monitor_backup_subprocess(proc)) + return {"ok": True, "status": "started", "target": selected_target} # ── Feature Manager endpoints ───────────────────────────────────── diff --git a/app/sovran_systemsos_web/static/js/support.js b/app/sovran_systemsos_web/static/js/support.js index 6078888..f6dd50c 100644 --- a/app/sovran_systemsos_web/static/js/support.js +++ b/app/sovran_systemsos_web/static/js/support.js @@ -491,7 +491,7 @@ function renderBackupReady(drives) { '
Requirements
', '
    ', '
  1. USB hard drive plugged into one of the open USB ports on your Sovran Pro
    2. ', - '
  2. At least 500 GB of free space on the drive
    3. ', + '
  3. Enough free space for your selected backup data (the backup checks this before starting)
    4. ', '
  4. Drive must be formatted as exFAT
    5. ', '
', '', @@ -584,9 +584,10 @@ async function pollBackupStatus() { logDiv.scrollTop = logDiv.scrollHeight; } _backupLogOffset = data.offset; - if (!data.running) { + var result = (data.result || "").toLowerCase(); + if (result === "success" || result === "failed") { stopBackupPoll(); - renderBackupDone(data.result === "success"); + renderBackupDone(result === "success"); } } catch (_) {} } diff --git a/app/tests/test_manual_backup_workflow.py b/app/tests/test_manual_backup_workflow.py new file mode 100644 index 0000000..109bc8c --- /dev/null +++ b/app/tests/test_manual_backup_workflow.py @@ -0,0 +1,41 @@ +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +BACKUP_SCRIPT = REPO_ROOT / "app" / "sovran_systemsos_web" / "scripts" / "sovran-hub-backup.sh" +SERVER_FILE = REPO_ROOT / "app" / "sovran_systemsos_web" / "server.py" +SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "support.js" + + +class ManualBackupWorkflowTests(unittest.TestCase): + def test_backup_script_uses_tar_archives_with_checksums_and_exclusions(self): + source = BACKUP_SCRIPT.read_text() + + self.assertIn("tar \\", source) + self.assertIn("--create", source) + self.assertIn("--one-file-system", source) + self.assertIn("sha256sum", source) + self.assertIn("export_postgresql_dumps", source) + self.assertIn("export_mariadb_dumps", source) + self.assertIn("export_lnd_scb_if_possible", source) + self.assertIn("--exclude='var/lib/bitcoind'", source) + self.assertIn("--exclude='var/lib/electrs'", source) + self.assertIn("--exclude='var/lib/lnd'", source) + self.assertIn("set_status \"RUNNING\"", source) + self.assertIn("set_status \"SUCCESS\"", source) + self.assertIn("set_status \"FAILED\"", source) + self.assertNotIn("rsync -a", source) + + def test_backend_and_frontend_use_explicit_backup_terminal_states(self): + server_source = SERVER_FILE.read_text() + support_source = SUPPORT_JS.read_text() + + self.assertIn("_write_backup_status, \"RUNNING\"", server_source) + self.assertIn("_monitor_backup_subprocess", server_source) + self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source) + self.assertIn("result === \"success\" || result === \"failed\"", support_source) + + +if __name__ == "__main__": + unittest.main() diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index 719d90f..af01ed7 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -388,7 +388,16 @@ in pkgs.nftables pkgs.iptables pkgs.hostname - ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package; + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.gnutar + pkgs.util-linux + ] + ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package + ++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ] + ++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ] + ++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ]; }; systemd.services.sovran-hub-update = {