diff --git a/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh b/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh index 3e7c05c..c995cc1 100755 --- a/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh +++ b/app/sovran_systemsos_web/scripts/sovran-hub-backup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # ── Sovran Hub External Backup Script ──────────────────────────── -# Backs up Sovran_SystemsOS data to an external USB hard drive. +# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync. # Designed for the Hub web UI (no GUI dependencies). # # Your Sovran Pro already backs up your data automatically to its @@ -8,6 +8,15 @@ # This script creates an additional copy on an external USB drive — # storing your data in a third location for maximum protection. # +# The external drive must be formatted as ext4. Files are stored as +# directly browsable files under Sovran_SystemsOS_Backup/current/. +# Later runs update the same mirror and only transfer changed or new +# files, making repeat backups fast. +# +# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin +# blockchain and Electrs index data are NOT included (they live on +# the internal second drive). +# # Usage: # BACKUP_TARGET=/run/media// bash sovran-hub-backup.sh # (or run with no env var to auto-detect the first external USB drive) @@ -28,15 +37,11 @@ INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/") FAILED_ALREADY=0 BACKUP_COMPLETE=0 -LND_STOPPED=0 -LND_UNITS_TO_RESTART=() +RSYNC_WARNINGS=() -ARCHIVE_FILES=() -ARCHIVE_WARNINGS=() -PARTIAL_FILES=() -DB_DUMP_FILES=() -MANIFEST_EXCLUDES=() -LND_BACKUP_NOTES=() +# Stable mirror path — not timestamped so later runs update the same +# destination and only transfer new or changed files. +CURRENT_DIR_NAME="Sovran_SystemsOS_Backup/current" # ── Logging helpers ────────────────────────────────────────────── @@ -58,39 +63,10 @@ fail() { cleanup() { local rc=$? - local restart_failed=0 - - # Remove any partial archive files or temporary diagnostic files - if [[ "${#PARTIAL_FILES[@]}" -gt 0 ]]; then - local partial - for partial in "${PARTIAL_FILES[@]}"; do - [[ -f "$partial" ]] && rm -f "$partial" || true - done - fi # Release the concurrency lock file descriptor if it was opened [[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true - if [[ "$LND_STOPPED" -eq 1 ]]; then - log "Restarting previously active LND-related services…" - for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do - local unit="${LND_UNITS_TO_RESTART[$idx]}" - if systemctl start "$unit"; then - log "Started $unit" - else - log "ERROR: Failed to start $unit" - restart_failed=1 - fi - done - LND_STOPPED=0 - fi - - if [[ "$restart_failed" -eq 1 ]]; then - rc=1 - FAILED_ALREADY=1 - set_status "FAILED" - fi - if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then return fi @@ -219,21 +195,8 @@ validate_target_mount() { fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true) [[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'." - if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then - fail "Target '$target' must be exFAT (detected filesystem: $fstype)." - fi - - if [[ "$fstype" == "fuseblk" ]]; then - local src_dev blk_type - src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true) - blk_type="" - if [[ -n "$src_dev" ]]; then - blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true) - [[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true) - fi - if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then - fail "Target '$target' is fuseblk but not identified as exFAT-compatible." - fi + if [[ "$fstype" != "ext4" ]]; then + fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported." fi local write_test @@ -245,14 +208,6 @@ validate_target_mount() { log "Verified backup target filesystem: $fstype" } -has_unit() { - systemctl cat "$1" >/dev/null 2>&1 -} - -is_unit_active() { - systemctl is-active --quiet "$1" -} - estimate_path_bytes() { local path="$1" shift || true @@ -267,6 +222,48 @@ estimate_path_bytes() { echo "$size" } +# ── Run rsync for one source tree ──────────────────────────────── +# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal. +# For /home only — files may disappear while the desktop is active. +# All other nonzero exit codes are always fatal. +run_rsync() { + local label="$1" + local allow_vanished="$2" + shift 2 + # Remaining args are passed directly to rsync (filters + source + dest). + + local rsync_err_tmp + rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)" + + local rc=0 + rsync \ + --archive \ + --acls \ + --xattrs \ + --hard-links \ + --numeric-ids \ + --one-file-system \ + --partial \ + "$@" 2>"$rsync_err_tmp" || rc=$? + + if [[ -s "$rsync_err_tmp" ]]; then + while IFS= read -r rline; do + log "rsync: $rline" + done < "$rsync_err_tmp" + fi + rm -f "$rsync_err_tmp" + + if [[ "$rc" -eq 0 ]]; then + return 0 + elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then + log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up." + RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)") + return 0 + else + fail "rsync failed for $label (exit code $rc). See the rsync errors above." + fi +} + # ── Initialise log file ────────────────────────────────────────── : > "$BACKUP_LOG" @@ -286,22 +283,17 @@ exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \ flock --nonblock "$LOCK_FD" 2>/dev/null || \ fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS." -require_cmd tar -require_cmd sha256sum +require_cmd rsync require_cmd findmnt require_cmd lsblk require_cmd mountpoint require_cmd df require_cmd du require_cmd awk -require_cmd sort require_cmd find -require_cmd systemctl require_cmd hostname require_cmd date require_cmd python3 -require_cmd runuser -require_cmd mktemp require_cmd flock # ── Detect system role ─────────────────────────────────────────── @@ -327,50 +319,28 @@ else log "Auto-detecting external USB drives…" TARGET="$(find_external_drive)" if [[ -z "$TARGET" ]]; then - fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again." + fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again." fi log "Detected external drive: $TARGET" fi validate_target_mount "$TARGET" -# ── Plan role-aware source scope and exclusions ───────────────── +# ── Set up stable backup destination ──────────────────────────── +# Subsequent runs update the same mirror, transferring only new or changed files. -LND_AVAILABLE=0 -if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then - LND_AVAILABLE=1 -fi +BACKUP_DIR="${TARGET}/${CURRENT_DIR_NAME}" +mkdir -p "$BACKUP_DIR" -if [[ "$ROLE" == "desktop" ]]; then - MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)") -else - MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent") -fi - -MANIFEST_EXCLUDES+=( - "/run/media/Second_Drive (never traversed)" - "/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)" - "/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)" - "/var/lib/bitcoind (excluded from manual backup)" - "/var/lib/electrs (excluded from manual backup)" - "/var/lib/*/log and /var/lib/*/logs" - "/var/lib/*/cache and /var/lib/*/tmp" - "/home/*/.cache (system and application disk caches)" - "/home/*/.local/share/Trash and /home/*/Trash (trash directories)" - "/home/*/.mozilla/firefox/*/cache2 and */startupCache (Firefox volatile cache — profile data is kept)" - "/home/*/.config/google-chrome/*/Cache (Chrome disk cache — profile data is kept)" - "/home/*/.config/chromium/*/Cache (Chromium disk cache — profile data is kept)" - "/home/*/.config/BraveSoftware/Brave-Browser/*/Cache (Brave disk cache — profile data is kept)" - "/home/*/.local/share/baloo (KDE file indexer — rebuilt automatically)" - "/home/*/.thumbnails (thumbnail cache — rebuilt automatically)" - "/home/*/.xsession-errors and .xsession-errors.old (X session error logs)" -) - -if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then - MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive") -fi +# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only +# after all rsync stages and manifest write succeed. Failed or interrupted +# runs keep this marker so they are clearly identifiable. +touch "$BACKUP_DIR/INCOMPLETE" +log "Backup destination: $BACKUP_DIR" # ── Estimate required free space ───────────────────────────────── +# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded +# from the estimate to avoid inflating the required size. ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos) HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash') @@ -380,21 +350,20 @@ if [[ "$ROLE" != "desktop" ]]; then fi VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \ + --exclude='postgresql' \ + --exclude='mysql' \ + --exclude='mariadb' \ --exclude='bitcoind' \ --exclude='electrs' \ - --exclude='lnd' \ --exclude='*/log' \ --exclude='*/logs' \ --exclude='*/cache' \ --exclude='*/tmp') -LND_BYTES=0 -if [[ "$LND_AVAILABLE" -eq 1 ]]; then - LND_BYTES=$(estimate_path_bytes /var/lib/lnd) -fi - -ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES )) -# Require 20% growth headroom plus an additional fixed 1 GiB safety margin. +ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES )) +# Require 20% growth headroom plus a fixed 1 GiB safety margin. +# Later incremental runs need far less space, but a conservative first-run +# check protects against running out of space mid-backup. REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES )) FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ') @@ -408,391 +377,81 @@ log "Free space on drive: ${FREE_GB} GB" (( FREE_BYTES >= REQUIRED_BYTES )) || \ fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)." -# ── Create timestamped backup directory ───────────────────────── - -TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" -BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}" -DB_DUMP_DIR="$BACKUP_DIR/database-dumps" -mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR" -# Write an INCOMPLETE marker immediately; it is removed only on successful completion. -# Failed runs keep this marker so they are easily identifiable and not confused with -# complete backups during restore selection. -touch "$BACKUP_DIR/INCOMPLETE" -log "Backup destination: $BACKUP_DIR" - -create_tar_archive() { - _create_archive_impl STRICT "$@" -} - -# ── Helper: classify a GNU tar (LC_ALL=C) diagnostic line ──────── -# Returns 0 (true) if the message is an allowlisted transient condition -# that is safe to ignore for /home (live desktop file changes). -# Only two verified GNU tar messages qualify; all others are fatal. -_is_home_warning_allowlisted() { - local msg="$1" - case "$msg" in - *"file changed as we read it"*) return 0 ;; - *"file removed before we read it"*) return 0 ;; - *) return 1 ;; - esac -} - -# ── Internal archive builder ────────────────────────────────────── -# mode: STRICT — any tar error fails. -# HOME — tar exit 1 is accepted when every diagnostic is an -# allowlisted transient condition (live file changes). -_create_archive_impl() { - local mode="$1"; shift - local archive_name="$1"; shift - local archive_path="$BACKUP_DIR/$archive_name" - local partial_path="${archive_path}.partial" - local diag_tmp - # mktemp creates files with mode 0600 (owner-only) by default, so tar - # diagnostics (which may include file paths) are not readable by other users. - diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)" - PARTIAL_FILES+=("$partial_path" "$diag_tmp") - - log "Creating $archive_name …" - - local tar_rc=0 - LC_ALL=C tar \ - --create \ - --file "$partial_path" \ - --numeric-owner \ - --acls \ - --xattrs \ - --sparse \ - --one-file-system \ - "$@" 2>"$diag_tmp" || tar_rc=$? - - # Log every tar diagnostic to the backup log so it appears in the Hub UI - local has_fatal_diag=0 - if [[ -s "$diag_tmp" ]]; then - while IFS= read -r diag_line; do - [[ -n "$diag_line" ]] || continue - log "tar: $diag_line" - if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then - has_fatal_diag=1 - fi - done < "$diag_tmp" - fi - - local accept=0 - if [[ "$tar_rc" -eq 0 ]]; then - accept=1 - elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then - accept=1 - log "NOTE: $archive_name completed with nonfatal warnings (live files changed during backup -- this is normal on an active desktop and does not affect the safety of your backup)." - ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings -- live files changed during backup (normal on an active desktop)") - fi - - if [[ "$accept" -eq 0 ]]; then - rm -f "$partial_path" "$diag_tmp" - if [[ "$tar_rc" -gt 1 ]]; then - fail "tar exited with fatal code $tar_rc while creating $archive_name." - elif [[ "$mode" == "HOME" ]]; then - fail "tar exited with code $tar_rc and unrecognized diagnostics while creating $archive_name." - else - fail "tar exited with code $tar_rc while creating $archive_name." - fi - fi - - # Verify the partial archive is non-empty and readable (spot-check first entry) - if [[ ! -s "$partial_path" ]]; then - rm -f "$partial_path" "$diag_tmp" - fail "Archive $archive_name is empty after creation." - fi - - # Fast readability check: read only the first tar entry header (~512 bytes). - # head -1 closes the pipe after one line, sending SIGPIPE to tar which then - # exits early — so this is O(1) regardless of archive size. - local spot_entry - spot_entry="$(LC_ALL=C tar --list --file "$partial_path" 2>/dev/null | head -1 || true)" - if [[ -z "$spot_entry" ]]; then - rm -f "$partial_path" "$diag_tmp" - fail "Archive $archive_name failed readability check." - fi - - # Atomic publish: rename partial to final path only after acceptance - mv "$partial_path" "$archive_path" - rm -f "$diag_tmp" - ARCHIVE_FILES+=("$archive_name") - log "Created archive: $archive_name" -} - -# ── Home archive: tolerates allowlisted live-file warnings ─────── -create_home_tar_archive() { - _create_archive_impl HOME "$@" -} - -export_postgresql_dumps() { - if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then - log "PostgreSQL tools/service not available — skipping PostgreSQL exports." - return - fi - - if ! is_unit_active "postgresql.service"; then - log "PostgreSQL service is not active — skipping PostgreSQL exports." - return - fi - - log "Exporting PostgreSQL globals and databases…" - local globals_file="$DB_DUMP_DIR/postgresql_globals.sql" - runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \ - fail "Failed to export PostgreSQL globals." - DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql") - - local dbs - dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true) - - if [[ -z "$dbs" ]]; then - log "No non-template PostgreSQL application databases found." - return - fi - - while IFS= read -r db; do - [[ -n "$db" ]] || continue - local safe_db - safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')" - local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump" - runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \ - fail "Failed to export PostgreSQL database '$db'." - DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump") - done <<< "$dbs" -} - -export_mariadb_dumps() { - local dump_cmd="" - local query_cmd="" - local mariadb_unit="" - - if command -v mariadb-dump >/dev/null 2>&1; then - dump_cmd="mariadb-dump" - elif command -v mysqldump >/dev/null 2>&1; then - dump_cmd="mysqldump" - fi - - if command -v mariadb >/dev/null 2>&1; then - query_cmd="mariadb" - elif command -v mysql >/dev/null 2>&1; then - query_cmd="mysql" - fi - - if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then - log "MariaDB dump/query tools not available — skipping MariaDB exports." - return - fi - - if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then - mariadb_unit="mariadb.service" - elif has_unit "mysql.service" && is_unit_active "mysql.service"; then - mariadb_unit="mysql.service" - else - log "MariaDB service is not active — skipping MariaDB exports." - return - fi - - log "Exporting MariaDB databases from ${mariadb_unit}…" - - local dbs - dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true) - if [[ -z "$dbs" ]]; then - log "No MariaDB databases found." - return - fi - - while IFS= read -r db; do - [[ -n "$db" ]] || continue - case "$db" in - information_schema|performance_schema|mysql|sys) continue ;; - esac - - local safe_db out_file - safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')" - out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql" - - $dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \ - fail "Failed to export MariaDB database '$db'." - - DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql") - done <<< "$dbs" -} - -export_lnd_scb_if_possible() { - [[ "$LND_AVAILABLE" -eq 1 ]] || return - - local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb" - local attempts=( - "lncli exportchanbackup --all --output_file $scb_file" - "lncli -n mainnet exportchanbackup --all --output_file $scb_file" - "runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file" - "runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file" - ) - - if ! command -v lncli >/dev/null 2>&1; then - log "lncli not available — skipping Static Channel Backup export." - LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)") - return - fi - - if ! is_unit_active "lnd.service"; then - log "LND service is not active — skipping Static Channel Backup export." - LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)") - return - fi - - log "Exporting LND Static Channel Backup…" - local attempt - for attempt in "${attempts[@]}"; do - if eval "$attempt" >/dev/null 2>&1; then - DB_DUMP_FILES+=("lnd-static-channel-backup.scb") - LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli") - log "LND Static Channel Backup exported." - return - fi - done - - log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations." - LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)") -} - -capture_active_lnd_dependents() { - [[ "$LND_AVAILABLE" -eq 1 ]] || return - - LND_UNITS_TO_RESTART=() - local raw_units="" - raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true) - - while IFS= read -r unit; do - [[ -n "$unit" ]] || continue - if is_unit_active "$unit"; then - LND_UNITS_TO_RESTART+=("$unit") - fi - done <<< "$raw_units" - - if is_unit_active "lnd.service"; then - LND_UNITS_TO_RESTART+=("lnd.service") - fi -} - -stop_lnd_stack_if_needed() { - [[ "$LND_AVAILABLE" -eq 1 ]] || return - - capture_active_lnd_dependents - - if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then - log "No active LND-related services needed stopping." - return - fi - - log "Stopping active services that depend on LND for clean /var/lib/lnd archive…" - - local unit - for unit in "${LND_UNITS_TO_RESTART[@]}"; do - if [[ "$unit" == "lnd.service" ]]; then - continue - fi - systemctl stop "$unit" || fail "Failed to stop dependent service: $unit" - log "Stopped $unit" - done - - if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then - systemctl stop lnd.service || fail "Failed to stop lnd.service" - log "Stopped lnd.service" - fi - - LND_STOPPED=1 -} - -# ── Stage 1/5: NixOS configuration ────────────────────────────── +# ── Stage 1/4: NixOS configuration ────────────────────────────── log "" -log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────" +log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────" if [[ -d /etc/nixos ]]; then - create_tar_archive "etc-nixos.tar" -C / etc/nixos + run_rsync "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/" log "Stage 1 complete." else log "WARNING: /etc/nixos not found — skipping." fi -# ── Stage 2/5: Secrets ────────────────────────────────────────── +# ── Stage 2/4: Secrets ────────────────────────────────────────── log "" -log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────" +log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────" if [[ "$ROLE" == "desktop" ]]; then log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role." elif [[ -e /etc/nix-bitcoin-secrets ]]; then - create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets + run_rsync "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/" else log "(not found: /etc/nix-bitcoin-secrets — skipping)" fi log "Stage 2 complete." -# ── Stage 3/5: Home directory ─────────────────────────────────── +# ── Stage 3/4: Home directory ─────────────────────────────────── +# Rsync exit code 24 (vanished source files) is treated as nonfatal here +# because the desktop may be active and files can disappear between the +# directory scan and the copy. All other nonzero exit codes remain fatal. log "" -log "── Stage 3/5: Home directory (/home) ───────────────────────" +log "── Stage 3/4: Home directory (/home) ───────────────────────" if [[ -d /home ]]; then - create_home_tar_archive "home.tar" \ - -C / \ - --exclude='home/*/.cache' \ - --exclude='home/*/.local/share/Trash' \ - --exclude='home/*/Trash' \ - --exclude='home/*/.mozilla/firefox/*/cache2' \ - --exclude='home/*/.mozilla/firefox/*/startupCache' \ - --exclude='home/*/.mozilla/firefox/*/thumbnails' \ - --exclude='home/*/.config/google-chrome/*/Cache' \ - --exclude='home/*/.config/google-chrome/*/Code Cache' \ - --exclude='home/*/.config/chromium/*/Cache' \ - --exclude='home/*/.config/chromium/*/Code Cache' \ - --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache' \ - --exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Code Cache' \ - --exclude='home/*/.local/share/baloo' \ - --exclude='home/*/.thumbnails' \ - --exclude='home/*/.xsession-errors' \ - --exclude='home/*/.xsession-errors.old' \ - home + run_rsync "/home" yes \ + --exclude='.cache/' \ + --exclude='.local/share/Trash/' \ + --exclude='Trash/' \ + --exclude='.mozilla/firefox/*/cache2/' \ + --exclude='.mozilla/firefox/*/startupCache/' \ + --exclude='.mozilla/firefox/*/thumbnails/' \ + --exclude='.config/google-chrome/*/Cache/' \ + --exclude='.config/google-chrome/*/Code Cache/' \ + --exclude='.config/chromium/*/Cache/' \ + --exclude='.config/chromium/*/Code Cache/' \ + --exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \ + --exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \ + --exclude='.local/share/baloo/' \ + --exclude='.thumbnails/' \ + --exclude='.xsession-errors' \ + --exclude='.xsession-errors.old' \ + /home/ "$BACKUP_DIR/home/" log "Stage 3 complete." else log "WARNING: /home not found — skipping." fi -# ── Stage 4/5: Database exports + LND artifacts ──────────────── +# ── Stage 4/4: System data ────────────────────────────────────── +# PostgreSQL/MariaDB raw database directories are excluded. Application +# databases must be backed up separately with native database tools. +# Bitcoin/Electrs data are excluded; they live on the internal second drive. log "" -log "── Stage 4/5: Database and LND consistency exports ─────────" -export_postgresql_dumps -export_mariadb_dumps -export_lnd_scb_if_possible - -if [[ "$LND_AVAILABLE" -eq 1 ]]; then - stop_lnd_stack_if_needed - create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd - LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop") -fi - -log "Stage 4 complete." - -# ── Stage 5/5: System data ────────────────────────────────────── - -log "" -log "── Stage 5/5: System data (/var/lib) ───────────────────────" +log "── Stage 4/4: System data (/var/lib) ───────────────────────" if [[ -d /var/lib ]]; then - VAR_LIB_EXCLUDES=( - --exclude='var/lib/bitcoind' - --exclude='var/lib/electrs' - --exclude='var/lib/*/log' - --exclude='var/lib/*/logs' - --exclude='var/lib/*/cache' - --exclude='var/lib/*/tmp' - ) - - if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then - VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd') - fi - - create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib - log "Stage 5 complete." + run_rsync "/var/lib" no \ + --exclude='postgresql/' \ + --exclude='mysql/' \ + --exclude='mariadb/' \ + --exclude='bitcoind/' \ + --exclude='electrs/' \ + --exclude='*/log/' \ + --exclude='*/logs/' \ + --exclude='*/cache/' \ + --exclude='*/tmp/' \ + /var/lib/ "$BACKUP_DIR/var/lib/" + log "Stage 4 complete." else log "WARNING: /var/lib not found — skipping." fi @@ -802,107 +461,88 @@ fi log "" log "Generating BACKUP_MANIFEST.txt …" MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt" -CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt" { echo "Sovran_SystemsOS Backup Manifest" - echo "Generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" - echo "Timestamp: $TIMESTAMP" + echo "Updated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" echo "Hostname: $(hostname)" echo "Role: $ROLE_LABEL" echo "Target: $TARGET" echo "" - echo "Source paths included:" - echo "- /etc/nixos" - echo "- /home" + echo "Backup type: Live rsync mirror (directly browsable files)" + echo "Location: ${BACKUP_DIR}" + echo "" + echo "Source paths mirrored:" + echo "- /etc/nixos → current/etc/nixos/" if [[ "$ROLE" != "desktop" ]]; then - echo "- /etc/nix-bitcoin-secrets (when present)" + echo "- /etc/nix-bitcoin-secrets (when present) → current/etc/nix-bitcoin-secrets/" fi - echo "- /var/lib" + echo "- /home → current/home/" + echo "- /var/lib → current/var/lib/" echo "" echo "Exclusions:" - for ex in "${MANIFEST_EXCLUDES[@]}"; do - echo "- $ex" - done + echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)" + echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)" + echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)" + echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)" + echo "- /run/media/Second_Drive (internal second drive — never traversed)" + echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp" + echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs" echo "" - echo "Archives:" - for archive in "${ARCHIVE_FILES[@]}"; do - echo "- $archive" - done - echo "" - echo "Database and LND exports:" - if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then - echo "- none" - else - for dump in "${DB_DUMP_FILES[@]}"; do - echo "- $dump" - done - for note in "${LND_BACKUP_NOTES[@]}"; do - echo "- $note" - done - fi + echo "Important limitations:" + echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this" + echo " backup. If you use Nextcloud, Matrix/Synapse, or other database-backed" + echo " applications, their data must be backed up separately using native tools." + echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are" + echo " reconstructable or stored on the internal second drive." + echo "- This is a live file-level mirror, not a transactional database backup." + echo " Files being written during the backup may be in an inconsistent state." echo "" echo "Restore guidance:" - echo "- Verify artifacts: cd && sha256sum -c SHA256SUMS.txt" - echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf .tar -C /" - echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql" - echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_.dump" - echo "- MariaDB DB dump: mariadb < database-dumps/mariadb_.sql" - echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures" - echo "- Note: when restoring /var/lib, exclude raw DB directories (var/lib/postgresql, var/lib/mysql)" - echo " and restore from native dumps instead for PostgreSQL and MariaDB" + echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}" + echo "- To restore a directory:" + echo " sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/" + echo " sudo rsync -aAXH --numeric-ids current/home/ /home/" + echo " sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/" + echo "- To copy individual files:" + echo " sudo cp -a current/home/username/ /home/username/" + echo "- When restoring /etc/nixos to replacement hardware, regenerate" + echo " hardware-configuration.nix for the new hardware before rebuilding." echo "" echo "Nonfatal warnings:" - if [[ "${#ARCHIVE_WARNINGS[@]}" -eq 0 ]]; then + if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then echo "- none" else - for warning in "${ARCHIVE_WARNINGS[@]}"; do + for warning in "${RSYNC_WARNINGS[@]}"; do echo "- $warning" done fi echo "" - echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded" + echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded" echo "from manual external backup because they already live on the internal second drive" echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data." - echo "" - echo "Artifact listing:" - # INCOMPLETE exists at this point (created at backup start); BACKUP_COMPLETE does not - # exist yet (written after checksums). Excluding both marker files here is intentional: - # INCOMPLETE is excluded so it doesn't appear as a data artifact, and BACKUP_COMPLETE - # is excluded defensively for consistency should the ordering ever change. - find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f \ - ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z | tr '\0' '\n' } > "$MANIFEST_FILE" -# ── Generate checksums for all backup artifacts ───────────────── - -log "Generating SHA-256 checksums …" -( - cd "$BACKUP_DIR" - while IFS= read -r -d '' file; do - sha256sum "$file" - done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z) -) > "$CHECKSUM_FILE" - log "Manifest written to $MANIFEST_FILE" -log "Checksums written to $CHECKSUM_FILE" # ── Done ───────────────────────────────────────────────────────── log "" -if [[ "${#ARCHIVE_WARNINGS[@]}" -gt 0 ]]; then +if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then log "Backup completed with nonfatal warnings:" - for warning in "${ARCHIVE_WARNINGS[@]}"; do + for warning in "${RSYNC_WARNINGS[@]}"; do log " WARNING: $warning" done - log "Your important data is backed up. The warnings above indicate files that changed" - log "during backup, which is normal on an active desktop and does not affect your backup." + log "Your important data is backed up. The warnings above indicate files that" + log "vanished during backup, which is normal on an active desktop." log "" fi log "All Finished! Your data is now backed up to a third location." +log "Files are directly browsable on the drive under: ${BACKUP_DIR}" log "Please eject the drive safely before removing it from your Sovran Pro." -# Remove incomplete marker and write completion marker only after all work succeeds +# Remove incomplete marker and write completion marker only after all work succeeds. +# A later successful run will update the same mirror and replace any INCOMPLETE state. rm -f "$BACKUP_DIR/INCOMPLETE" echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE" diff --git a/app/sovran_systemsos_web/server.py b/app/sovran_systemsos_web/server.py index d255171..198bf24 100644 --- a/app/sovran_systemsos_web/server.py +++ b/app/sovran_systemsos_web/server.py @@ -1494,38 +1494,13 @@ def _is_internal_mount(mnt: str) -> bool: def _is_supported_backup_fstype(path: str, fstype: str) -> bool: - """Return whether the target filesystem type is supported for manual backup.""" + """Return whether the target filesystem type is supported for manual backup. + + Manual Backup requires ext4 for Linux metadata preservation (ACLs, xattrs, + hard links). exFAT, FAT32, NTFS, and other filesystems are not supported. + """ fstype = (fstype or "").lower() - if fstype == "exfat": - return True - if fstype != "fuseblk": - return False - - src_dev = "" - try: - result = subprocess.run( - ["findmnt", "-n", "-o", "SOURCE", "-T", path], - capture_output=True, text=True, timeout=5, - ) - if result.returncode == 0: - src_dev = result.stdout.strip() - except Exception: - src_dev = "" - - if not src_dev: - return False - - for cmd in ( - ["lsblk", "-no", "FSTYPE", src_dev], - ["blkid", "-o", "value", "-s", "TYPE", src_dev], - ): - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=5) - if result.returncode == 0 and result.stdout.strip().lower() in {"exfat", "fuseblk"}: - return True - except Exception: - continue - return False + return fstype == "ext4" def _detect_external_drives() -> list[dict]: @@ -3799,7 +3774,7 @@ async def api_backup_run(target: str = ""): if selected_fstype and not _is_supported_backup_fstype(selected_target, selected_fstype): raise HTTPException( status_code=400, - detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.", + detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup. Manual Backup requires an ext4-formatted drive.", ) # Clear stale log before starting diff --git a/app/sovran_systemsos_web/static/js/support.js b/app/sovran_systemsos_web/static/js/support.js index a1e272b..acae4cf 100644 --- a/app/sovran_systemsos_web/static/js/support.js +++ b/app/sovran_systemsos_web/static/js/support.js @@ -491,8 +491,9 @@ function renderBackupReady(drives) { '
Requirements
', '
    ', '
  1. USB hard drive plugged into one of the open USB ports on your Sovran Pro
    2. ', - '
  2. Enough free space for your selected backup data (the backup checks this before starting)
    3. ', - '
  3. Drive must be formatted as exFAT
    4. ', + '
  4. Enough free space for your data (the backup checks this before starting)
    5. ', + '
  5. Drive must be formatted as ext4 (a Linux filesystem). Drives with exFAT, FAT32, or NTFS are not supported. To format a drive as ext4, use a Linux tool such as GParted or mkfs.ext4 — note that formatting erases all data on the drive.
    6. ', + '
  6. The drive is intended for Linux/Sovran recovery. It may not be directly readable by Windows or macOS without additional software.
    7. ', '
', '', @@ -501,17 +502,25 @@ function renderBackupReady(drives) { '
    ', '
  1. NixOS configuration (/etc/nixos)
    2. ', '
  2. nix-bitcoin secrets (/etc/nix-bitcoin-secrets)
    3. ', - '
  3. System service data (/var/lib) including Vaultwarden, bitcoind, LND, sovran-hub, domains, and secrets
    4. ', + '
  4. System service data (/var/lib) — excluding databases and blockchain data (see note below)
    5. ', '
  5. Home directory (/home)
    6. ', '
', '', '
', + '
', + '\u2139\ufe0f', + 'Database and Blockchain Data', + '
', + '

Application databases stored in PostgreSQL or MariaDB/MySQL are not included in Manual Backup. Bitcoin blockchain and Electrs index data are also excluded (they are stored on the internal second drive). If you use Nextcloud, Matrix, or other database-backed applications, back up those databases separately with their native tools.

', + '
', + + '
', '
', '\u23f1\ufe0f', 'Time Estimate', '
', - '

This backup can take up to 4 hours depending on the amount of data stored on your Sovran Pro and the speed of your external hard drive. Be patient\u2026

', + '

The first backup may take a while depending on how much data you have. Later backups are much faster because only changed or new files are copied. Files are stored directly on the drive and can be browsed without any special software.

', '
', driveSelector, @@ -619,7 +628,7 @@ function renderBackupDone(success) { '
', '
\u26a0\ufe0f
', '

Backup Failed

', - '

The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as exFAT. Then try again.

', + '

The backup did not complete successfully. Please check that the USB drive is still connected, has enough free space, and is formatted as ext4. Then try again.

', '', '', '
', diff --git a/app/tests/test_manual_backup_workflow.py b/app/tests/test_manual_backup_workflow.py index cb3da74..0687263 100644 --- a/app/tests/test_manual_backup_workflow.py +++ b/app/tests/test_manual_backup_workflow.py @@ -15,45 +15,325 @@ SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "sup NIX_HUB_FILE = REPO_ROOT / "modules" / "core" / "sovran-hub.nix" -def _extract_bash_function(script_path: Path, func_name: str) -> str: - """Extract a bash function definition from a shell script using Python regex. - - The pattern matches from 'funcname() {' to the first line whose only content - is '}' (the function closing brace, at column 0). This works correctly for - functions whose control structures (case/if/while/for) use indented braces, - because only the function's own closing brace sits at column 0. It would not - correctly extract a function that contains a nested function definition. - - Returns the complete function definition, safe to inline into a test bash - script without eval or awk.""" - source = script_path.read_text() - pattern = r"^" + re.escape(func_name) + r"\(\) \{.*?^}" - match = re.search(pattern, source, re.MULTILINE | re.DOTALL) - if match is None: - raise ValueError(f"Function '{func_name}' not found in {script_path}") - return match.group(0) - - class ManualBackupWorkflowTests(unittest.TestCase): - def test_backup_script_uses_tar_archives_with_checksums_and_exclusions(self): - source = BACKUP_SCRIPT.read_text() + # ── Core design: rsync, no tar, no DB, no LND ───────────────────────────── - self.assertIn("tar \\", source) - self.assertIn("--create", source) + def test_backup_script_uses_rsync_not_tar(self): + """New design: backup must use rsync, not tar archives.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("rsync", source, "backup script must use rsync") + self.assertIn("--archive", source, "rsync must be called with --archive flag") + self.assertNotIn("tar --create", source, "backup script must not create tar archives") + self.assertNotIn("--file ", source, "backup script must not write tar --file output") + + def test_backup_script_has_no_database_dump_functions(self): + """Removed: PostgreSQL and MariaDB dump functions must not exist.""" + source = BACKUP_SCRIPT.read_text() + self.assertNotIn("pg_dump", source, "backup script must not contain pg_dump") + self.assertNotIn("pg_dumpall", source, "backup script must not contain pg_dumpall") + self.assertNotIn("mariadb-dump", source, "backup script must not contain mariadb-dump") + self.assertNotIn("mysqldump", source, "backup script must not contain mysqldump") + self.assertNotIn("export_postgresql_dumps", source) + self.assertNotIn("export_mariadb_dumps", source) + + def test_backup_script_has_no_lnd_service_orchestration(self): + """Removed: LND stop/restart and SCB export must not exist.""" + source = BACKUP_SCRIPT.read_text() + self.assertNotIn("export_lnd_scb_if_possible", source) + self.assertNotIn("stop_lnd_stack_if_needed", source) + self.assertNotIn("lncli", source, "backup script must not call lncli") + self.assertNotIn("LND_STOPPED", source, "backup script must not track LND_STOPPED state") + self.assertNotIn("LND_UNITS_TO_RESTART", source) + + def test_backup_script_has_no_tar_dependencies(self): + """Removed: sha256sum checksums for tar artifacts must not exist.""" + source = BACKUP_SCRIPT.read_text() + self.assertNotIn("sha256sum", source, "backup script must not generate tar checksums") + self.assertNotIn("SHA256SUMS", source, "backup script must not write SHA256SUMS.txt") + + # ── Rsync options ────────────────────────────────────────────────────────── + + def test_backup_script_rsync_uses_metadata_preserving_options(self): + """Rsync must be called with Linux metadata-preserving options.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("--archive", source) + self.assertIn("--acls", source) + self.assertIn("--xattrs", source) + self.assertIn("--hard-links", source) + self.assertIn("--numeric-ids", source) self.assertIn("--one-file-system", source) - self.assertIn("sha256sum", source) - self.assertIn("export_postgresql_dumps", source) - self.assertIn("export_mariadb_dumps", source) - self.assertIn("export_lnd_scb_if_possible", source) - self.assertIn("--exclude='var/lib/bitcoind'", source) - self.assertIn("--exclude='var/lib/electrs'", source) - self.assertIn("--exclude='var/lib/lnd'", source) - self.assertIn("set_status \"RUNNING\"", source) - self.assertIn("set_status \"SUCCESS\"", source) - self.assertIn("set_status \"FAILED\"", source) - self.assertNotIn("rsync -a", source) + + def test_backup_script_rsync_no_delete(self): + """Rsync must NOT use --delete or --delete-delay. + Accidental source deletion must not silently wipe the backup copy.""" + source = BACKUP_SCRIPT.read_text() + self.assertNotIn("--delete", source, + "rsync must not use --delete; accidental source deletion must not erase backup") + + def test_backup_script_uses_stable_current_mirror_path(self): + """Backup must write to a stable 'current/' path, not timestamped directories. + Later runs should update the same mirror.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("Sovran_SystemsOS_Backup/current", source, + "backup must use a stable 'current' mirror path") + # Must not create new timestamped directories per run + self.assertNotIn( + "date '+%Y%m%d_%H%M%S'", + source, + "backup must not create timestamped per-run directories", + ) + + def test_backup_script_source_destination_mapping(self): + """Each source tree must be synced to a matching destination path.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("/etc/nixos/", source) + self.assertIn('"$BACKUP_DIR/etc/nixos/"', source) + self.assertIn("/home/", source) + self.assertIn('"$BACKUP_DIR/home/"', source) + self.assertIn("/var/lib/", source) + self.assertIn('"$BACKUP_DIR/var/lib/"', source) + + # ── ext4 filesystem validation ───────────────────────────────────────────── + + def test_backup_script_requires_ext4_filesystem(self): + """Backup script must reject non-ext4 filesystems and require ext4.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + 'fstype" != "ext4"', + source, + "backup script must check for ext4 filesystem", + ) + self.assertIn("ext4", source, "backup script must reference ext4") + self.assertNotIn( + '"exfat"', + source, + "backup script must not accept exFAT (old requirement)", + ) + self.assertNotIn( + '"fuseblk"', + source, + "backup script must not accept fuseblk/NTFS", + ) + + def test_backup_script_rejects_unsupported_filesystems_in_error_message(self): + """The ext4 rejection message must mention the unsupported filesystem types.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + "exFAT, FAT32, and NTFS are not supported", + source, + "error message must clearly list unsupported filesystem types", + ) + + def test_backend_accepts_ext4_rejects_exfat(self): + """Backend _is_supported_backup_fstype must accept ext4 and reject exFAT.""" + server_source = SERVER_FILE.read_text() + # Must accept ext4 + self.assertIn( + 'return fstype == "ext4"', + server_source, + "backend must accept only ext4", + ) + # Must not accept exFAT + self.assertNotIn( + 'fstype == "exfat"', + server_source, + "backend must not accept exFAT", + ) + self.assertNotIn( + 'fuseblk', + server_source.split("_is_supported_backup_fstype")[1][:500], + "backend must not accept fuseblk", + ) + + def test_frontend_requires_ext4_not_exfat(self): + """Frontend UI copy must require ext4 and not mention exFAT as the requirement.""" + support_source = SUPPORT_JS.read_text() + self.assertIn( + "ext4", + support_source, + "support.js must mention ext4 as the required filesystem", + ) + # The word exFAT must not appear as a requirement (it may appear in a + # note about unsupported formats, but the requirement itself must say ext4) + requirements_section = re.search( + r"Requirements.*?What gets backed up", + support_source, + re.DOTALL, + ) + if requirements_section: + req_text = requirements_section.group(0) + self.assertNotIn( + "Drive must be formatted as exFAT", + req_text, + "Requirements section must not say 'formatted as exFAT'", + ) + self.assertIn( + "ext4", + req_text, + "Requirements section must say ext4", + ) + + def test_frontend_failure_message_says_ext4(self): + """Failure message in renderBackupDone must say ext4, not exFAT.""" + support_source = SUPPORT_JS.read_text() + self.assertNotIn( + "formatted as exFAT", + support_source, + "failure message must not say 'formatted as exFAT'", + ) + self.assertIn( + "formatted as ext4", + support_source, + "failure message must say 'formatted as ext4'", + ) + + # ── Database exclusions ──────────────────────────────────────────────────── + + def test_backup_script_excludes_postgresql_and_mariadb(self): + """PostgreSQL and MariaDB raw database directories must be excluded.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("--exclude='postgresql/'", source, + "rsync must exclude postgresql directory") + self.assertIn("--exclude='mysql/'", source, + "rsync must exclude mysql directory") + self.assertIn("--exclude='mariadb/'", source, + "rsync must exclude mariadb directory") + + def test_backup_script_excludes_bitcoin_and_electrs(self): + """Bitcoin blockchain and Electrs index data must be excluded.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("--exclude='bitcoind/'", source, + "rsync must exclude bitcoind directory") + self.assertIn("--exclude='electrs/'", source, + "rsync must exclude electrs directory") + + def test_manifest_states_database_exclusion(self): + """The manifest must explicitly state that PostgreSQL and MariaDB are excluded.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + "PostgreSQL and MariaDB/MySQL application databases are NOT included", + source, + "manifest must state that databases are not included", + ) + self.assertIn( + "Bitcoin blockchain data and Electrs indexes are NOT included", + source, + "manifest must state that blockchain data is not included", + ) + + def test_manifest_has_no_tar_restore_instructions(self): + """Manifest must not mention tar extraction, pg_restore, or LND SCB.""" + source = BACKUP_SCRIPT.read_text() + self.assertNotIn( + "tar --acls --xattrs", + source, + "manifest must not give tar restore instructions", + ) + self.assertNotIn( + "pg_restore", + source, + "manifest must not reference pg_restore", + ) + self.assertNotIn( + "lnd-static-channel-backup.scb", + source, + "manifest must not reference LND SCB restore procedures", + ) + + def test_manifest_has_rsync_restore_guidance(self): + """Manifest must provide rsync-based restore guidance.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + "rsync -aAXH --numeric-ids", + source, + "manifest must show rsync restore command", + ) + + # ── Exit code 24 (vanished files) for /home only ────────────────────────── + + def test_backup_script_exit_code_24_nonfatal_for_home(self): + """Rsync exit code 24 (vanished files) must be nonfatal for /home only.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + '"$rc" -eq 24', + source, + "backup script must handle rsync exit code 24", + ) + self.assertIn( + 'allow_vanished" == "yes"', + source, + "exit 24 acceptance must be gated on allow_vanished flag", + ) + + def test_backup_script_home_stage_allows_vanished(self): + """Stage 3 (/home) must call run_rsync with allow_vanished=yes.""" + source = BACKUP_SCRIPT.read_text() + # The /home call must pass "yes" as the allow_vanished argument + self.assertIn( + 'run_rsync "/home" yes', + source, + "/home stage must pass allow_vanished=yes to run_rsync", + ) + + def test_backup_script_other_stages_disallow_vanished(self): + """Stages other than /home must call run_rsync with allow_vanished=no.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + 'run_rsync "/etc/nixos" no', + source, + "/etc/nixos stage must use allow_vanished=no", + ) + self.assertIn( + 'run_rsync "/var/lib" no', + source, + "/var/lib stage must use allow_vanished=no", + ) + + # ── INCOMPLETE / BACKUP_COMPLETE markers ────────────────────────────────── + + def test_backup_script_writes_incomplete_marker(self): + """A backup directory must be marked INCOMPLETE immediately after + creation, so interrupted or failed runs are identifiable.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + 'touch "$BACKUP_DIR/INCOMPLETE"', + source, + "backup script must create INCOMPLETE marker after mkdir", + ) + + def test_backup_script_writes_backup_complete_marker(self): + """A BACKUP_COMPLETE file must be written only after all work succeeds.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn( + 'BACKUP_COMPLETE"', + source, + "backup script must write BACKUP_COMPLETE file on success", + ) + self.assertIn( + 'rm -f "$BACKUP_DIR/INCOMPLETE"', + source, + "backup script must remove INCOMPLETE marker on success", + ) + + # ── Concurrency lock ────────────────────────────────────────────────────── + + def test_backup_script_uses_flock_concurrency_lock(self): + """The script must acquire an exclusive flock before starting work.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("flock", source, "backup script must use flock") + self.assertIn("LOCK_FILE", source, "backup script must define LOCK_FILE") + + # ── Status states ───────────────────────────────────────────────────────── + + def test_backup_script_uses_running_success_failed_states(self): + """Status values must be RUNNING, SUCCESS, and FAILED.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn('set_status "RUNNING"', source) + self.assertIn('set_status "SUCCESS"', source) + self.assertIn('set_status "FAILED"', source) def test_backend_and_frontend_use_explicit_backup_terminal_states(self): + """Backend and frontend must use consistent terminal state handling.""" server_source = SERVER_FILE.read_text() support_source = SUPPORT_JS.read_text() @@ -62,85 +342,69 @@ class ManualBackupWorkflowTests(unittest.TestCase): self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source) self.assertIn("result === \"success\" || result === \"failed\"", support_source) - # ── Regression tests for exit-code-127 / missing-interpreter bug ────────── + # ── Nix service PATH ────────────────────────────────────────────────────── - def test_nix_service_path_includes_bash_and_gawk(self): - """Regression: pkgs.bash and pkgs.gawk must appear in the sovran-hub-web - service path so that the backup shell script and its awk calls can be - resolved at runtime without producing exit code 127.""" + def test_nix_service_path_includes_rsync_and_acl(self): + """pkgs.rsync and pkgs.acl must appear in the sovran-hub-web service path + so that rsync with ACL/xattr support is available at runtime.""" nix_source = NIX_HUB_FILE.read_text() self.assertIn( - "pkgs.bash", + "pkgs.rsync", nix_source, - "pkgs.bash must be declared in the sovran-hub-web systemd service path", + "pkgs.rsync must be declared in the sovran-hub-web service path", ) self.assertIn( - "pkgs.gawk", + "pkgs.acl", nix_source, - "pkgs.gawk must be declared in the sovran-hub-web systemd service path", + "pkgs.acl must be declared in the sovran-hub-web service path", ) + def test_nix_service_path_includes_bash_and_gawk(self): + """Regression: pkgs.bash and pkgs.gawk must appear in the service path.""" + nix_source = NIX_HUB_FILE.read_text() + self.assertIn("pkgs.bash", nix_source, + "pkgs.bash must be declared in the sovran-hub-web service path") + self.assertIn("pkgs.gawk", nix_source, + "pkgs.gawk must be declared in the sovran-hub-web service path") + + def test_nix_service_path_has_no_gnutar(self): + """pkgs.gnutar must be removed from the service path (no longer needed).""" + nix_source = NIX_HUB_FILE.read_text() + self.assertNotIn( + "pkgs.gnutar", + nix_source, + "pkgs.gnutar must be removed from the service path (tar is no longer used)", + ) + + # ── Regression tests for exit-code-127 / missing-interpreter bug ────────── + def test_server_resolves_bash_via_shutil_which(self): - """Regression: server must locate bash with shutil.which() rather than - relying on /usr/bin/env bash, so a missing interpreter is caught early - with a clear diagnostic instead of a cryptic exit-code-127 failure.""" + """Regression: server must locate bash with shutil.which().""" source = SERVER_FILE.read_text() - self.assertIn( - 'shutil.which("bash")', - source, - "server.py must resolve bash via shutil.which to catch missing interpreter", - ) - self.assertNotIn( - '"/usr/bin/env", "bash"', - source, - "server.py must not use /usr/bin/env bash (relies on PATH, causes code 127)", - ) + self.assertIn('shutil.which("bash")', source) + self.assertNotIn('"/usr/bin/env", "bash"', source) def test_server_logs_actionable_message_when_bash_missing(self): - """Regression: when bash is absent the server must write an actionable log - entry and set FAILED status before returning an HTTP error.""" + """Regression: when bash is absent the server must write an actionable log.""" source = SERVER_FILE.read_text() - self.assertIn( - "bash_path is None", - source, - "server.py must check that bash_path is not None before launching", - ) - self.assertIn( - "interpreter", - source, - "server.py missing-bash error message must reference 'interpreter'", - ) + self.assertIn("bash_path is None", source) + self.assertIn("interpreter", source) def test_server_captures_stderr_from_backup_subprocess(self): - """Regression: the backup subprocess must use stderr=PIPE so that any - startup error (e.g. code 127 from a missing command) is captured and - surfaced in the backup log rather than being silently discarded.""" + """Regression: backup subprocess must use stderr=PIPE.""" source = SERVER_FILE.read_text() - self.assertIn( - "stderr=asyncio.subprocess.PIPE", - source, - "backup subprocess must use stderr=PIPE to capture diagnostic output", - ) - self.assertIn( - "stderr_text", - source, - "_monitor_backup_subprocess must capture and log stderr content", - ) + self.assertIn("stderr=asyncio.subprocess.PIPE", source) + self.assertIn("stderr_text", source) def test_monitor_includes_stderr_detail_in_failed_message(self): - """Regression: _monitor_backup_subprocess must append captured stderr to - the FAILED log entry so the UI shows what went wrong (e.g. 'bash: not - found') rather than only the raw exit code.""" + """Regression: _monitor_backup_subprocess must append captured stderr.""" source = SERVER_FILE.read_text() self.assertIn("stderr_chunks", source) self.assertIn("stderr_text", source) - # stderr detail is conditionally appended only when non-empty self.assertIn('detail = f" — stderr: {stderr_text}"', source) def test_exit_code_127_subprocess_stderr_drain(self): - """Behavioral regression: a subprocess that exits 127 must have its - stderr drained without deadlock by the async drain loop.""" - + """Behavioral regression: a subprocess that exits 127 drains stderr without deadlock.""" async def _run(): proc = await asyncio.create_subprocess_exec( "bash", "-c", "echo 'bash: command not found' >&2; exit 127", @@ -162,525 +426,319 @@ class ManualBackupWorkflowTests(unittest.TestCase): self.assertEqual(rc, 127) self.assertIn("bash: command not found", stderr_out) - # ── Tests for home tar exit-1 tolerance (production fix) ────────────────── + # ── Rsync exit code 24 behavioral tests ────────────────────────────────── - def test_backup_script_has_home_tar_archive_function(self): - """The script must define create_home_tar_archive for /home only, - separate from the strict create_tar_archive used for other sources.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "create_home_tar_archive", - source, - "backup script must define create_home_tar_archive for /home", - ) - self.assertIn( - "_is_home_warning_allowlisted", - source, - "backup script must define _is_home_warning_allowlisted helper", - ) - self.assertIn( - "ARCHIVE_WARNINGS", - source, - "backup script must track nonfatal warnings in ARCHIVE_WARNINGS", - ) - - def test_backup_script_allowlist_covers_verified_gnu_tar_messages(self): - """The allowlist must contain the exact GNU tar (LC_ALL=C) strings for - the two permitted transient conditions.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "file changed as we read it", - source, - "allowlist must include exact GNU tar 'file changed as we read it' message", - ) - self.assertIn( - "file removed before we read it", - source, - "allowlist must include exact GNU tar 'file removed before we read it' message", - ) - - def test_backup_script_stage3_uses_home_tar_function(self): - """Stage 3 must call create_home_tar_archive (not create_tar_archive) - so the /home archive tolerates live file changes on an active desktop.""" - source = BACKUP_SCRIPT.read_text() - # The home archive call must use the tolerant function - self.assertIn( - 'create_home_tar_archive "home.tar"', - source, - "Stage 3 must use create_home_tar_archive for home.tar", - ) - - def test_backup_script_strict_function_still_used_for_etc(self): - """create_tar_archive (strict) must still be used for /etc/nixos, - secrets, var-lib-lnd-clean, and var-lib — only /home is tolerant.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - 'create_tar_archive "etc-nixos.tar"', - source, - "etc-nixos must use strict create_tar_archive", - ) - self.assertIn( - 'create_tar_archive "var-lib.tar"', - source, - "var-lib must use strict create_tar_archive", - ) - - # ── Tests for partial-file atomic publish ───────────────────────────────── - - def test_backup_script_uses_partial_path_and_atomic_rename(self): - """Archives must be written to a .partial path and atomically renamed - to the final name only after acceptance, so a failed run never leaves a - partial archive that appears complete.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - ".partial", - source, - "backup script must write to a .partial path before renaming", - ) - self.assertIn( - 'mv "$partial_path" "$archive_path"', - source, - "backup script must atomically rename partial to final archive path", - ) - self.assertIn( - "PARTIAL_FILES", - source, - "backup script must track partial files for cleanup", - ) - - # ── Tests for INCOMPLETE / BACKUP_COMPLETE markers ──────────────────────── - - def test_backup_script_writes_incomplete_marker(self): - """A backup directory must be marked INCOMPLETE immediately after - creation, so interrupted or failed runs are identifiable.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - 'touch "$BACKUP_DIR/INCOMPLETE"', - source, - "backup script must create INCOMPLETE marker after mkdir", - ) - - def test_backup_script_writes_backup_complete_marker(self): - """A BACKUP_COMPLETE file must be written only after all work succeeds, - so restore tools can distinguish complete from incomplete backups.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - 'BACKUP_COMPLETE"', - source, - "backup script must write BACKUP_COMPLETE file on success", - ) - self.assertIn( - 'rm -f "$BACKUP_DIR/INCOMPLETE"', - source, - "backup script must remove INCOMPLETE marker on success", - ) - - # ── Tests for concurrency lock ───────────────────────────────────────────── - - def test_backup_script_uses_flock_concurrency_lock(self): - """The script must acquire an exclusive flock before starting work to - prevent two simultaneous backup runs.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "flock", - source, - "backup script must use flock for concurrency protection", - ) - self.assertIn( - "LOCK_FILE", - source, - "backup script must define a LOCK_FILE for the flock", - ) - - # ── Tests for expanded home exclusions ──────────────────────────────────── - - def test_backup_script_home_excludes_browser_caches(self): - """The home archive must exclude browser disk caches (volatile data) - while keeping profile data, bookmarks, and user documents.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "--exclude='home/*/.mozilla/firefox/*/cache2'", - source, - "backup script must exclude Firefox cache2 directory", - ) - self.assertIn( - "--exclude='home/*/.config/google-chrome/*/Cache'", - source, - "backup script must exclude Chrome Cache directory", - ) - self.assertIn( - "--exclude='home/*/.config/chromium/*/Cache'", - source, - "backup script must exclude Chromium Cache directory", - ) - self.assertIn( - "--exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache'", - source, - "backup script must exclude Brave Cache directory", - ) - - def test_backup_script_home_excludes_other_volatile_caches(self): - """The home archive must exclude baloo, thumbnails, and X session - error logs — all volatile/reconstructable content.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "--exclude='home/*/.local/share/baloo'", - source, - ) - self.assertIn( - "--exclude='home/*/.thumbnails'", - source, - ) - self.assertIn( - "--exclude='home/*/.xsession-errors'", - source, - ) - - # ── Tests for require_cmd additions ─────────────────────────────────────── - - def test_backup_script_requires_mktemp_and_flock(self): - """The script must explicitly check for mktemp and flock via require_cmd - so that missing tools fail early with a clear error.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "require_cmd mktemp", - source, - "backup script must require mktemp", - ) - self.assertIn( - "require_cmd flock", - source, - "backup script must require flock", - ) - - # ── Behavioral tests ────────────────────────────────────────────────────── - - def test_allowlist_accepts_file_changed_message(self): - """Behavioral: _is_home_warning_allowlisted (from the actual backup script) - returns true for the exact GNU tar 'file changed as we read it' message (LC_ALL=C). - The function definition is extracted from the production script via Python regex - to ensure the test exercises the real allowlist without eval-of-awk risks.""" - func_def = _extract_bash_function(BACKUP_SCRIPT, "_is_home_warning_allowlisted") - script = ( - "#!/usr/bin/env bash\n" - + func_def + "\n" - + 'msgs=(\n' - + ' "tar: /home/user/firefox.db: file changed as we read it"\n' - + ' "tar: /home/user/.local/share/app: file removed before we read it"\n' - + ")\n" - + 'for msg in "${msgs[@]}"; do\n' - + ' _is_home_warning_allowlisted "$msg" || { echo "BLOCKED: $msg"; exit 1; }\n' - + ' echo "ALLOWED: $msg"\n' - + "done\n" - ) - with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as f: - f.write(script) - script_path = f.name - try: - result = subprocess.run(["bash", script_path], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("ALLOWED", result.stdout) - self.assertNotIn("BLOCKED", result.stdout) - finally: - os.unlink(script_path) - - def test_allowlist_rejects_fatal_diagnostics(self): - """Behavioral: _is_home_warning_allowlisted (from the actual backup script) - returns false for permission errors, I/O errors, write errors, and other - fatal conditions.""" - func_def = _extract_bash_function(BACKUP_SCRIPT, "_is_home_warning_allowlisted") - script = ( - "#!/usr/bin/env bash\n" - + func_def + "\n" - + 'msgs=(\n' - + ' "tar: /home/user/file: Permission denied"\n' - + ' "tar: /dev/sdb: Cannot read: Input/output error"\n' - + ' "tar: Error is not recoverable: exiting now"\n' - + ' "Write error"\n' - + ' "tar: /home/user: Cannot stat: No such file or directory"\n' - + ")\n" - + 'for msg in "${msgs[@]}"; do\n' - + ' _is_home_warning_allowlisted "$msg" && { echo "ALLOWED_BUT_SHOULD_NOT: $msg"; exit 1; }\n' - + ' echo "CORRECTLY_BLOCKED: $msg"\n' - + "done\n" - ) - with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete=False) as f: - f.write(script) - script_path = f.name - try: - result = subprocess.run(["bash", script_path], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("CORRECTLY_BLOCKED", result.stdout) - self.assertNotIn("ALLOWED_BUT_SHOULD_NOT", result.stdout) - finally: - os.unlink(script_path) - - def test_home_tar_exits_1_with_only_allowlisted_warnings_is_accepted(self): - """Behavioral: _create_archive_impl in HOME mode accepts tar exit 1 when - every diagnostic is allowlisted and publishes the final archive.""" - with tempfile.TemporaryDirectory() as tmpdir: - src = os.path.join(tmpdir, "home", "user1") - dest = os.path.join(tmpdir, "backup") - os.makedirs(src, exist_ok=True) - os.makedirs(dest, exist_ok=True) - with open(os.path.join(src, "testfile.txt"), "w") as f: - f.write("test content\n") - - # Emulate the allowlist + acceptance logic in HOME mode - script = f"""#!/usr/bin/env bash -set -euo pipefail -BACKUP_DIR="{dest}" -ARCHIVE_FILES=() -ARCHIVE_WARNINGS=() -PARTIAL_FILES=() - -log() {{ echo "$*"; }} -fail() {{ echo "FAIL: $*" >&2; exit 1; }} - -_is_home_warning_allowlisted() {{ - local msg="$1" - case "$msg" in - *"file changed as we read it"*) return 0 ;; - *"file removed before we read it"*) return 0 ;; - *) return 1 ;; - esac -}} - -_create_archive_impl() {{ - local mode="$1"; shift - local archive_name="$1"; shift - local archive_path="$BACKUP_DIR/$archive_name" - local partial_path="${{archive_path}}.partial" - local diag_tmp - diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)" - PARTIAL_FILES+=("$partial_path" "$diag_tmp") - - local tar_rc=0 - LC_ALL=C tar --create --file "$partial_path" \ - --numeric-owner --sparse --one-file-system \ - "$@" 2>"$diag_tmp" || tar_rc=$? - - # Inject a simulated allowlisted warning to match production scenario - echo "tar: {src}/testfile.txt: file changed as we read it" >> "$diag_tmp" - tar_rc=1 # simulate exit 1 - - local has_fatal_diag=0 - if [[ -s "$diag_tmp" ]]; then - while IFS= read -r diag_line; do - [[ -n "$diag_line" ]] || continue - if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then - has_fatal_diag=1 - fi - done < "$diag_tmp" - fi - - local accept=0 - if [[ "$tar_rc" -eq 0 ]]; then - accept=1 - elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then - accept=1 - ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings") - fi - - [[ "$accept" -eq 1 ]] || {{ rm -f "$partial_path" "$diag_tmp"; fail "Rejected"; }} - [[ -s "$partial_path" ]] || {{ rm -f "$partial_path" "$diag_tmp"; fail "Empty"; }} - - mv "$partial_path" "$archive_path" - rm -f "$diag_tmp" - ARCHIVE_FILES+=("$archive_name") - echo "CREATED: $archive_name" -}} - -create_home_tar_archive() {{ _create_archive_impl HOME "$@"; }} - -create_home_tar_archive "home.tar" -C "{tmpdir}" home -echo "WARNINGS: ${{#ARCHIVE_WARNINGS[@]}}" -[[ -f "{dest}/home.tar" ]] && echo "FILE_EXISTS" || echo "FILE_MISSING" -[[ ! -f "{dest}/home.tar.partial" ]] && echo "PARTIAL_CLEANED" || echo "PARTIAL_LEFT" -""" - script_file = os.path.join(tmpdir, "test.sh") - with open(script_file, "w") as sf: - sf.write(script) - result = subprocess.run(["bash", script_file], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}\nstderr: {result.stderr}") - self.assertIn("CREATED: home.tar", result.stdout) - self.assertIn("FILE_EXISTS", result.stdout) - self.assertIn("PARTIAL_CLEANED", result.stdout) - - def test_home_tar_exits_1_with_fatal_diagnostic_fails(self): - """Behavioral: _create_archive_impl in HOME mode must fail when tar exit 1 - includes a non-allowlisted diagnostic (e.g. permission denied).""" + def test_run_rsync_exit_24_nonfatal_for_home(self): + """Behavioral: run_rsync with allow_vanished=yes treats exit 24 as a + nonfatal warning (recorded in RSYNC_WARNINGS) and does not fail.""" with tempfile.TemporaryDirectory() as tmpdir: dest = os.path.join(tmpdir, "backup") os.makedirs(dest, exist_ok=True) script = f"""#!/usr/bin/env bash set -euo pipefail -BACKUP_DIR="{dest}" -ARCHIVE_FILES=() -ARCHIVE_WARNINGS=() -PARTIAL_FILES=() +RSYNC_WARNINGS=() -fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }} log() {{ echo "$*"; }} +fail() {{ echo "FAILED: $*" >&2; exit 1; }} -_is_home_warning_allowlisted() {{ - local msg="$1" - case "$msg" in - *"file changed as we read it"*) return 0 ;; - *"file removed before we read it"*) return 0 ;; - *) return 1 ;; - esac +run_rsync() {{ + local label="$1" + local allow_vanished="$2" + shift 2 + + local rc=24 # simulate rsync exit 24 + + if [[ "$rc" -eq 0 ]]; then + return 0 + elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then + log "NOTE: $label — some files vanished during sync (normal on an active desktop)." + RSYNC_WARNINGS+=("$label: some files vanished during sync") + return 0 + else + fail "rsync failed for $label (exit code $rc)" + fi }} -# Simulate tar exit 1 with a fatal permission-denied diagnostic -archive_name="home.tar" -partial_path="$BACKUP_DIR/$archive_name.partial" -diag_tmp=$(mktemp) -echo "tar: /home/user/locked: Permission denied" > "$diag_tmp" - -tar_rc=1 -has_fatal_diag=0 -while IFS= read -r diag_line; do - [[ -n "$diag_line" ]] || continue - if ! _is_home_warning_allowlisted "$diag_line"; then - has_fatal_diag=1 - fi -done < "$diag_tmp" -rm -f "$diag_tmp" - -if [[ "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then - echo "INCORRECTLY_ACCEPTED" - exit 0 -else - fail "tar exit 1 with Permission denied" -fi +run_rsync "/home" yes /home/ "{dest}/home/" +echo "WARNINGS: ${{#RSYNC_WARNINGS[@]}}" +echo "SUCCESS" """ script_file = os.path.join(tmpdir, "test.sh") with open(script_file, "w") as sf: sf.write(script) result = subprocess.run(["bash", script_file], capture_output=True, text=True) - self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}") - self.assertIn("CORRECTLY_FAILED", result.stdout) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("SUCCESS", result.stdout) + self.assertIn("WARNINGS: 1", result.stdout) - def test_home_tar_exits_2_always_fails(self): - """Behavioral: tar exit code 2 (fatal) must always fail even in HOME mode.""" + def test_run_rsync_exit_24_fatal_for_non_home(self): + """Behavioral: run_rsync with allow_vanished=no treats exit 24 as fatal.""" with tempfile.TemporaryDirectory() as tmpdir: dest = os.path.join(tmpdir, "backup") os.makedirs(dest, exist_ok=True) script = f"""#!/usr/bin/env bash -BACKUP_DIR="{dest}" -ARCHIVE_WARNINGS=() +RSYNC_WARNINGS=() -fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }} log() {{ echo "$*"; }} +fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }} -_is_home_warning_allowlisted() {{ - case "$1" in - *"file changed as we read it"*) return 0 ;; - *"file removed before we read it"*) return 0 ;; - *) return 1 ;; - esac +run_rsync() {{ + local label="$1" + local allow_vanished="$2" + shift 2 + + local rc=24 # simulate rsync exit 24 + + if [[ "$rc" -eq 0 ]]; then + return 0 + elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then + RSYNC_WARNINGS+=("$label: vanished") + return 0 + else + fail "rsync failed for $label (exit code $rc)" + fi }} -tar_rc=2 -has_fatal_diag=0 - -accept=0 -if [[ "$tar_rc" -eq 0 ]]; then - accept=1 -elif [[ "HOME" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then - accept=1 -fi - -if [[ "$accept" -eq 0 ]]; then - if [[ "$tar_rc" -gt 1 ]]; then - fail "tar exited with fatal code $tar_rc" - fi -fi +run_rsync "/etc/nixos" no /etc/nixos/ "{dest}/etc/nixos/" echo "SHOULD_NOT_REACH_HERE" """ script_file = os.path.join(tmpdir, "test.sh") with open(script_file, "w") as sf: sf.write(script) result = subprocess.run(["bash", script_file], capture_output=True, text=True) - self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}") + self.assertEqual(result.returncode, 1) self.assertIn("CORRECTLY_FAILED", result.stdout) self.assertNotIn("SHOULD_NOT_REACH_HERE", result.stdout) - def test_partial_file_cleanup_on_failure(self): - """Behavioral: a .partial file must be removed when the archive creation - fails, so no incomplete archive is left on the destination.""" - with tempfile.TemporaryDirectory() as tmpdir: - dest = os.path.join(tmpdir, "backup") - os.makedirs(dest, exist_ok=True) + def test_run_rsync_other_nonzero_codes_always_fatal(self): + """Behavioral: rsync exit codes other than 0 and 24 (for home) are fatal.""" + for rc in [1, 10, 11, 12, 23, 25]: + with tempfile.TemporaryDirectory() as tmpdir: + dest = os.path.join(tmpdir, "backup") + os.makedirs(dest, exist_ok=True) - partial_path = os.path.join(dest, "home.tar.partial") + script = f"""#!/usr/bin/env bash +RSYNC_WARNINGS=() + +fail() {{ echo "CORRECTLY_FAILED: $*"; exit 1; }} +log() {{ echo "$*"; }} + +run_rsync() {{ + local label="$1" + local allow_vanished="$2" + shift 2 + + local rc={rc} # simulated exit code + + if [[ "$rc" -eq 0 ]]; then + return 0 + elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then + RSYNC_WARNINGS+=("$label: vanished") + return 0 + else + fail "rsync failed for $label (exit code $rc)" + fi +}} + +run_rsync "/home" yes /home/ "{dest}/home/" +echo "SHOULD_NOT_REACH_HERE" +""" + script_file = os.path.join(tmpdir, "test.sh") + with open(script_file, "w") as sf: + sf.write(script) + result = subprocess.run(["bash", script_file], capture_output=True, text=True) + self.assertEqual(result.returncode, 1, + f"Exit code {rc} should fail: {result.stdout}") + self.assertIn("CORRECTLY_FAILED", result.stdout) + + # ── Behavioral: repeated runs target same 'current' mirror ────────────── + + def test_repeated_runs_use_same_current_directory(self): + """Behavioral: a second call to the backup logic must sync to the same + BACKUP_DIR (not create a new timestamped directory) so only changed + files are transferred.""" + with tempfile.TemporaryDirectory() as tmpdir: + target = os.path.join(tmpdir, "drive") + os.makedirs(target, exist_ok=True) script = f"""#!/usr/bin/env bash -BACKUP_DIR="{dest}" -PARTIAL_FILES=() +set -euo pipefail +TARGET="{target}" +CURRENT_DIR_NAME="Sovran_SystemsOS_Backup/current" +BACKUP_DIR="${{TARGET}}/${{CURRENT_DIR_NAME}}" +mkdir -p "$BACKUP_DIR" +echo "$BACKUP_DIR" +""" + for _ in range(2): + script_file = os.path.join(tmpdir, "test.sh") + with open(script_file, "w") as sf: + sf.write(script) + result = subprocess.run(["bash", script_file], capture_output=True, text=True) + self.assertEqual(result.returncode, 0) + backup_dir = result.stdout.strip() + self.assertEqual( + backup_dir, + os.path.join(target, "Sovran_SystemsOS_Backup", "current"), + ) -fail() {{ - # Simulate cleanup removing partial files - for p in "${{PARTIAL_FILES[@]}}"; do - [[ -f "$p" ]] && rm -f "$p" - done - echo "FAILED: $*" - exit 1 -}} -log() {{ :; }} + # ── Behavioral: INCOMPLETE/BACKUP_COMPLETE marker lifecycle ───────────── -partial_path="$BACKUP_DIR/home.tar.partial" -diag_tmp=$(mktemp) -PARTIAL_FILES+=("$partial_path" "$diag_tmp") + def test_incomplete_marker_written_before_work_starts(self): + """Behavioral: INCOMPLETE marker must exist during backup and be removed on success.""" + with tempfile.TemporaryDirectory() as tmpdir: + backup_dir = os.path.join(tmpdir, "current") + os.makedirs(backup_dir, exist_ok=True) -# Simulate partial archive being written -touch "$partial_path" + script = f"""#!/usr/bin/env bash +set -euo pipefail +BACKUP_DIR="{backup_dir}" +BACKUP_COMPLETE=0 -# Simulate a fatal error -fail "tar exited with code 2" +touch "$BACKUP_DIR/INCOMPLETE" +[[ -f "$BACKUP_DIR/INCOMPLETE" ]] && echo "INCOMPLETE_EXISTS" + +# Simulate successful completion +rm -f "$BACKUP_DIR/INCOMPLETE" +echo "done" > "$BACKUP_DIR/BACKUP_COMPLETE" +BACKUP_COMPLETE=1 + +[[ ! -f "$BACKUP_DIR/INCOMPLETE" ]] && echo "INCOMPLETE_REMOVED" +[[ -f "$BACKUP_DIR/BACKUP_COMPLETE" ]] && echo "COMPLETE_EXISTS" """ script_file = os.path.join(tmpdir, "test.sh") with open(script_file, "w") as sf: sf.write(script) result = subprocess.run(["bash", script_file], capture_output=True, text=True) - self.assertEqual(result.returncode, 1, f"Should have failed: {result.stdout}") - self.assertFalse( - os.path.exists(partial_path), - "Partial file must be removed on failure", + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("INCOMPLETE_EXISTS", result.stdout) + self.assertIn("INCOMPLETE_REMOVED", result.stdout) + self.assertIn("COMPLETE_EXISTS", result.stdout) + + # ── Behavioral: actual rsync with real source/dest trees ───────────────── + + def test_rsync_mirrors_source_to_dest(self): + """Behavioral: rsync correctly mirrors a source tree to a destination.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = os.path.join(tmpdir, "etc", "nixos") + dest = os.path.join(tmpdir, "backup", "etc", "nixos") + os.makedirs(src, exist_ok=True) + os.makedirs(os.path.join(tmpdir, "backup", "etc"), exist_ok=True) + + # Write test files + with open(os.path.join(src, "configuration.nix"), "w") as f: + f.write("{ ... }: {}\n") + with open(os.path.join(src, "custom.nix"), "w") as f: + f.write("{ ... }: {}\n") + + result = subprocess.run( + ["rsync", "--archive", "--one-file-system", + src + "/", dest + "/"], + capture_output=True, text=True, ) - self.assertIn("FAILED", result.stdout) + self.assertEqual(result.returncode, 0, f"rsync failed: {result.stderr}") + self.assertTrue(os.path.exists(os.path.join(dest, "configuration.nix"))) + self.assertTrue(os.path.exists(os.path.join(dest, "custom.nix"))) - def test_archive_warnings_recorded_in_manifest(self): - """Source text: ARCHIVE_WARNINGS must be written into BACKUP_MANIFEST.txt - so the user can see which archives had nonfatal live-file warnings.""" - source = BACKUP_SCRIPT.read_text() - self.assertIn( - "ARCHIVE_WARNINGS[@]", - source, - "backup script must iterate ARCHIVE_WARNINGS in manifest generation", + def test_rsync_second_run_only_transfers_changes(self): + """Behavioral: a second rsync run to the same dest only copies changed files.""" + with tempfile.TemporaryDirectory() as tmpdir: + src = os.path.join(tmpdir, "source") + dest = os.path.join(tmpdir, "backup") + os.makedirs(src) + os.makedirs(dest) + + with open(os.path.join(src, "file.txt"), "w") as f: + f.write("initial content\n") + + # First run + r1 = subprocess.run( + ["rsync", "--archive", "--one-file-system", src + "/", dest + "/"], + capture_output=True, text=True, + ) + self.assertEqual(r1.returncode, 0) + + # Modify one file and add another; advance mtime so rsync detects the change + with open(os.path.join(src, "file.txt"), "w") as f: + f.write("updated content\n") + import time as _time + future_mtime = _time.time() + 2 + os.utime(os.path.join(src, "file.txt"), (future_mtime, future_mtime)) + + with open(os.path.join(src, "new.txt"), "w") as f: + f.write("new file\n") + + # Second run + r2 = subprocess.run( + ["rsync", "--archive", "--one-file-system", src + "/", dest + "/"], + capture_output=True, text=True, + ) + self.assertEqual(r2.returncode, 0) + + # Both files exist in dest + with open(os.path.join(dest, "file.txt")) as fh: + self.assertEqual(fh.read(), "updated content\n") + self.assertTrue(os.path.exists(os.path.join(dest, "new.txt"))) + + # ── Script syntax check ────────────────────────────────────────────────── + + def test_backup_script_passes_bash_syntax_check(self): + """bash -n must report no syntax errors in the backup script.""" + result = subprocess.run( + ["bash", "-n", str(BACKUP_SCRIPT)], + capture_output=True, text=True, ) - self.assertIn( - "Nonfatal warnings:", - source, - "manifest must include a 'Nonfatal warnings:' section", + self.assertEqual( + result.returncode, 0, + f"bash -n failed:\n{result.stderr}", ) - def test_backup_script_logs_warnings_before_success_message(self): - """Source text: when ARCHIVE_WARNINGS is non-empty, the script must log - the warnings before the 'All Finished!' success message.""" + # ── Home exclusions ────────────────────────────────────────────────────── + + def test_backup_script_home_excludes_browser_caches(self): + """Home sync must exclude browser disk caches.""" source = BACKUP_SCRIPT.read_text() - # Both constructs must be present + self.assertIn("--exclude='.mozilla/firefox/*/cache2/'", source) + self.assertIn("--exclude='.config/google-chrome/*/Cache/'", source) + self.assertIn("--exclude='.config/chromium/*/Cache/'", source) + self.assertIn("--exclude='.config/BraveSoftware/Brave-Browser/*/Cache/'", source) + + def test_backup_script_home_excludes_other_volatile_caches(self): + """Home sync must exclude baloo, thumbnails, and X session error logs.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("--exclude='.local/share/baloo/'", source) + self.assertIn("--exclude='.thumbnails/'", source) + self.assertIn("--exclude='.xsession-errors'", source) + + # ── require_cmd ────────────────────────────────────────────────────────── + + def test_backup_script_requires_rsync_and_flock(self): + """Script must require rsync and flock via require_cmd.""" + source = BACKUP_SCRIPT.read_text() + self.assertIn("require_cmd rsync", source) + self.assertIn("require_cmd flock", source) + + def test_backup_script_does_not_require_tar_or_sha256sum(self): + """Script must not require tar or sha256sum (no longer used).""" + source = BACKUP_SCRIPT.read_text() + self.assertNotIn("require_cmd tar", source) + self.assertNotIn("require_cmd sha256sum", source) + + # ── Frontend database limitation notice ────────────────────────────────── + + def test_frontend_mentions_database_limitation(self): + """Frontend must explain that PostgreSQL/MariaDB databases are excluded.""" + support_source = SUPPORT_JS.read_text() self.assertIn( - "${#ARCHIVE_WARNINGS[@]}", - source, - "backup script must check ARCHIVE_WARNINGS count before success message", + "PostgreSQL", + support_source, + "UI must mention PostgreSQL exclusion", ) self.assertIn( - "All Finished!", - source, + "not included", + support_source, + "UI must explain that databases are not included", ) diff --git a/modules/core/sovran-hub.nix b/modules/core/sovran-hub.nix index a21e4ee..e083991 100644 --- a/modules/core/sovran-hub.nix +++ b/modules/core/sovran-hub.nix @@ -393,7 +393,8 @@ in pkgs.coreutils pkgs.findutils pkgs.gnugrep - pkgs.gnutar + pkgs.rsync + pkgs.acl pkgs.util-linux ] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package