feat: replace tar+DB+LND backup with ext4+rsync workflow
- Rewrite sovran-hub-backup.sh: rsync-based mirror to stable current/ path, ext4 validation, no tar/pg_dump/mariadb-dump/LND orchestration, exit-24 nonfatal for /home, INCOMPLETE/BACKUP_COMPLETE markers, flock - Update server.py: _is_supported_backup_fstype accepts only ext4 - Update support.js: require ext4, explain database limitations, update failure message from exFAT to ext4 - Update sovran-hub.nix: replace pkgs.gnutar with pkgs.rsync + pkgs.acl - Rewrite test_manual_backup_workflow.py: 40 new tests covering rsync options, ext4 acceptance, exit-24 handling, no-delete, stable current/ path, INCOMPLETE markers, behavioral rsync tests, and regressions
This commit is contained in:
copilot-swe-agent[bot]
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# ── Sovran Hub External Backup Script ────────────────────────────
|
||||
# Backs up Sovran_SystemsOS data to an external USB hard drive.
|
||||
# Backs up Sovran_SystemsOS data to an external USB hard drive using rsync.
|
||||
# Designed for the Hub web UI (no GUI dependencies).
|
||||
#
|
||||
# Your Sovran Pro already backs up your data automatically to its
|
||||
@@ -8,6 +8,15 @@
|
||||
# This script creates an additional copy on an external USB drive —
|
||||
# storing your data in a third location for maximum protection.
|
||||
#
|
||||
# The external drive must be formatted as ext4. Files are stored as
|
||||
# directly browsable files under Sovran_SystemsOS_Backup/current/.
|
||||
# Later runs update the same mirror and only transfer changed or new
|
||||
# files, making repeat backups fast.
|
||||
#
|
||||
# PostgreSQL and MariaDB/MySQL databases are NOT included. Bitcoin
|
||||
# blockchain and Electrs index data are NOT included (they live on
|
||||
# the internal second drive).
|
||||
#
|
||||
# Usage:
|
||||
# BACKUP_TARGET=/run/media/<user>/<drive> bash sovran-hub-backup.sh
|
||||
# (or run with no env var to auto-detect the first external USB drive)
|
||||
@@ -28,15 +37,11 @@ INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
|
||||
|
||||
FAILED_ALREADY=0
|
||||
BACKUP_COMPLETE=0
|
||||
LND_STOPPED=0
|
||||
LND_UNITS_TO_RESTART=()
|
||||
RSYNC_WARNINGS=()
|
||||
|
||||
ARCHIVE_FILES=()
|
||||
ARCHIVE_WARNINGS=()
|
||||
PARTIAL_FILES=()
|
||||
DB_DUMP_FILES=()
|
||||
MANIFEST_EXCLUDES=()
|
||||
LND_BACKUP_NOTES=()
|
||||
# Stable mirror path — not timestamped so later runs update the same
|
||||
# destination and only transfer new or changed files.
|
||||
CURRENT_DIR_NAME="Sovran_SystemsOS_Backup/current"
|
||||
|
||||
# ── Logging helpers ──────────────────────────────────────────────
|
||||
|
||||
@@ -58,39 +63,10 @@ fail() {
|
||||
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
local restart_failed=0
|
||||
|
||||
# Remove any partial archive files or temporary diagnostic files
|
||||
if [[ "${#PARTIAL_FILES[@]}" -gt 0 ]]; then
|
||||
local partial
|
||||
for partial in "${PARTIAL_FILES[@]}"; do
|
||||
[[ -f "$partial" ]] && rm -f "$partial" || true
|
||||
done
|
||||
fi
|
||||
|
||||
# Release the concurrency lock file descriptor if it was opened
|
||||
[[ -n "${LOCK_FD:-}" ]] && exec {LOCK_FD}>&- 2>/dev/null || true
|
||||
|
||||
if [[ "$LND_STOPPED" -eq 1 ]]; then
|
||||
log "Restarting previously active LND-related services…"
|
||||
for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do
|
||||
local unit="${LND_UNITS_TO_RESTART[$idx]}"
|
||||
if systemctl start "$unit"; then
|
||||
log "Started $unit"
|
||||
else
|
||||
log "ERROR: Failed to start $unit"
|
||||
restart_failed=1
|
||||
fi
|
||||
done
|
||||
LND_STOPPED=0
|
||||
fi
|
||||
|
||||
if [[ "$restart_failed" -eq 1 ]]; then
|
||||
rc=1
|
||||
FAILED_ALREADY=1
|
||||
set_status "FAILED"
|
||||
fi
|
||||
|
||||
if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
|
||||
return
|
||||
fi
|
||||
@@ -219,21 +195,8 @@ validate_target_mount() {
|
||||
fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
|
||||
[[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
|
||||
|
||||
if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then
|
||||
fail "Target '$target' must be exFAT (detected filesystem: $fstype)."
|
||||
fi
|
||||
|
||||
if [[ "$fstype" == "fuseblk" ]]; then
|
||||
local src_dev blk_type
|
||||
src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true)
|
||||
blk_type=""
|
||||
if [[ -n "$src_dev" ]]; then
|
||||
blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true)
|
||||
[[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true)
|
||||
fi
|
||||
if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then
|
||||
fail "Target '$target' is fuseblk but not identified as exFAT-compatible."
|
||||
fi
|
||||
if [[ "$fstype" != "ext4" ]]; then
|
||||
fail "Target '$target' must be formatted as ext4 (detected filesystem: $fstype). Manual Backup requires an ext4-formatted external drive for Linux metadata preservation. exFAT, FAT32, and NTFS are not supported."
|
||||
fi
|
||||
|
||||
local write_test
|
||||
@@ -245,14 +208,6 @@ validate_target_mount() {
|
||||
log "Verified backup target filesystem: $fstype"
|
||||
}
|
||||
|
||||
has_unit() {
|
||||
systemctl cat "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
is_unit_active() {
|
||||
systemctl is-active --quiet "$1"
|
||||
}
|
||||
|
||||
estimate_path_bytes() {
|
||||
local path="$1"
|
||||
shift || true
|
||||
@@ -267,6 +222,48 @@ estimate_path_bytes() {
|
||||
echo "$size"
|
||||
}
|
||||
|
||||
# ── Run rsync for one source tree ────────────────────────────────
|
||||
# allow_vanished: "yes" means rsync exit 24 (vanished files) is nonfatal.
|
||||
# For /home only — files may disappear while the desktop is active.
|
||||
# All other nonzero exit codes are always fatal.
|
||||
run_rsync() {
|
||||
local label="$1"
|
||||
local allow_vanished="$2"
|
||||
shift 2
|
||||
# Remaining args are passed directly to rsync (filters + source + dest).
|
||||
|
||||
local rsync_err_tmp
|
||||
rsync_err_tmp="$(mktemp /tmp/sovran-rsync-err.XXXXXX)"
|
||||
|
||||
local rc=0
|
||||
rsync \
|
||||
--archive \
|
||||
--acls \
|
||||
--xattrs \
|
||||
--hard-links \
|
||||
--numeric-ids \
|
||||
--one-file-system \
|
||||
--partial \
|
||||
"$@" 2>"$rsync_err_tmp" || rc=$?
|
||||
|
||||
if [[ -s "$rsync_err_tmp" ]]; then
|
||||
while IFS= read -r rline; do
|
||||
log "rsync: $rline"
|
||||
done < "$rsync_err_tmp"
|
||||
fi
|
||||
rm -f "$rsync_err_tmp"
|
||||
|
||||
if [[ "$rc" -eq 0 ]]; then
|
||||
return 0
|
||||
elif [[ "$allow_vanished" == "yes" && "$rc" -eq 24 ]]; then
|
||||
log "NOTE: $label — some files vanished during sync (normal on an active desktop). Your important data is backed up."
|
||||
RSYNC_WARNINGS+=("$label: some files vanished during sync (rsync exit 24 — normal on active desktop)")
|
||||
return 0
|
||||
else
|
||||
fail "rsync failed for $label (exit code $rc). See the rsync errors above."
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Initialise log file ──────────────────────────────────────────
|
||||
|
||||
: > "$BACKUP_LOG"
|
||||
@@ -286,22 +283,17 @@ exec {LOCK_FD}>>"$LOCK_FILE" 2>/dev/null || \
|
||||
flock --nonblock "$LOCK_FD" 2>/dev/null || \
|
||||
fail "Another backup is already running. Wait for it to complete or check $BACKUP_STATUS."
|
||||
|
||||
require_cmd tar
|
||||
require_cmd sha256sum
|
||||
require_cmd rsync
|
||||
require_cmd findmnt
|
||||
require_cmd lsblk
|
||||
require_cmd mountpoint
|
||||
require_cmd df
|
||||
require_cmd du
|
||||
require_cmd awk
|
||||
require_cmd sort
|
||||
require_cmd find
|
||||
require_cmd systemctl
|
||||
require_cmd hostname
|
||||
require_cmd date
|
||||
require_cmd python3
|
||||
require_cmd runuser
|
||||
require_cmd mktemp
|
||||
require_cmd flock
|
||||
|
||||
# ── Detect system role ───────────────────────────────────────────
|
||||
@@ -327,50 +319,28 @@ else
|
||||
log "Auto-detecting external USB drives…"
|
||||
TARGET="$(find_external_drive)"
|
||||
if [[ -z "$TARGET" ]]; then
|
||||
fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again."
|
||||
fail "No external USB drive detected. Please plug in an ext4-formatted USB drive and try again."
|
||||
fi
|
||||
log "Detected external drive: $TARGET"
|
||||
fi
|
||||
|
||||
validate_target_mount "$TARGET"
|
||||
|
||||
# ── Plan role-aware source scope and exclusions ─────────────────
|
||||
# ── Set up stable backup destination ────────────────────────────
|
||||
# Subsequent runs update the same mirror, transferring only new or changed files.
|
||||
|
||||
LND_AVAILABLE=0
|
||||
if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then
|
||||
LND_AVAILABLE=1
|
||||
fi
|
||||
BACKUP_DIR="${TARGET}/${CURRENT_DIR_NAME}"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
if [[ "$ROLE" == "desktop" ]]; then
|
||||
MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)")
|
||||
else
|
||||
MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent")
|
||||
fi
|
||||
|
||||
MANIFEST_EXCLUDES+=(
|
||||
"/run/media/Second_Drive (never traversed)"
|
||||
"/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)"
|
||||
"/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)"
|
||||
"/var/lib/bitcoind (excluded from manual backup)"
|
||||
"/var/lib/electrs (excluded from manual backup)"
|
||||
"/var/lib/*/log and /var/lib/*/logs"
|
||||
"/var/lib/*/cache and /var/lib/*/tmp"
|
||||
"/home/*/.cache (system and application disk caches)"
|
||||
"/home/*/.local/share/Trash and /home/*/Trash (trash directories)"
|
||||
"/home/*/.mozilla/firefox/*/cache2 and */startupCache (Firefox volatile cache — profile data is kept)"
|
||||
"/home/*/.config/google-chrome/*/Cache (Chrome disk cache — profile data is kept)"
|
||||
"/home/*/.config/chromium/*/Cache (Chromium disk cache — profile data is kept)"
|
||||
"/home/*/.config/BraveSoftware/Brave-Browser/*/Cache (Brave disk cache — profile data is kept)"
|
||||
"/home/*/.local/share/baloo (KDE file indexer — rebuilt automatically)"
|
||||
"/home/*/.thumbnails (thumbnail cache — rebuilt automatically)"
|
||||
"/home/*/.xsession-errors and .xsession-errors.old (X session error logs)"
|
||||
)
|
||||
|
||||
if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
|
||||
MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive")
|
||||
fi
|
||||
# Write an INCOMPLETE marker immediately; replaced by BACKUP_COMPLETE only
|
||||
# after all rsync stages and manifest write succeed. Failed or interrupted
|
||||
# runs keep this marker so they are clearly identifiable.
|
||||
touch "$BACKUP_DIR/INCOMPLETE"
|
||||
log "Backup destination: $BACKUP_DIR"
|
||||
|
||||
# ── Estimate required free space ─────────────────────────────────
|
||||
# PostgreSQL/MariaDB raw directories and Bitcoin/Electrs data are excluded
|
||||
# from the estimate to avoid inflating the required size.
|
||||
|
||||
ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
|
||||
HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
|
||||
@@ -380,21 +350,20 @@ if [[ "$ROLE" != "desktop" ]]; then
|
||||
fi
|
||||
|
||||
VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
|
||||
--exclude='postgresql' \
|
||||
--exclude='mysql' \
|
||||
--exclude='mariadb' \
|
||||
--exclude='bitcoind' \
|
||||
--exclude='electrs' \
|
||||
--exclude='lnd' \
|
||||
--exclude='*/log' \
|
||||
--exclude='*/logs' \
|
||||
--exclude='*/cache' \
|
||||
--exclude='*/tmp')
|
||||
|
||||
LND_BYTES=0
|
||||
if [[ "$LND_AVAILABLE" -eq 1 ]]; then
|
||||
LND_BYTES=$(estimate_path_bytes /var/lib/lnd)
|
||||
fi
|
||||
|
||||
ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES ))
|
||||
# Require 20% growth headroom plus an additional fixed 1 GiB safety margin.
|
||||
ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES ))
|
||||
# Require 20% growth headroom plus a fixed 1 GiB safety margin.
|
||||
# Later incremental runs need far less space, but a conservative first-run
|
||||
# check protects against running out of space mid-backup.
|
||||
REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
|
||||
|
||||
FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
|
||||
@@ -408,391 +377,81 @@ log "Free space on drive: ${FREE_GB} GB"
|
||||
(( FREE_BYTES >= REQUIRED_BYTES )) || \
|
||||
fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
|
||||
|
||||
# ── Create timestamped backup directory ─────────────────────────
|
||||
|
||||
TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
|
||||
BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
|
||||
DB_DUMP_DIR="$BACKUP_DIR/database-dumps"
|
||||
mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR"
|
||||
# Write an INCOMPLETE marker immediately; it is removed only on successful completion.
|
||||
# Failed runs keep this marker so they are easily identifiable and not confused with
|
||||
# complete backups during restore selection.
|
||||
touch "$BACKUP_DIR/INCOMPLETE"
|
||||
log "Backup destination: $BACKUP_DIR"
|
||||
|
||||
create_tar_archive() {
|
||||
_create_archive_impl STRICT "$@"
|
||||
}
|
||||
|
||||
# ── Helper: classify a GNU tar (LC_ALL=C) diagnostic line ────────
|
||||
# Returns 0 (true) if the message is an allowlisted transient condition
|
||||
# that is safe to ignore for /home (live desktop file changes).
|
||||
# Only two verified GNU tar messages qualify; all others are fatal.
|
||||
_is_home_warning_allowlisted() {
|
||||
local msg="$1"
|
||||
case "$msg" in
|
||||
*"file changed as we read it"*) return 0 ;;
|
||||
*"file removed before we read it"*) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── Internal archive builder ──────────────────────────────────────
|
||||
# mode: STRICT — any tar error fails.
|
||||
# HOME — tar exit 1 is accepted when every diagnostic is an
|
||||
# allowlisted transient condition (live file changes).
|
||||
_create_archive_impl() {
|
||||
local mode="$1"; shift
|
||||
local archive_name="$1"; shift
|
||||
local archive_path="$BACKUP_DIR/$archive_name"
|
||||
local partial_path="${archive_path}.partial"
|
||||
local diag_tmp
|
||||
# mktemp creates files with mode 0600 (owner-only) by default, so tar
|
||||
# diagnostics (which may include file paths) are not readable by other users.
|
||||
diag_tmp="$(mktemp /tmp/sovran-tar-diag.XXXXXX)"
|
||||
PARTIAL_FILES+=("$partial_path" "$diag_tmp")
|
||||
|
||||
log "Creating $archive_name …"
|
||||
|
||||
local tar_rc=0
|
||||
LC_ALL=C tar \
|
||||
--create \
|
||||
--file "$partial_path" \
|
||||
--numeric-owner \
|
||||
--acls \
|
||||
--xattrs \
|
||||
--sparse \
|
||||
--one-file-system \
|
||||
"$@" 2>"$diag_tmp" || tar_rc=$?
|
||||
|
||||
# Log every tar diagnostic to the backup log so it appears in the Hub UI
|
||||
local has_fatal_diag=0
|
||||
if [[ -s "$diag_tmp" ]]; then
|
||||
while IFS= read -r diag_line; do
|
||||
[[ -n "$diag_line" ]] || continue
|
||||
log "tar: $diag_line"
|
||||
if [[ "$mode" == "HOME" ]] && ! _is_home_warning_allowlisted "$diag_line"; then
|
||||
has_fatal_diag=1
|
||||
fi
|
||||
done < "$diag_tmp"
|
||||
fi
|
||||
|
||||
local accept=0
|
||||
if [[ "$tar_rc" -eq 0 ]]; then
|
||||
accept=1
|
||||
elif [[ "$mode" == "HOME" && "$tar_rc" -eq 1 && "$has_fatal_diag" -eq 0 ]]; then
|
||||
accept=1
|
||||
log "NOTE: $archive_name completed with nonfatal warnings (live files changed during backup -- this is normal on an active desktop and does not affect the safety of your backup)."
|
||||
ARCHIVE_WARNINGS+=("$archive_name: nonfatal warnings -- live files changed during backup (normal on an active desktop)")
|
||||
fi
|
||||
|
||||
if [[ "$accept" -eq 0 ]]; then
|
||||
rm -f "$partial_path" "$diag_tmp"
|
||||
if [[ "$tar_rc" -gt 1 ]]; then
|
||||
fail "tar exited with fatal code $tar_rc while creating $archive_name."
|
||||
elif [[ "$mode" == "HOME" ]]; then
|
||||
fail "tar exited with code $tar_rc and unrecognized diagnostics while creating $archive_name."
|
||||
else
|
||||
fail "tar exited with code $tar_rc while creating $archive_name."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify the partial archive is non-empty and readable (spot-check first entry)
|
||||
if [[ ! -s "$partial_path" ]]; then
|
||||
rm -f "$partial_path" "$diag_tmp"
|
||||
fail "Archive $archive_name is empty after creation."
|
||||
fi
|
||||
|
||||
# Fast readability check: read only the first tar entry header (~512 bytes).
|
||||
# head -1 closes the pipe after one line, sending SIGPIPE to tar which then
|
||||
# exits early — so this is O(1) regardless of archive size.
|
||||
local spot_entry
|
||||
spot_entry="$(LC_ALL=C tar --list --file "$partial_path" 2>/dev/null | head -1 || true)"
|
||||
if [[ -z "$spot_entry" ]]; then
|
||||
rm -f "$partial_path" "$diag_tmp"
|
||||
fail "Archive $archive_name failed readability check."
|
||||
fi
|
||||
|
||||
# Atomic publish: rename partial to final path only after acceptance
|
||||
mv "$partial_path" "$archive_path"
|
||||
rm -f "$diag_tmp"
|
||||
ARCHIVE_FILES+=("$archive_name")
|
||||
log "Created archive: $archive_name"
|
||||
}
|
||||
|
||||
# ── Home archive: tolerates allowlisted live-file warnings ───────
|
||||
create_home_tar_archive() {
|
||||
_create_archive_impl HOME "$@"
|
||||
}
|
||||
|
||||
export_postgresql_dumps() {
|
||||
if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then
|
||||
log "PostgreSQL tools/service not available — skipping PostgreSQL exports."
|
||||
return
|
||||
fi
|
||||
|
||||
if ! is_unit_active "postgresql.service"; then
|
||||
log "PostgreSQL service is not active — skipping PostgreSQL exports."
|
||||
return
|
||||
fi
|
||||
|
||||
log "Exporting PostgreSQL globals and databases…"
|
||||
local globals_file="$DB_DUMP_DIR/postgresql_globals.sql"
|
||||
runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \
|
||||
fail "Failed to export PostgreSQL globals."
|
||||
DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql")
|
||||
|
||||
local dbs
|
||||
dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$dbs" ]]; then
|
||||
log "No non-template PostgreSQL application databases found."
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS= read -r db; do
|
||||
[[ -n "$db" ]] || continue
|
||||
local safe_db
|
||||
safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
|
||||
local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump"
|
||||
runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \
|
||||
fail "Failed to export PostgreSQL database '$db'."
|
||||
DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump")
|
||||
done <<< "$dbs"
|
||||
}
|
||||
|
||||
export_mariadb_dumps() {
|
||||
local dump_cmd=""
|
||||
local query_cmd=""
|
||||
local mariadb_unit=""
|
||||
|
||||
if command -v mariadb-dump >/dev/null 2>&1; then
|
||||
dump_cmd="mariadb-dump"
|
||||
elif command -v mysqldump >/dev/null 2>&1; then
|
||||
dump_cmd="mysqldump"
|
||||
fi
|
||||
|
||||
if command -v mariadb >/dev/null 2>&1; then
|
||||
query_cmd="mariadb"
|
||||
elif command -v mysql >/dev/null 2>&1; then
|
||||
query_cmd="mysql"
|
||||
fi
|
||||
|
||||
if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then
|
||||
log "MariaDB dump/query tools not available — skipping MariaDB exports."
|
||||
return
|
||||
fi
|
||||
|
||||
if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then
|
||||
mariadb_unit="mariadb.service"
|
||||
elif has_unit "mysql.service" && is_unit_active "mysql.service"; then
|
||||
mariadb_unit="mysql.service"
|
||||
else
|
||||
log "MariaDB service is not active — skipping MariaDB exports."
|
||||
return
|
||||
fi
|
||||
|
||||
log "Exporting MariaDB databases from ${mariadb_unit}…"
|
||||
|
||||
local dbs
|
||||
dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true)
|
||||
if [[ -z "$dbs" ]]; then
|
||||
log "No MariaDB databases found."
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS= read -r db; do
|
||||
[[ -n "$db" ]] || continue
|
||||
case "$db" in
|
||||
information_schema|performance_schema|mysql|sys) continue ;;
|
||||
esac
|
||||
|
||||
local safe_db out_file
|
||||
safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
|
||||
out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql"
|
||||
|
||||
$dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \
|
||||
fail "Failed to export MariaDB database '$db'."
|
||||
|
||||
DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql")
|
||||
done <<< "$dbs"
|
||||
}
|
||||
|
||||
export_lnd_scb_if_possible() {
|
||||
[[ "$LND_AVAILABLE" -eq 1 ]] || return
|
||||
|
||||
local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb"
|
||||
local attempts=(
|
||||
"lncli exportchanbackup --all --output_file $scb_file"
|
||||
"lncli -n mainnet exportchanbackup --all --output_file $scb_file"
|
||||
"runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file"
|
||||
"runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file"
|
||||
)
|
||||
|
||||
if ! command -v lncli >/dev/null 2>&1; then
|
||||
log "lncli not available — skipping Static Channel Backup export."
|
||||
LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)")
|
||||
return
|
||||
fi
|
||||
|
||||
if ! is_unit_active "lnd.service"; then
|
||||
log "LND service is not active — skipping Static Channel Backup export."
|
||||
LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)")
|
||||
return
|
||||
fi
|
||||
|
||||
log "Exporting LND Static Channel Backup…"
|
||||
local attempt
|
||||
for attempt in "${attempts[@]}"; do
|
||||
if eval "$attempt" >/dev/null 2>&1; then
|
||||
DB_DUMP_FILES+=("lnd-static-channel-backup.scb")
|
||||
LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli")
|
||||
log "LND Static Channel Backup exported."
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations."
|
||||
LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)")
|
||||
}
|
||||
|
||||
capture_active_lnd_dependents() {
|
||||
[[ "$LND_AVAILABLE" -eq 1 ]] || return
|
||||
|
||||
LND_UNITS_TO_RESTART=()
|
||||
local raw_units=""
|
||||
raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true)
|
||||
|
||||
while IFS= read -r unit; do
|
||||
[[ -n "$unit" ]] || continue
|
||||
if is_unit_active "$unit"; then
|
||||
LND_UNITS_TO_RESTART+=("$unit")
|
||||
fi
|
||||
done <<< "$raw_units"
|
||||
|
||||
if is_unit_active "lnd.service"; then
|
||||
LND_UNITS_TO_RESTART+=("lnd.service")
|
||||
fi
|
||||
}
|
||||
|
||||
stop_lnd_stack_if_needed() {
|
||||
[[ "$LND_AVAILABLE" -eq 1 ]] || return
|
||||
|
||||
capture_active_lnd_dependents
|
||||
|
||||
if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then
|
||||
log "No active LND-related services needed stopping."
|
||||
return
|
||||
fi
|
||||
|
||||
log "Stopping active services that depend on LND for clean /var/lib/lnd archive…"
|
||||
|
||||
local unit
|
||||
for unit in "${LND_UNITS_TO_RESTART[@]}"; do
|
||||
if [[ "$unit" == "lnd.service" ]]; then
|
||||
continue
|
||||
fi
|
||||
systemctl stop "$unit" || fail "Failed to stop dependent service: $unit"
|
||||
log "Stopped $unit"
|
||||
done
|
||||
|
||||
if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then
|
||||
systemctl stop lnd.service || fail "Failed to stop lnd.service"
|
||||
log "Stopped lnd.service"
|
||||
fi
|
||||
|
||||
LND_STOPPED=1
|
||||
}
|
||||
|
||||
# ── Stage 1/5: NixOS configuration ──────────────────────────────
|
||||
# ── Stage 1/4: NixOS configuration ──────────────────────────────
|
||||
|
||||
log ""
|
||||
log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────"
|
||||
log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────"
|
||||
if [[ -d /etc/nixos ]]; then
|
||||
create_tar_archive "etc-nixos.tar" -C / etc/nixos
|
||||
run_rsync "/etc/nixos" no /etc/nixos/ "$BACKUP_DIR/etc/nixos/"
|
||||
log "Stage 1 complete."
|
||||
else
|
||||
log "WARNING: /etc/nixos not found — skipping."
|
||||
fi
|
||||
|
||||
# ── Stage 2/5: Secrets ──────────────────────────────────────────
|
||||
# ── Stage 2/4: Secrets ──────────────────────────────────────────
|
||||
|
||||
log ""
|
||||
log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────"
|
||||
log "── Stage 2/4: Secrets (/etc/nix-bitcoin-secrets) ───────────"
|
||||
if [[ "$ROLE" == "desktop" ]]; then
|
||||
log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
|
||||
elif [[ -e /etc/nix-bitcoin-secrets ]]; then
|
||||
create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets
|
||||
run_rsync "/etc/nix-bitcoin-secrets" no /etc/nix-bitcoin-secrets/ "$BACKUP_DIR/etc/nix-bitcoin-secrets/"
|
||||
else
|
||||
log "(not found: /etc/nix-bitcoin-secrets — skipping)"
|
||||
fi
|
||||
log "Stage 2 complete."
|
||||
|
||||
# ── Stage 3/5: Home directory ───────────────────────────────────
|
||||
# ── Stage 3/4: Home directory ───────────────────────────────────
|
||||
# Rsync exit code 24 (vanished source files) is treated as nonfatal here
|
||||
# because the desktop may be active and files can disappear between the
|
||||
# directory scan and the copy. All other nonzero exit codes remain fatal.
|
||||
|
||||
log ""
|
||||
log "── Stage 3/5: Home directory (/home) ───────────────────────"
|
||||
log "── Stage 3/4: Home directory (/home) ───────────────────────"
|
||||
if [[ -d /home ]]; then
|
||||
create_home_tar_archive "home.tar" \
|
||||
-C / \
|
||||
--exclude='home/*/.cache' \
|
||||
--exclude='home/*/.local/share/Trash' \
|
||||
--exclude='home/*/Trash' \
|
||||
--exclude='home/*/.mozilla/firefox/*/cache2' \
|
||||
--exclude='home/*/.mozilla/firefox/*/startupCache' \
|
||||
--exclude='home/*/.mozilla/firefox/*/thumbnails' \
|
||||
--exclude='home/*/.config/google-chrome/*/Cache' \
|
||||
--exclude='home/*/.config/google-chrome/*/Code Cache' \
|
||||
--exclude='home/*/.config/chromium/*/Cache' \
|
||||
--exclude='home/*/.config/chromium/*/Code Cache' \
|
||||
--exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Cache' \
|
||||
--exclude='home/*/.config/BraveSoftware/Brave-Browser/*/Code Cache' \
|
||||
--exclude='home/*/.local/share/baloo' \
|
||||
--exclude='home/*/.thumbnails' \
|
||||
--exclude='home/*/.xsession-errors' \
|
||||
--exclude='home/*/.xsession-errors.old' \
|
||||
home
|
||||
run_rsync "/home" yes \
|
||||
--exclude='.cache/' \
|
||||
--exclude='.local/share/Trash/' \
|
||||
--exclude='Trash/' \
|
||||
--exclude='.mozilla/firefox/*/cache2/' \
|
||||
--exclude='.mozilla/firefox/*/startupCache/' \
|
||||
--exclude='.mozilla/firefox/*/thumbnails/' \
|
||||
--exclude='.config/google-chrome/*/Cache/' \
|
||||
--exclude='.config/google-chrome/*/Code Cache/' \
|
||||
--exclude='.config/chromium/*/Cache/' \
|
||||
--exclude='.config/chromium/*/Code Cache/' \
|
||||
--exclude='.config/BraveSoftware/Brave-Browser/*/Cache/' \
|
||||
--exclude='.config/BraveSoftware/Brave-Browser/*/Code Cache/' \
|
||||
--exclude='.local/share/baloo/' \
|
||||
--exclude='.thumbnails/' \
|
||||
--exclude='.xsession-errors' \
|
||||
--exclude='.xsession-errors.old' \
|
||||
/home/ "$BACKUP_DIR/home/"
|
||||
log "Stage 3 complete."
|
||||
else
|
||||
log "WARNING: /home not found — skipping."
|
||||
fi
|
||||
|
||||
# ── Stage 4/5: Database exports + LND artifacts ────────────────
|
||||
# ── Stage 4/4: System data ──────────────────────────────────────
|
||||
# PostgreSQL/MariaDB raw database directories are excluded. Application
|
||||
# databases must be backed up separately with native database tools.
|
||||
# Bitcoin/Electrs data are excluded; they live on the internal second drive.
|
||||
|
||||
log ""
|
||||
log "── Stage 4/5: Database and LND consistency exports ─────────"
|
||||
export_postgresql_dumps
|
||||
export_mariadb_dumps
|
||||
export_lnd_scb_if_possible
|
||||
|
||||
if [[ "$LND_AVAILABLE" -eq 1 ]]; then
|
||||
stop_lnd_stack_if_needed
|
||||
create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd
|
||||
LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop")
|
||||
fi
|
||||
|
||||
log "Stage 4 complete."
|
||||
|
||||
# ── Stage 5/5: System data ──────────────────────────────────────
|
||||
|
||||
log ""
|
||||
log "── Stage 5/5: System data (/var/lib) ───────────────────────"
|
||||
log "── Stage 4/4: System data (/var/lib) ───────────────────────"
|
||||
if [[ -d /var/lib ]]; then
|
||||
VAR_LIB_EXCLUDES=(
|
||||
--exclude='var/lib/bitcoind'
|
||||
--exclude='var/lib/electrs'
|
||||
--exclude='var/lib/*/log'
|
||||
--exclude='var/lib/*/logs'
|
||||
--exclude='var/lib/*/cache'
|
||||
--exclude='var/lib/*/tmp'
|
||||
)
|
||||
|
||||
if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
|
||||
VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd')
|
||||
fi
|
||||
|
||||
create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib
|
||||
log "Stage 5 complete."
|
||||
run_rsync "/var/lib" no \
|
||||
--exclude='postgresql/' \
|
||||
--exclude='mysql/' \
|
||||
--exclude='mariadb/' \
|
||||
--exclude='bitcoind/' \
|
||||
--exclude='electrs/' \
|
||||
--exclude='*/log/' \
|
||||
--exclude='*/logs/' \
|
||||
--exclude='*/cache/' \
|
||||
--exclude='*/tmp/' \
|
||||
/var/lib/ "$BACKUP_DIR/var/lib/"
|
||||
log "Stage 4 complete."
|
||||
else
|
||||
log "WARNING: /var/lib not found — skipping."
|
||||
fi
|
||||
@@ -802,107 +461,88 @@ fi
|
||||
log ""
|
||||
log "Generating BACKUP_MANIFEST.txt …"
|
||||
MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
|
||||
CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt"
|
||||
|
||||
{
|
||||
echo "Sovran_SystemsOS Backup Manifest"
|
||||
echo "Generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
echo "Timestamp: $TIMESTAMP"
|
||||
echo "Updated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
echo "Hostname: $(hostname)"
|
||||
echo "Role: $ROLE_LABEL"
|
||||
echo "Target: $TARGET"
|
||||
echo ""
|
||||
echo "Source paths included:"
|
||||
echo "- /etc/nixos"
|
||||
echo "- /home"
|
||||
echo "Backup type: Live rsync mirror (directly browsable files)"
|
||||
echo "Location: ${BACKUP_DIR}"
|
||||
echo ""
|
||||
echo "Source paths mirrored:"
|
||||
echo "- /etc/nixos → current/etc/nixos/"
|
||||
if [[ "$ROLE" != "desktop" ]]; then
|
||||
echo "- /etc/nix-bitcoin-secrets (when present)"
|
||||
echo "- /etc/nix-bitcoin-secrets (when present) → current/etc/nix-bitcoin-secrets/"
|
||||
fi
|
||||
echo "- /var/lib"
|
||||
echo "- /home → current/home/"
|
||||
echo "- /var/lib → current/var/lib/"
|
||||
echo ""
|
||||
echo "Exclusions:"
|
||||
for ex in "${MANIFEST_EXCLUDES[@]}"; do
|
||||
echo "- $ex"
|
||||
done
|
||||
echo "- /var/lib/postgresql (PostgreSQL raw database files — not included)"
|
||||
echo "- /var/lib/mysql, /var/lib/mariadb (MariaDB raw database files — not included)"
|
||||
echo "- /var/lib/bitcoind (Bitcoin blockchain — excluded; lives on internal second drive)"
|
||||
echo "- /var/lib/electrs (Electrs index — excluded; lives on internal second drive)"
|
||||
echo "- /run/media/Second_Drive (internal second drive — never traversed)"
|
||||
echo "- /var/lib/*/log, /var/lib/*/logs, /var/lib/*/cache, /var/lib/*/tmp"
|
||||
echo "- Browser disk caches, thumbnail caches, trash directories, X session error logs"
|
||||
echo ""
|
||||
echo "Archives:"
|
||||
for archive in "${ARCHIVE_FILES[@]}"; do
|
||||
echo "- $archive"
|
||||
done
|
||||
echo ""
|
||||
echo "Database and LND exports:"
|
||||
if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then
|
||||
echo "- none"
|
||||
else
|
||||
for dump in "${DB_DUMP_FILES[@]}"; do
|
||||
echo "- $dump"
|
||||
done
|
||||
for note in "${LND_BACKUP_NOTES[@]}"; do
|
||||
echo "- $note"
|
||||
done
|
||||
fi
|
||||
echo "Important limitations:"
|
||||
echo "- PostgreSQL and MariaDB/MySQL application databases are NOT included in this"
|
||||
echo " backup. If you use Nextcloud, Matrix/Synapse, or other database-backed"
|
||||
echo " applications, their data must be backed up separately using native tools."
|
||||
echo "- Bitcoin blockchain data and Electrs indexes are NOT included; they are"
|
||||
echo " reconstructable or stored on the internal second drive."
|
||||
echo "- This is a live file-level mirror, not a transactional database backup."
|
||||
echo " Files being written during the backup may be in an inconsistent state."
|
||||
echo ""
|
||||
echo "Restore guidance:"
|
||||
echo "- Verify artifacts: cd <backup_dir> && sha256sum -c SHA256SUMS.txt"
|
||||
echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf <archive>.tar -C /"
|
||||
echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql"
|
||||
echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_<db>.dump"
|
||||
echo "- MariaDB DB dump: mariadb <db_name> < database-dumps/mariadb_<db>.sql"
|
||||
echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures"
|
||||
echo "- Note: when restoring /var/lib, exclude raw DB directories (var/lib/postgresql, var/lib/mysql)"
|
||||
echo " and restore from native dumps instead for PostgreSQL and MariaDB"
|
||||
echo "- Files are directly browsable on the backup drive under: ${BACKUP_DIR}"
|
||||
echo "- To restore a directory:"
|
||||
echo " sudo rsync -aAXH --numeric-ids current/etc/nixos/ /etc/nixos/"
|
||||
echo " sudo rsync -aAXH --numeric-ids current/home/ /home/"
|
||||
echo " sudo rsync -aAXH --numeric-ids current/var/lib/ /var/lib/"
|
||||
echo "- To copy individual files:"
|
||||
echo " sudo cp -a current/home/username/ /home/username/"
|
||||
echo "- When restoring /etc/nixos to replacement hardware, regenerate"
|
||||
echo " hardware-configuration.nix for the new hardware before rebuilding."
|
||||
echo ""
|
||||
echo "Nonfatal warnings:"
|
||||
if [[ "${#ARCHIVE_WARNINGS[@]}" -eq 0 ]]; then
|
||||
if [[ "${#RSYNC_WARNINGS[@]}" -eq 0 ]]; then
|
||||
echo "- none"
|
||||
else
|
||||
for warning in "${ARCHIVE_WARNINGS[@]}"; do
|
||||
for warning in "${RSYNC_WARNINGS[@]}"; do
|
||||
echo "- $warning"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded"
|
||||
echo "Note: Bitcoin blockchain and Electrs index data are intentionally excluded"
|
||||
echo "from manual external backup because they already live on the internal second drive"
|
||||
echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
|
||||
echo ""
|
||||
echo "Artifact listing:"
|
||||
# INCOMPLETE exists at this point (created at backup start); BACKUP_COMPLETE does not
|
||||
# exist yet (written after checksums). Excluding both marker files here is intentional:
|
||||
# INCOMPLETE is excluded so it doesn't appear as a data artifact, and BACKUP_COMPLETE
|
||||
# is excluded defensively for consistency should the ordering ever change.
|
||||
find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f \
|
||||
! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z | tr '\0' '\n'
|
||||
} > "$MANIFEST_FILE"
|
||||
|
||||
# ── Generate checksums for all backup artifacts ─────────────────
|
||||
|
||||
log "Generating SHA-256 checksums …"
|
||||
(
|
||||
cd "$BACKUP_DIR"
|
||||
while IFS= read -r -d '' file; do
|
||||
sha256sum "$file"
|
||||
done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' ! -name 'INCOMPLETE' ! -name 'BACKUP_COMPLETE' -print0 | sort -z)
|
||||
) > "$CHECKSUM_FILE"
|
||||
|
||||
log "Manifest written to $MANIFEST_FILE"
|
||||
log "Checksums written to $CHECKSUM_FILE"
|
||||
|
||||
# ── Done ─────────────────────────────────────────────────────────
|
||||
|
||||
log ""
|
||||
if [[ "${#ARCHIVE_WARNINGS[@]}" -gt 0 ]]; then
|
||||
if [[ "${#RSYNC_WARNINGS[@]}" -gt 0 ]]; then
|
||||
log "Backup completed with nonfatal warnings:"
|
||||
for warning in "${ARCHIVE_WARNINGS[@]}"; do
|
||||
for warning in "${RSYNC_WARNINGS[@]}"; do
|
||||
log " WARNING: $warning"
|
||||
done
|
||||
log "Your important data is backed up. The warnings above indicate files that changed"
|
||||
log "during backup, which is normal on an active desktop and does not affect your backup."
|
||||
log "Your important data is backed up. The warnings above indicate files that"
|
||||
log "vanished during backup, which is normal on an active desktop."
|
||||
log ""
|
||||
fi
|
||||
log "All Finished! Your data is now backed up to a third location."
|
||||
log "Files are directly browsable on the drive under: ${BACKUP_DIR}"
|
||||
log "Please eject the drive safely before removing it from your Sovran Pro."
|
||||
|
||||
# Remove incomplete marker and write completion marker only after all work succeeds
|
||||
# Remove incomplete marker and write completion marker only after all work succeeds.
|
||||
# A later successful run will update the same mirror and replace any INCOMPLETE state.
|
||||
rm -f "$BACKUP_DIR/INCOMPLETE"
|
||||
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$BACKUP_DIR/BACKUP_COMPLETE"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user