Sovran_Systems/Sovran_SystemsOS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement reliable exFAT manual backup workflow

Browse Source
This commit is contained in:
copilot-swe-agent[bot]
2026-07-17 16:56:08 +00:00
committed by GitHub
parent 0ec8203557
commit 9f3d3e7670
5 changed files with 676 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/sovran_systemsos_web/scripts/sovran-hub-backup.sh
+535 -115
View File
@@ -17,13 +17,24 @@ set -euo pipefail
BACKUP_LOG="/var/log/sovran-hub-backup.log" BACKUP_LOG="/var/log/sovran-hub-backup.log"
BACKUP_STATUS="/var/log/sovran-hub-backup.status" BACKUP_STATUS="/var/log/sovran-hub-backup.status"
MEDIA_ROOT="/run/media" MEDIA_ROOT="/run/media"
MIN_FREE_GB=10
HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json" HUB_CONFIG_JSON="/var/lib/sovran-hub/config.json"
ROLE_STATE_NIX="/etc/nixos/role-state.nix" ROLE_STATE_NIX="/etc/nixos/role-state.nix"
SECOND_DRIVE_MOUNT="/run/media/Second_Drive"
SAFETY_MARGIN_BYTES=$((1024 * 1024 * 1024))
# ── Internal drive labels/paths to NEVER use as backup targets ─── # ── Internal drive labels/paths to NEVER use as backup targets ───
INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos") INTERNAL_LABELS=("BTCEcoandBackup" "sovran_systemsos")
INTERNAL_MOUNTS=("/run/media/Second_Drive" "/boot/efi" "/") INTERNAL_MOUNTS=("$SECOND_DRIVE_MOUNT" "/boot/efi" "/")
FAILED_ALREADY=0
BACKUP_COMPLETE=0
LND_STOPPED=0
LND_UNITS_TO_RESTART=()
ARCHIVE_FILES=()
DB_DUMP_FILES=()
MANIFEST_EXCLUDES=()
LND_BACKUP_NOTES=()
# ── Logging helpers ────────────────────────────────────────────── # ── Logging helpers ──────────────────────────────────────────────
@@ -37,16 +48,58 @@ set_status() {
} }
fail() { fail() {
FAILED_ALREADY=1
log "ERROR: $*" log "ERROR: $*"
set_status "FAILED" set_status "FAILED"
exit 1 exit 1
} }
cleanup() {
local rc=$?
local restart_failed=0
if [[ "$LND_STOPPED" -eq 1 ]]; then
log "Restarting previously active LND-related services…"
for (( idx=${#LND_UNITS_TO_RESTART[@]}-1 ; idx>=0 ; idx-- )); do
local unit="${LND_UNITS_TO_RESTART[$idx]}"
if systemctl start "$unit"; then
log "Started $unit"
else
log "ERROR: Failed to start $unit"
restart_failed=1
fi
done
LND_STOPPED=0
fi
if [[ "$restart_failed" -eq 1 ]]; then
rc=1
FAILED_ALREADY=1
set_status "FAILED"
fi
if [[ "$BACKUP_COMPLETE" -eq 1 && "$rc" -eq 0 ]]; then
return
fi
if [[ "$FAILED_ALREADY" -eq 0 ]]; then
log "ERROR: Backup terminated unexpectedly (exit code $rc)."
set_status "FAILED"
fi
}
trap cleanup EXIT
trap 'exit 1' INT TERM
require_cmd() {
local cmd="$1"
command -v "$cmd" >/dev/null 2>&1 || fail "Required command not found: $cmd"
}
# ── Check whether a mount point is an internal drive ──────────── # ── Check whether a mount point is an internal drive ────────────
is_internal() { is_internal() {
local mnt="$1" local mnt="$1"
# Reject known internal mount points and their subdirectories
for internal in "${INTERNAL_MOUNTS[@]}"; do for internal in "${INTERNAL_MOUNTS[@]}"; do
if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then if [[ "$mnt" == "$internal" || "$mnt" == "${internal}/"* ]]; then
return 0 return 0
@@ -59,39 +112,37 @@ is_internal() {
find_external_drive() { find_external_drive() {
local target="" local target=""
# lsblk JSON output: NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE
if command -v lsblk &>/dev/null; then
while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
# Must be a partition or disk, and be removable/hotplug
[[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
[[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
[[ -n "$mountpoint" ]] || continue
# Filter out internal labels while IFS=$'\t' read -r dev_type hotplug removable label mountpoint; do
local skip=0 [[ "$dev_type" == "part" || "$dev_type" == "disk" ]] || continue
for lbl in "${INTERNAL_LABELS[@]}"; do [[ "$hotplug" == "1" || "$removable" == "1" ]] || continue
[[ "$label" == "$lbl" ]] && skip=1 && break [[ -n "$mountpoint" ]] || continue
done
[[ "$skip" -eq 1 ]] && continue
# Filter out internal mount points local skip=0
is_internal "$mountpoint" && continue for lbl in "${INTERNAL_LABELS[@]}"; do
[[ "$label" == "$lbl" ]] && skip=1 && break
done
[[ "$skip" -eq 1 ]] && continue
if mountpoint -q "$mountpoint" 2>/dev/null; then is_internal "$mountpoint" && continue
target="$mountpoint"
break if mountpoint -q "$mountpoint" 2>/dev/null; then
fi target="$mountpoint"
done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \ break
python3 -c " fi
done < <(lsblk -J -o NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE 2>/dev/null | \
python3 -c "
import sys, json import sys, json
data = json.load(sys.stdin)
def flatten(devs): def flatten(devs):
for d in devs: for d in devs:
yield d yield d
for c in d.get('children', []): for c in d.get('children', []):
yield from flatten([c]) yield from flatten([c])
data = json.load(sys.stdin)
for d in flatten(data.get('blockdevices', [])): for d in flatten(data.get('blockdevices', [])):
print('\t'.join([ print('\\t'.join([
d.get('type') or '', d.get('type') or '',
str(d.get('hotplug') or '0'), str(d.get('hotplug') or '0'),
str(d.get('rm') or '0'), str(d.get('rm') or '0'),
@@ -99,24 +150,10 @@ for d in flatten(data.get('blockdevices', [])):
d.get('mountpoint') or '', d.get('mountpoint') or '',
])) ]))
" 2>/dev/null || true) " 2>/dev/null || true)
fi
# Fallback: walk /run/media/ if lsblk produced nothing
if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then if [[ -z "$target" && -d "$MEDIA_ROOT" ]]; then
while IFS= read -r -d '' mnt; do while IFS= read -r -d '' mnt; do
is_internal "$mnt" && continue is_internal "$mnt" && continue
# Check label via lsblk on the device backing this mount
local dev
dev=$(findmnt -n -o SOURCE "$mnt" 2>/dev/null || true)
if [[ -n "$dev" ]]; then
local lbl
lbl=$(lsblk -n -o LABEL "$dev" 2>/dev/null || true)
local skip=0
for internal_lbl in "${INTERNAL_LABELS[@]}"; do
[[ "$lbl" == "$internal_lbl" ]] && skip=1 && break
done
[[ "$skip" -eq 1 ]] && continue
fi
if mountpoint -q "$mnt" 2>/dev/null; then if mountpoint -q "$mnt" 2>/dev/null; then
target="$mnt" target="$mnt"
break break
@@ -128,16 +165,10 @@ for d in flatten(data.get('blockdevices', [])):
} }
# ── Detect the configured system role ─────────────────────────── # ── Detect the configured system role ───────────────────────────
#
# Priority:
# 1. Hub config JSON (/var/lib/sovran-hub/config.json) — "role" key
# 2. role-state.nix (/etc/nixos/role-state.nix) — grep for true flag
# 3. Default: server_plus_desktop
detect_role() { detect_role() {
local role="server_plus_desktop" local role="server_plus_desktop"
# 1. Try the Hub config JSON
if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then if [[ -f "$HUB_CONFIG_JSON" ]] && command -v python3 &>/dev/null; then
local r local r
r=$(python3 -c \ r=$(python3 -c \
@@ -149,7 +180,6 @@ detect_role() {
fi fi
fi fi
# 2. Fall back to parsing role-state.nix
if [[ -f "$ROLE_STATE_NIX" ]]; then if [[ -f "$ROLE_STATE_NIX" ]]; then
if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then if grep -q 'roles\.desktop = lib\.mkDefault true' "$ROLE_STATE_NIX" 2>/dev/null; then
role="desktop" role="desktop"
@@ -161,6 +191,64 @@ detect_role() {
echo "$role" echo "$role"
} }
validate_target_mount() {
local target="$1"
[[ "$target" == "${MEDIA_ROOT}/"* ]] || fail "Target '$target' must be mounted under $MEDIA_ROOT."
[[ -d "$target" ]] || fail "Target path '$target' does not exist."
mountpoint -q "$target" || fail "Target path '$target' is not a mount point."
local fstype=""
fstype=$(findmnt -n -o FSTYPE -T "$target" 2>/dev/null || true)
[[ -n "$fstype" ]] || fail "Could not determine filesystem type for '$target'."
if [[ "$fstype" != "exfat" && "$fstype" != "fuseblk" ]]; then
fail "Target '$target' must be exFAT (detected filesystem: $fstype)."
fi
if [[ "$fstype" == "fuseblk" ]]; then
local src_dev blk_type
src_dev=$(findmnt -n -o SOURCE -T "$target" 2>/dev/null || true)
blk_type=""
if [[ -n "$src_dev" ]]; then
blk_type=$(lsblk -no FSTYPE "$src_dev" 2>/dev/null || true)
[[ -z "$blk_type" ]] && blk_type=$(blkid -o value -s TYPE "$src_dev" 2>/dev/null || true)
fi
if [[ "$blk_type" != "exfat" && "$blk_type" != "fuseblk" ]]; then
fail "Target '$target' is fuseblk but not identified as exFAT-compatible."
fi
fi
local write_test
write_test="$target/.sovran-write-test-$$"
if ! ( : > "$write_test" && echo "ok" >> "$write_test" && rm -f "$write_test" ); then
fail "Target '$target' is not writable."
fi
log "Verified backup target filesystem: $fstype"
}
has_unit() {
systemctl cat "$1" >/dev/null 2>&1
}
is_unit_active() {
systemctl is-active --quiet "$1"
}
estimate_path_bytes() {
local path="$1"
shift || true
[[ -e "$path" ]] || {
echo 0
return
}
local size
size=$(du -s -B1 -x "$@" "$path" 2>/dev/null | awk '{print $1}' || true)
[[ -n "$size" ]] || size=0
echo "$size"
}
# ── Initialise log file ────────────────────────────────────────── # ── Initialise log file ──────────────────────────────────────────
: > "$BACKUP_LOG" : > "$BACKUP_LOG"
@@ -169,14 +257,30 @@ set_status "RUNNING"
log "=== Sovran_SystemsOS External Hub Backup ===" log "=== Sovran_SystemsOS External Hub Backup ==="
log "Starting backup process…" log "Starting backup process…"
require_cmd tar
require_cmd sha256sum
require_cmd findmnt
require_cmd lsblk
require_cmd mountpoint
require_cmd df
require_cmd du
require_cmd awk
require_cmd sort
require_cmd find
require_cmd systemctl
require_cmd hostname
require_cmd date
require_cmd python3
require_cmd runuser
# ── Detect system role ─────────────────────────────────────────── # ── Detect system role ───────────────────────────────────────────
ROLE="$(detect_role)" ROLE="$(detect_role)"
case "$ROLE" in case "$ROLE" in
desktop) ROLE_LABEL="Desktop Only" ;; desktop) ROLE_LABEL="Desktop Only" ;;
node) ROLE_LABEL="Node (Bitcoin-only)" ;; node) ROLE_LABEL="Node (Bitcoin-only)" ;;
server_plus_desktop) ROLE_LABEL="Server + Desktop" ;; server_plus_desktop) ROLE_LABEL="Server + Desktop" ;;
*) ROLE_LABEL="$ROLE" ;; *) ROLE_LABEL="$ROLE" ;;
esac esac
log "Detected role: $ROLE_LABEL" log "Detected role: $ROLE_LABEL"
@@ -184,7 +288,6 @@ log "Detected role: $ROLE_LABEL"
if [[ -n "${BACKUP_TARGET:-}" ]]; then if [[ -n "${BACKUP_TARGET:-}" ]]; then
TARGET="$BACKUP_TARGET" TARGET="$BACKUP_TARGET"
# Safety: never allow internal drives even if explicitly passed
if is_internal "$TARGET"; then if is_internal "$TARGET"; then
fail "Target '$TARGET' is an internal system drive and cannot be used for external backup." fail "Target '$TARGET' is an internal system drive and cannot be used for external backup."
fi fi
@@ -193,106 +296,361 @@ else
log "Auto-detecting external USB drives…" log "Auto-detecting external USB drives…"
TARGET="$(find_external_drive)" TARGET="$(find_external_drive)"
if [[ -z "$TARGET" ]]; then if [[ -z "$TARGET" ]]; then
fail "No external USB drive detected. " \ fail "No external USB drive detected. Please plug in an exFAT-formatted USB drive and try again."
"Please plug in an exFAT-formatted USB drive (≥500 GB) and try again."
fi fi
log "Detected external drive: $TARGET" log "Detected external drive: $TARGET"
fi fi
# ── Verify mount point ─────────────────────────────────────────── validate_target_mount "$TARGET"
[[ -d "$TARGET" ]] || fail "Target path '$TARGET' does not exist." # ── Plan role-aware source scope and exclusions ─────────────────
mountpoint -q "$TARGET" || fail "Target path '$TARGET' is not a mount point."
# ── Check free disk space (require ≥ 10 GB) ────────────────────── LND_AVAILABLE=0
if [[ "$ROLE" != "desktop" ]] && [[ -d /var/lib/lnd ]] && has_unit "lnd.service"; then
LND_AVAILABLE=1
fi
FREE_KB=$(df -k --output=avail "$TARGET" | tail -1) if [[ "$ROLE" == "desktop" ]]; then
FREE_GB=$(( FREE_KB / 1024 / 1024 )) MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets (not applicable for Desktop Only role)")
else
MANIFEST_EXCLUDES+=("/etc/nix-bitcoin-secrets skipped when path absent")
fi
MANIFEST_EXCLUDES+=(
"/run/media/Second_Drive (never traversed)"
"/run/media/Second_Drive/BTCEcoandBackup/Bitcoin_Node (excluded; internal second-drive data)"
"/run/media/Second_Drive/BTCEcoandBackup/Electrs_Data (excluded; internal second-drive data)"
"/var/lib/bitcoind (excluded from manual backup)"
"/var/lib/electrs (excluded from manual backup)"
"/var/lib/*/log and /var/lib/*/logs"
"/var/lib/*/cache and /var/lib/*/tmp"
"/home/*/.cache and /home/*/.local/share/Trash"
)
if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
MANIFEST_EXCLUDES+=("/var/lib/lnd from general /var/lib archive")
fi
# ── Estimate required free space ─────────────────────────────────
ETC_NIXOS_BYTES=$(estimate_path_bytes /etc/nixos)
HOME_BYTES=$(estimate_path_bytes /home --exclude='*/.cache' --exclude='*/.local/share/Trash' --exclude='*/Trash')
SECRETS_BYTES=0
if [[ "$ROLE" != "desktop" ]]; then
SECRETS_BYTES=$(estimate_path_bytes /etc/nix-bitcoin-secrets)
fi
VAR_LIB_BYTES=$(estimate_path_bytes /var/lib \
--exclude='bitcoind' \
--exclude='electrs' \
--exclude='lnd' \
--exclude='*/log' \
--exclude='*/logs' \
--exclude='*/cache' \
--exclude='*/tmp')
LND_BYTES=0
if [[ "$LND_AVAILABLE" -eq 1 ]]; then
LND_BYTES=$(estimate_path_bytes /var/lib/lnd)
fi
ESTIMATED_BYTES=$(( ETC_NIXOS_BYTES + HOME_BYTES + SECRETS_BYTES + VAR_LIB_BYTES + LND_BYTES ))
REQUIRED_BYTES=$(( ESTIMATED_BYTES + (ESTIMATED_BYTES / 5) + SAFETY_MARGIN_BYTES ))
FREE_BYTES=$(df -B1 --output=avail "$TARGET" | tail -1 | tr -d ' ')
FREE_GB=$(( FREE_BYTES / 1024 / 1024 / 1024 ))
REQUIRED_GB=$(( REQUIRED_BYTES / 1024 / 1024 / 1024 ))
log "Estimated backup size: $(( ESTIMATED_BYTES / 1024 / 1024 / 1024 )) GB"
log "Required free space (with safety margin): ${REQUIRED_GB} GB"
log "Free space on drive: ${FREE_GB} GB" log "Free space on drive: ${FREE_GB} GB"
(( FREE_GB >= MIN_FREE_GB )) || \
fail "Not enough free space on drive (${FREE_GB} GB available, ${MIN_FREE_GB} GB required)." (( FREE_BYTES >= REQUIRED_BYTES )) || \
fail "Not enough free space on drive (${FREE_GB} GB available, ${REQUIRED_GB} GB required)."
# ── Create timestamped backup directory ───────────────────────── # ── Create timestamped backup directory ─────────────────────────
TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}" BACKUP_DIR="${TARGET}/Sovran_SystemsOS_Backup/${TIMESTAMP}"
mkdir -p "$BACKUP_DIR" DB_DUMP_DIR="$BACKUP_DIR/database-dumps"
mkdir -p "$BACKUP_DIR" "$DB_DUMP_DIR"
log "Backup destination: $BACKUP_DIR" log "Backup destination: $BACKUP_DIR"
# ── Stage 1/4: NixOS configuration ────────────────────────────── create_tar_archive() {
local archive_name="$1"
shift
local archive_path="$BACKUP_DIR/$archive_name"
log "Creating $archive_name …"
tar \
--create \
--file "$archive_path" \
--numeric-owner \
--acls \
--xattrs \
--sparse \
--one-file-system \
"$@"
ARCHIVE_FILES+=("$archive_name")
log "Created archive: $archive_name"
}
export_postgresql_dumps() {
if ! command -v pg_dump >/dev/null 2>&1 || ! has_unit "postgresql.service"; then
log "PostgreSQL tools/service not available — skipping PostgreSQL exports."
return
fi
if ! is_unit_active "postgresql.service"; then
log "PostgreSQL service is not active — skipping PostgreSQL exports."
return
fi
log "Exporting PostgreSQL globals and databases…"
local globals_file="$DB_DUMP_DIR/postgresql_globals.sql"
runuser -u postgres -- pg_dumpall --globals-only > "$globals_file" || \
fail "Failed to export PostgreSQL globals."
DB_DUMP_FILES+=("database-dumps/postgresql_globals.sql")
local dbs
dbs=$(runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datistemplate = false AND datallowconn AND datname <> 'postgres';" 2>/dev/null || true)
if [[ -z "$dbs" ]]; then
log "No non-template PostgreSQL application databases found."
return
fi
while IFS= read -r db; do
[[ -n "$db" ]] || continue
local safe_db
safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
local out_file="$DB_DUMP_DIR/postgresql_${safe_db}.dump"
runuser -u postgres -- pg_dump --format=custom --file "$out_file" "$db" || \
fail "Failed to export PostgreSQL database '$db'."
DB_DUMP_FILES+=("database-dumps/postgresql_${safe_db}.dump")
done <<< "$dbs"
}
export_mariadb_dumps() {
local dump_cmd=""
local query_cmd=""
local mariadb_unit=""
if command -v mariadb-dump >/dev/null 2>&1; then
dump_cmd="mariadb-dump"
elif command -v mysqldump >/dev/null 2>&1; then
dump_cmd="mysqldump"
fi
if command -v mariadb >/dev/null 2>&1; then
query_cmd="mariadb"
elif command -v mysql >/dev/null 2>&1; then
query_cmd="mysql"
fi
if [[ -z "$dump_cmd" || -z "$query_cmd" ]]; then
log "MariaDB dump/query tools not available — skipping MariaDB exports."
return
fi
if has_unit "mariadb.service" && is_unit_active "mariadb.service"; then
mariadb_unit="mariadb.service"
elif has_unit "mysql.service" && is_unit_active "mysql.service"; then
mariadb_unit="mysql.service"
else
log "MariaDB service is not active — skipping MariaDB exports."
return
fi
log "Exporting MariaDB databases from ${mariadb_unit}…"
local dbs
dbs=$($query_cmd -N -e "SHOW DATABASES" 2>/dev/null || true)
if [[ -z "$dbs" ]]; then
log "No MariaDB databases found."
return
fi
while IFS= read -r db; do
[[ -n "$db" ]] || continue
case "$db" in
information_schema|performance_schema|mysql|sys) continue ;;
esac
local safe_db out_file
safe_db="$(echo "$db" | tr -c '[:alnum:]_.-' '_')"
out_file="$DB_DUMP_DIR/mariadb_${safe_db}.sql"
$dump_cmd --single-transaction --quick --routines --events --triggers "$db" > "$out_file" || \
fail "Failed to export MariaDB database '$db'."
DB_DUMP_FILES+=("database-dumps/mariadb_${safe_db}.sql")
done <<< "$dbs"
}
export_lnd_scb_if_possible() {
[[ "$LND_AVAILABLE" -eq 1 ]] || return
local scb_file="$BACKUP_DIR/lnd-static-channel-backup.scb"
local attempts=(
"lncli exportchanbackup --all --output_file $scb_file"
"lncli -n mainnet exportchanbackup --all --output_file $scb_file"
"runuser -u lnd -- lncli exportchanbackup --all --output_file $scb_file"
"runuser -u lnd -- lncli -n mainnet exportchanbackup --all --output_file $scb_file"
)
if ! command -v lncli >/dev/null 2>&1; then
log "lncli not available — skipping Static Channel Backup export."
LND_BACKUP_NOTES+=("Static Channel Backup skipped (lncli unavailable)")
return
fi
if ! is_unit_active "lnd.service"; then
log "LND service is not active — skipping Static Channel Backup export."
LND_BACKUP_NOTES+=("Static Channel Backup skipped (lnd.service inactive)")
return
fi
log "Exporting LND Static Channel Backup…"
local attempt
for attempt in "${attempts[@]}"; do
if eval "$attempt" >/dev/null 2>&1; then
DB_DUMP_FILES+=("lnd-static-channel-backup.scb")
LND_BACKUP_NOTES+=("Static Channel Backup exported via lncli")
log "LND Static Channel Backup exported."
return
fi
done
log "WARNING: Unable to export LND Static Channel Backup with available lncli invocations."
LND_BACKUP_NOTES+=("Static Channel Backup export failed (no compatible lncli invocation succeeded)")
}
capture_active_lnd_dependents() {
[[ "$LND_AVAILABLE" -eq 1 ]] || return
LND_UNITS_TO_RESTART=()
local raw_units=""
raw_units=$(systemctl show lnd.service -p RequiredBy -p WantedBy --value 2>/dev/null | tr ' ' '\n' | grep '\.service$' | sort -u || true)
while IFS= read -r unit; do
[[ -n "$unit" ]] || continue
if is_unit_active "$unit"; then
LND_UNITS_TO_RESTART+=("$unit")
fi
done <<< "$raw_units"
if is_unit_active "lnd.service"; then
LND_UNITS_TO_RESTART+=("lnd.service")
fi
}
stop_lnd_stack_if_needed() {
[[ "$LND_AVAILABLE" -eq 1 ]] || return
capture_active_lnd_dependents
if [[ "${#LND_UNITS_TO_RESTART[@]}" -eq 0 ]]; then
log "No active LND-related services needed stopping."
return
fi
log "Stopping active services that depend on LND for clean /var/lib/lnd archive…"
local unit
for unit in "${LND_UNITS_TO_RESTART[@]}"; do
if [[ "$unit" == "lnd.service" ]]; then
continue
fi
systemctl stop "$unit" || fail "Failed to stop dependent service: $unit"
log "Stopped $unit"
done
if printf '%s\n' "${LND_UNITS_TO_RESTART[@]}" | grep -qx 'lnd.service'; then
systemctl stop lnd.service || fail "Failed to stop lnd.service"
log "Stopped lnd.service"
fi
LND_STOPPED=1
}
# ── Stage 1/5: NixOS configuration ──────────────────────────────
log "" log ""
log "── Stage 1/4: NixOS configuration (/etc/nixos) ──────────────" log "── Stage 1/5: NixOS configuration (/etc/nixos) ──────────────"
if [[ -d /etc/nixos ]]; then if [[ -d /etc/nixos ]]; then
rsync -a --info=progress2 /etc/nixos/ "$BACKUP_DIR/nixos/" 2>&1 | tee -a "$BACKUP_LOG" || \ create_tar_archive "etc-nixos.tar" -C / etc/nixos
fail "Stage 1 failed while copying /etc/nixos"
log "Stage 1 complete." log "Stage 1 complete."
else else
log "WARNING: /etc/nixos not found — skipping." log "WARNING: /etc/nixos not found — skipping."
fi fi
# ── Stage 2/4: Secrets ────────────────────────────────────────── # ── Stage 2/5: Secrets ──────────────────────────────────────────
log "" log ""
log "── Stage 2/4: Secrets ───────────────────────────────────────" log "── Stage 2/5: Secrets (/etc/nix-bitcoin-secrets) ───────────"
mkdir -p "$BACKUP_DIR/secrets"
if [[ "$ROLE" == "desktop" ]]; then if [[ "$ROLE" == "desktop" ]]; then
log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role." log "Skipping /etc/nix-bitcoin-secrets — not applicable for Desktop Only role."
elif [[ -e /etc/nix-bitcoin-secrets ]]; then
create_tar_archive "etc-nix-bitcoin-secrets.tar" -C / etc/nix-bitcoin-secrets
else else
if [[ -e /etc/nix-bitcoin-secrets ]]; then log "(not found: /etc/nix-bitcoin-secrets — skipping)"
rsync -a --info=progress2 /etc/nix-bitcoin-secrets "$BACKUP_DIR/secrets/" 2>&1 | tee -a "$BACKUP_LOG" || \
log "WARNING: Could not copy /etc/nix-bitcoin-secrets — continuing."
else
log " (not found: /etc/nix-bitcoin-secrets — skipping)"
fi
fi fi
log "Stage 2 complete." log "Stage 2 complete."
# ── Stage 3/4: Home directory ──────────────────────────────────── # ── Stage 3/5: Home directory ───────────────────────────────────
log "" log ""
log "── Stage 3/4: Home directory (/home) ───────────────────────" log "── Stage 3/5: Home directory (/home) ───────────────────────"
if [[ -d /home ]]; then if [[ -d /home ]]; then
rsync -a --info=progress2 \ create_tar_archive "home.tar" \
--exclude='.cache/' \ -C / \
--exclude='.local/share/Trash/' \ --exclude='home/*/.cache' \
--exclude='*/Trash/' \ --exclude='home/*/.local/share/Trash' \
/home/ "$BACKUP_DIR/home/" 2>&1 | tee -a "$BACKUP_LOG" || \ --exclude='home/*/Trash' \
fail "Stage 3 failed while copying /home" home
log "Stage 3 complete." log "Stage 3 complete."
else else
log "WARNING: /home not found — skipping." log "WARNING: /home not found — skipping."
fi fi
# ── Stage 4/4: System data ─────────────────────────────────────── # ── Stage 4/5: Database exports + LND artifacts ────────────────
log "" log ""
log "── Stage 4/4: System data (/var/lib) ────────────────────────" log "── Stage 4/5: Database and LND consistency exports ─────────"
if [[ "$ROLE" == "desktop" ]]; then export_postgresql_dumps
if [[ -d /var/lib ]]; then export_mariadb_dumps
rsync -a --info=progress2 \ export_lnd_scb_if_possible
--filter='- /lnd/***' \
--exclude='logs/' \ if [[ "$LND_AVAILABLE" -eq 1 ]]; then
--exclude='log/' \ stop_lnd_stack_if_needed
--exclude='*/logs/' \ create_tar_archive "var-lib-lnd-clean.tar" -C / var/lib/lnd
--exclude='*/log/' \ LND_BACKUP_NOTES+=("Created clean raw /var/lib/lnd archive after controlled service stop")
/var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \ fi
fail "Stage 4 failed while copying /var/lib for Desktop Only role"
log "Stage 4 complete (Desktop Only role excludes /var/lib/lnd)." log "Stage 4 complete."
else
log "WARNING: /var/lib not found — skipping." # ── Stage 5/5: System data ──────────────────────────────────────
log ""
log "── Stage 5/5: System data (/var/lib) ───────────────────────"
if [[ -d /var/lib ]]; then
VAR_LIB_EXCLUDES=(
--exclude='var/lib/bitcoind'
--exclude='var/lib/electrs'
--exclude='var/lib/*/log'
--exclude='var/lib/*/logs'
--exclude='var/lib/*/cache'
--exclude='var/lib/*/tmp'
)
if [[ "$ROLE" == "desktop" || "$LND_AVAILABLE" -eq 1 ]]; then
VAR_LIB_EXCLUDES+=(--exclude='var/lib/lnd')
fi fi
elif [[ -d /var/lib ]]; then
rsync -a --info=progress2 \ create_tar_archive "var-lib.tar" -C / "${VAR_LIB_EXCLUDES[@]}" var/lib
--exclude='logs/' \ log "Stage 5 complete."
--exclude='log/' \
--exclude='*/logs/' \
--exclude='*/log/' \
/var/lib/ "$BACKUP_DIR/var-lib/" 2>&1 | tee -a "$BACKUP_LOG" || \
fail "Stage 4 failed while copying /var/lib"
log "Stage 4 complete."
else else
log "WARNING: /var/lib not found — skipping." log "WARNING: /var/lib not found — skipping."
fi fi
@@ -301,21 +659,83 @@ fi
log "" log ""
log "Generating BACKUP_MANIFEST.txt …" log "Generating BACKUP_MANIFEST.txt …"
MANIFEST_FILE="$BACKUP_DIR/BACKUP_MANIFEST.txt"
CHECKSUM_FILE="$BACKUP_DIR/SHA256SUMS.txt"
{ {
echo "Sovran_SystemsOS Backup Manifest" echo "Sovran_SystemsOS Backup Manifest"
echo "Generated: $(date)" echo "Generated: $(date -Is)"
echo "Timestamp: $TIMESTAMP"
echo "Hostname: $(hostname)" echo "Hostname: $(hostname)"
echo "Role: $ROLE_LABEL" echo "Role: $ROLE_LABEL"
echo "Target: $TARGET" echo "Target: $TARGET"
echo "" echo ""
echo "Contents:" echo "Source paths included:"
find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 | sort echo "- /etc/nixos"
} > "$BACKUP_DIR/BACKUP_MANIFEST.txt" echo "- /home"
log "Manifest written to $BACKUP_DIR/BACKUP_MANIFEST.txt" if [[ "$ROLE" != "desktop" ]]; then
echo "- /etc/nix-bitcoin-secrets (when present)"
fi
echo "- /var/lib"
echo ""
echo "Exclusions:"
for ex in "${MANIFEST_EXCLUDES[@]}"; do
echo "- $ex"
done
echo ""
echo "Archives:"
for archive in "${ARCHIVE_FILES[@]}"; do
echo "- $archive"
done
echo ""
echo "Database and LND exports:"
if [[ "${#DB_DUMP_FILES[@]}" -eq 0 && "${#LND_BACKUP_NOTES[@]}" -eq 0 ]]; then
echo "- none"
else
for dump in "${DB_DUMP_FILES[@]}"; do
echo "- $dump"
done
for note in "${LND_BACKUP_NOTES[@]}"; do
echo "- $note"
done
fi
echo ""
echo "Restore guidance:"
echo "- Verify artifacts: cd '$BACKUP_DIR' && sha256sum -c SHA256SUMS.txt"
echo "- Extract a tar archive: sudo tar --acls --xattrs --numeric-owner -xpf <archive>.tar -C /"
echo "- PostgreSQL globals: sudo -u postgres psql -f database-dumps/postgresql_globals.sql"
echo "- PostgreSQL DB dump: sudo -u postgres pg_restore --create --clean --if-exists -d postgres database-dumps/postgresql_<db>.dump"
echo "- MariaDB DB dump: mariadb <db_name> < database-dumps/mariadb_<db>.sql"
echo "- LND SCB: keep lnd-static-channel-backup.scb with wallet seed for channel recovery procedures"
echo ""
echo "Important note: Bitcoin blockchain and Electrs index data are intentionally excluded"
echo "from manual external backup because they already live on the internal second drive"
echo "(/run/media/Second_Drive) and are reconstructable/internal-backup data."
echo ""
echo "Artifact listing:"
find "$BACKUP_DIR" -mindepth 1 -maxdepth 2 -type f | sort
} > "$MANIFEST_FILE"
ARCHIVE_FILES+=("BACKUP_MANIFEST.txt")
# ── Generate checksums for all backup artifacts ─────────────────
log "Generating SHA-256 checksums …"
(
cd "$BACKUP_DIR"
while IFS= read -r -d '' file; do
sha256sum "$file"
done < <(find . -mindepth 1 -maxdepth 2 -type f ! -name 'SHA256SUMS.txt' -print0 | sort -z)
) > "$CHECKSUM_FILE"
log "Manifest written to $MANIFEST_FILE"
log "Checksums written to $CHECKSUM_FILE"
# ── Done ───────────────────────────────────────────────────────── # ── Done ─────────────────────────────────────────────────────────
log "" log ""
log "All Finished! Your data is now backed up to a third location." log "All Finished! Your data is now backed up to a third location."
log "Please eject the drive safely before removing it from your Sovran Pro." log "Please eject the drive safely before removing it from your Sovran Pro."
BACKUP_COMPLETE=1
set_status "SUCCESS" set_status "SUCCESS"
app/sovran_systemsos_web/server.py
+86 -14
View File
@@ -1451,6 +1451,12 @@ def _read_backup_status() -> str:
return "IDLE" return "IDLE"
def _write_backup_status(value: str) -> None:
"""Write backup status file."""
with open(BACKUP_STATUS, "w") as f:
f.write(value)
def _read_backup_log(offset: int = 0) -> tuple[str, int]: def _read_backup_log(offset: int = 0) -> tuple[str, int]:
"""Read the backup log file from the given byte offset. """Read the backup log file from the given byte offset.
Returns (new_text, new_offset).""" Returns (new_text, new_offset)."""
@@ -1467,6 +1473,12 @@ def _read_backup_log(offset: int = 0) -> tuple[str, int]:
return "", 0 return "", 0
def _append_backup_log(line: str) -> None:
"""Append one line to backup log."""
with open(BACKUP_LOG, "a") as f:
f.write(line.rstrip("\n") + "\n")
_INTERNAL_LABELS = {"BTCEcoandBackup", "sovran_systemsos"} _INTERNAL_LABELS = {"BTCEcoandBackup", "sovran_systemsos"}
_INTERNAL_MOUNTS = {"/", "/boot/efi"} _INTERNAL_MOUNTS = {"/", "/boot/efi"}
_INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive" _INTERNAL_MOUNT_PREFIX = "/run/media/Second_Drive"
@@ -1490,7 +1502,7 @@ def _detect_external_drives() -> list[dict]:
/run/media/ directly if lsblk is unavailable, applying the same /run/media/ directly if lsblk is unavailable, applying the same
label/path filters. label/path filters.
Returns a list of dicts with name, path, free_gb, total_gb. Returns a list of dicts with name, path, free_gb, total_gb, fstype.
""" """
import json as _json import json as _json
import subprocess as _subprocess import subprocess as _subprocess
@@ -1501,7 +1513,7 @@ def _detect_external_drives() -> list[dict]:
# ── Primary path: lsblk JSON ──────────────────────────────── # ── Primary path: lsblk JSON ────────────────────────────────
try: try:
result = _subprocess.run( result = _subprocess.run(
["lsblk", "-J", "-o", "NAME,LABEL,MOUNTPOINT,HOTPLUG,RM,TYPE"], ["lsblk", "-J", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT,HOTPLUG,RM,TYPE"],
capture_output=True, text=True, timeout=10 capture_output=True, text=True, timeout=10
) )
if result.returncode == 0: if result.returncode == 0:
@@ -1519,6 +1531,7 @@ def _detect_external_drives() -> list[dict]:
hotplug = str(dev.get("hotplug", "0")) hotplug = str(dev.get("hotplug", "0"))
rm = str(dev.get("rm", "0")) rm = str(dev.get("rm", "0"))
label = dev.get("label") or "" label = dev.get("label") or ""
fstype = (dev.get("fstype") or "").lower()
mountpoint = dev.get("mountpoint") or "" mountpoint = dev.get("mountpoint") or ""
if dev_type not in ("part", "disk"): if dev_type not in ("part", "disk"):
@@ -1544,6 +1557,7 @@ def _detect_external_drives() -> list[dict]:
"path": mountpoint, "path": mountpoint,
"free_gb": free_gb, "free_gb": free_gb,
"total_gb": total_gb, "total_gb": total_gb,
"fstype": fstype,
}) })
seen_paths.add(mountpoint) seen_paths.add(mountpoint)
except OSError: except OSError:
@@ -1577,11 +1591,20 @@ def _detect_external_drives() -> list[dict]:
st = os.statvfs(drive_path) st = os.statvfs(drive_path)
total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1) total_gb = round((st.f_blocks * st.f_frsize) / (1024 ** 3), 1)
free_gb = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1) free_gb = round((st.f_bavail * st.f_frsize) / (1024 ** 3), 1)
fstype = ""
try:
fstype = _subprocess.run(
["findmnt", "-n", "-o", "FSTYPE", "-T", drive_path],
capture_output=True, text=True, timeout=5
).stdout.strip().lower()
except Exception:
fstype = ""
drives.append({ drives.append({
"name": drive_name, "name": drive_name,
"path": drive_path, "path": drive_path,
"free_gb": free_gb, "free_gb": free_gb,
"total_gb": total_gb, "total_gb": total_gb,
"fstype": fstype,
}) })
seen_paths.add(drive_path) seen_paths.add(drive_path)
except OSError: except OSError:
@@ -3682,6 +3705,22 @@ async def api_backup_drives():
return {"drives": drives} return {"drives": drives}
async def _monitor_backup_subprocess(proc: asyncio.subprocess.Process) -> None:
"""Mark status FAILED if backup subprocess exits unexpectedly."""
rc = await proc.wait()
if rc == 0:
return
loop = asyncio.get_event_loop()
status = await loop.run_in_executor(None, _read_backup_status)
if status in {"SUCCESS", "FAILED"}:
return
msg = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Backup subprocess exited unexpectedly (code {rc})."
await loop.run_in_executor(None, _append_backup_log, msg)
await loop.run_in_executor(None, _write_backup_status, "FAILED")
@app.post("/api/backup/run") @app.post("/api/backup/run")
async def api_backup_run(target: str = ""): async def api_backup_run(target: str = ""):
"""Start the backup script as a background subprocess. """Start the backup script as a background subprocess.
@@ -3692,6 +3731,26 @@ async def api_backup_run(target: str = ""):
if status == "RUNNING": if status == "RUNNING":
return {"ok": True, "status": "already_running"} return {"ok": True, "status": "already_running"}
drives = await loop.run_in_executor(None, _detect_external_drives)
if not drives:
raise HTTPException(status_code=400, detail="No external backup drive detected.")
drive_map = {d.get("path", ""): d for d in drives if d.get("path")}
if target:
if target not in drive_map:
raise HTTPException(status_code=400, detail="Selected backup target is not an available external drive.")
selected = drive_map[target]
else:
selected = drives[0]
selected_target = selected.get("path", "")
selected_fstype = (selected.get("fstype") or "").lower()
if selected_fstype and selected_fstype not in {"exfat", "fuseblk"}:
raise HTTPException(
status_code=400,
detail=f"Selected drive filesystem '{selected_fstype}' is not supported for manual backup.",
)
# Clear stale log before starting # Clear stale log before starting
try: try:
with open(BACKUP_LOG, "w") as f: with open(BACKUP_LOG, "w") as f:
@@ -3699,21 +3758,34 @@ async def api_backup_run(target: str = ""):
except OSError: except OSError:
pass pass
env = dict(os.environ) try:
if target: await loop.run_in_executor(None, _write_backup_status, "RUNNING")
env["BACKUP_TARGET"] = target except OSError as exc:
raise HTTPException(status_code=500, detail=f"Could not set backup status: {exc}")
# Fire-and-forget: the script writes its own status/log files. await loop.run_in_executor(
# Progress is read by the client via /api/backup/status (same pattern None,
# as /api/updates/run and the rebuild feature). _append_backup_log,
await asyncio.create_subprocess_exec( f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Starting backup process…",
"/usr/bin/env", "bash", BACKUP_SCRIPT,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
env=env,
) )
return {"ok": True, "status": "started"} env = dict(os.environ)
env["BACKUP_TARGET"] = selected_target
try:
proc = await asyncio.create_subprocess_exec(
"/usr/bin/env", "bash", BACKUP_SCRIPT,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
env=env,
)
except Exception as exc:
await loop.run_in_executor(None, _append_backup_log, f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ERROR: Failed to launch backup script: {exc}")
await loop.run_in_executor(None, _write_backup_status, "FAILED")
raise HTTPException(status_code=500, detail="Failed to launch backup process.")
asyncio.create_task(_monitor_backup_subprocess(proc))
return {"ok": True, "status": "started", "target": selected_target}
# ── Feature Manager endpoints ───────────────────────────────────── # ── Feature Manager endpoints ─────────────────────────────────────
app/sovran_systemsos_web/static/js/support.js
+4 -3
View File
@@ -491,7 +491,7 @@ function renderBackupReady(drives) {
'<div class="support-steps-title">Requirements</div>', '<div class="support-steps-title">Requirements</div>',
'<ol class="support-backup-steps">', '<ol class="support-backup-steps">',
'<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>', '<li>USB hard drive plugged into one of the open USB ports on your Sovran Pro</li>',
'<li>At least 500 GB of free space on the drive</li>', '<li>Enough free space for your selected backup data (the backup checks this before starting)</li>',
'<li>Drive must be formatted as <strong>exFAT</strong></li>', '<li>Drive must be formatted as <strong>exFAT</strong></li>',
'</ol>', '</ol>',
'</div>', '</div>',
@@ -584,9 +584,10 @@ async function pollBackupStatus() {
logDiv.scrollTop = logDiv.scrollHeight; logDiv.scrollTop = logDiv.scrollHeight;
} }
_backupLogOffset = data.offset; _backupLogOffset = data.offset;
if (!data.running) { var result = (data.result || "").toLowerCase();
if (result === "success" || result === "failed") {
stopBackupPoll(); stopBackupPoll();
renderBackupDone(data.result === "success"); renderBackupDone(result === "success");
} }
} catch (_) {} } catch (_) {}
} }
app/tests/test_manual_backup_workflow.py
+41
View File
@@ -0,0 +1,41 @@
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
BACKUP_SCRIPT = REPO_ROOT / "app" / "sovran_systemsos_web" / "scripts" / "sovran-hub-backup.sh"
SERVER_FILE = REPO_ROOT / "app" / "sovran_systemsos_web" / "server.py"
SUPPORT_JS = REPO_ROOT / "app" / "sovran_systemsos_web" / "static" / "js" / "support.js"
class ManualBackupWorkflowTests(unittest.TestCase):
def test_backup_script_uses_tar_archives_with_checksums_and_exclusions(self):
source = BACKUP_SCRIPT.read_text()
self.assertIn("tar \\", source)
self.assertIn("--create", source)
self.assertIn("--one-file-system", source)
self.assertIn("sha256sum", source)
self.assertIn("export_postgresql_dumps", source)
self.assertIn("export_mariadb_dumps", source)
self.assertIn("export_lnd_scb_if_possible", source)
self.assertIn("--exclude='var/lib/bitcoind'", source)
self.assertIn("--exclude='var/lib/electrs'", source)
self.assertIn("--exclude='var/lib/lnd'", source)
self.assertIn("set_status \"RUNNING\"", source)
self.assertIn("set_status \"SUCCESS\"", source)
self.assertIn("set_status \"FAILED\"", source)
self.assertNotIn("rsync -a", source)
def test_backend_and_frontend_use_explicit_backup_terminal_states(self):
server_source = SERVER_FILE.read_text()
support_source = SUPPORT_JS.read_text()
self.assertIn("_write_backup_status, \"RUNNING\"", server_source)
self.assertIn("_monitor_backup_subprocess", server_source)
self.assertIn("asyncio.create_task(_monitor_backup_subprocess(proc))", server_source)
self.assertIn("result === \"success\" || result === \"failed\"", support_source)
if __name__ == "__main__":
unittest.main()
modules/core/sovran-hub.nix
+10 -1
View File
@@ -388,7 +388,16 @@ in
pkgs.nftables pkgs.nftables
pkgs.iptables pkgs.iptables
pkgs.hostname pkgs.hostname
] ++ lib.optional cfg.services.bitcoin config.services.bitcoind.package; pkgs.coreutils
pkgs.findutils
pkgs.gnugrep
pkgs.gnutar
pkgs.util-linux
]
++ lib.optional cfg.services.bitcoin config.services.bitcoind.package
++ lib.optionals cfg.services.bitcoin [ pkgs.lnd ]
++ lib.optionals (cfg.services.nextcloud || cfg.services.synapse) [ config.services.postgresql.package ]
++ lib.optionals config.services.mysql.enable [ config.services.mysql.package ];
}; };
systemd.services.sovran-hub-update = { systemd.services.sovran-hub-update = {