From a3d6a0df3dbc5aa67153ea35cd5e29b90d496961 Mon Sep 17 00:00:00 2001 From: Outis Date: Thu, 4 Jun 2026 17:24:11 -0400 Subject: [PATCH] Add backup test scripts and enhance workers with timing, counts, and pre-flight checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - extras/test_backup_kopia.sh: stop → restore → compare → roll back test for Kopia - extras/test_backup_borg.sh: stop → extract → compare → roll back test for Borg - backup workers: timing (duration), service count, and pre-flight disk check (< 512 MB warns) - backup workers: ntfy notifications include count, duration, and per-failure detail - services/backup.sh: install test_backup_kopia.sh + optional weekly test timer - services/borg-backup.sh: install test_backup_borg.sh + optional weekly test timer --- extras/backup_borg.sh | 29 +++- extras/backup_gaming.sh | 28 +++- extras/backup_kopia.sh | 29 +++- extras/test_backup_borg.sh | 265 +++++++++++++++++++++++++++++++++++ extras/test_backup_kopia.sh | 271 ++++++++++++++++++++++++++++++++++++ services/backup.sh | 62 ++++++++- services/borg-backup.sh | 62 ++++++++- 7 files changed, 719 insertions(+), 27 deletions(-) create mode 100755 extras/test_backup_borg.sh create mode 100755 extras/test_backup_kopia.sh diff --git a/extras/backup_borg.sh b/extras/backup_borg.sh index bc23866..651dc3b 100644 --- a/extras/backup_borg.sh +++ b/extras/backup_borg.sh @@ -86,6 +86,21 @@ TS="$(date +%Y-%m-%dT%H-%M-%S)" declare -a FAILED_SVCS=() _ERR="$(mktemp)" trap 'rm -f "$_ERR"' EXIT +START_TS="$(date +%s)" +BACKUP_COUNT=0 + +for _pf_dest in ${DEST_NAMES:-default}; do + _pf_var="DEST_${_pf_dest}_REPO"; _pf_repo="${!_pf_var:-}" + if [ -n "$_pf_repo" ] && [[ "$_pf_repo" != *@*:* ]] && [[ "$_pf_repo" != ssh://* ]]; then + _pf_dir="$([ -d "$_pf_repo" ] && echo "$_pf_repo" || dirname "$_pf_repo")" + _pf_avail="$(df -m "$_pf_dir" 2>/dev/null | awk 'NR==2{print $4}')" + if [ -n "$_pf_avail" ] && [ "$_pf_avail" -lt 512 ]; then + log "WARNING: Low disk for '$_pf_dest' — ${_pf_avail}MB free at $_pf_repo" + FAILED_SVCS+=("$_pf_dest: low disk (${_pf_avail}MB free)") + rc=1 + fi + fi +done for svc_dir in "$DOCKER_DIR"/*/; do [ -f "${svc_dir}docker-compose.yml" ] || continue @@ -110,6 +125,7 @@ for svc_dir in "$DOCKER_DIR"/*/; do --compression=zstd,6 --exclude-caches --stats \ "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log " $line"; done; then log "OK $svc (Minecraft, no downtime)" + BACKUP_COUNT=$((BACKUP_COUNT+1)) else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: archive failed for $svc — $_reason" @@ -131,6 +147,7 @@ for svc_dir in "$DOCKER_DIR"/*/; do --compression=zstd,6 --exclude-caches --stats \ "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log " $line"; done; then log "OK $svc" + BACKUP_COUNT=$((BACKUP_COUNT+1)) else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: archive failed for $svc — $_reason" @@ -162,13 +179,17 @@ for dest in ${DEST_NAMES:-default}; do b_for "$dest" compact 2>/dev/null || true done +DURATION=$(( $(date +%s) - START_TS )) +DURATION_STR="$((DURATION/60))m $((DURATION%60))s" + if [ "$rc" -eq 0 ]; then - log "===== Borg backup complete =====" - ntfy_send "✓ Borg backup complete" "$HOST: all services archived successfully" \ + log "===== Borg backup complete — $BACKUP_COUNT service(s) in $DURATION_STR =====" + ntfy_send "✓ Borg backup complete" \ + "$HOST: $BACKUP_COUNT service(s) archived in $DURATION_STR" \ "low" "white_check_mark" else - log "===== Borg backup finished WITH WARNINGS (see above) =====" - _ntfy_msg="$HOST: Borg backup failures:" + log "===== Borg backup finished WITH WARNINGS — $BACKUP_COUNT/$((BACKUP_COUNT+${#FAILED_SVCS[@]})) succeeded in $DURATION_STR =====" + _ntfy_msg="$HOST: Borg backup failures (${#FAILED_SVCS[@]}):" for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done ntfy_send "✗ Borg backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi diff --git a/extras/backup_gaming.sh b/extras/backup_gaming.sh index 5233d8b..b2952bc 100644 --- a/extras/backup_gaming.sh +++ b/extras/backup_gaming.sh @@ -82,6 +82,18 @@ rc=0 declare -a FAILED_LABELS=() _ERR="$(mktemp)" trap 'rm -f "$_ERR"' EXIT +START_TS="$(date +%s)" +BACKUP_COUNT=0 + +if [ -n "${KOPIA_REPO:-}" ] && [[ "${KOPIA_REPO:-}" != *@*:* ]] && [[ "${KOPIA_REPO:-}" != ssh://* ]]; then + _pf_dir="$([ -d "$KOPIA_REPO" ] && echo "$KOPIA_REPO" || dirname "$KOPIA_REPO")" + _pf_avail="$(df -m "$_pf_dir" 2>/dev/null | awk 'NR==2{print $4}')" + if [ -n "$_pf_avail" ] && [ "$_pf_avail" -lt 512 ]; then + log "WARNING: Low disk space — ${_pf_avail}MB free at $KOPIA_REPO" + FAILED_LABELS+=("repo: low disk (${_pf_avail}MB free)") + rc=1 + fi +fi snap() { local label="$1" path="$2" @@ -89,7 +101,9 @@ snap() { log "skip $label — not found: ${path:-}"; return fi log "Snapshotting $label: $path" - if ! k snapshot create --description="gaming: $label" "$path" 2>"$_ERR"; then + if k snapshot create --description="gaming: $label" "$path" 2>"$_ERR"; then + BACKUP_COUNT=$((BACKUP_COUNT+1)) + else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: snapshot failed for $label — $_reason" FAILED_LABELS+=("$label: $_reason") @@ -121,13 +135,17 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then fi fi +DURATION=$(( $(date +%s) - START_TS )) +DURATION_STR="$((DURATION/60))m $((DURATION%60))s" + if [ "$rc" -eq 0 ]; then - log "===== Gaming backup complete =====" - ntfy_send "✓ Gaming backup complete" "$HOST: all saves backed up successfully" \ + log "===== Gaming backup complete — $BACKUP_COUNT snapshot(s) in $DURATION_STR =====" + ntfy_send "✓ Gaming backup complete" \ + "$HOST: $BACKUP_COUNT snapshot(s) saved in $DURATION_STR" \ "low" "white_check_mark" else - log "===== Gaming backup finished WITH WARNINGS =====" - _ntfy_msg="$HOST: gaming backup failures:" + log "===== Gaming backup finished WITH WARNINGS — $BACKUP_COUNT/$((BACKUP_COUNT+${#FAILED_LABELS[@]})) succeeded in $DURATION_STR =====" + _ntfy_msg="$HOST: gaming backup failures (${#FAILED_LABELS[@]}):" for _s in "${FAILED_LABELS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done ntfy_send "✗ Gaming backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi diff --git a/extras/backup_kopia.sh b/extras/backup_kopia.sh index a9e48ca..5a9dac0 100644 --- a/extras/backup_kopia.sh +++ b/extras/backup_kopia.sh @@ -86,6 +86,21 @@ rc=0 declare -a FAILED_SVCS=() _ERR="$(mktemp)" trap 'rm -f "$_ERR"' EXIT +START_TS="$(date +%s)" +BACKUP_COUNT=0 + +for _pf_dest in ${DEST_NAMES:-default}; do + _pf_var="DEST_${_pf_dest}_REPO"; _pf_repo="${!_pf_var:-}" + if [ -n "$_pf_repo" ] && [[ "$_pf_repo" != *@*:* ]] && [[ "$_pf_repo" != ssh://* ]]; then + _pf_dir="$([ -d "$_pf_repo" ] && echo "$_pf_repo" || dirname "$_pf_repo")" + _pf_avail="$(df -m "$_pf_dir" 2>/dev/null | awk 'NR==2{print $4}')" + if [ -n "$_pf_avail" ] && [ "$_pf_avail" -lt 512 ]; then + log "WARNING: Low disk for '$_pf_dest' — ${_pf_avail}MB free at $_pf_repo" + FAILED_SVCS+=("$_pf_dest: low disk (${_pf_avail}MB free)") + rc=1 + fi + fi +done for svc_dir in "$DOCKER_DIR"/*/; do [ -f "${svc_dir}docker-compose.yml" ] || continue @@ -106,6 +121,7 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "Snapshotting $svc (dest: $dest)..." if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then log "OK $svc (Minecraft, no downtime)" + BACKUP_COUNT=$((BACKUP_COUNT+1)) else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: snapshot failed for $svc — $_reason" @@ -125,6 +141,7 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "Snapshotting $svc (dest: $dest)..." if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then log "OK $svc" + BACKUP_COUNT=$((BACKUP_COUNT+1)) else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: snapshot failed for $svc — $_reason" @@ -153,13 +170,17 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then done fi +DURATION=$(( $(date +%s) - START_TS )) +DURATION_STR="$((DURATION/60))m $((DURATION%60))s" + if [ "$rc" -eq 0 ]; then - log "===== Backup complete =====" - ntfy_send "✓ Backup complete" "$HOST: all services backed up successfully" \ + log "===== Backup complete — $BACKUP_COUNT service(s) in $DURATION_STR =====" + ntfy_send "✓ Backup complete" \ + "$HOST: $BACKUP_COUNT service(s) backed up in $DURATION_STR" \ "low" "white_check_mark" else - log "===== Backup finished WITH WARNINGS (see above) =====" - _ntfy_msg="$HOST: backup failures:" + log "===== Backup finished WITH WARNINGS — $BACKUP_COUNT/$((BACKUP_COUNT+${#FAILED_SVCS[@]})) succeeded in $DURATION_STR =====" + _ntfy_msg="$HOST: backup failures (${#FAILED_SVCS[@]}):" for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done ntfy_send "✗ Backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi diff --git a/extras/test_backup_borg.sh b/extras/test_backup_borg.sh new file mode 100755 index 0000000..2176860 --- /dev/null +++ b/extras/test_backup_borg.sh @@ -0,0 +1,265 @@ +#!/bin/bash +# extras/test_backup_borg.sh — Restore-verify test for Borg-backed services. +# Installed to ~/docker/borg-backup/ by the borg-backup service installer. +# +# sudo ./test_backup_borg.sh test all services +# sudo ./test_backup_borg.sh --service test one service +# sudo ./test_backup_borg.sh --list list services and archive counts +# +# For each service: +# 1. Checks the latest archive for integrity (borg check) +# 2. Stops the container +# 3. Moves live data aside → .test-aside-TIMESTAMP +# 4. Extracts latest archive → +# 5. Compares file inventory: restored vs original (informational) +# 6. Rolls back: original data returns, container restarts +# 7. Reports PASS / FAIL +# +# PASS criteria: +# • borg check exits 0 (no corruption) +# • borg extract exits 0 +# • Restored directory is non-empty and contains docker-compose.yml +# +# Sends ntfy notification on completion if NTFY_URL / NTFY_TOPIC are set in backup.conf. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONF="${BACKUP_CONF:-$HERE/backup.conf}" +[ -f "$CONF" ] || { echo "Config not found: $CONF (re-run: sudo setup.sh borg-backup)"; exit 1; } +# shellcheck source=/dev/null +source "$CONF" + +[ "${EUID:-$(id -u)}" -eq 0 ] || { echo "Run as root: sudo $0"; exit 1; } +command -v borg >/dev/null 2>&1 || { echo "borg required: sudo apt install borgbackup"; exit 1; } + +ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" +ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" +DOCKER_DIR="$ACTUAL_HOME/docker" +TS="$(date +%Y%m%d-%H%M%S)" +LOG="/var/log/post-install-borg-test.log" + +log() { printf "[%s] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +info() { printf "[%s] [INFO] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +pass() { printf "[%s] [PASS] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +fail() { printf "[%s] [FAIL] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +warn() { printf "[%s] [WARN] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } + +repo_for() { local v="DEST_${1}_REPO"; echo "${!v:-}"; } +pass_for() { local v="DEST_${1}_PASSPHRASE"; echo "${!v:-}"; } +dest_for_svc() { local v="SVC_${1//-/_}"; echo "${!v:-${DEST_DEFAULT:-default}}"; } + +b_for() { + local dest="$1"; shift + local repo; repo="$(repo_for "$dest")" + local pass; pass="$(pass_for "$dest")" + [ -n "$repo" ] || { log "Unknown destination: $dest"; return 1; } + BORG_PASSPHRASE="$pass" BORG_REPO="$repo" "$BORG" "$@" +} + +ntfy_send() { + local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" + [ -z "${NTFY_URL:-}" ] && return 0 + local -a args=(-fsSL -o /dev/null) + args+=(-H "Title: ${title}" -H "Priority: ${priority}") + [ -n "${tags}" ] && args+=(-H "Tags: ${tags}") + [ -n "${NTFY_TOKEN:-}" ] && args+=(-H "Authorization: Bearer ${NTFY_TOKEN}") + args+=(-d "${msg}") + curl "${args[@]}" "$NTFY_URL" 2>/dev/null || true +} + +# ── Args ────────────────────────────────────────────────────────────────────── +FILTER_SVC=""; LIST_ONLY=false +while [ "$#" -gt 0 ]; do + case "$1" in + --service|-s) FILTER_SVC="${2:-}"; shift 2 ;; + --list|-l) LIST_ONLY=true; shift ;; + *) shift ;; + esac +done + +# ── File inventory comparison (informational only) ──────────────────────────── +compare_dirs() { + local restored="${1%/}" original="${2%/}" + local tmp_r tmp_o + tmp_r="$(mktemp /tmp/bktest-XXXXXX)"; tmp_o="$(mktemp /tmp/bktest-XXXXXX)" + find "$restored" -type f -printf "%P\n" 2>/dev/null | sort > "$tmp_r" + find "$original" -type f -printf "%P\n" 2>/dev/null | sort > "$tmp_o" + local r_files o_files added deleted + r_files="$(wc -l < "$tmp_r")"; o_files="$(wc -l < "$tmp_o")" + added="$(comm -23 "$tmp_o" "$tmp_r" | wc -l)" + deleted="$(comm -13 "$tmp_o" "$tmp_r" | wc -l)" + rm -f "$tmp_r" "$tmp_o" + printf "restored=%d original=%d added_since_backup=%d deleted_since_backup=%d" \ + "$r_files" "$o_files" "$added" "$deleted" +} + +# ── Collect services ────────────────────────────────────────────────────────── +declare -a SVCS_TO_TEST=() +for _sd in "$DOCKER_DIR"/*/; do + [ -f "${_sd}docker-compose.yml" ] || continue + _sv="$(basename "$_sd")" + [[ "$_sv" == "backup" || "$_sv" == "borg-backup" || "$_sv" == "gaming-backup" ]] && continue + [ -n "$FILTER_SVC" ] && [ "$_sv" != "$FILTER_SVC" ] && continue + SVCS_TO_TEST+=("$_sv") +done + +[ "${#SVCS_TO_TEST[@]}" -gt 0 ] || { + echo "No services found$([ -n "$FILTER_SVC" ] && echo " matching: $FILTER_SVC" || echo " under $DOCKER_DIR")" + exit 1 +} + +# ── List mode ───────────────────────────────────────────────────────────────── +if [ "$LIST_ONLY" = true ]; then + echo "" + printf " %-20s %-14s %s\n" "SERVICE" "DESTINATION" "ARCHIVES" + echo " ──────────────────────────────────────────────────" + declare -A _DEST_LISTS=() + for _sv in "${SVCS_TO_TEST[@]}"; do + dest="$(dest_for_svc "$_sv")" + if [ -z "${_DEST_LISTS[$dest]+x}" ]; then + _DEST_LISTS["$dest"]="$(b_for "$dest" list --format '{archive}{NL}' 2>/dev/null || echo '')" + fi + n="$(echo "${_DEST_LISTS[$dest]}" | grep -c "^${_sv}-" 2>/dev/null || echo '0')" + printf " %-20s %-14s %s\n" "$_sv" "$dest" "$n" + done + echo ""; exit 0 +fi + +# ── Pre-cache archive lists per destination ─────────────────────────────────── +declare -A DEST_ARCHIVE_CACHE=() +for _sv in "${SVCS_TO_TEST[@]}"; do + dest="$(dest_for_svc "$_sv")" + if [ -z "${DEST_ARCHIVE_CACHE[$dest]+x}" ]; then + _repo="$(repo_for "$dest")" + if [ -n "$_repo" ]; then + DEST_ARCHIVE_CACHE["$dest"]="$(b_for "$dest" list --format '{archive}{NL}' 2>/dev/null || echo '')" + else + DEST_ARCHIVE_CACHE["$dest"]="" + fi + fi +done + +# ── Test run ────────────────────────────────────────────────────────────────── +log "===== Borg backup restore test starting (${#SVCS_TO_TEST[@]} service(s)) =====" +PASS_N=0; FAIL_N=0; SKIP_N=0 +declare -a FAIL_LIST=() + +for svc in "${SVCS_TO_TEST[@]}"; do + svc_dir="${DOCKER_DIR}/${svc}" + compose_file="${svc_dir}/docker-compose.yml" + dest="$(dest_for_svc "$svc")" + repo="$(repo_for "$dest")" + + if [ -z "$repo" ]; then + info "SKIP $svc — destination '$dest' not configured"; SKIP_N=$((SKIP_N+1)); continue + fi + + log "── Testing: $svc (dest: $dest)" + + # ── Find latest archive ────────────────────────────────────────────────── + archive_list="${DEST_ARCHIVE_CACHE[$dest]:-}" + latest_archive="$(echo "$archive_list" | grep "^${svc}-" | sort -r | head -1 || true)" + + if [ -z "${latest_archive:-}" ]; then + warn "SKIP $svc — no archives found in destination '$dest'"; SKIP_N=$((SKIP_N+1)); continue + fi + + archive_ts="${latest_archive#${svc}-}" + info "$svc: latest archive ${archive_ts} ($latest_archive)" + + # ── Step 1: Check archive integrity ───────────────────────────────────── + info "$svc: checking archive integrity..." + if ! b_for "$dest" check --archives-only "::$latest_archive" >/dev/null 2>&1; then + fail "$svc: FAIL — borg check failed (archive may be corrupted)" + FAIL_N=$((FAIL_N+1)); FAIL_LIST+=("$svc (archive corruption)"); continue + fi + info "$svc: archive check OK" + + # ── Step 2: Stop container ─────────────────────────────────────────────── + STOPPED=false + if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$svc"; then + info "$svc: stopping container..." + if docker compose -f "$compose_file" down 2>/dev/null \ + || docker stop "$svc" 2>/dev/null; then + STOPPED=true + else + warn "$svc: could not stop container — testing live (consistency not guaranteed)" + fi + fi + + # ── Step 3: Move live data aside ───────────────────────────────────────── + ASIDE="${svc_dir}.test-aside-${TS}" + if ! mv "$svc_dir" "$ASIDE" 2>/dev/null; then + fail "$svc: FAIL — could not move data aside from $svc_dir" + [ "$STOPPED" = true ] && \ + docker compose -f "${ASIDE}/docker-compose.yml" up -d 2>/dev/null || true + FAIL_N=$((FAIL_N+1)); FAIL_LIST+=("$svc (move failed)"); continue + fi + + # ── Step 4: Extract archive ────────────────────────────────────────────── + # Borg archives absolute paths; extracting from / restores them in place. + mkdir -p "$svc_dir" + TEST_PASS=true; FAIL_REASON=""; RESTORED_FILES=0 + EXTRACT_PATH="${svc_dir#/}" + if ( cd / && b_for "$dest" extract "::$latest_archive" "$EXTRACT_PATH" ) >/dev/null 2>&1; then + RESTORED_FILES="$(find "$svc_dir" -type f 2>/dev/null | wc -l)" + else + TEST_PASS=false; FAIL_REASON="borg extract failed" + fi + + # ── Step 5: Sanity checks ──────────────────────────────────────────────── + if [ "$TEST_PASS" = true ] && [ "$RESTORED_FILES" -eq 0 ]; then + TEST_PASS=false; FAIL_REASON="restored directory is empty" + fi + if [ "$TEST_PASS" = true ] && [ ! -f "${svc_dir}/docker-compose.yml" ]; then + TEST_PASS=false; FAIL_REASON="docker-compose.yml missing from restore" + fi + + # ── Step 6: File inventory comparison (informational) ──────────────────── + if [ "$TEST_PASS" = true ]; then + info "$svc: comparing file inventory..." + cmp_out="$(compare_dirs "$svc_dir" "$ASIDE" 2>/dev/null || echo "comparison error")" + info "$svc: $cmp_out" + fi + + # ── Step 7: Roll back to original data ─────────────────────────────────── + info "$svc: rolling back to live data..." + rm -rf "$svc_dir" 2>/dev/null || true + if mv "$ASIDE" "$svc_dir" 2>/dev/null; then + info "$svc: live data restored" + else + warn "$svc: could not restore original — manual action needed:" + warn "$svc: mv \"$ASIDE\" \"$svc_dir\"" + fi + if [ "$STOPPED" = true ]; then + docker compose -f "$compose_file" up -d 2>/dev/null \ + || warn "$svc: restart failed — run: docker compose -f $compose_file up -d" + fi + + # ── Result ──────────────────────────────────────────────────────────────── + if [ "$TEST_PASS" = true ]; then + pass "$svc: PASS (archive: $archive_ts, restored $RESTORED_FILES file(s))" + PASS_N=$((PASS_N+1)) + else + fail "$svc: FAIL — $FAIL_REASON" + FAIL_N=$((FAIL_N+1)); FAIL_LIST+=("$svc ($FAIL_REASON)") + fi +done + +# ── Summary & notification ──────────────────────────────────────────────────── +TOTAL=$((PASS_N + FAIL_N)) +log "===== Test complete: ${PASS_N}/${TOTAL} passed, ${SKIP_N} skipped =====" +[ -n "${NTFY_URL:-}" ] && log "Log: $LOG" + +if [ "$FAIL_N" -eq 0 ]; then + ntfy_send "Borg backup test passed (${PASS_N}/${TOTAL})" \ + "$(date '+%F %T') — All ${PASS_N} service(s) restore-tested OK. Log: $LOG" \ + "low" "white_check_mark,microscope" + exit 0 +else + fail_str="$(IFS=', '; echo "${FAIL_LIST[*]}")" + ntfy_send "Borg backup test FAILED (${FAIL_N}/${TOTAL} failed)" \ + "$(date '+%F %T') — FAILED: ${fail_str}. Log: $LOG" \ + "high" "rotating_light,microscope" + exit 1 +fi diff --git a/extras/test_backup_kopia.sh b/extras/test_backup_kopia.sh new file mode 100755 index 0000000..65593f5 --- /dev/null +++ b/extras/test_backup_kopia.sh @@ -0,0 +1,271 @@ +#!/bin/bash +# extras/test_backup_kopia.sh — Restore-verify test for Kopia-backed services. +# Installed to ~/docker/backup/ by the backup service installer. +# +# sudo ./test_backup_kopia.sh test all services +# sudo ./test_backup_kopia.sh --service test one service +# sudo ./test_backup_kopia.sh --list list services and snapshot counts +# +# For each service: +# 1. Verifies the latest snapshot (kopia snapshot verify — catches corruption) +# 2. Stops the container +# 3. Moves live data aside → .test-aside-TIMESTAMP +# 4. Restores latest snapshot → +# 5. Compares file inventory: restored vs original (informational) +# 6. Rolls back: original data returns, container restarts +# 7. Reports PASS / FAIL +# +# PASS criteria: +# • snapshot verify exits 0 (no data corruption) +# • kopia restore exits 0 +# • Restored directory is non-empty and contains docker-compose.yml +# +# The file comparison (step 5) is informational: any delta between restored and +# current data represents normal writes since the last backup — not a failure. +# +# Sends ntfy notification on completion if NTFY_URL / NTFY_TOPIC are set in backup.conf. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONF="${BACKUP_CONF:-$HERE/backup.conf}" +[ -f "$CONF" ] || { echo "Config not found: $CONF (re-run: sudo setup.sh backup)"; exit 1; } +# shellcheck source=/dev/null +source "$CONF" + +[ "${EUID:-$(id -u)}" -eq 0 ] || { echo "Run as root: sudo $0"; exit 1; } +command -v jq >/dev/null 2>&1 || { echo "jq required: sudo apt install jq"; exit 1; } + +ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" +ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" +DOCKER_DIR="$ACTUAL_HOME/docker" +TS="$(date +%Y%m%d-%H%M%S)" +LOG="/var/log/post-install-backup-test.log" + +log() { printf "[%s] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +info() { printf "[%s] [INFO] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +pass() { printf "[%s] [PASS] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +fail() { printf "[%s] [FAIL] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } +warn() { printf "[%s] [WARN] %s\n" "$(date '+%F %T')" "$*" | tee -a "$LOG"; } + +kp_for() { + local dest="$1"; shift + local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD" + local cfg="${!cfg_var:-}" pw="${!pw_var:-}" + [ -n "$cfg" ] || { log "Unknown destination: $dest"; return 1; } + env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@" +} + +dest_for_svc() { local v="SVC_${1//-/_}"; echo "${!v:-${DEST_DEFAULT:-default}}"; } + +ntfy_send() { + local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" + [ -z "${NTFY_URL:-}" ] && return 0 + local -a args=(-fsSL -o /dev/null) + args+=(-H "Title: ${title}" -H "Priority: ${priority}") + [ -n "${tags}" ] && args+=(-H "Tags: ${tags}") + [ -n "${NTFY_TOKEN:-}" ] && args+=(-H "Authorization: Bearer ${NTFY_TOKEN}") + args+=(-d "${msg}") + curl "${args[@]}" "$NTFY_URL" 2>/dev/null || true +} + +# ── Args ────────────────────────────────────────────────────────────────────── +FILTER_SVC=""; LIST_ONLY=false +while [ "$#" -gt 0 ]; do + case "$1" in + --service|-s) FILTER_SVC="${2:-}"; shift 2 ;; + --list|-l) LIST_ONLY=true; shift ;; + *) shift ;; + esac +done + +# ── File inventory comparison (informational only) ──────────────────────────── +compare_dirs() { + local restored="${1%/}" original="${2%/}" + local tmp_r tmp_o + tmp_r="$(mktemp /tmp/bktest-XXXXXX)"; tmp_o="$(mktemp /tmp/bktest-XXXXXX)" + find "$restored" -type f -printf "%P\n" 2>/dev/null | sort > "$tmp_r" + find "$original" -type f -printf "%P\n" 2>/dev/null | sort > "$tmp_o" + local r_files o_files added deleted + r_files="$(wc -l < "$tmp_r")"; o_files="$(wc -l < "$tmp_o")" + added="$(comm -23 "$tmp_o" "$tmp_r" | wc -l)" # in original, not in restored + deleted="$(comm -13 "$tmp_o" "$tmp_r" | wc -l)" # in restored, not in original (deleted since backup) + rm -f "$tmp_r" "$tmp_o" + printf "restored=%d original=%d added_since_backup=%d deleted_since_backup=%d" \ + "$r_files" "$o_files" "$added" "$deleted" +} + +# ── Collect services ────────────────────────────────────────────────────────── +declare -a SVCS_TO_TEST=() +for _sd in "$DOCKER_DIR"/*/; do + [ -f "${_sd}docker-compose.yml" ] || continue + _sv="$(basename "$_sd")" + [[ "$_sv" == "backup" || "$_sv" == "borg-backup" || "$_sv" == "gaming-backup" ]] && continue + [ -n "$FILTER_SVC" ] && [ "$_sv" != "$FILTER_SVC" ] && continue + SVCS_TO_TEST+=("$_sv") +done + +[ "${#SVCS_TO_TEST[@]}" -gt 0 ] || { + echo "No services found$([ -n "$FILTER_SVC" ] && echo " matching: $FILTER_SVC" || echo " under $DOCKER_DIR")" + exit 1 +} + +# ── List mode ───────────────────────────────────────────────────────────────── +if [ "$LIST_ONLY" = true ]; then + echo "" + printf " %-20s %-14s %s\n" "SERVICE" "DESTINATION" "SNAPSHOTS" + echo " ──────────────────────────────────────────────────" + # Cache snapshot JSON per destination + declare -A _DEST_SNAPS=() + for _sv in "${SVCS_TO_TEST[@]}"; do + dest="$(dest_for_svc "$_sv")" + if [ -z "${_DEST_SNAPS[$dest]+x}" ]; then + _DEST_SNAPS["$dest"]="$(kp_for "$dest" snapshot list --all --json 2>/dev/null || echo '[]')" + fi + n="$(echo "${_DEST_SNAPS[$dest]}" | jq -r --arg p "${DOCKER_DIR}/${_sv}/" \ + '[.[] | select(.source.path == $p)] | length' 2>/dev/null || echo '?')" + printf " %-20s %-14s %s\n" "$_sv" "$dest" "$n" + done + echo ""; exit 0 +fi + +# ── Pre-cache snapshot JSON per destination ─────────────────────────────────── +declare -A DEST_SNAP_CACHE=() +for _sv in "${SVCS_TO_TEST[@]}"; do + dest="$(dest_for_svc "$_sv")" + if [ -z "${DEST_SNAP_CACHE[$dest]+x}" ]; then + _repo_var="DEST_${dest}_REPO" + if [ -n "${!_repo_var:-}" ]; then + DEST_SNAP_CACHE["$dest"]="$(kp_for "$dest" snapshot list --all --json 2>/dev/null || echo '[]')" + else + DEST_SNAP_CACHE["$dest"]="[]" + fi + fi +done + +# ── Test run ────────────────────────────────────────────────────────────────── +log "===== Backup restore test starting (${#SVCS_TO_TEST[@]} service(s)) =====" +PASS_N=0; FAIL_N=0; SKIP_N=0 +declare -a FAIL_LIST=() + +for svc in "${SVCS_TO_TEST[@]}"; do + svc_dir="${DOCKER_DIR}/${svc}" + compose_file="${svc_dir}/docker-compose.yml" + dest="$(dest_for_svc "$svc")" + + # Destination must be configured + _repo_var="DEST_${dest}_REPO" + if [ -z "${!_repo_var:-}" ]; then + info "SKIP $svc — destination '$dest' not configured"; SKIP_N=$((SKIP_N+1)); continue + fi + + log "── Testing: $svc (dest: $dest)" + + # ── Find latest snapshot ───────────────────────────────────────────────── + svc_path="${svc_dir}/" + all_snaps="${DEST_SNAP_CACHE[$dest]:-[]}" + latest_id="$(echo "$all_snaps" | jq -r --arg p "$svc_path" \ + '[.[] | select(.source.path == $p)] | sort_by(.startTime) | reverse | .[0].id // empty' \ + 2>/dev/null || true)" + latest_time="$(echo "$all_snaps" | jq -r --arg p "$svc_path" \ + '[.[] | select(.source.path == $p)] | sort_by(.startTime) | reverse | + .[0].startTime | split("T") | "\(.[0]) \(.[1][:8]) UTC"' 2>/dev/null || true)" + + if [ -z "${latest_id:-}" ]; then + warn "SKIP $svc — no snapshots found in destination '$dest'"; SKIP_N=$((SKIP_N+1)); continue + fi + info "$svc: latest snapshot $latest_time (id ${latest_id:0:12}...)" + + # ── Step 1: Verify snapshot integrity ─────────────────────────────────── + info "$svc: verifying snapshot..." + if ! kp_for "$dest" snapshot verify "$latest_id" >/dev/null 2>&1; then + fail "$svc: FAIL — snapshot verify failed (data may be corrupted in backup)" + FAIL_N=$((FAIL_N+1)); FAIL_LIST+=("$svc (snapshot corruption)"); continue + fi + info "$svc: snapshot verified OK" + + # ── Step 2: Stop container ─────────────────────────────────────────────── + STOPPED=false + if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$svc"; then + info "$svc: stopping container..." + if docker compose -f "$compose_file" down 2>/dev/null \ + || docker stop "$svc" 2>/dev/null; then + STOPPED=true + else + warn "$svc: could not stop container — testing live (consistency not guaranteed)" + fi + fi + + # ── Step 3: Move live data aside ───────────────────────────────────────── + ASIDE="${svc_dir}.test-aside-${TS}" + if ! mv "$svc_dir" "$ASIDE" 2>/dev/null; then + fail "$svc: FAIL — could not move data aside from $svc_dir" + [ "$STOPPED" = true ] && \ + docker compose -f "${ASIDE}/docker-compose.yml" up -d 2>/dev/null || true + FAIL_N=$((FAIL_N+1)); FAIL_LIST+=("$svc (move failed)"); continue + fi + + # ── Step 4: Restore snapshot ───────────────────────────────────────────── + mkdir -p "$svc_dir" + TEST_PASS=true; FAIL_REASON=""; RESTORED_FILES=0 + if kp_for "$dest" restore "$latest_id" "$svc_dir" >/dev/null 2>&1; then + RESTORED_FILES="$(find "$svc_dir" -type f 2>/dev/null | wc -l)" + else + TEST_PASS=false; FAIL_REASON="kopia restore failed" + fi + + # ── Step 5: Sanity checks ──────────────────────────────────────────────── + if [ "$TEST_PASS" = true ] && [ "$RESTORED_FILES" -eq 0 ]; then + TEST_PASS=false; FAIL_REASON="restored directory is empty" + fi + if [ "$TEST_PASS" = true ] && [ ! -f "${svc_dir}/docker-compose.yml" ]; then + TEST_PASS=false; FAIL_REASON="docker-compose.yml missing from restore" + fi + + # ── Step 6: File inventory comparison (informational) ──────────────────── + if [ "$TEST_PASS" = true ]; then + info "$svc: comparing file inventory..." + cmp_out="$(compare_dirs "$svc_dir" "$ASIDE" 2>/dev/null || echo "comparison error")" + info "$svc: $cmp_out" + fi + + # ── Step 7: Roll back to original data ─────────────────────────────────── + info "$svc: rolling back to live data..." + rm -rf "$svc_dir" 2>/dev/null || true + if mv "$ASIDE" "$svc_dir" 2>/dev/null; then + info "$svc: live data restored" + else + warn "$svc: could not restore original — manual action needed:" + warn "$svc: mv \"$ASIDE\" \"$svc_dir\"" + fi + if [ "$STOPPED" = true ]; then + docker compose -f "$compose_file" up -d 2>/dev/null \ + || warn "$svc: restart failed — run: docker compose -f $compose_file up -d" + fi + + # ── Result ──────────────────────────────────────────────────────────────── + if [ "$TEST_PASS" = true ]; then + pass "$svc: PASS (snapshot: $latest_time, restored $RESTORED_FILES file(s))" + PASS_N=$((PASS_N+1)) + else + fail "$svc: FAIL — $FAIL_REASON" + FAIL_N=$((FAIL_N+1)); FAIL_LIST+=("$svc ($FAIL_REASON)") + fi +done + +# ── Summary & notification ──────────────────────────────────────────────────── +TOTAL=$((PASS_N + FAIL_N)) +log "===== Test complete: ${PASS_N}/${TOTAL} passed, ${SKIP_N} skipped =====" +[ -n "${NTFY_URL:-}" ] && log "Log: $LOG" + +if [ "$FAIL_N" -eq 0 ]; then + ntfy_send "Backup test passed (${PASS_N}/${TOTAL})" \ + "$(date '+%F %T') — All ${PASS_N} service(s) restore-tested OK. Log: $LOG" \ + "low" "white_check_mark,microscope" + exit 0 +else + fail_str="$(IFS=', '; echo "${FAIL_LIST[*]}")" + ntfy_send "Backup test FAILED (${FAIL_N}/${TOTAL} failed)" \ + "$(date '+%F %T') — FAILED: ${fail_str}. Log: $LOG" \ + "high" "rotating_light,microscope" + exit 1 +fi diff --git a/services/backup.sh b/services/backup.sh index 810ebd5..dd8d0d5 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -364,16 +364,64 @@ install_backup() { log_warning "Copy it manually: cp extras/restore_kopia.sh $RESTORE" fi - # ── 11. Install test script ────────────────────────────────────────────── - local TEST_SCRIPT="$DIR/test_backup.sh" - local TEST_SRC="${HERE:-}/extras/test_backup.sh" + # ── 11. Install test scripts ───────────────────────────────────────────── + local TEST_SCRIPT="$DIR/test_backup_kopia.sh" + local TEST_SRC="${HERE:-}/extras/test_backup_kopia.sh" if [ -f "$TEST_SRC" ]; then cp "$TEST_SRC" "$TEST_SCRIPT" chmod +x "$TEST_SCRIPT" chown root:root "$TEST_SCRIPT" 2>/dev/null || true - log_success "test_backup.sh installed" + log_success "test_backup_kopia.sh installed" else - log_warning "extras/test_backup.sh not found — test script not installed" + log_warning "extras/test_backup_kopia.sh not found — test script not installed" + fi + + local TEST_UNIFIED="$DIR/test_backup.sh" + local TEST_UNIFIED_SRC="${HERE:-}/extras/test_backup.sh" + if [ -f "$TEST_UNIFIED_SRC" ]; then + cp "$TEST_UNIFIED_SRC" "$TEST_UNIFIED" + chmod +x "$TEST_UNIFIED" + chown root:root "$TEST_UNIFIED" 2>/dev/null || true + log_success "test_backup.sh installed" + fi + + # ── 11b. Weekly backup test timer ──────────────────────────────────────── + local TEST_SVC_NAME="post-install-backup-test" + local _add_test="" + prompt_yn " Schedule a weekly automated backup test? (y/N):" "n" _add_test + if [[ "$_add_test" =~ ^[Yy]$ ]]; then + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + tee "/etc/systemd/system/${TEST_SVC_NAME}.service" >/dev/null << SVCEOF +[Unit] +Description=Weekly restore test for Kopia backup +After=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash $TEST_SCRIPT +SVCEOF + + tee "/etc/systemd/system/${TEST_SVC_NAME}.timer" >/dev/null << SVCEOF +[Unit] +Description=Weekly Kopia backup restore test (Saturday 03:00) + +[Timer] +OnCalendar=Sat *-*-* 03:00:00 +Persistent=true +RandomizedDelaySec=600 + +[Install] +WantedBy=timers.target +SVCEOF + + systemctl daemon-reload + systemctl enable --now "${TEST_SVC_NAME}.timer" + log_success "Weekly test timer enabled (Saturday 03:00)" + else + echo "0 3 * * 6 root /bin/bash $TEST_SCRIPT >> /var/log/${TEST_SVC_NAME}.log 2>&1" \ + > "/etc/cron.d/${TEST_SVC_NAME}" + log_success "Weekly test cron installed (Saturday 03:00)" + fi fi # ── 12. Systemd timer ──────────────────────────────────────────────────── @@ -460,9 +508,9 @@ SVCEOF echo " sudo $RESTORE --list" echo "" echo " Backup test (stop/restore/compare/restore-back):" - echo " sudo $TEST_SCRIPT test most recent backup" + echo " sudo $TEST_SCRIPT test most recent backup (all services)" echo " sudo $TEST_SCRIPT --list list testable services" - echo " sudo $TEST_SCRIPT test a specific service" + echo " sudo $TEST_SCRIPT --service test a specific service" [ -n "${NTFY_URL:-}" ] && echo "" && echo " Notifications: $NTFY_URL" echo "" [ -n "$AUTORUN" ] && echo " $AUTORUN" && echo "" diff --git a/services/borg-backup.sh b/services/borg-backup.sh index 5f1aef9..a45c430 100644 --- a/services/borg-backup.sh +++ b/services/borg-backup.sh @@ -308,16 +308,64 @@ install_borg_backup() { log_warning "Copy it manually: cp extras/restore_borg.sh $RESTORE" fi - # ── 11. Install test script ────────────────────────────────────────────── - local TEST_SCRIPT="$DIR/test_backup.sh" - local TEST_SRC="${HERE:-}/extras/test_backup.sh" + # ── 11. Install test scripts ───────────────────────────────────────────── + local TEST_SCRIPT="$DIR/test_backup_borg.sh" + local TEST_SRC="${HERE:-}/extras/test_backup_borg.sh" if [ -f "$TEST_SRC" ]; then cp "$TEST_SRC" "$TEST_SCRIPT" chmod +x "$TEST_SCRIPT" chown root:root "$TEST_SCRIPT" 2>/dev/null || true - log_success "test_backup.sh installed" + log_success "test_backup_borg.sh installed" else - log_warning "extras/test_backup.sh not found — test script not installed" + log_warning "extras/test_backup_borg.sh not found — test script not installed" + fi + + local TEST_UNIFIED="$DIR/test_backup.sh" + local TEST_UNIFIED_SRC="${HERE:-}/extras/test_backup.sh" + if [ -f "$TEST_UNIFIED_SRC" ]; then + cp "$TEST_UNIFIED_SRC" "$TEST_UNIFIED" + chmod +x "$TEST_UNIFIED" + chown root:root "$TEST_UNIFIED" 2>/dev/null || true + log_success "test_backup.sh installed" + fi + + # ── 11b. Weekly backup test timer ──────────────────────────────────────── + local TEST_SVC_NAME="post-install-borg-backup-test" + local _add_test="" + prompt_yn " Schedule a weekly automated backup test? (y/N):" "n" _add_test + if [[ "$_add_test" =~ ^[Yy]$ ]]; then + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + tee "/etc/systemd/system/${TEST_SVC_NAME}.service" >/dev/null << SVCEOF +[Unit] +Description=Weekly restore test for Borg backup +After=docker.service + +[Service] +Type=oneshot +ExecStart=/bin/bash $TEST_SCRIPT +SVCEOF + + tee "/etc/systemd/system/${TEST_SVC_NAME}.timer" >/dev/null << SVCEOF +[Unit] +Description=Weekly Borg backup restore test (Saturday 03:00) + +[Timer] +OnCalendar=Sat *-*-* 03:00:00 +Persistent=true +RandomizedDelaySec=600 + +[Install] +WantedBy=timers.target +SVCEOF + + systemctl daemon-reload + systemctl enable --now "${TEST_SVC_NAME}.timer" + log_success "Weekly test timer enabled (Saturday 03:00)" + else + echo "0 3 * * 6 root /bin/bash $TEST_SCRIPT >> /var/log/${TEST_SVC_NAME}.log 2>&1" \ + > "/etc/cron.d/${TEST_SVC_NAME}" + log_success "Weekly test cron installed (Saturday 03:00)" + fi fi # ── 12. Systemd timer ───────────────────────────────────────────────────── @@ -403,9 +451,9 @@ SVCEOF echo " sudo $RESTORE --list" echo "" echo " Backup test (stop/restore/compare/restore-back):" - echo " sudo $TEST_SCRIPT test most recent backup" + echo " sudo $TEST_SCRIPT test most recent backup (all services)" echo " sudo $TEST_SCRIPT --list list testable services" - echo " sudo $TEST_SCRIPT test a specific service" + echo " sudo $TEST_SCRIPT --service test a specific service" [ -n "${NTFY_URL:-}" ] && echo "" && echo " Notifications: $NTFY_URL" echo "" log_warning "IMPORTANT — back up your Borg key and passphrase now."