From 862ecf10e949d3ae1c92643562dc9e61cec69bff Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Jun 2026 20:03:30 +0000 Subject: [PATCH] feat: backup test script, ntfy notifications, and error categorization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extras/test_backup.sh — new unified test script (Kopia + Borg): • Stops container, moves live data aside, restores latest backup, compares restored vs live with diff -rq (content, not timestamps), moves live data back and restarts container • PASS = restore succeeded; diff output is informational (files changed since last backup are normal) • FAIL = restore command failed or target empty after restore • --list flag, CLI service arg, interactive picker • Handles both full-service dirs and sub-path sources (gaming-backup) • Cleanup trap always restores live data even on error • Sends ntfy notification on pass and fail extras/backup_kopia.sh, backup_borg.sh, backup_gaming.sh: • ntfy_send() + categorize_error() helpers added • Each snapshot/archive failure captures stderr and categorizes: disk full, remote unreachable, repository not found, wrong passphrase, permission denied, unknown error • Single ntfy notification at end: success (low priority) or failure (urgent) with per-service failure reasons listed • backup_borg.sh: changed 2>&1 | pipe to 2>"$_ERR" | so stdout logs cleanly and stderr is captured for error categorization services/backup.sh, borg-backup.sh, gaming-backup.sh: • New ntfy prompt section in installer (URL + optional token) • NTFY_URL / NTFY_TOKEN written to backup.conf • test_backup.sh copied from extras/ into service dir • Summary updated to show test_backup.sh commands and ntfy URL https://claude.ai/code/session_019XgsQ13XKm4Zj3cNsDNwHj --- extras/backup_borg.sh | 56 +++++- extras/backup_gaming.sh | 54 +++++- extras/backup_kopia.sh | 58 ++++++- extras/test_backup.sh | 355 ++++++++++++++++++++++++++++++++++++++ services/backup.sh | 42 ++++- services/borg-backup.sh | 42 ++++- services/gaming-backup.sh | 40 +++++ 7 files changed, 627 insertions(+), 20 deletions(-) create mode 100644 extras/test_backup.sh diff --git a/extras/backup_borg.sh b/extras/backup_borg.sh index e9d090d..bc23866 100644 --- a/extras/backup_borg.sh +++ b/extras/backup_borg.sh @@ -20,12 +20,40 @@ source "$CONF" ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" DOCKER_DIR="$ACTUAL_HOME/docker" +HOST="$(hostname -s 2>/dev/null || hostname)" log() { echo "[$(date '+%F %T')] $*"; } -repo_for() { local var="DEST_${1}_REPO"; echo "${!var:-}"; } -pass_for() { local var="DEST_${1}_PASSPHRASE"; echo "${!var:-}"; } -dest_for_svc() { local var="SVC_${1//-/_}"; echo "${!var:-${DEST_DEFAULT:-default}}"; } +ntfy_send() { + local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" + [ -z "${NTFY_URL:-}" ] && return 0 + local -a _args=(-fsS -o /dev/null) + _args+=(-H "Title: $title" -H "Priority: $priority") + [ -n "$tags" ] && _args+=(-H "Tags: $tags") + [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN") + curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true +} + +categorize_error() { + local txt="$1" + if echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then + echo "disk full — backup destination is out of space" + elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then + echo "remote unreachable — check network / destination host" + elif echo "$txt" | grep -qi "repository.*does not exist\|not a borg\|is not a valid"; then + echo "repository not found — re-run the backup installer" + elif echo "$txt" | grep -qi "passphrase\|wrong key\|bad key\|cannot decrypt"; then + echo "wrong passphrase — check backup.conf" + elif echo "$txt" | grep -qi "permission denied\|access denied"; then + echo "permission denied — check file permissions" + else + echo "error — see system logs on $HOST" + fi +} + +repo_for() { local var="DEST_${1}_REPO"; echo "${!var:-}"; } +pass_for() { local var="DEST_${1}_PASSPHRASE"; echo "${!var:-}"; } +dest_for_svc() { local var="SVC_${1//-/_}"; echo "${!var:-${DEST_DEFAULT:-default}}"; } b_for() { local dest="$1"; shift @@ -55,6 +83,9 @@ esac log "===== Borg backup starting =====" rc=0 TS="$(date +%Y-%m-%dT%H-%M-%S)" +declare -a FAILED_SVCS=() +_ERR="$(mktemp)" +trap 'rm -f "$_ERR"' EXIT for svc_dir in "$DOCKER_DIR"/*/; do [ -f "${svc_dir}docker-compose.yml" ] || continue @@ -77,10 +108,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "Archiving $svc → $dest::$ARCHIVE ..." if b_for "$dest" create \ --compression=zstd,6 --exclude-caches --stats \ - "::$ARCHIVE" "$svc_dir" 2>&1 | while IFS= read -r line; do log " $line"; done; then + "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log " $line"; done; then log "OK $svc (Minecraft, no downtime)" else - log "WARNING: archive failed for $svc"; rc=1 + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: archive failed for $svc — $_reason" + FAILED_SVCS+=("$svc: $_reason") + rc=1 fi else STOPPED=false @@ -95,10 +129,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "Archiving $svc → $dest::$ARCHIVE ..." if b_for "$dest" create \ --compression=zstd,6 --exclude-caches --stats \ - "::$ARCHIVE" "$svc_dir" 2>&1 | while IFS= read -r line; do log " $line"; done; then + "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log " $line"; done; then log "OK $svc" else - log "WARNING: archive failed for $svc"; rc=1 + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: archive failed for $svc — $_reason" + FAILED_SVCS+=("$svc: $_reason") + rc=1 fi if [ "$STOPPED" = true ]; then @@ -127,7 +164,12 @@ done if [ "$rc" -eq 0 ]; then log "===== Borg backup complete =====" + ntfy_send "✓ Borg backup complete" "$HOST: all services archived successfully" \ + "low" "white_check_mark" else log "===== Borg backup finished WITH WARNINGS (see above) =====" + _ntfy_msg="$HOST: Borg backup failures:" + for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done + ntfy_send "✗ Borg backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi exit "$rc" diff --git a/extras/backup_gaming.sh b/extras/backup_gaming.sh index 9513caa..5233d8b 100644 --- a/extras/backup_gaming.sh +++ b/extras/backup_gaming.sh @@ -17,11 +17,42 @@ CONF="${BACKUP_CONF:-$HERE/backup.conf}" source "$CONF" export KOPIA_PASSWORD +HOST="$(hostname -s 2>/dev/null || hostname)" log() { echo "[$(date '+%F %T')] $*"; } k() { "$KOPIA" --config-file="$KOPIA_CONFIG" "$@"; } +ntfy_send() { + local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" + [ -z "${NTFY_URL:-}" ] && return 0 + local -a _args=(-fsS -o /dev/null) + _args+=(-H "Title: $title" -H "Priority: $priority") + [ -n "$tags" ] && _args+=(-H "Tags: $tags") + [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN") + curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true +} + +categorize_error() { + local txt="$1" + if echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then + echo "disk full — backup destination is out of space" + elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then + echo "remote unreachable — check network / destination host" + elif echo "$txt" | grep -qi "repository.*not.*exist\|not a valid kopia\|not connected"; then + echo "repository not found — re-run the gaming-backup installer" + elif echo "$txt" | grep -qi "passphrase\|wrong key\|cannot decrypt"; then + echo "wrong passphrase — check backup.conf" + elif echo "$txt" | grep -qi "permission denied\|access denied"; then + echo "permission denied — check file permissions" + else + echo "error — see system logs on $HOST" + fi +} + if ! k repository status >/dev/null 2>&1; then log "ERROR: not connected to a repository — re-run the gaming-backup service" + ntfy_send "✗ Gaming backup FAILED" \ + "$HOST: cannot connect to Kopia repository — re-run gaming-backup installer" \ + "urgent" "rotating_light" exit 1 fi @@ -48,14 +79,21 @@ if [ -n "${MC_BASE_DIR:-}" ] && command -v docker >/dev/null 2>&1; then fi rc=0 +declare -a FAILED_LABELS=() +_ERR="$(mktemp)" +trap 'rm -f "$_ERR"' EXIT + snap() { local label="$1" path="$2" if [ -z "$path" ] || [ ! -e "$path" ]; then log "skip $label — not found: ${path:-}"; return fi log "Snapshotting $label: $path" - if ! k snapshot create --description="gaming: $label" "$path"; then - log "WARNING: snapshot failed for $label"; rc=1 + if ! k snapshot create --description="gaming: $label" "$path" 2>"$_ERR"; then + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: snapshot failed for $label — $_reason" + FAILED_LABELS+=("$label: $_reason") + rc=1 fi } @@ -75,14 +113,22 @@ fi if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then log "Mirroring repository to remote ($REMOTE_TYPE)..." # shellcheck disable=SC2086 - if ! k repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS; then - log "WARNING: remote mirror failed"; rc=1 + if ! k repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: remote mirror failed — $_reason" + FAILED_LABELS+=("mirror: $_reason") + rc=1 fi fi if [ "$rc" -eq 0 ]; then log "===== Gaming backup complete =====" + ntfy_send "✓ Gaming backup complete" "$HOST: all saves backed up successfully" \ + "low" "white_check_mark" else log "===== Gaming backup finished WITH WARNINGS =====" + _ntfy_msg="$HOST: gaming backup failures:" + for _s in "${FAILED_LABELS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done + ntfy_send "✗ Gaming backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi exit "$rc" diff --git a/extras/backup_kopia.sh b/extras/backup_kopia.sh index a9e05a6..a9e48ca 100644 --- a/extras/backup_kopia.sh +++ b/extras/backup_kopia.sh @@ -20,9 +20,37 @@ source "$CONF" ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" DOCKER_DIR="$ACTUAL_HOME/docker" +HOST="$(hostname -s 2>/dev/null || hostname)" log() { echo "[$(date '+%F %T')] $*"; } +ntfy_send() { + local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" + [ -z "${NTFY_URL:-}" ] && return 0 + local -a _args=(-fsS -o /dev/null) + _args+=(-H "Title: $title" -H "Priority: $priority") + [ -n "$tags" ] && _args+=(-H "Tags: $tags") + [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN") + curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true +} + +categorize_error() { + local txt="$1" + if echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then + echo "disk full — backup destination is out of space" + elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then + echo "remote unreachable — check network / destination host" + elif echo "$txt" | grep -qi "repository.*not.*exist\|not a valid kopia\|not connected"; then + echo "repository not found — re-run the backup installer" + elif echo "$txt" | grep -qi "passphrase\|wrong key\|cannot decrypt"; then + echo "wrong passphrase — check backup.conf" + elif echo "$txt" | grep -qi "permission denied\|access denied"; then + echo "permission denied — check file permissions" + else + echo "error — see system logs on $HOST" + fi +} + kp_for() { local dest="$1"; shift local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD" @@ -55,6 +83,9 @@ esac log "===== Backup starting =====" rc=0 +declare -a FAILED_SVCS=() +_ERR="$(mktemp)" +trap 'rm -f "$_ERR"' EXIT for svc_dir in "$DOCKER_DIR"/*/; do [ -f "${svc_dir}docker-compose.yml" ] || continue @@ -73,10 +104,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do sleep 5 fi log "Snapshotting $svc (dest: $dest)..." - if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir"; then + if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then log "OK $svc (Minecraft, no downtime)" else - log "WARNING: snapshot failed for $svc"; rc=1 + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: snapshot failed for $svc — $_reason" + FAILED_SVCS+=("$svc: $_reason") + rc=1 fi else STOPPED=false @@ -89,10 +123,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do fi log "Snapshotting $svc (dest: $dest)..." - if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir"; then + if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then log "OK $svc" else - log "WARNING: snapshot failed for $svc"; rc=1 + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: snapshot failed for $svc — $_reason" + FAILED_SVCS+=("$svc: $_reason") + rc=1 fi if [ "$STOPPED" = true ]; then @@ -107,14 +144,23 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then for dest in ${DEST_NAMES:-default}; do log "Mirroring '$dest' offsite ($REMOTE_TYPE)..." # shellcheck disable=SC2086 - kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS \ - || { log "WARNING: mirror failed for '$dest'"; rc=1; } + if ! kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: mirror failed for '$dest' — $_reason" + FAILED_SVCS+=("mirror[$dest]: $_reason") + rc=1 + fi done fi if [ "$rc" -eq 0 ]; then log "===== Backup complete =====" + ntfy_send "✓ Backup complete" "$HOST: all services backed up successfully" \ + "low" "white_check_mark" else log "===== Backup finished WITH WARNINGS (see above) =====" + _ntfy_msg="$HOST: backup failures:" + for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done + ntfy_send "✗ Backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi exit "$rc" diff --git a/extras/test_backup.sh b/extras/test_backup.sh new file mode 100644 index 0000000..ca88823 --- /dev/null +++ b/extras/test_backup.sh @@ -0,0 +1,355 @@ +#!/bin/bash +# extras/test_backup.sh — Verify a backup: restore latest snapshot, compare to live data. +# Installed alongside the backup worker by the backup service installers. +# +# Run as root: +# sudo ./test_backup.sh pick service interactively +# sudo ./test_backup.sh test specific service (most recent backup) +# sudo ./test_backup.sh --list list testable services and exit +# +# What it does: +# 1. Identifies the most recent backup for the chosen service +# 2. Stops the container briefly (so data is stable during comparison) +# 3. Moves live data aside — nothing is deleted until the test completes +# 4. Restores the backup into the original location +# 5. Compares restored data vs live data (content, not timestamps) +# 6. Moves live data back and restarts the container +# 7. Reports PASS/FAIL and sends ntfy notification if NTFY_URL is set +# +# PASS = restore succeeded (diff output is informational — files changed since backup are normal) +# FAIL = restore command failed or target was empty after restore +# +# Detects Kopia or Borg automatically from backup.conf. +# Requires: rsync, diff; plus jq (Kopia) or borg (Borg). +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONF="${BACKUP_CONF:-$HERE/backup.conf}" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +info() { echo -e "${BLUE}[INFO]${NC} $*"; } +ok() { echo -e "${GREEN}[OK]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +err() { echo -e "${RED}[ERROR]${NC} $*" >&2; } +die() { err "$*"; exit 1; } + +[ "${EUID:-$(id -u)}" -eq 0 ] || die "Run as root: sudo $0" +[ -f "$CONF" ] || die "backup.conf not found: $CONF" +command -v rsync >/dev/null 2>&1 || die "rsync required — install: sudo apt install rsync" + +# shellcheck source=/dev/null +source "$CONF" + +ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" +ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" +DOCKER_BASE="$ACTUAL_HOME/docker" +HOST="$(hostname -s 2>/dev/null || hostname)" + +# ── ntfy ─────────────────────────────────────────────────────────────────────── +ntfy_send() { + local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" + [ -z "${NTFY_URL:-}" ] && return 0 + local -a _args=(-fsS -o /dev/null) + _args+=(-H "Title: $title" -H "Priority: $priority") + [ -n "$tags" ] && _args+=(-H "Tags: $tags") + [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN") + curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true +} + +# ── Detect backend ───────────────────────────────────────────────────────────── +if [ -n "${BORG:-}" ]; then + BACKEND="borg" + BORG_BIN="$BORG" + command -v "$BORG_BIN" >/dev/null 2>&1 || die "borg not found at $BORG_BIN" +elif [ -n "${KOPIA:-}" ]; then + BACKEND="kopia" + KOPIA_BIN="$KOPIA" + command -v jq >/dev/null 2>&1 || die "jq required — install: sudo apt install jq" +else + die "Cannot detect backup backend — backup.conf must set KOPIA= or BORG=" +fi + +# ── Normalise Kopia single-dest (gaming) → multi-dest format ────────────────── +if [ "$BACKEND" = "kopia" ] && [ -z "${DEST_NAMES:-}" ]; then + DEST_NAMES="default" + DEST_default_CONFIG="${KOPIA_CONFIG:-}" + DEST_default_PASSWORD="${KOPIA_PASSWORD:-}" +fi + +# ── Destination picker (skipped for single dest) ─────────────────────────────── +read -ra _DEST_ARR <<< "${DEST_NAMES:-default}" + +if [ "${#_DEST_ARR[@]}" -gt 1 ]; then + echo "" + echo "Select backup destination:" + echo "" + for i in "${!_DEST_ARR[@]}"; do + printf " %d) %s\n" "$((i+1))" "${_DEST_ARR[$i]}" + done + echo "" + read -rp "Destination [1-${#_DEST_ARR[@]}]: " _DEST_SEL + [[ "$_DEST_SEL" =~ ^[0-9]+$ ]] && [ "$_DEST_SEL" -ge 1 ] && [ "$_DEST_SEL" -le "${#_DEST_ARR[@]}" ] \ + || die "Invalid selection" + ACTIVE_DEST="${_DEST_ARR[$((_DEST_SEL-1))]}" +else + ACTIVE_DEST="${_DEST_ARR[0]}" +fi + +# ── Connect to backend ───────────────────────────────────────────────────────── +if [ "$BACKEND" = "kopia" ]; then + _CFG_VAR="DEST_${ACTIVE_DEST}_CONFIG" + _PW_VAR="DEST_${ACTIVE_DEST}_PASSWORD" + K_CFG="${!_CFG_VAR:-}" + K_PW="${!_PW_VAR:-}" + [ -n "$K_CFG" ] || die "No Kopia config found for destination '$ACTIVE_DEST'" + export KOPIA_PASSWORD="$K_PW" + k() { "$KOPIA_BIN" --config-file="$K_CFG" "$@"; } + k repository status >/dev/null 2>&1 \ + || die "Cannot connect to Kopia repo '$ACTIVE_DEST' — check backup.conf" +else + _REPO_VAR="DEST_${ACTIVE_DEST}_REPO" + _PASS_VAR="DEST_${ACTIVE_DEST}_PASSPHRASE" + export BORG_REPO="${!_REPO_VAR:-}" + export BORG_PASSPHRASE="${!_PASS_VAR:-}" + [ -n "$BORG_REPO" ] || die "No Borg repo found for destination '$ACTIVE_DEST'" + b() { "$BORG_BIN" "$@"; } + b info >/dev/null 2>&1 \ + || die "Cannot connect to Borg repo at $BORG_REPO — check backup.conf" +fi + +# ── Build list of testable services ─────────────────────────────────────────── +declare -a SERVICES=() +declare -A SVC_SNAP_SRC=() # service name → backup source path +SNAP_JSON="" + +if [ "$BACKEND" = "kopia" ]; then + SNAP_JSON=$(k snapshot list --all --json 2>/dev/null) + [ -z "$SNAP_JSON" ] || [ "$SNAP_JSON" = "[]" ] || [ "$SNAP_JSON" = "null" ] \ + && die "No snapshots found. Run a backup first." + while IFS= read -r src_path; do + [[ "$src_path" == "$DOCKER_BASE/"* ]] || continue + rel="${src_path#"$DOCKER_BASE/"}" + svc="${rel%%/*}" + [[ "$svc" == "backup" || "$svc" == "borg-backup" || "$svc" == "gaming-backup" ]] && continue + _already=false + for _s in "${SERVICES[@]+"${SERVICES[@]}"}"; do [ "$_s" = "$svc" ] && _already=true && break; done + if [ "$_already" = false ]; then + SERVICES+=("$svc") + SVC_SNAP_SRC["$svc"]="$src_path" + fi + done < <(echo "$SNAP_JSON" | jq -r 'group_by(.source.path)[] | .[0].source.path') +else + _seen="" + while IFS= read -r arch; do + svc="${arch%-[0-9][0-9][0-9][0-9]-*}" + [[ "$svc" == "backup" || "$svc" == "borg-backup" || "$svc" == "gaming-backup" ]] && continue + [ -z "$svc" ] && continue + [[ "$_seen" == *"|${svc}|"* ]] && continue + _seen+="|${svc}|" + SERVICES+=("$svc") + done < <(b list --format '{archive}{NL}' 2>/dev/null | sort) +fi + +[ "${#SERVICES[@]}" -eq 0 ] && die "No backed-up services found. Run a backup first." + +# ── --list ───────────────────────────────────────────────────────────────────── +if [ "${1:-}" = "--list" ]; then + echo "" + info "Testable services (dest: $ACTIVE_DEST, backend: $BACKEND):" + echo "" + for svc in "${SERVICES[@]}"; do printf " • %s\n" "$svc"; done + echo "" + exit 0 +fi + +# ── Service selection ────────────────────────────────────────────────────────── +SELECTED_SVC="" +if [ -n "${1:-}" ] && [ "${1:-}" != "--list" ]; then + SELECTED_SVC="${1:-}" + _found=false + for _s in "${SERVICES[@]}"; do [ "$_s" = "$SELECTED_SVC" ] && _found=true && break; done + [ "$_found" = true ] || die "Service '$SELECTED_SVC' has no backups. Use --list." +else + echo "" + echo "╔═══════════════════════════════════════════════════════╗" + echo "║ Backup Verification Test ║" + echo "╚═══════════════════════════════════════════════════════╝" + echo "" + info "Destination: $ACTIVE_DEST (backend: $BACKEND)" + echo "" + echo "Services with backups:" + echo "" + for i in "${!SERVICES[@]}"; do + printf " %2d) %s\n" "$((i+1))" "${SERVICES[$i]}" + done + echo "" + read -rp "Select service [1-${#SERVICES[@]}] or q to quit: " SEL + [[ "$SEL" =~ ^[qQ]$ ]] && echo "Cancelled." && exit 0 + [[ "$SEL" =~ ^[0-9]+$ ]] && [ "$SEL" -ge 1 ] && [ "$SEL" -le "${#SERVICES[@]}" ] \ + || die "Invalid selection: $SEL" + SELECTED_SVC="${SERVICES[$((SEL-1))]}" +fi +ok "Testing service: $SELECTED_SVC" + +# ── Find most recent backup ──────────────────────────────────────────────────── +SNAP_ID="" SNAP_DESC="" ARCHIVE_NAME="" SOURCE_PATH="" + +if [ "$BACKEND" = "kopia" ]; then + SOURCE_PATH="${SVC_SNAP_SRC[$SELECTED_SVC]:-$DOCKER_BASE/$SELECTED_SVC}" + SNAP_ID=$(echo "$SNAP_JSON" | jq -r --arg p "$SOURCE_PATH" \ + '[.[] | select(.source.path == $p)] | sort_by(.startTime) | last | .id // empty') + SNAP_DESC=$(echo "$SNAP_JSON" | jq -r --arg p "$SOURCE_PATH" \ + '[.[] | select(.source.path == $p)] | sort_by(.startTime) | last | + .startTime | split("T") | "\(.[0]) \(.[1][:8]) UTC"' 2>/dev/null || echo "unknown") + [ -n "$SNAP_ID" ] || die "No Kopia snapshot for '$SELECTED_SVC' at $SOURCE_PATH" +else + SOURCE_PATH="$DOCKER_BASE/$SELECTED_SVC" + ARCHIVE_NAME=$(b list --format '{archive}{NL}' 2>/dev/null \ + | grep "^${SELECTED_SVC}-" | sort | tail -1) + [ -n "$ARCHIVE_NAME" ] || die "No Borg archive found for '$SELECTED_SVC'" + SNAP_DESC="${ARCHIVE_NAME#"${SELECTED_SVC}-"}" +fi +info "Most recent backup: $SNAP_DESC" + +# ── Resolve paths ────────────────────────────────────────────────────────────── +# TARGET_DIR = the exact path that was backed up (may be a sub-path for gaming) +# SVC_DIR = top-level docker service dir (for container stop/start) +TARGET_DIR="$SOURCE_PATH" +SVC_DIR="$DOCKER_BASE/$SELECTED_SVC" +COMPOSE_FILE="$SVC_DIR/docker-compose.yml" + +TS="$(date +%Y%m%d-%H%M%S)" +ASIDE_DIR="${TARGET_DIR}.test-aside-${TS}" +ERR_LOG="$(mktemp)" +STOPPED=false + +# ── Cleanup trap — always restores live data ─────────────────────────────────── +cleanup() { + rm -f "$ERR_LOG" 2>/dev/null || true + if [ -d "$ASIDE_DIR" ]; then + warn "Restoring live data from aside copy..." + rm -rf "$TARGET_DIR" 2>/dev/null || true + mv "$ASIDE_DIR" "$TARGET_DIR" + ok "Live data restored to $TARGET_DIR" + fi + if [ "$STOPPED" = true ] && [ -f "$COMPOSE_FILE" ]; then + info "Restarting $SELECTED_SVC..." + docker compose -f "$COMPOSE_FILE" up -d 2>/dev/null \ + && ok "$SELECTED_SVC restarted." \ + || warn "Auto-restart failed — run: docker compose -f $COMPOSE_FILE up -d" + fi +} +trap cleanup EXIT + +echo "" +warn "This test will briefly stop '$SELECTED_SVC', move its data aside, restore" +warn "from the most recent backup, compare, then move everything back." +echo "" +read -rp "Proceed? (y/N): " _CONFIRM +[[ "$_CONFIRM" =~ ^[Yy]$ ]] || { echo "Cancelled."; exit 0; } + +# ── Stop container ───────────────────────────────────────────────────────────── +if [ -f "$COMPOSE_FILE" ] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$SELECTED_SVC"; then + info "Stopping $SELECTED_SVC..." + docker compose -f "$COMPOSE_FILE" down 2>/dev/null \ + || docker stop "$SELECTED_SVC" 2>/dev/null \ + || warn "Could not stop container — data may be inconsistent." + STOPPED=true + ok "$SELECTED_SVC stopped." +fi + +# ── Move live data aside ─────────────────────────────────────────────────────── +if [ -e "$TARGET_DIR" ]; then + info "Moving live data aside → $(basename "$ASIDE_DIR") ..." + mv "$TARGET_DIR" "$ASIDE_DIR" + ok "Live data saved at: $ASIDE_DIR" +else + warn "$TARGET_DIR not found — testing fresh restore (no live data to compare)." +fi + +# ── Restore from backup ──────────────────────────────────────────────────────── +mkdir -p "$TARGET_DIR" +info "Restoring from backup..." + +if [ "$BACKEND" = "kopia" ]; then + if ! k restore "$SNAP_ID" "$TARGET_DIR" 2>"$ERR_LOG"; then + FAIL_MSG="Restore failed: $(head -3 "$ERR_LOG" | tr '\n' ' ')" + err "$FAIL_MSG" + ntfy_send "✗ Backup test FAILED: $SELECTED_SVC" "$HOST\n$FAIL_MSG" \ + "urgent" "rotating_light" + exit 1 + fi +else + EXTRACT_PATH="${SOURCE_PATH#/}" + if ! ( cd / && b extract "$BORG_REPO::$ARCHIVE_NAME" "$EXTRACT_PATH" 2>"$ERR_LOG" ); then + FAIL_MSG="Restore failed: $(head -3 "$ERR_LOG" | tr '\n' ' ')" + err "$FAIL_MSG" + ntfy_send "✗ Backup test FAILED: $SELECTED_SVC" "$HOST\n$FAIL_MSG" \ + "urgent" "rotating_light" + exit 1 + fi +fi +ok "Restore complete." + +# Sanity check — something must have been restored +RESTORED_COUNT=$(find "$TARGET_DIR" -mindepth 1 -maxdepth 2 2>/dev/null | wc -l) +if [ "$RESTORED_COUNT" -eq 0 ]; then + FAIL_MSG="Restore succeeded but target directory is empty" + err "$FAIL_MSG" + ntfy_send "✗ Backup test FAILED: $SELECTED_SVC" "$HOST\n$FAIL_MSG" \ + "urgent" "rotating_light" + exit 1 +fi + +# ── Compare restored vs live data ───────────────────────────────────────────── +DIFF_COUNT=0 +DIFF_SAMPLE="" + +if [ -d "$ASIDE_DIR" ]; then + info "Comparing restored vs live data (content, not timestamps)..." + DIFF_OUT="$(diff -rq "$ASIDE_DIR" "$TARGET_DIR" 2>/dev/null || true)" + DIFF_COUNT=$(printf '%s' "$DIFF_OUT" | grep -c '^' 2>/dev/null || echo 0) + + if [ "$DIFF_COUNT" -eq 0 ]; then + ok "Perfect match — restored data is identical to live data." + else + DIFF_SAMPLE="$(printf '%s' "$DIFF_OUT" | head -10)" + warn "$DIFF_COUNT file(s) differ between backup and live data." + warn "(Normal — these files changed between the last backup and now.)" + echo "" + echo " Files changed since last backup (up to 10):" + printf '%s\n' "$DIFF_SAMPLE" | while IFS= read -r line; do echo " $line"; done + [ "$DIFF_COUNT" -gt 10 ] && echo " ... and $((DIFF_COUNT - 10)) more" + fi +else + warn "No live data to compare. Restored $RESTORED_COUNT item(s) from backup." +fi + +# ── Result ───────────────────────────────────────────────────────────────────── +echo "" +echo "═══════════════════════════════════════════════════════" +if [ "$DIFF_COUNT" -eq 0 ]; then + echo " BACKUP TEST PASSED ✓ (perfect match)" +else + echo " BACKUP TEST PASSED ✓ ($DIFF_COUNT file(s) changed since last backup)" +fi +echo "═══════════════════════════════════════════════════════" +echo "" +echo " Service : $SELECTED_SVC" +echo " Backend : $BACKEND" +echo " Backup : $SNAP_DESC" +echo " Restored : $RESTORED_COUNT item(s)" +[ "$DIFF_COUNT" -gt 0 ] && echo " Changed : $DIFF_COUNT file(s) modified since last backup (normal)" +echo "" + +if [ "$DIFF_COUNT" -eq 0 ]; then + ntfy_send "✓ Backup test PASSED: $SELECTED_SVC" \ + "$HOST: $SELECTED_SVC backup verified — perfect match (${RESTORED_COUNT} items)" \ + "low" "white_check_mark" +else + ntfy_send "✓ Backup test PASSED: $SELECTED_SVC" \ + "$HOST: $SELECTED_SVC backup OK — ${DIFF_COUNT} file(s) changed since last backup" \ + "default" "white_check_mark" +fi + +# cleanup trap handles data restoration and container restart diff --git a/services/backup.sh b/services/backup.sh index 403610e..810ebd5 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -246,7 +246,21 @@ install_backup() { prompt_text " Snapshots to keep (latest)? [7]:" "7" KEEP_LATEST KEEP_LATEST="${KEEP_LATEST:-7}" - # ── 7. Create dirs + init Kopia repos ──────────────────────────────────── + # ── Notifications (ntfy) ───────────────────────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " NOTIFICATIONS (optional)" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Receive a push notification after every backup (and on failures)." + echo " Uses ntfy — free and self-hostable. Create a topic at https://ntfy.sh" + echo " Example URL: https://ntfy.sh/my-backup-alerts" + echo "" + local NTFY_URL="" NTFY_TOKEN="" + prompt_text " ntfy topic URL (blank to skip):" "" NTFY_URL + if [ -n "$NTFY_URL" ]; then + prompt_text " ntfy access token (blank if public/no auth):" "" NTFY_TOKEN + fi mkdir -p "$DIR" ensure_docker_dir_ownership "$DIR" @@ -320,6 +334,12 @@ install_backup() { echo "# Example SFTP: REMOTE_TYPE=sftp REMOTE_ARGS=\"--host H --username U --path /srv/...\"" echo "REMOTE_TYPE=\"none\"" echo "REMOTE_ARGS=\"\"" + echo "" + echo "# ── Notifications (ntfy) ─────────────────────────────────────────────────────" + echo "# Set NTFY_URL to receive backup success/failure alerts." + echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)." + printf "NTFY_URL='%s'\n" "${NTFY_URL:-}" + printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}" } > "$CONF_FILE" chown root:root "$CONF_FILE" 2>/dev/null || true chmod 600 "$CONF_FILE" @@ -344,7 +364,19 @@ install_backup() { log_warning "Copy it manually: cp extras/restore_kopia.sh $RESTORE" fi - # ── 11. Systemd timer ──────────────────────────────────────────────────── + # ── 11. Install test script ────────────────────────────────────────────── + local TEST_SCRIPT="$DIR/test_backup.sh" + local TEST_SRC="${HERE:-}/extras/test_backup.sh" + if [ -f "$TEST_SRC" ]; then + cp "$TEST_SRC" "$TEST_SCRIPT" + chmod +x "$TEST_SCRIPT" + chown root:root "$TEST_SCRIPT" 2>/dev/null || true + log_success "test_backup.sh installed" + else + log_warning "extras/test_backup.sh not found — test script not installed" + fi + + # ── 12. Systemd timer ──────────────────────────────────────────────────── log_info "Installing systemd timer ($SCHED_LABEL)..." local AUTORUN="" if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then @@ -427,6 +459,12 @@ SVCEOF echo " sudo $RESTORE" echo " sudo $RESTORE --list" echo "" + echo " Backup test (stop/restore/compare/restore-back):" + echo " sudo $TEST_SCRIPT test most recent backup" + echo " sudo $TEST_SCRIPT --list list testable services" + echo " sudo $TEST_SCRIPT test a specific service" + [ -n "${NTFY_URL:-}" ] && echo "" && echo " Notifications: $NTFY_URL" + echo "" [ -n "$AUTORUN" ] && echo " $AUTORUN" && echo "" log_warning "Save your passwords (in backup.conf) somewhere safe —" log_warning "without them the encrypted repos cannot be restored." diff --git a/services/borg-backup.sh b/services/borg-backup.sh index 369110c..5f1aef9 100644 --- a/services/borg-backup.sh +++ b/services/borg-backup.sh @@ -191,6 +191,22 @@ install_borg_backup() { KEEP_WEEKLY="${KEEP_WEEKLY:-4}" KEEP_MONTHLY="${KEEP_MONTHLY:-3}" + # ── Notifications (ntfy) ───────────────────────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " NOTIFICATIONS (optional)" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Receive a push notification after every backup (and on failures)." + echo " Uses ntfy — free and self-hostable. Create a topic at https://ntfy.sh" + echo " Example URL: https://ntfy.sh/my-backup-alerts" + echo "" + local NTFY_URL="" NTFY_TOKEN="" + prompt_text " ntfy topic URL (blank to skip):" "" NTFY_URL + if [ -n "$NTFY_URL" ]; then + prompt_text " ntfy access token (blank if public/no auth):" "" NTFY_TOKEN + fi + # ── 7. Create dirs + init Borg repos ───────────────────────────────────── mkdir -p "$DIR" ensure_docker_dir_ownership "$DIR" @@ -262,6 +278,12 @@ install_borg_backup() { echo "# SVC_${svc_var}=\"default\"" fi done + echo "" + echo "# ── Notifications (ntfy) ─────────────────────────────────────────────────────" + echo "# Set NTFY_URL to receive backup success/failure alerts." + echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)." + printf "NTFY_URL='%s'\n" "${NTFY_URL:-}" + printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}" } > "$CONF_FILE" chown root:root "$CONF_FILE" 2>/dev/null || true chmod 600 "$CONF_FILE" @@ -286,7 +308,19 @@ install_borg_backup() { log_warning "Copy it manually: cp extras/restore_borg.sh $RESTORE" fi - # ── 11. Systemd timer ───────────────────────────────────────────────────── + # ── 11. Install test script ────────────────────────────────────────────── + local TEST_SCRIPT="$DIR/test_backup.sh" + local TEST_SRC="${HERE:-}/extras/test_backup.sh" + if [ -f "$TEST_SRC" ]; then + cp "$TEST_SRC" "$TEST_SCRIPT" + chmod +x "$TEST_SCRIPT" + chown root:root "$TEST_SCRIPT" 2>/dev/null || true + log_success "test_backup.sh installed" + else + log_warning "extras/test_backup.sh not found — test script not installed" + fi + + # ── 12. Systemd timer ───────────────────────────────────────────────────── log_info "Installing systemd timer ($SCHED_LABEL)..." if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then tee "/etc/systemd/system/${SVC_NAME}.service" >/dev/null << SVCEOF @@ -368,6 +402,12 @@ SVCEOF echo " sudo $RESTORE" echo " sudo $RESTORE --list" echo "" + echo " Backup test (stop/restore/compare/restore-back):" + echo " sudo $TEST_SCRIPT test most recent backup" + echo " sudo $TEST_SCRIPT --list list testable services" + echo " sudo $TEST_SCRIPT test a specific service" + [ -n "${NTFY_URL:-}" ] && echo "" && echo " Notifications: $NTFY_URL" + echo "" log_warning "IMPORTANT — back up your Borg key and passphrase now." echo " The key is stored in the repo itself (repokey-blake2 encryption)." echo " Export it to a safe location:" diff --git a/services/gaming-backup.sh b/services/gaming-backup.sh index 82e502b..6399b0f 100644 --- a/services/gaming-backup.sh +++ b/services/gaming-backup.sh @@ -192,6 +192,21 @@ install_gaming_backup() { prompt_text " How many recent snapshots to keep (latest)? [24]:" "24" KEEP_LATEST local KEEP_DAILY=7 KEEP_WEEKLY=4 KEEP_MONTHLY=6 + # ── Notifications (ntfy) ───────────────────────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " NOTIFICATIONS (optional)" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Receive a push notification after every backup (and on failures)." + echo " Uses ntfy — free and self-hostable. Create a topic at https://ntfy.sh" + echo "" + local NTFY_URL="" NTFY_TOKEN="" + prompt_text " ntfy topic URL (blank to skip):" "" NTFY_URL + if [ -n "$NTFY_URL" ]; then + prompt_text " ntfy access token (blank if public/no auth):" "" NTFY_TOKEN + fi + # ── 5. Write backup.conf ────────────────────────────────────────────────── log_info "Writing $CONF_FILE ..." tee "$CONF_FILE" >/dev/null << CONFEOF @@ -226,6 +241,12 @@ BACKUP_WOLF="$BACKUP_WOLF" # /etc/wolf — config + profile_data # REMOTE_TYPE="none" REMOTE_ARGS="" + +# ── Notifications (ntfy) ───────────────────────────────────────────────────── +# Set NTFY_URL to receive backup success/failure alerts. +# Leave blank to disable. NTFY_TOKEN is optional (for private topics). +NTFY_URL="$NTFY_URL" +NTFY_TOKEN="$NTFY_TOKEN" CONFEOF chown root:root "$CONF_FILE" 2>/dev/null || true chmod 600 "$CONF_FILE" @@ -292,6 +313,18 @@ CONFEOF log_warning "extras/restore_kopia.sh not found — restore script not installed" fi + # ── Install test script ─────────────────────────────────────────────────── + local TEST_SCRIPT="$BACKUP_DIR/test_backup.sh" + local TEST_SRC="${HERE:-}/extras/test_backup.sh" + if [ -f "$TEST_SRC" ]; then + cp "$TEST_SRC" "$TEST_SCRIPT" + chmod +x "$TEST_SCRIPT" + chown root:root "$TEST_SCRIPT" 2>/dev/null || true + log_success "test_backup.sh installed" + else + log_warning "extras/test_backup.sh not found — test script not installed" + fi + # ── 8. Install systemd timer (fallback: cron) ──────────────────────────── local AUTORUN="" if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then @@ -369,6 +402,13 @@ UNITEOF echo " sudo $WORKER snapshots list snapshots" echo " sudo $RESTORE_DEST interactive restore" echo " sudo $RESTORE_DEST --list list all snapshot sources" + echo "" + echo " Backup test (restore latest, compare, restore-back):" + echo " sudo $TEST_SCRIPT test most recent backup" + echo " sudo $TEST_SCRIPT --list list testable sources" + echo " sudo $TEST_SCRIPT test a specific source" + [ -n "${NTFY_URL:-}" ] && echo "" && echo " Notifications: $NTFY_URL" + echo "" echo " $AUTORUN" echo "" echo " Tip: also install the 'backup' service for nightly full-service recovery."