feat: backup test script, ntfy notifications, and error categorization

extras/test_backup.sh — new unified test script (Kopia + Borg):
  • Stops container, moves live data aside, restores latest backup,
    compares restored vs live with diff -rq (content, not timestamps),
    moves live data back and restarts container
  • PASS = restore succeeded; diff output is informational (files changed
    since last backup are normal)
  • FAIL = restore command failed or target empty after restore
  • --list flag, CLI service arg, interactive picker
  • Handles both full-service dirs and sub-path sources (gaming-backup)
  • Cleanup trap always restores live data even on error
  • Sends ntfy notification on pass and fail

extras/backup_kopia.sh, backup_borg.sh, backup_gaming.sh:
  • ntfy_send() + categorize_error() helpers added
  • Each snapshot/archive failure captures stderr and categorizes:
    disk full, remote unreachable, repository not found, wrong passphrase,
    permission denied, unknown error
  • Single ntfy notification at end: success (low priority) or failure
    (urgent) with per-service failure reasons listed
  • backup_borg.sh: changed 2>&1 | pipe to 2>"$_ERR" | so stdout logs
    cleanly and stderr is captured for error categorization

services/backup.sh, borg-backup.sh, gaming-backup.sh:
  • New ntfy prompt section in installer (URL + optional token)
  • NTFY_URL / NTFY_TOKEN written to backup.conf
  • test_backup.sh copied from extras/ into service dir
  • Summary updated to show test_backup.sh commands and ntfy URL

https://claude.ai/code/session_019XgsQ13XKm4Zj3cNsDNwHj

extras/backup_borg.sh

@@ -20,12 +20,40 @@ source "$CONF"
 ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}"
 ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")"
 DOCKER_DIR="$ACTUAL_HOME/docker"
+HOST="$(hostname -s 2>/dev/null || hostname)"
 
 log() { echo "[$(date '+%F %T')] $*"; }
 
-repo_for() { local var="DEST_${1}_REPO"; echo "${!var:-}"; }
-pass_for() { local var="DEST_${1}_PASSPHRASE"; echo "${!var:-}"; }
-dest_for_svc() { local var="SVC_${1//-/_}"; echo "${!var:-${DEST_DEFAULT:-default}}"; }
+ntfy_send() {
+    local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}"
+    [ -z "${NTFY_URL:-}" ] && return 0
+    local -a _args=(-fsS -o /dev/null)
+    _args+=(-H "Title: $title" -H "Priority: $priority")
+    [ -n "$tags" ]           && _args+=(-H "Tags: $tags")
+    [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN")
+    curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true
+}
+
+categorize_error() {
+    local txt="$1"
+    if   echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then
+        echo "disk full — backup destination is out of space"
+    elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then
+        echo "remote unreachable — check network / destination host"
+    elif echo "$txt" | grep -qi "repository.*does not exist\|not a borg\|is not a valid"; then
+        echo "repository not found — re-run the backup installer"
+    elif echo "$txt" | grep -qi "passphrase\|wrong key\|bad key\|cannot decrypt"; then
+        echo "wrong passphrase — check backup.conf"
+    elif echo "$txt" | grep -qi "permission denied\|access denied"; then
+        echo "permission denied — check file permissions"
+    else
+        echo "error — see system logs on $HOST"
+    fi
+}
+
+repo_for()    { local var="DEST_${1}_REPO";       echo "${!var:-}"; }
+pass_for()    { local var="DEST_${1}_PASSPHRASE"; echo "${!var:-}"; }
+dest_for_svc() { local var="SVC_${1//-/_}";       echo "${!var:-${DEST_DEFAULT:-default}}"; }
 
 b_for() {
     local dest="$1"; shift
@@ -55,6 +83,9 @@ esac
 log "===== Borg backup starting ====="
 rc=0
 TS="$(date +%Y-%m-%dT%H-%M-%S)"
+declare -a FAILED_SVCS=()
+_ERR="$(mktemp)"
+trap 'rm -f "$_ERR"' EXIT
 
 for svc_dir in "$DOCKER_DIR"/*/; do
     [ -f "${svc_dir}docker-compose.yml" ] || continue
@@ -77,10 +108,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do
         log "Archiving $svc → $dest::$ARCHIVE ..."
         if b_for "$dest" create \
             --compression=zstd,6 --exclude-caches --stats \
-            "::$ARCHIVE" "$svc_dir" 2>&1 | while IFS= read -r line; do log "  $line"; done; then
+            "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log "  $line"; done; then
             log "OK $svc (Minecraft, no downtime)"
         else
-            log "WARNING: archive failed for $svc"; rc=1
+            _reason="$(categorize_error "$(cat "$_ERR")")"
+            log "WARNING: archive failed for $svc — $_reason"
+            FAILED_SVCS+=("$svc: $_reason")
+            rc=1
         fi
     else
         STOPPED=false
@@ -95,10 +129,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do
         log "Archiving $svc → $dest::$ARCHIVE ..."
         if b_for "$dest" create \
             --compression=zstd,6 --exclude-caches --stats \
-            "::$ARCHIVE" "$svc_dir" 2>&1 | while IFS= read -r line; do log "  $line"; done; then
+            "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log "  $line"; done; then
             log "OK $svc"
         else
-            log "WARNING: archive failed for $svc"; rc=1
+            _reason="$(categorize_error "$(cat "$_ERR")")"
+            log "WARNING: archive failed for $svc — $_reason"
+            FAILED_SVCS+=("$svc: $_reason")
+            rc=1
         fi
 
         if [ "$STOPPED" = true ]; then
@@ -127,7 +164,12 @@ done
 
 if [ "$rc" -eq 0 ]; then
     log "===== Borg backup complete ====="
+    ntfy_send "✓ Borg backup complete" "$HOST: all services archived successfully" \
+        "low" "white_check_mark"
 else
     log "===== Borg backup finished WITH WARNINGS (see above) ====="
+    _ntfy_msg="$HOST: Borg backup failures:"
+    for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done
+    ntfy_send "✗ Borg backup FAILED" "$_ntfy_msg" "urgent" "rotating_light"
 fi
 exit "$rc"

extras/backup_gaming.sh

@@ -17,11 +17,42 @@ CONF="${BACKUP_CONF:-$HERE/backup.conf}"
 source "$CONF"
 
 export KOPIA_PASSWORD
+HOST="$(hostname -s 2>/dev/null || hostname)"
 log() { echo "[$(date '+%F %T')] $*"; }
 k()   { "$KOPIA" --config-file="$KOPIA_CONFIG" "$@"; }
 
+ntfy_send() {
+    local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}"
+    [ -z "${NTFY_URL:-}" ] && return 0
+    local -a _args=(-fsS -o /dev/null)
+    _args+=(-H "Title: $title" -H "Priority: $priority")
+    [ -n "$tags" ]           && _args+=(-H "Tags: $tags")
+    [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN")
+    curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true
+}
+
+categorize_error() {
+    local txt="$1"
+    if   echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then
+        echo "disk full — backup destination is out of space"
+    elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then
+        echo "remote unreachable — check network / destination host"
+    elif echo "$txt" | grep -qi "repository.*not.*exist\|not a valid kopia\|not connected"; then
+        echo "repository not found — re-run the gaming-backup installer"
+    elif echo "$txt" | grep -qi "passphrase\|wrong key\|cannot decrypt"; then
+        echo "wrong passphrase — check backup.conf"
+    elif echo "$txt" | grep -qi "permission denied\|access denied"; then
+        echo "permission denied — check file permissions"
+    else
+        echo "error — see system logs on $HOST"
+    fi
+}
+
 if ! k repository status >/dev/null 2>&1; then
     log "ERROR: not connected to a repository — re-run the gaming-backup service"
+    ntfy_send "✗ Gaming backup FAILED" \
+        "$HOST: cannot connect to Kopia repository — re-run gaming-backup installer" \
+        "urgent" "rotating_light"
     exit 1
 fi
 
@@ -48,14 +79,21 @@ if [ -n "${MC_BASE_DIR:-}" ] && command -v docker >/dev/null 2>&1; then
 fi
 
 rc=0
+declare -a FAILED_LABELS=()
+_ERR="$(mktemp)"
+trap 'rm -f "$_ERR"' EXIT
+
 snap() {
     local label="$1" path="$2"
     if [ -z "$path" ] || [ ! -e "$path" ]; then
         log "skip $label — not found: ${path:-<unset>}"; return
     fi
     log "Snapshotting $label: $path"
-    if ! k snapshot create --description="gaming: $label" "$path"; then
-        log "WARNING: snapshot failed for $label"; rc=1
+    if ! k snapshot create --description="gaming: $label" "$path" 2>"$_ERR"; then
+        _reason="$(categorize_error "$(cat "$_ERR")")"
+        log "WARNING: snapshot failed for $label — $_reason"
+        FAILED_LABELS+=("$label: $_reason")
+        rc=1
     fi
 }
 
@@ -75,14 +113,22 @@ fi
 if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
     log "Mirroring repository to remote ($REMOTE_TYPE)..."
     # shellcheck disable=SC2086
-    if ! k repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS; then
-        log "WARNING: remote mirror failed"; rc=1
+    if ! k repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then
+        _reason="$(categorize_error "$(cat "$_ERR")")"
+        log "WARNING: remote mirror failed — $_reason"
+        FAILED_LABELS+=("mirror: $_reason")
+        rc=1
     fi
 fi
 
 if [ "$rc" -eq 0 ]; then
     log "===== Gaming backup complete ====="
+    ntfy_send "✓ Gaming backup complete" "$HOST: all saves backed up successfully" \
+        "low" "white_check_mark"
 else
     log "===== Gaming backup finished WITH WARNINGS ====="
+    _ntfy_msg="$HOST: gaming backup failures:"
+    for _s in "${FAILED_LABELS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done
+    ntfy_send "✗ Gaming backup FAILED" "$_ntfy_msg" "urgent" "rotating_light"
 fi
 exit "$rc"

extras/backup_kopia.sh

@@ -20,9 +20,37 @@ source "$CONF"
 ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}"
 ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")"
 DOCKER_DIR="$ACTUAL_HOME/docker"
+HOST="$(hostname -s 2>/dev/null || hostname)"
 
 log() { echo "[$(date '+%F %T')] $*"; }
 
+ntfy_send() {
+    local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}"
+    [ -z "${NTFY_URL:-}" ] && return 0
+    local -a _args=(-fsS -o /dev/null)
+    _args+=(-H "Title: $title" -H "Priority: $priority")
+    [ -n "$tags" ]           && _args+=(-H "Tags: $tags")
+    [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN")
+    curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true
+}
+
+categorize_error() {
+    local txt="$1"
+    if   echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then
+        echo "disk full — backup destination is out of space"
+    elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then
+        echo "remote unreachable — check network / destination host"
+    elif echo "$txt" | grep -qi "repository.*not.*exist\|not a valid kopia\|not connected"; then
+        echo "repository not found — re-run the backup installer"
+    elif echo "$txt" | grep -qi "passphrase\|wrong key\|cannot decrypt"; then
+        echo "wrong passphrase — check backup.conf"
+    elif echo "$txt" | grep -qi "permission denied\|access denied"; then
+        echo "permission denied — check file permissions"
+    else
+        echo "error — see system logs on $HOST"
+    fi
+}
+
 kp_for() {
     local dest="$1"; shift
     local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD"
@@ -55,6 +83,9 @@ esac
 
 log "===== Backup starting ====="
 rc=0
+declare -a FAILED_SVCS=()
+_ERR="$(mktemp)"
+trap 'rm -f "$_ERR"' EXIT
 
 for svc_dir in "$DOCKER_DIR"/*/; do
     [ -f "${svc_dir}docker-compose.yml" ] || continue
@@ -73,10 +104,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do
             sleep 5
         fi
         log "Snapshotting $svc (dest: $dest)..."
-        if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir"; then
+        if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then
             log "OK $svc (Minecraft, no downtime)"
         else
-            log "WARNING: snapshot failed for $svc"; rc=1
+            _reason="$(categorize_error "$(cat "$_ERR")")"
+            log "WARNING: snapshot failed for $svc — $_reason"
+            FAILED_SVCS+=("$svc: $_reason")
+            rc=1
         fi
     else
         STOPPED=false
@@ -89,10 +123,13 @@ for svc_dir in "$DOCKER_DIR"/*/; do
         fi
 
         log "Snapshotting $svc (dest: $dest)..."
-        if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir"; then
+        if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then
             log "OK $svc"
         else
-            log "WARNING: snapshot failed for $svc"; rc=1
+            _reason="$(categorize_error "$(cat "$_ERR")")"
+            log "WARNING: snapshot failed for $svc — $_reason"
+            FAILED_SVCS+=("$svc: $_reason")
+            rc=1
         fi
 
         if [ "$STOPPED" = true ]; then
@@ -107,14 +144,23 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
     for dest in ${DEST_NAMES:-default}; do
         log "Mirroring '$dest' offsite ($REMOTE_TYPE)..."
         # shellcheck disable=SC2086
-        kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS \
-            || { log "WARNING: mirror failed for '$dest'"; rc=1; }
+        if ! kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then
+            _reason="$(categorize_error "$(cat "$_ERR")")"
+            log "WARNING: mirror failed for '$dest' — $_reason"
+            FAILED_SVCS+=("mirror[$dest]: $_reason")
+            rc=1
+        fi
     done
 fi
 
 if [ "$rc" -eq 0 ]; then
     log "===== Backup complete ====="
+    ntfy_send "✓ Backup complete" "$HOST: all services backed up successfully" \
+        "low" "white_check_mark"
 else
     log "===== Backup finished WITH WARNINGS (see above) ====="
+    _ntfy_msg="$HOST: backup failures:"
+    for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done
+    ntfy_send "✗ Backup FAILED" "$_ntfy_msg" "urgent" "rotating_light"
 fi
 exit "$rc"

extras/test_backup.sh

@@ -0,0 +1,355 @@
+#!/bin/bash
+# extras/test_backup.sh — Verify a backup: restore latest snapshot, compare to live data.
+# Installed alongside the backup worker by the backup service installers.
+#
+# Run as root:
+#   sudo ./test_backup.sh                  pick service interactively
+#   sudo ./test_backup.sh <service-name>   test specific service (most recent backup)
+#   sudo ./test_backup.sh --list           list testable services and exit
+#
+# What it does:
+#   1. Identifies the most recent backup for the chosen service
+#   2. Stops the container briefly (so data is stable during comparison)
+#   3. Moves live data aside — nothing is deleted until the test completes
+#   4. Restores the backup into the original location
+#   5. Compares restored data vs live data (content, not timestamps)
+#   6. Moves live data back and restarts the container
+#   7. Reports PASS/FAIL and sends ntfy notification if NTFY_URL is set
+#
+# PASS = restore succeeded (diff output is informational — files changed since backup are normal)
+# FAIL = restore command failed or target was empty after restore
+#
+# Detects Kopia or Borg automatically from backup.conf.
+# Requires: rsync, diff; plus jq (Kopia) or borg (Borg).
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CONF="${BACKUP_CONF:-$HERE/backup.conf}"
+
+RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
+info() { echo -e "${BLUE}[INFO]${NC}  $*"; }
+ok()   { echo -e "${GREEN}[OK]${NC}    $*"; }
+warn() { echo -e "${YELLOW}[WARN]${NC}  $*"; }
+err()  { echo -e "${RED}[ERROR]${NC} $*" >&2; }
+die()  { err "$*"; exit 1; }
+
+[ "${EUID:-$(id -u)}" -eq 0 ] || die "Run as root: sudo $0"
+[ -f "$CONF" ]                 || die "backup.conf not found: $CONF"
+command -v rsync >/dev/null 2>&1 || die "rsync required — install: sudo apt install rsync"
+
+# shellcheck source=/dev/null
+source "$CONF"
+
+ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}"
+ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")"
+DOCKER_BASE="$ACTUAL_HOME/docker"
+HOST="$(hostname -s 2>/dev/null || hostname)"
+
+# ── ntfy ───────────────────────────────────────────────────────────────────────
+ntfy_send() {
+    local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}"
+    [ -z "${NTFY_URL:-}" ] && return 0
+    local -a _args=(-fsS -o /dev/null)
+    _args+=(-H "Title: $title" -H "Priority: $priority")
+    [ -n "$tags" ]           && _args+=(-H "Tags: $tags")
+    [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN")
+    curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true
+}
+
+# ── Detect backend ─────────────────────────────────────────────────────────────
+if [ -n "${BORG:-}" ]; then
+    BACKEND="borg"
+    BORG_BIN="$BORG"
+    command -v "$BORG_BIN" >/dev/null 2>&1 || die "borg not found at $BORG_BIN"
+elif [ -n "${KOPIA:-}" ]; then
+    BACKEND="kopia"
+    KOPIA_BIN="$KOPIA"
+    command -v jq >/dev/null 2>&1 || die "jq required — install: sudo apt install jq"
+else
+    die "Cannot detect backup backend — backup.conf must set KOPIA= or BORG="
+fi
+
+# ── Normalise Kopia single-dest (gaming) → multi-dest format ──────────────────
+if [ "$BACKEND" = "kopia" ] && [ -z "${DEST_NAMES:-}" ]; then
+    DEST_NAMES="default"
+    DEST_default_CONFIG="${KOPIA_CONFIG:-}"
+    DEST_default_PASSWORD="${KOPIA_PASSWORD:-}"
+fi
+
+# ── Destination picker (skipped for single dest) ───────────────────────────────
+read -ra _DEST_ARR <<< "${DEST_NAMES:-default}"
+
+if [ "${#_DEST_ARR[@]}" -gt 1 ]; then
+    echo ""
+    echo "Select backup destination:"
+    echo ""
+    for i in "${!_DEST_ARR[@]}"; do
+        printf "  %d)  %s\n" "$((i+1))" "${_DEST_ARR[$i]}"
+    done
+    echo ""
+    read -rp "Destination [1-${#_DEST_ARR[@]}]: " _DEST_SEL
+    [[ "$_DEST_SEL" =~ ^[0-9]+$ ]] && [ "$_DEST_SEL" -ge 1 ] && [ "$_DEST_SEL" -le "${#_DEST_ARR[@]}" ] \
+        || die "Invalid selection"
+    ACTIVE_DEST="${_DEST_ARR[$((_DEST_SEL-1))]}"
+else
+    ACTIVE_DEST="${_DEST_ARR[0]}"
+fi
+
+# ── Connect to backend ─────────────────────────────────────────────────────────
+if [ "$BACKEND" = "kopia" ]; then
+    _CFG_VAR="DEST_${ACTIVE_DEST}_CONFIG"
+    _PW_VAR="DEST_${ACTIVE_DEST}_PASSWORD"
+    K_CFG="${!_CFG_VAR:-}"
+    K_PW="${!_PW_VAR:-}"
+    [ -n "$K_CFG" ] || die "No Kopia config found for destination '$ACTIVE_DEST'"
+    export KOPIA_PASSWORD="$K_PW"
+    k() { "$KOPIA_BIN" --config-file="$K_CFG" "$@"; }
+    k repository status >/dev/null 2>&1 \
+        || die "Cannot connect to Kopia repo '$ACTIVE_DEST' — check backup.conf"
+else
+    _REPO_VAR="DEST_${ACTIVE_DEST}_REPO"
+    _PASS_VAR="DEST_${ACTIVE_DEST}_PASSPHRASE"
+    export BORG_REPO="${!_REPO_VAR:-}"
+    export BORG_PASSPHRASE="${!_PASS_VAR:-}"
+    [ -n "$BORG_REPO" ] || die "No Borg repo found for destination '$ACTIVE_DEST'"
+    b() { "$BORG_BIN" "$@"; }
+    b info >/dev/null 2>&1 \
+        || die "Cannot connect to Borg repo at $BORG_REPO — check backup.conf"
+fi
+
+# ── Build list of testable services ───────────────────────────────────────────
+declare -a SERVICES=()
+declare -A SVC_SNAP_SRC=()   # service name → backup source path
+SNAP_JSON=""
+
+if [ "$BACKEND" = "kopia" ]; then
+    SNAP_JSON=$(k snapshot list --all --json 2>/dev/null)
+    [ -z "$SNAP_JSON" ] || [ "$SNAP_JSON" = "[]" ] || [ "$SNAP_JSON" = "null" ] \
+        && die "No snapshots found. Run a backup first."
+    while IFS= read -r src_path; do
+        [[ "$src_path" == "$DOCKER_BASE/"* ]] || continue
+        rel="${src_path#"$DOCKER_BASE/"}"
+        svc="${rel%%/*}"
+        [[ "$svc" == "backup" || "$svc" == "borg-backup" || "$svc" == "gaming-backup" ]] && continue
+        _already=false
+        for _s in "${SERVICES[@]+"${SERVICES[@]}"}"; do [ "$_s" = "$svc" ] && _already=true && break; done
+        if [ "$_already" = false ]; then
+            SERVICES+=("$svc")
+            SVC_SNAP_SRC["$svc"]="$src_path"
+        fi
+    done < <(echo "$SNAP_JSON" | jq -r 'group_by(.source.path)[] | .[0].source.path')
+else
+    _seen=""
+    while IFS= read -r arch; do
+        svc="${arch%-[0-9][0-9][0-9][0-9]-*}"
+        [[ "$svc" == "backup" || "$svc" == "borg-backup" || "$svc" == "gaming-backup" ]] && continue
+        [ -z "$svc" ] && continue
+        [[ "$_seen" == *"|${svc}|"* ]] && continue
+        _seen+="|${svc}|"
+        SERVICES+=("$svc")
+    done < <(b list --format '{archive}{NL}' 2>/dev/null | sort)
+fi
+
+[ "${#SERVICES[@]}" -eq 0 ] && die "No backed-up services found. Run a backup first."
+
+# ── --list ─────────────────────────────────────────────────────────────────────
+if [ "${1:-}" = "--list" ]; then
+    echo ""
+    info "Testable services (dest: $ACTIVE_DEST, backend: $BACKEND):"
+    echo ""
+    for svc in "${SERVICES[@]}"; do printf "  • %s\n" "$svc"; done
+    echo ""
+    exit 0
+fi
+
+# ── Service selection ──────────────────────────────────────────────────────────
+SELECTED_SVC=""
+if [ -n "${1:-}" ] && [ "${1:-}" != "--list" ]; then
+    SELECTED_SVC="${1:-}"
+    _found=false
+    for _s in "${SERVICES[@]}"; do [ "$_s" = "$SELECTED_SVC" ] && _found=true && break; done
+    [ "$_found" = true ] || die "Service '$SELECTED_SVC' has no backups. Use --list."
+else
+    echo ""
+    echo "╔═══════════════════════════════════════════════════════╗"
+    echo "║   Backup Verification Test                            ║"
+    echo "╚═══════════════════════════════════════════════════════╝"
+    echo ""
+    info "Destination: $ACTIVE_DEST  (backend: $BACKEND)"
+    echo ""
+    echo "Services with backups:"
+    echo ""
+    for i in "${!SERVICES[@]}"; do
+        printf "  %2d)  %s\n" "$((i+1))" "${SERVICES[$i]}"
+    done
+    echo ""
+    read -rp "Select service [1-${#SERVICES[@]}] or q to quit: " SEL
+    [[ "$SEL" =~ ^[qQ]$ ]] && echo "Cancelled." && exit 0
+    [[ "$SEL" =~ ^[0-9]+$ ]] && [ "$SEL" -ge 1 ] && [ "$SEL" -le "${#SERVICES[@]}" ] \
+        || die "Invalid selection: $SEL"
+    SELECTED_SVC="${SERVICES[$((SEL-1))]}"
+fi
+ok "Testing service: $SELECTED_SVC"
+
+# ── Find most recent backup ────────────────────────────────────────────────────
+SNAP_ID="" SNAP_DESC="" ARCHIVE_NAME="" SOURCE_PATH=""
+
+if [ "$BACKEND" = "kopia" ]; then
+    SOURCE_PATH="${SVC_SNAP_SRC[$SELECTED_SVC]:-$DOCKER_BASE/$SELECTED_SVC}"
+    SNAP_ID=$(echo "$SNAP_JSON" | jq -r --arg p "$SOURCE_PATH" \
+        '[.[] | select(.source.path == $p)] | sort_by(.startTime) | last | .id // empty')
+    SNAP_DESC=$(echo "$SNAP_JSON" | jq -r --arg p "$SOURCE_PATH" \
+        '[.[] | select(.source.path == $p)] | sort_by(.startTime) | last |
+         .startTime | split("T") | "\(.[0]) \(.[1][:8]) UTC"' 2>/dev/null || echo "unknown")
+    [ -n "$SNAP_ID" ] || die "No Kopia snapshot for '$SELECTED_SVC' at $SOURCE_PATH"
+else
+    SOURCE_PATH="$DOCKER_BASE/$SELECTED_SVC"
+    ARCHIVE_NAME=$(b list --format '{archive}{NL}' 2>/dev/null \
+        | grep "^${SELECTED_SVC}-" | sort | tail -1)
+    [ -n "$ARCHIVE_NAME" ] || die "No Borg archive found for '$SELECTED_SVC'"
+    SNAP_DESC="${ARCHIVE_NAME#"${SELECTED_SVC}-"}"
+fi
+info "Most recent backup: $SNAP_DESC"
+
+# ── Resolve paths ──────────────────────────────────────────────────────────────
+# TARGET_DIR = the exact path that was backed up (may be a sub-path for gaming)
+# SVC_DIR    = top-level docker service dir (for container stop/start)
+TARGET_DIR="$SOURCE_PATH"
+SVC_DIR="$DOCKER_BASE/$SELECTED_SVC"
+COMPOSE_FILE="$SVC_DIR/docker-compose.yml"
+
+TS="$(date +%Y%m%d-%H%M%S)"
+ASIDE_DIR="${TARGET_DIR}.test-aside-${TS}"
+ERR_LOG="$(mktemp)"
+STOPPED=false
+
+# ── Cleanup trap — always restores live data ───────────────────────────────────
+cleanup() {
+    rm -f "$ERR_LOG" 2>/dev/null || true
+    if [ -d "$ASIDE_DIR" ]; then
+        warn "Restoring live data from aside copy..."
+        rm -rf "$TARGET_DIR" 2>/dev/null || true
+        mv "$ASIDE_DIR" "$TARGET_DIR"
+        ok "Live data restored to $TARGET_DIR"
+    fi
+    if [ "$STOPPED" = true ] && [ -f "$COMPOSE_FILE" ]; then
+        info "Restarting $SELECTED_SVC..."
+        docker compose -f "$COMPOSE_FILE" up -d 2>/dev/null \
+            && ok "$SELECTED_SVC restarted." \
+            || warn "Auto-restart failed — run: docker compose -f $COMPOSE_FILE up -d"
+    fi
+}
+trap cleanup EXIT
+
+echo ""
+warn "This test will briefly stop '$SELECTED_SVC', move its data aside, restore"
+warn "from the most recent backup, compare, then move everything back."
+echo ""
+read -rp "Proceed? (y/N): " _CONFIRM
+[[ "$_CONFIRM" =~ ^[Yy]$ ]] || { echo "Cancelled."; exit 0; }
+
+# ── Stop container ─────────────────────────────────────────────────────────────
+if [ -f "$COMPOSE_FILE" ] && docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$SELECTED_SVC"; then
+    info "Stopping $SELECTED_SVC..."
+    docker compose -f "$COMPOSE_FILE" down 2>/dev/null \
+        || docker stop "$SELECTED_SVC" 2>/dev/null \
+        || warn "Could not stop container — data may be inconsistent."
+    STOPPED=true
+    ok "$SELECTED_SVC stopped."
+fi
+
+# ── Move live data aside ───────────────────────────────────────────────────────
+if [ -e "$TARGET_DIR" ]; then
+    info "Moving live data aside → $(basename "$ASIDE_DIR") ..."
+    mv "$TARGET_DIR" "$ASIDE_DIR"
+    ok "Live data saved at: $ASIDE_DIR"
+else
+    warn "$TARGET_DIR not found — testing fresh restore (no live data to compare)."
+fi
+
+# ── Restore from backup ────────────────────────────────────────────────────────
+mkdir -p "$TARGET_DIR"
+info "Restoring from backup..."
+
+if [ "$BACKEND" = "kopia" ]; then
+    if ! k restore "$SNAP_ID" "$TARGET_DIR" 2>"$ERR_LOG"; then
+        FAIL_MSG="Restore failed: $(head -3 "$ERR_LOG" | tr '\n' ' ')"
+        err "$FAIL_MSG"
+        ntfy_send "✗ Backup test FAILED: $SELECTED_SVC" "$HOST\n$FAIL_MSG" \
+            "urgent" "rotating_light"
+        exit 1
+    fi
+else
+    EXTRACT_PATH="${SOURCE_PATH#/}"
+    if ! ( cd / && b extract "$BORG_REPO::$ARCHIVE_NAME" "$EXTRACT_PATH" 2>"$ERR_LOG" ); then
+        FAIL_MSG="Restore failed: $(head -3 "$ERR_LOG" | tr '\n' ' ')"
+        err "$FAIL_MSG"
+        ntfy_send "✗ Backup test FAILED: $SELECTED_SVC" "$HOST\n$FAIL_MSG" \
+            "urgent" "rotating_light"
+        exit 1
+    fi
+fi
+ok "Restore complete."
+
+# Sanity check — something must have been restored
+RESTORED_COUNT=$(find "$TARGET_DIR" -mindepth 1 -maxdepth 2 2>/dev/null | wc -l)
+if [ "$RESTORED_COUNT" -eq 0 ]; then
+    FAIL_MSG="Restore succeeded but target directory is empty"
+    err "$FAIL_MSG"
+    ntfy_send "✗ Backup test FAILED: $SELECTED_SVC" "$HOST\n$FAIL_MSG" \
+        "urgent" "rotating_light"
+    exit 1
+fi
+
+# ── Compare restored vs live data ─────────────────────────────────────────────
+DIFF_COUNT=0
+DIFF_SAMPLE=""
+
+if [ -d "$ASIDE_DIR" ]; then
+    info "Comparing restored vs live data (content, not timestamps)..."
+    DIFF_OUT="$(diff -rq "$ASIDE_DIR" "$TARGET_DIR" 2>/dev/null || true)"
+    DIFF_COUNT=$(printf '%s' "$DIFF_OUT" | grep -c '^' 2>/dev/null || echo 0)
+
+    if [ "$DIFF_COUNT" -eq 0 ]; then
+        ok "Perfect match — restored data is identical to live data."
+    else
+        DIFF_SAMPLE="$(printf '%s' "$DIFF_OUT" | head -10)"
+        warn "$DIFF_COUNT file(s) differ between backup and live data."
+        warn "(Normal — these files changed between the last backup and now.)"
+        echo ""
+        echo "  Files changed since last backup (up to 10):"
+        printf '%s\n' "$DIFF_SAMPLE" | while IFS= read -r line; do echo "    $line"; done
+        [ "$DIFF_COUNT" -gt 10 ] && echo "    ... and $((DIFF_COUNT - 10)) more"
+    fi
+else
+    warn "No live data to compare. Restored $RESTORED_COUNT item(s) from backup."
+fi
+
+# ── Result ─────────────────────────────────────────────────────────────────────
+echo ""
+echo "═══════════════════════════════════════════════════════"
+if [ "$DIFF_COUNT" -eq 0 ]; then
+    echo "  BACKUP TEST PASSED ✓  (perfect match)"
+else
+    echo "  BACKUP TEST PASSED ✓  ($DIFF_COUNT file(s) changed since last backup)"
+fi
+echo "═══════════════════════════════════════════════════════"
+echo ""
+echo "  Service  : $SELECTED_SVC"
+echo "  Backend  : $BACKEND"
+echo "  Backup   : $SNAP_DESC"
+echo "  Restored : $RESTORED_COUNT item(s)"
+[ "$DIFF_COUNT" -gt 0 ] && echo "  Changed  : $DIFF_COUNT file(s) modified since last backup (normal)"
+echo ""
+
+if [ "$DIFF_COUNT" -eq 0 ]; then
+    ntfy_send "✓ Backup test PASSED: $SELECTED_SVC" \
+        "$HOST: $SELECTED_SVC backup verified — perfect match (${RESTORED_COUNT} items)" \
+        "low" "white_check_mark"
+else
+    ntfy_send "✓ Backup test PASSED: $SELECTED_SVC" \
+        "$HOST: $SELECTED_SVC backup OK — ${DIFF_COUNT} file(s) changed since last backup" \
+        "default" "white_check_mark"
+fi
+
+# cleanup trap handles data restoration and container restart

services/backup.sh

@@ -246,7 +246,21 @@ install_backup() {
     prompt_text "  Snapshots to keep (latest)? [7]:" "7" KEEP_LATEST
     KEEP_LATEST="${KEEP_LATEST:-7}"
 
-    # ── 7. Create dirs + init Kopia repos ────────────────────────────────────
+    # ── Notifications (ntfy) ─────────────────────────────────────────────────
+    echo ""
+    echo "═══════════════════════════════════════════════════════"
+    echo "  NOTIFICATIONS (optional)"
+    echo "═══════════════════════════════════════════════════════"
+    echo ""
+    echo "  Receive a push notification after every backup (and on failures)."
+    echo "  Uses ntfy — free and self-hostable. Create a topic at https://ntfy.sh"
+    echo "  Example URL: https://ntfy.sh/my-backup-alerts"
+    echo ""
+    local NTFY_URL="" NTFY_TOKEN=""
+    prompt_text "  ntfy topic URL (blank to skip):" "" NTFY_URL
+    if [ -n "$NTFY_URL" ]; then
+        prompt_text "  ntfy access token (blank if public/no auth):" "" NTFY_TOKEN
+    fi
     mkdir -p "$DIR"
     ensure_docker_dir_ownership "$DIR"
 
@@ -320,6 +334,12 @@ install_backup() {
         echo "# Example SFTP: REMOTE_TYPE=sftp  REMOTE_ARGS=\"--host H --username U --path /srv/...\""
         echo "REMOTE_TYPE=\"none\""
         echo "REMOTE_ARGS=\"\""
+        echo ""
+        echo "# ── Notifications (ntfy) ─────────────────────────────────────────────────────"
+        echo "# Set NTFY_URL to receive backup success/failure alerts."
+        echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)."
+        printf "NTFY_URL='%s'\n" "${NTFY_URL:-}"
+        printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}"
     } > "$CONF_FILE"
     chown root:root "$CONF_FILE" 2>/dev/null || true
     chmod 600 "$CONF_FILE"
@@ -344,7 +364,19 @@ install_backup() {
         log_warning "Copy it manually: cp extras/restore_kopia.sh $RESTORE"
     fi
 
-    # ── 11. Systemd timer ────────────────────────────────────────────────────
+    # ── 11. Install test script ──────────────────────────────────────────────
+    local TEST_SCRIPT="$DIR/test_backup.sh"
+    local TEST_SRC="${HERE:-}/extras/test_backup.sh"
+    if [ -f "$TEST_SRC" ]; then
+        cp "$TEST_SRC" "$TEST_SCRIPT"
+        chmod +x "$TEST_SCRIPT"
+        chown root:root "$TEST_SCRIPT" 2>/dev/null || true
+        log_success "test_backup.sh installed"
+    else
+        log_warning "extras/test_backup.sh not found — test script not installed"
+    fi
+
+    # ── 12. Systemd timer ────────────────────────────────────────────────────
     log_info "Installing systemd timer ($SCHED_LABEL)..."
     local AUTORUN=""
     if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
@@ -427,6 +459,12 @@ SVCEOF
     echo "    sudo $RESTORE"
     echo "    sudo $RESTORE --list"
     echo ""
+    echo "  Backup test (stop/restore/compare/restore-back):"
+    echo "    sudo $TEST_SCRIPT                test most recent backup"
+    echo "    sudo $TEST_SCRIPT --list         list testable services"
+    echo "    sudo $TEST_SCRIPT <service>      test a specific service"
+    [ -n "${NTFY_URL:-}" ] && echo "" && echo "  Notifications: $NTFY_URL"
+    echo ""
     [ -n "$AUTORUN" ] && echo "  $AUTORUN" && echo ""
     log_warning "Save your passwords (in backup.conf) somewhere safe —"
     log_warning "without them the encrypted repos cannot be restored."

services/borg-backup.sh

@@ -191,6 +191,22 @@ install_borg_backup() {
     KEEP_WEEKLY="${KEEP_WEEKLY:-4}"
     KEEP_MONTHLY="${KEEP_MONTHLY:-3}"
 
+    # ── Notifications (ntfy) ─────────────────────────────────────────────────
+    echo ""
+    echo "═══════════════════════════════════════════════════════"
+    echo "  NOTIFICATIONS (optional)"
+    echo "═══════════════════════════════════════════════════════"
+    echo ""
+    echo "  Receive a push notification after every backup (and on failures)."
+    echo "  Uses ntfy — free and self-hostable. Create a topic at https://ntfy.sh"
+    echo "  Example URL: https://ntfy.sh/my-backup-alerts"
+    echo ""
+    local NTFY_URL="" NTFY_TOKEN=""
+    prompt_text "  ntfy topic URL (blank to skip):" "" NTFY_URL
+    if [ -n "$NTFY_URL" ]; then
+        prompt_text "  ntfy access token (blank if public/no auth):" "" NTFY_TOKEN
+    fi
+
     # ── 7. Create dirs + init Borg repos ─────────────────────────────────────
     mkdir -p "$DIR"
     ensure_docker_dir_ownership "$DIR"
@@ -262,6 +278,12 @@ install_borg_backup() {
                 echo "# SVC_${svc_var}=\"default\""
             fi
         done
+        echo ""
+        echo "# ── Notifications (ntfy) ─────────────────────────────────────────────────────"
+        echo "# Set NTFY_URL to receive backup success/failure alerts."
+        echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)."
+        printf "NTFY_URL='%s'\n" "${NTFY_URL:-}"
+        printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}"
     } > "$CONF_FILE"
     chown root:root "$CONF_FILE" 2>/dev/null || true
     chmod 600 "$CONF_FILE"
@@ -286,7 +308,19 @@ install_borg_backup() {
         log_warning "Copy it manually: cp extras/restore_borg.sh $RESTORE"
     fi
 
-    # ── 11. Systemd timer ─────────────────────────────────────────────────────
+    # ── 11. Install test script ──────────────────────────────────────────────
+    local TEST_SCRIPT="$DIR/test_backup.sh"
+    local TEST_SRC="${HERE:-}/extras/test_backup.sh"
+    if [ -f "$TEST_SRC" ]; then
+        cp "$TEST_SRC" "$TEST_SCRIPT"
+        chmod +x "$TEST_SCRIPT"
+        chown root:root "$TEST_SCRIPT" 2>/dev/null || true
+        log_success "test_backup.sh installed"
+    else
+        log_warning "extras/test_backup.sh not found — test script not installed"
+    fi
+
+    # ── 12. Systemd timer ─────────────────────────────────────────────────────
     log_info "Installing systemd timer ($SCHED_LABEL)..."
     if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
         tee "/etc/systemd/system/${SVC_NAME}.service" >/dev/null << SVCEOF
@@ -368,6 +402,12 @@ SVCEOF
     echo "    sudo $RESTORE"
     echo "    sudo $RESTORE --list"
     echo ""
+    echo "  Backup test (stop/restore/compare/restore-back):"
+    echo "    sudo $TEST_SCRIPT                test most recent backup"
+    echo "    sudo $TEST_SCRIPT --list         list testable services"
+    echo "    sudo $TEST_SCRIPT <service>      test a specific service"
+    [ -n "${NTFY_URL:-}" ] && echo "" && echo "  Notifications: $NTFY_URL"
+    echo ""
     log_warning "IMPORTANT — back up your Borg key and passphrase now."
     echo "  The key is stored in the repo itself (repokey-blake2 encryption)."
     echo "  Export it to a safe location:"

services/gaming-backup.sh

@@ -192,6 +192,21 @@ install_gaming_backup() {
     prompt_text "  How many recent snapshots to keep (latest)? [24]:" "24" KEEP_LATEST
     local KEEP_DAILY=7 KEEP_WEEKLY=4 KEEP_MONTHLY=6
 
+    # ── Notifications (ntfy) ─────────────────────────────────────────────────
+    echo ""
+    echo "═══════════════════════════════════════════════════════"
+    echo "  NOTIFICATIONS (optional)"
+    echo "═══════════════════════════════════════════════════════"
+    echo ""
+    echo "  Receive a push notification after every backup (and on failures)."
+    echo "  Uses ntfy — free and self-hostable. Create a topic at https://ntfy.sh"
+    echo ""
+    local NTFY_URL="" NTFY_TOKEN=""
+    prompt_text "  ntfy topic URL (blank to skip):" "" NTFY_URL
+    if [ -n "$NTFY_URL" ]; then
+        prompt_text "  ntfy access token (blank if public/no auth):" "" NTFY_TOKEN
+    fi
+
     # ── 5. Write backup.conf ──────────────────────────────────────────────────
     log_info "Writing $CONF_FILE ..."
     tee "$CONF_FILE" >/dev/null << CONFEOF
@@ -226,6 +241,12 @@ BACKUP_WOLF="$BACKUP_WOLF"     # /etc/wolf — config + profile_data
 #
 REMOTE_TYPE="none"
 REMOTE_ARGS=""
+
+# ── Notifications (ntfy) ─────────────────────────────────────────────────────
+# Set NTFY_URL to receive backup success/failure alerts.
+# Leave blank to disable. NTFY_TOKEN is optional (for private topics).
+NTFY_URL="$NTFY_URL"
+NTFY_TOKEN="$NTFY_TOKEN"
 CONFEOF
     chown root:root "$CONF_FILE" 2>/dev/null || true
     chmod 600 "$CONF_FILE"
@@ -292,6 +313,18 @@ CONFEOF
         log_warning "extras/restore_kopia.sh not found — restore script not installed"
     fi
 
+    # ── Install test script ───────────────────────────────────────────────────
+    local TEST_SCRIPT="$BACKUP_DIR/test_backup.sh"
+    local TEST_SRC="${HERE:-}/extras/test_backup.sh"
+    if [ -f "$TEST_SRC" ]; then
+        cp "$TEST_SRC" "$TEST_SCRIPT"
+        chmod +x "$TEST_SCRIPT"
+        chown root:root "$TEST_SCRIPT" 2>/dev/null || true
+        log_success "test_backup.sh installed"
+    else
+        log_warning "extras/test_backup.sh not found — test script not installed"
+    fi
+
     # ── 8. Install systemd timer (fallback: cron) ────────────────────────────
     local AUTORUN=""
     if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
@@ -369,6 +402,13 @@ UNITEOF
     echo "    sudo $WORKER snapshots               list snapshots"
     echo "    sudo $RESTORE_DEST                  interactive restore"
     echo "    sudo $RESTORE_DEST --list           list all snapshot sources"
+    echo ""
+    echo "  Backup test (restore latest, compare, restore-back):"
+    echo "    sudo $TEST_SCRIPT                test most recent backup"
+    echo "    sudo $TEST_SCRIPT --list         list testable sources"
+    echo "    sudo $TEST_SCRIPT <source>       test a specific source"
+    [ -n "${NTFY_URL:-}" ] && echo "" && echo "  Notifications: $NTFY_URL"
+    echo ""
     echo "    $AUTORUN"
     echo ""
     echo "  Tip: also install the 'backup' service for nightly full-service recovery."