Add backup test scripts and enhance workers with timing, counts, and pre-flight checks

- extras/test_backup_kopia.sh: stop → restore → compare → roll back test for Kopia
- extras/test_backup_borg.sh:  stop → extract → compare → roll back test for Borg
- backup workers: add START_TS/BACKUP_COUNT/DURATION_STR to completion log and ntfy
- backup workers: pre-flight disk check warns and records failure if < 512 MB free
- services/backup.sh: install test_backup_kopia.sh + optional weekly test timer
- services/borg-backup.sh: install test_backup_borg.sh + optional weekly test timer
- ntfy notifications include service count, duration, and per-failure detail lines

extras/backup_kopia.sh

@@ -86,6 +86,21 @@ rc=0
 declare -a FAILED_SVCS=()
 _ERR="$(mktemp)"
 trap 'rm -f "$_ERR"' EXIT
+START_TS="$(date +%s)"
+BACKUP_COUNT=0
+
+for _pf_dest in ${DEST_NAMES:-default}; do
+    _pf_var="DEST_${_pf_dest}_REPO"; _pf_repo="${!_pf_var:-}"
+    if [ -n "$_pf_repo" ] && [[ "$_pf_repo" != *@*:* ]] && [[ "$_pf_repo" != ssh://* ]]; then
+        _pf_dir="$([ -d "$_pf_repo" ] && echo "$_pf_repo" || dirname "$_pf_repo")"
+        _pf_avail="$(df -m "$_pf_dir" 2>/dev/null | awk 'NR==2{print $4}')"
+        if [ -n "$_pf_avail" ] && [ "$_pf_avail" -lt 512 ]; then
+            log "WARNING: Low disk for '$_pf_dest' — ${_pf_avail}MB free at $_pf_repo"
+            FAILED_SVCS+=("$_pf_dest: low disk (${_pf_avail}MB free)")
+            rc=1
+        fi
+    fi
+done
 
 for svc_dir in "$DOCKER_DIR"/*/; do
     [ -f "${svc_dir}docker-compose.yml" ] || continue
@@ -106,6 +121,7 @@ for svc_dir in "$DOCKER_DIR"/*/; do
         log "Snapshotting $svc (dest: $dest)..."
         if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then
             log "OK $svc (Minecraft, no downtime)"
+            BACKUP_COUNT=$((BACKUP_COUNT+1))
         else
             _reason="$(categorize_error "$(cat "$_ERR")")"
             log "WARNING: snapshot failed for $svc — $_reason"
@@ -125,6 +141,7 @@ for svc_dir in "$DOCKER_DIR"/*/; do
         log "Snapshotting $svc (dest: $dest)..."
         if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then
             log "OK $svc"
+            BACKUP_COUNT=$((BACKUP_COUNT+1))
         else
             _reason="$(categorize_error "$(cat "$_ERR")")"
             log "WARNING: snapshot failed for $svc — $_reason"
@@ -153,13 +170,17 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
     done
 fi
 
+DURATION=$(( $(date +%s) - START_TS ))
+DURATION_STR="$((DURATION/60))m $((DURATION%60))s"
+
 if [ "$rc" -eq 0 ]; then
-    log "===== Backup complete ====="
-    ntfy_send "✓ Backup complete" "$HOST: all services backed up successfully" \
+    log "===== Backup complete — $BACKUP_COUNT service(s) in $DURATION_STR ====="
+    ntfy_send "✓ Backup complete" \
+        "$HOST: $BACKUP_COUNT service(s) backed up in $DURATION_STR" \
         "low" "white_check_mark"
 else
-    log "===== Backup finished WITH WARNINGS (see above) ====="
-    _ntfy_msg="$HOST: backup failures:"
+    log "===== Backup finished WITH WARNINGS — $BACKUP_COUNT/$((BACKUP_COUNT+${#FAILED_SVCS[@]})) succeeded in $DURATION_STR ====="
+    _ntfy_msg="$HOST: backup failures (${#FAILED_SVCS[@]}):"
     for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done
     ntfy_send "✗ Backup FAILED" "$_ntfy_msg" "urgent" "rotating_light"
 fi