#!/bin/bash # extras/backup_borg.sh — Borg backup worker for all Docker services. # Installed to ~/docker/borg-backup/backup_borg.sh by the borg-backup installer. # # sudo ./backup_borg.sh run a full backup cycle # sudo ./backup_borg.sh list list all archives in all repos # sudo ./backup_borg.sh info show repo info for all destinations # # Minecraft instances: flush to disk (save-all) then archive — no downtime. # All other services: stop → archive → restart for consistency. # Reads backup.conf from the same directory. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONF="${BACKUP_CONF:-$HERE/backup.conf}" [ -f "$CONF" ] || { echo "Config not found: $CONF (re-run: sudo setup.sh borg-backup)"; exit 1; } # shellcheck source=/dev/null source "$CONF" ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" DOCKER_DIR="$ACTUAL_HOME/docker" HOST="$(hostname -s 2>/dev/null || hostname)" log() { echo "[$(date '+%F %T')] $*"; } ntfy_send() { local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}" [ -z "${NTFY_URL:-}" ] && return 0 local -a _args=(-fsS -o /dev/null) _args+=(-H "Title: $title" -H "Priority: $priority") [ -n "$tags" ] && _args+=(-H "Tags: $tags") [ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN") curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true } categorize_error() { local txt="$1" if echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then echo "disk full — backup destination is out of space" elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then echo "remote unreachable — check network / destination host" elif echo "$txt" | grep -qi "repository.*does not exist\|not a borg\|is not a valid"; then echo "repository not found — re-run the backup installer" elif echo "$txt" | grep -qi "passphrase\|wrong key\|bad key\|cannot decrypt"; then echo "wrong passphrase — check backup.conf" elif echo "$txt" | grep -qi "permission denied\|access denied"; then echo "permission denied — check file permissions" else echo "error — see system logs on $HOST" fi } repo_for() { local var="DEST_${1}_REPO"; echo "${!var:-}"; } pass_for() { local var="DEST_${1}_PASSPHRASE"; echo "${!var:-}"; } dest_for_svc() { local var="SVC_${1//-/_}"; echo "${!var:-${DEST_DEFAULT:-default}}"; } b_for() { local dest="$1"; shift local repo; repo="$(repo_for "$dest")" local pass; pass="$(pass_for "$dest")" [ -n "$repo" ] || { log "Unknown destination: $dest"; return 1; } BORG_PASSPHRASE="$pass" BORG_REPO="$repo" "$BORG" "$@" } is_minecraft() { [ -f "${1}Dockerfile" ] && grep -qs itzg "${1}Dockerfile"; } case "${1:-run}" in list) for dest in ${DEST_NAMES:-default}; do echo ""; echo "── dest: $dest ($(repo_for "$dest")) ──" b_for "$dest" list 2>/dev/null | sort -r | head -30 || true done exit 0 ;; info) for dest in ${DEST_NAMES:-default}; do echo ""; echo "── dest: $dest ──" b_for "$dest" info 2>/dev/null || true done exit 0 ;; esac log "===== Borg backup starting =====" rc=0 TS="$(date +%Y-%m-%dT%H-%M-%S)" declare -a FAILED_SVCS=() _ERR="$(mktemp)" trap 'rm -f "$_ERR"' EXIT START_TS="$(date +%s)" BACKUP_COUNT=0 for _pf_dest in ${DEST_NAMES:-default}; do _pf_var="DEST_${_pf_dest}_REPO"; _pf_repo="${!_pf_var:-}" if [ -n "$_pf_repo" ] && [[ "$_pf_repo" != *@*:* ]] && [[ "$_pf_repo" != ssh://* ]]; then _pf_dir="$([ -d "$_pf_repo" ] && echo "$_pf_repo" || dirname "$_pf_repo")" _pf_avail="$(df -m "$_pf_dir" 2>/dev/null | awk 'NR==2{print $4}')" if [ -n "$_pf_avail" ] && [ "$_pf_avail" -lt 512 ]; then log "WARNING: Low disk for '$_pf_dest' — ${_pf_avail}MB free at $_pf_repo" FAILED_SVCS+=("$_pf_dest: low disk (${_pf_avail}MB free)") rc=1 fi fi done for svc_dir in "$DOCKER_DIR"/*/; do [ -f "${svc_dir}docker-compose.yml" ] || continue svc="$(basename "$svc_dir")" [[ "$svc" == "borg-backup" || "$svc" == "backup" || "$svc" == "gaming-backup" ]] && continue dest="$(dest_for_svc "$svc")" repo="$(repo_for "$dest")" [ -n "$repo" ] || { log "SKIP $svc — dest '$dest' not configured"; continue; } ARCHIVE="${svc}-${TS}" if is_minecraft "$svc_dir"; then if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$svc"; then log "Flushing Minecraft world '$svc' (save-all, no downtime)..." docker exec "$svc" mc-send-to-console save-all flush 2>/dev/null \ || docker exec "$svc" rcon-cli save-all 2>/dev/null || true sleep 5 fi log "Archiving $svc → $dest::$ARCHIVE ..." if b_for "$dest" create \ --compression=zstd,6 --exclude-caches --stats \ "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log " $line"; done; then log "OK $svc (Minecraft, no downtime)" BACKUP_COUNT=$((BACKUP_COUNT+1)) else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: archive failed for $svc — $_reason" FAILED_SVCS+=("$svc: $_reason") rc=1 fi else STOPPED=false if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$svc"; then log "Stopping $svc..." docker compose -f "${svc_dir}docker-compose.yml" down 2>/dev/null \ || docker stop "$svc" 2>/dev/null \ || log "WARNING: could not stop $svc — archiving live (consistency not guaranteed)" STOPPED=true fi log "Archiving $svc → $dest::$ARCHIVE ..." if b_for "$dest" create \ --compression=zstd,6 --exclude-caches --stats \ "::$ARCHIVE" "$svc_dir" 2>"$_ERR" | while IFS= read -r line; do log " $line"; done; then log "OK $svc" BACKUP_COUNT=$((BACKUP_COUNT+1)) else _reason="$(categorize_error "$(cat "$_ERR")")" log "WARNING: archive failed for $svc — $_reason" FAILED_SVCS+=("$svc: $_reason") rc=1 fi if [ "$STOPPED" = true ]; then log "Starting $svc..." docker compose -f "${svc_dir}docker-compose.yml" up -d 2>/dev/null \ || log "WARNING: could not restart $svc — run: docker compose -f ${svc_dir}docker-compose.yml up -d" fi fi log "Pruning old archives for $svc in '$dest'..." b_for "$dest" prune \ --keep-daily="${KEEP_DAILY:-7}" \ --keep-weekly="${KEEP_WEEKLY:-4}" \ --keep-monthly="${KEEP_MONTHLY:-3}" \ --glob-archives="${svc}-*" \ --list 2>/dev/null \ || log "WARNING: prune failed for $svc (non-fatal)" done for dest in ${DEST_NAMES:-default}; do repo="$(repo_for "$dest")" [ -n "$repo" ] || continue log "Compacting repo '$dest'..." b_for "$dest" compact 2>/dev/null || true done DURATION=$(( $(date +%s) - START_TS )) DURATION_STR="$((DURATION/60))m $((DURATION%60))s" if [ "$rc" -eq 0 ]; then log "===== Borg backup complete — $BACKUP_COUNT service(s) in $DURATION_STR =====" ntfy_send "✓ Borg backup complete" \ "$HOST: $BACKUP_COUNT service(s) archived in $DURATION_STR" \ "low" "white_check_mark" else log "===== Borg backup finished WITH WARNINGS — $BACKUP_COUNT/$((BACKUP_COUNT+${#FAILED_SVCS[@]})) succeeded in $DURATION_STR =====" _ntfy_msg="$HOST: Borg backup failures (${#FAILED_SVCS[@]}):" for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done ntfy_send "✗ Borg backup FAILED" "$_ntfy_msg" "urgent" "rotating_light" fi exit "$rc"