Claude
f6e5bb4ea3
The freshly-added raw-error logging paid off immediately: the box's spare sync was failing every run with "scp: Connection closed" while plain ssh exec to the same host worked fine. That split (ssh exec OK, scp specifically rejected) matches modern OpenSSH's default scp-over-SFTP transfer hitting a restriction on the remote side that a plain exec or rsync's own protocol don't trigger. Swapped the scp step for rsync -a over the same ssh options, keeping the ssh mkdir -p before it (rsync doesn't create missing destination directories) and the ssh chmod after. Verified the exact command/quoting against mocked ssh/rsync binaries — array expansion and remote path handling both check out.
272 lines
12 KiB
Bash
272 lines
12 KiB
Bash
#!/bin/bash
|
|
# extras/backup_kopia.sh — Kopia backup worker for all Docker services.
|
|
# Installed to ~/docker/backup/backup_kopia.sh by the backup service installer.
|
|
#
|
|
# sudo ./backup_kopia.sh run a full backup cycle
|
|
# sudo ./backup_kopia.sh snapshots list all snapshots (all repos)
|
|
# sudo ./backup_kopia.sh policy show retention policies
|
|
#
|
|
# Minecraft instances: flush to disk (save-all) then snapshot — no downtime.
|
|
# All other services: stop → snapshot → restart for consistency.
|
|
# Reads backup.conf from the same directory.
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CONF="${BACKUP_CONF:-$HERE/backup.conf}"
|
|
[ -f "$CONF" ] || { echo "Config not found: $CONF (re-run: sudo setup.sh backup)"; exit 1; }
|
|
# shellcheck source=/dev/null
|
|
source "$CONF"
|
|
|
|
ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}"
|
|
ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")"
|
|
DOCKER_DIR="$ACTUAL_HOME/docker"
|
|
HOST="$(hostname -s 2>/dev/null || hostname)"
|
|
|
|
log() { echo "[$(date '+%F %T')] $*"; }
|
|
|
|
ntfy_send() {
|
|
local title="$1" msg="$2" priority="${3:-default}" tags="${4:-}"
|
|
[ -z "${NTFY_URL:-}" ] && return 0
|
|
local -a _args=(-fsS -o /dev/null)
|
|
_args+=(-H "Title: $title" -H "Priority: $priority")
|
|
[ -n "$tags" ] && _args+=(-H "Tags: $tags")
|
|
[ -n "${NTFY_TOKEN:-}" ] && _args+=(-H "Authorization: Bearer $NTFY_TOKEN")
|
|
curl "${_args[@]}" -d "$msg" "$NTFY_URL" 2>/dev/null || true
|
|
}
|
|
|
|
categorize_error() {
|
|
local txt="$1"
|
|
if echo "$txt" | grep -qi "no space left\|disk quota exceeded"; then
|
|
echo "disk full — backup destination is out of space"
|
|
elif echo "$txt" | grep -qi "connection refused\|network unreachable\|no route to host\|ssh.*connect\|timed out\|host unreachable"; then
|
|
echo "remote unreachable — check network / destination host"
|
|
elif echo "$txt" | grep -qi "repository.*not.*exist\|not a valid kopia\|not connected"; then
|
|
echo "repository not found — re-run the backup installer"
|
|
elif echo "$txt" | grep -qi "passphrase\|wrong key\|cannot decrypt"; then
|
|
echo "wrong passphrase — check backup.conf"
|
|
elif echo "$txt" | grep -qi "permission denied\|access denied"; then
|
|
echo "permission denied — check file permissions"
|
|
else
|
|
echo "error — see system logs on $HOST"
|
|
fi
|
|
}
|
|
|
|
# Logs the raw stderr text a failure produced, truncated to a sane length.
|
|
# categorize_error()'s bucket alone ("error — see system logs") used to be
|
|
# the end of the trail for anything that didn't match one of its known
|
|
# patterns — the actual message was in a mktemp'd file this script deletes
|
|
# on exit (trap ... EXIT), so nothing was ever really "in system logs" to
|
|
# go look at. This makes that claim true: the raw text now lands in the
|
|
# same log stream (journal, when run via the systemd timer) as everything
|
|
# else, surviving past the run that produced it.
|
|
#
|
|
# Takes the already-read error TEXT, not the file path. A live run showed
|
|
# categorize_error() correctly matching a specific pattern (so $_ERR had
|
|
# real content at that point) while a second, later read of the same file
|
|
# for this function came back completely empty — every failure that night
|
|
# logged its categorized reason but zero "Raw error:" lines. Whatever causes
|
|
# that (the file is reused across the whole script run and read twice per
|
|
# failure), reading it once and passing the string to both this function and
|
|
# categorize_error() removes the second read entirely, so there's nothing
|
|
# left to race.
|
|
log_raw_error() {
|
|
local raw
|
|
raw="$(printf '%s' "$1" | tr '\n' ' ' | head -c 500)"
|
|
[ -n "$raw" ] && log " Raw error: $raw"
|
|
}
|
|
|
|
kp_for() {
|
|
local dest="$1"; shift
|
|
local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD"
|
|
local cfg="${!cfg_var:-}" pw="${!pw_var:-}"
|
|
[ -n "$cfg" ] || { log "Unknown destination: $dest"; return 1; }
|
|
env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
|
|
}
|
|
|
|
dest_for_svc() {
|
|
local var="SVC_${1//-/_}"
|
|
echo "${!var:-${DEST_DEFAULT:-default}}"
|
|
}
|
|
|
|
is_minecraft() { [ -f "${1}Dockerfile" ] && grep -qs itzg "${1}Dockerfile"; }
|
|
|
|
case "${1:-run}" in
|
|
snapshots)
|
|
for dest in ${DEST_NAMES:-default}; do
|
|
echo ""; echo "── dest: $dest ──"
|
|
kp_for "$dest" snapshot list 2>/dev/null || true
|
|
done
|
|
exit 0 ;;
|
|
policy)
|
|
for dest in ${DEST_NAMES:-default}; do
|
|
echo ""; echo "── dest: $dest ──"
|
|
kp_for "$dest" policy show --global 2>/dev/null || true
|
|
done
|
|
exit 0 ;;
|
|
esac
|
|
|
|
log "===== Backup starting ====="
|
|
rc=0
|
|
declare -a FAILED_SVCS=()
|
|
_ERR="$(mktemp)"
|
|
trap 'rm -f "$_ERR"' EXIT
|
|
START_TS="$(date +%s)"
|
|
BACKUP_COUNT=0
|
|
|
|
for _pf_dest in ${DEST_NAMES:-default}; do
|
|
_pf_var="DEST_${_pf_dest}_REPO"; _pf_repo="${!_pf_var:-}"
|
|
if [ -n "$_pf_repo" ] && [[ "$_pf_repo" != *@*:* ]] && [[ "$_pf_repo" != ssh://* ]]; then
|
|
_pf_dir="$([ -d "$_pf_repo" ] && echo "$_pf_repo" || dirname "$_pf_repo")"
|
|
_pf_avail="$(df -m "$_pf_dir" 2>/dev/null | awk 'NR==2{print $4}')"
|
|
if [ -n "$_pf_avail" ] && [ "$_pf_avail" -lt 512 ]; then
|
|
log "WARNING: Low disk for '$_pf_dest' — ${_pf_avail}MB free at $_pf_repo"
|
|
FAILED_SVCS+=("$_pf_dest: low disk (${_pf_avail}MB free)")
|
|
rc=1
|
|
fi
|
|
fi
|
|
done
|
|
|
|
for svc_dir in "$DOCKER_DIR"/*/; do
|
|
[ -f "${svc_dir}docker-compose.yml" ] || continue
|
|
svc="$(basename "$svc_dir")"
|
|
[[ "$svc" == "backup" || "$svc" == "borg-backup" || "$svc" == "gaming-backup" ]] && continue
|
|
|
|
dest="$(dest_for_svc "$svc")"
|
|
_repo_var="DEST_${dest}_REPO"
|
|
[ -n "${!_repo_var:-}" ] || { log "SKIP $svc — dest '$dest' not configured in conf"; continue; }
|
|
|
|
if is_minecraft "$svc_dir"; then
|
|
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$svc"; then
|
|
log "Flushing Minecraft world '$svc' (save-all, no downtime)..."
|
|
docker exec "$svc" mc-send-to-console save-all flush 2>/dev/null \
|
|
|| docker exec "$svc" rcon-cli save-all 2>/dev/null || true
|
|
sleep 5
|
|
fi
|
|
log "Snapshotting $svc (dest: $dest)..."
|
|
if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then
|
|
log "OK $svc (Minecraft, no downtime)"
|
|
BACKUP_COUNT=$((BACKUP_COUNT+1))
|
|
else
|
|
_err_text="$(cat "$_ERR" 2>/dev/null)"
|
|
_reason="$(categorize_error "$_err_text")"
|
|
log "WARNING: snapshot failed for $svc — $_reason"
|
|
log_raw_error "$_err_text"
|
|
FAILED_SVCS+=("$svc: $_reason")
|
|
rc=1
|
|
fi
|
|
else
|
|
STOPPED=false
|
|
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$svc"; then
|
|
log "Stopping $svc..."
|
|
docker compose -f "${svc_dir}docker-compose.yml" down 2>/dev/null \
|
|
|| docker stop "$svc" 2>/dev/null \
|
|
|| log "WARNING: could not stop $svc — snapshotting live (consistency not guaranteed)"
|
|
STOPPED=true
|
|
fi
|
|
|
|
log "Snapshotting $svc (dest: $dest)..."
|
|
if kp_for "$dest" snapshot create --description="backup: $svc" "$svc_dir" 2>"$_ERR"; then
|
|
log "OK $svc"
|
|
BACKUP_COUNT=$((BACKUP_COUNT+1))
|
|
else
|
|
_err_text="$(cat "$_ERR" 2>/dev/null)"
|
|
_reason="$(categorize_error "$_err_text")"
|
|
log "WARNING: snapshot failed for $svc — $_reason"
|
|
log_raw_error "$_err_text"
|
|
FAILED_SVCS+=("$svc: $_reason")
|
|
rc=1
|
|
fi
|
|
|
|
if [ "$STOPPED" = true ]; then
|
|
log "Starting $svc..."
|
|
docker compose -f "${svc_dir}docker-compose.yml" up -d 2>/dev/null \
|
|
|| log "WARNING: could not restart $svc — run: docker compose -f ${svc_dir}docker-compose.yml up -d"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
|
|
for dest in ${DEST_NAMES:-default}; do
|
|
log "Mirroring '$dest' offsite ($REMOTE_TYPE)..."
|
|
# shellcheck disable=SC2086
|
|
if ! kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then
|
|
_err_text="$(cat "$_ERR" 2>/dev/null)"
|
|
_reason="$(categorize_error "$_err_text")"
|
|
log "WARNING: mirror failed for '$dest' — $_reason"
|
|
log_raw_error "$_err_text"
|
|
FAILED_SVCS+=("mirror[$dest]: $_reason")
|
|
rc=1
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Additional mirrors (EXTRA_MIRROR_NAMES) — run alongside REMOTE_TYPE above,
|
|
# not instead of it, so a box can mirror to e.g. Backblaze B2 AND directly
|
|
# to a spare over SSH/SFTP at the same time.
|
|
for mirror_name in ${EXTRA_MIRROR_NAMES:-}; do
|
|
_mtype_var="MIRROR_${mirror_name}_TYPE"
|
|
_margs_var="MIRROR_${mirror_name}_ARGS"
|
|
_mtype="${!_mtype_var:-}"
|
|
_margs="${!_margs_var:-}"
|
|
[ -n "$_mtype" ] || continue
|
|
for dest in ${DEST_NAMES:-default}; do
|
|
log "Mirroring '$dest' to '$mirror_name' ($_mtype)..."
|
|
# shellcheck disable=SC2086
|
|
if ! kp_for "$dest" repository sync-to "$_mtype" $_margs 2>"$_ERR"; then
|
|
_err_text="$(cat "$_ERR" 2>/dev/null)"
|
|
_reason="$(categorize_error "$_err_text")"
|
|
log "WARNING: mirror '$mirror_name' failed for '$dest' — $_reason"
|
|
log_raw_error "$_err_text"
|
|
FAILED_SVCS+=("mirror[$mirror_name/$dest]: $_reason")
|
|
rc=1
|
|
fi
|
|
done
|
|
done
|
|
|
|
# ── Keep a spare box's copy of backup.conf + README current ─────────────────
|
|
# Runs after the data itself is backed up (and mirrored, if configured) so a
|
|
# sync never ships config pointing at a repo state that isn't actually there
|
|
# yet. dr_bringup.sh on the spare only needs these two small files — the
|
|
# repo data itself already lives wherever REMOTE_TYPE mirrored it (or is
|
|
# local, if the spare IS that target).
|
|
if [ -n "${DR_SYNC_HOST:-}" ]; then
|
|
_dr_path="${DR_SYNC_PATH:-~/docker/backup}"
|
|
log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..."
|
|
_dr_files=("$CONF")
|
|
[ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md")
|
|
# rsync instead of scp: modern OpenSSH (9.0+) defaults scp to an
|
|
# SFTP-based transfer, and some remote-side setups (restricted shells,
|
|
# forced commands, older sshd) reject that with an immediate "Connection
|
|
# closed" while plain ssh exec and rsync's own protocol both still work
|
|
# fine over the same connection. Confirmed live: scp failing this way
|
|
# while `ssh "$DR_SYNC_HOST" true` succeeded, rsync doesn't hit it.
|
|
if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \
|
|
&& rsync -a -e 'ssh -o BatchMode=yes -o ConnectTimeout=10' "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \
|
|
&& ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then
|
|
log "OK spare sync ($DR_SYNC_HOST)"
|
|
else
|
|
_err_text="$(cat "$_ERR" 2>/dev/null)"
|
|
_reason="$(categorize_error "$_err_text")"
|
|
log "WARNING: spare sync failed — $_reason"
|
|
log_raw_error "$_err_text"
|
|
FAILED_SVCS+=("spare-sync: $_reason")
|
|
rc=1
|
|
fi
|
|
fi
|
|
|
|
DURATION=$(( $(date +%s) - START_TS ))
|
|
DURATION_STR="$((DURATION/60))m $((DURATION%60))s"
|
|
|
|
if [ "$rc" -eq 0 ]; then
|
|
log "===== Backup complete — $BACKUP_COUNT service(s) in $DURATION_STR ====="
|
|
ntfy_send "✓ Backup complete" \
|
|
"$HOST: $BACKUP_COUNT service(s) backed up in $DURATION_STR" \
|
|
"low" "white_check_mark"
|
|
else
|
|
log "===== Backup finished WITH WARNINGS — $BACKUP_COUNT/$((BACKUP_COUNT+${#FAILED_SVCS[@]})) succeeded in $DURATION_STR ====="
|
|
_ntfy_msg="$HOST: backup failures (${#FAILED_SVCS[@]}):"
|
|
for _s in "${FAILED_SVCS[@]}"; do _ntfy_msg+=$'\n'"• $_s"; done
|
|
ntfy_send "✗ Backup FAILED" "$_ntfy_msg" "urgent" "rotating_light"
|
|
fi
|
|
exit "$rc"
|