diff --git a/extras/backup_kopia.sh b/extras/backup_kopia.sh index 5a9dac0..39fe9ea 100644 --- a/extras/backup_kopia.sh +++ b/extras/backup_kopia.sh @@ -170,6 +170,29 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then done fi +# ── Keep a spare box's copy of backup.conf + README current ───────────────── +# Runs after the data itself is backed up (and mirrored, if configured) so a +# sync never ships config pointing at a repo state that isn't actually there +# yet. dr_bringup.sh on the spare only needs these two small files — the +# repo data itself already lives wherever REMOTE_TYPE mirrored it (or is +# local, if the spare IS that target). +if [ -n "${DR_SYNC_HOST:-}" ]; then + _dr_path="${DR_SYNC_PATH:-~/docker/backup}" + log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..." + _dr_files=("$CONF") + [ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md") + if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \ + && scp -o BatchMode=yes -o ConnectTimeout=10 "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \ + && ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then + log "OK spare sync ($DR_SYNC_HOST)" + else + _reason="$(categorize_error "$(cat "$_ERR")")" + log "WARNING: spare sync failed — $_reason" + FAILED_SVCS+=("spare-sync: $_reason") + rc=1 + fi +fi + DURATION=$(( $(date +%s) - START_TS )) DURATION_STR="$((DURATION/60))m $((DURATION%60))s" diff --git a/extras/dr_bringup_kopia.sh b/extras/dr_bringup_kopia.sh index 924fb24..48ca048 100644 --- a/extras/dr_bringup_kopia.sh +++ b/extras/dr_bringup_kopia.sh @@ -68,12 +68,16 @@ while [ $# -gt 0 ]; do esac done +# Bounded so a stalled repo connection (offsite mirror host unreachable +# mid-restore, etc.) can't hang forever and block every service behind it in +# the batch — 300s is generous for this stack's data sizes; raise it if a +# service's dataset genuinely needs longer. k_for() { local dest="$1"; shift local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD" local cfg="${!cfg_var:-}" pw="${!pw_var:-}" [ -n "$cfg" ] || return 1 - env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@" + timeout 300 env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@" } # ── Discover the latest restorable snapshot per service, across every @@ -172,8 +176,8 @@ for i in "${!SRC_PATH_LIST[@]}"; do mkdir -p "$path" if ! k_for "$dest" restore "$snap" "$path"; then - err " Restore failed for $svc" - FAILED_SVCS+=("$svc: restore failed") + err " Restore failed (or timed out) for $svc" + FAILED_SVCS+=("$svc: restore failed or timed out") continue fi chown -R "$ACTUAL_USER:$ACTUAL_USER" "$path" 2>/dev/null || true @@ -185,11 +189,15 @@ for i in "${!SRC_PATH_LIST[@]}"; do FAILED_SVCS+=("$svc: no compose file") continue fi - if docker compose -f "$compose" up -d 2>/dev/null; then + # Bounded so a stuck image pull or a compose file waiting on something + # (e.g. an interactive prompt) can't stall the rest of the batch — one + # bad service should never cost the others their spot in the 10-minute + # budget this script exists for. + if timeout 120 docker compose -f "$compose" up -d 2>/dev/null; then ok " Started" else - err " docker compose up -d failed for $svc" - FAILED_SVCS+=("$svc: compose up failed") + err " docker compose up -d failed (or timed out) for $svc" + FAILED_SVCS+=("$svc: compose up failed or timed out") continue fi fi @@ -218,9 +226,16 @@ if [ "${#UP_SVCS[@]}" -gt 0 ]; then fi if [ "${#FAILED_SVCS[@]}" -gt 0 ]; then - echo " Failed:" + echo " Failed (skipped — did not stop the rest of the batch):" for s in "${FAILED_SVCS[@]}"; do echo " ✗ $s"; done echo "" +fi + +# One bad service is a partial success, not a failed run — the whole point of +# this script is getting as much of the stack back up as possible. Only a +# fully empty result (nothing came up at all) counts as a failed exit code. +if [ "$DRY" = false ] && [ "${#UP_SVCS[@]}" -eq 0 ]; then + err "Nothing came up." exit 1 fi diff --git a/services/backup.sh b/services/backup.sh index 5535506..c32b388 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -452,6 +452,40 @@ install_backup() { if [ -n "$NTFY_URL" ]; then prompt_text " ntfy access token (blank if public/no auth):" "" NTFY_TOKEN fi + + # ── Disaster-recovery spare box (optional) ──────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " DISASTER-RECOVERY SPARE (optional)" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " If you keep a spare box ready to take over on failure (running" + echo " dr_bringup.sh), this can push backup.conf + README.md to it after" + echo " every successful backup, so it's always ready without a manual copy." + echo "" + echo " Requires passwordless SSH (key-based) from THIS box to the spare," + echo " as the account below. Since the backup timer runs as root, that" + echo " usually means a key in /root/.ssh authorized on the spare — set that" + echo " up first if you haven't (ssh-keygen, then ssh-copy-id to the spare)." + echo "" + local DR_SYNC_HOST="" DR_SYNC_PATH="" + prompt_text " Spare box SSH destination, user@host (blank to skip):" "" DR_SYNC_HOST + if [ -n "$DR_SYNC_HOST" ]; then + prompt_text " Path for backup.conf/README on the spare:" "~/docker/backup" DR_SYNC_PATH + DR_SYNC_PATH="${DR_SYNC_PATH:-~/docker/backup}" + + # Catch a missing/unauthorized key now, not at 2am during the first + # scheduled backup. Non-fatal either way — the setting is saved + # regardless, since the key may simply not be set up yet. + if ssh -o BatchMode=yes -o ConnectTimeout=5 "$DR_SYNC_HOST" true 2>/dev/null; then + log_success " SSH to $DR_SYNC_HOST works — spare sync will run after each backup." + else + log_warning " Couldn't SSH to $DR_SYNC_HOST without a password right now." + log_warning " Spare sync is saved but will fail until this works (as root, since" + log_warning " the backup timer runs as root): ssh-keygen; ssh-copy-id $DR_SYNC_HOST" + fi + fi + mkdir -p "$DIR" ensure_docker_dir_ownership "$DIR" @@ -531,6 +565,14 @@ install_backup() { echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)." printf "NTFY_URL='%s'\n" "${NTFY_URL:-}" printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}" + echo "" + echo "# ── Disaster-recovery spare sync ───────────────────────────────────────────" + echo "# If set, backup_kopia.sh scp's this backup.conf + README.md to" + echo "# DR_SYNC_HOST:DR_SYNC_PATH after every successful backup, so a spare box" + echo "# running dr_bringup.sh is always ready with no manual copy step. Requires" + echo "# passwordless SSH from this box to the spare (see README.md)." + printf "DR_SYNC_HOST='%s'\n" "${DR_SYNC_HOST:-}" + printf "DR_SYNC_PATH='%s'\n" "${DR_SYNC_PATH:-~/docker/backup}" } > "$CONF_FILE" chown root:root "$CONF_FILE" 2>/dev/null || true chmod 600 "$CONF_FILE" @@ -676,6 +718,106 @@ SVCEOF AUTORUN="cat /etc/cron.d/${SVC_NAME}" fi + # ── Write README ───────────────────────────────────────────────────────── + # Written before the optional first run below so, if DR sync is enabled, + # the very first backup already ships an up-to-date README to the spare. + local DEST_LIST_MD="" + for dn in "${DEST_NAMES_ARR[@]}"; do + DEST_LIST_MD+="- **${dn}**: ${DEST_REPOS[$dn]}"$'\n' + done + + local DR_SYNC_MD OFFSITE_MD + if [ -n "${DR_SYNC_HOST:-}" ]; then + DR_SYNC_MD="Configured: after every successful backup, this box copies backup.conf + this README to \`${DR_SYNC_HOST}:${DR_SYNC_PATH:-~/docker/backup}\` over SSH." + else + DR_SYNC_MD="Not configured. Re-run this installer to set it up, or copy backup.conf to the spare manually whenever it changes." + fi + if [ "${REMOTE_TYPE:-none}" != "none" ]; then + OFFSITE_MD="Configured: every backup also runs \`kopia repository sync-to ${REMOTE_TYPE}\` to mirror the repo off this box." + else + OFFSITE_MD="Not configured. Set REMOTE_TYPE/REMOTE_ARGS in backup.conf (see the comment above them) to mirror the repo off this box." + fi + + write_readme "$DIR" << MD +# Backup — Kopia + +Full recovery for every Docker service under \`$DOCKER_DIR\`: each service's +entire directory (compose file, \`.env\`, config, data, databases) is +snapshotted with Kopia — deduplicated, compressed (zstd), and encrypted. +Databases are captured consistently (container stopped briefly, snapshotted, +restarted); Minecraft instead gets a live save-all flush, no downtime. + +## Destinations + +$DEST_LIST_MD +## Schedule + +$SCHED_LABEL — keeps the latest $KEEP_LATEST snapshots (plus 7 daily / 4 +weekly / 3 monthly). + +## Commands + +\`\`\`bash +sudo $WORKER # back up now +sudo $WORKER snapshots # list all snapshots +sudo $WORKER policy # show retention policy +\`\`\` + +### Restore — interactive, one service at a time + +\`\`\`bash +sudo $RESTORE +sudo $RESTORE --list +\`\`\` + +### Disaster recovery — unattended, every service, for a cold spare box + +\`\`\`bash +sudo $DR_BRINGUP # restore + start everything +sudo $DR_BRINGUP --list # list what's restorable +sudo $DR_BRINGUP --dry-run # preview, touch nothing +sudo $DR_BRINGUP --service NAME # just one service +\`\`\` + +A single service failing to restore or start does not stop the rest of the +batch — it's logged and skipped so the run maximizes what actually comes +back up. The exit code is only non-zero if nothing came up at all. + +**On the spare box**, \`dr_bringup.sh\` needs \`backup.conf\` from this +directory to connect to the repo — see the DR spare sync section below. + +## Disaster-recovery spare sync + +$DR_SYNC_MD + +Requires passwordless SSH (key-based) from this box to the spare — since the +backup timer runs as root, generate/authorize a key for root: +\`ssh-keygen\`, then \`ssh-copy-id\` to the spare. + +## Offsite mirror + +$OFFSITE_MD + +## Backup test — stop / restore / compare / restore-back + +\`\`\`bash +sudo $TEST_SCRIPT # test most recent backup, all services +sudo $TEST_SCRIPT --list # list testable services +sudo $TEST_SCRIPT --service NAME +\`\`\` + +## Files + +- \`backup.conf\` — destinations, passwords, retention, DR-sync/offsite settings (chmod 600) +- \`backup_kopia.sh\` — the worker the systemd timer runs +- \`restore_kopia.sh\` — interactive restore +- \`dr_bringup.sh\` — unattended full-stack restore + start +- \`test_backup_kopia.sh\` / \`test_backup.sh\` — automated restore tests + +**Save the passwords in \`backup.conf\` somewhere safe** — without them the +encrypted repos cannot be restored. +MD + # ── 12. Optional first run ──────────────────────────────────────────────── echo "" local _now="" @@ -717,8 +859,15 @@ SVCEOF echo " sudo $DR_BRINGUP restore + start everything" echo " sudo $DR_BRINGUP --list list what's restorable" echo " sudo $DR_BRINGUP --dry-run preview, touch nothing" - echo " Copy backup.conf to the spare box first — it holds the repo path(s)" - echo " and password(s) this needs to connect." + if [ -n "${DR_SYNC_HOST:-}" ]; then + echo " backup.conf + README.md sync to $DR_SYNC_HOST after every backup — the" + echo " spare stays ready with no manual copy step." + else + echo " Copy backup.conf to the spare box first — it holds the repo path(s)" + echo " and password(s) this needs to connect." + fi + echo "" + echo " Full docs: $DIR/README.md" echo "" echo " Backup test (stop/restore/compare/restore-back):" echo " sudo $TEST_SCRIPT test most recent backup (all services)"