Sync backup.conf/README to a spare box; make DR bring-up fault-tolerant
- dr_bringup.sh: bound every kopia call and docker compose up with a timeout so one stuck service can't stall the rest of the batch, and only exit non-zero if literally nothing came up — a partial recovery is a partial success, not a failed run. - backup_kopia.sh: optional DR_SYNC_HOST/DR_SYNC_PATH in backup.conf scp's backup.conf + README.md to a spare box over SSH after every successful backup, so dr_bringup.sh is ready there with no manual copy step. - backup.sh: prompts for the spare's SSH destination, verifies connectivity at install time instead of failing silently at 2am, and writes ~/docker/backup/README.md (this service never had one) so the synced copy documents every command listed above. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NQkdAn3iG5A4WoqU9FHMaN
This commit is contained in:
Claude
@@ -170,6 +170,29 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Keep a spare box's copy of backup.conf + README current ─────────────────
|
||||
# Runs after the data itself is backed up (and mirrored, if configured) so a
|
||||
# sync never ships config pointing at a repo state that isn't actually there
|
||||
# yet. dr_bringup.sh on the spare only needs these two small files — the
|
||||
# repo data itself already lives wherever REMOTE_TYPE mirrored it (or is
|
||||
# local, if the spare IS that target).
|
||||
if [ -n "${DR_SYNC_HOST:-}" ]; then
|
||||
_dr_path="${DR_SYNC_PATH:-~/docker/backup}"
|
||||
log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..."
|
||||
_dr_files=("$CONF")
|
||||
[ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md")
|
||||
if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \
|
||||
&& scp -o BatchMode=yes -o ConnectTimeout=10 "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \
|
||||
&& ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then
|
||||
log "OK spare sync ($DR_SYNC_HOST)"
|
||||
else
|
||||
_reason="$(categorize_error "$(cat "$_ERR")")"
|
||||
log "WARNING: spare sync failed — $_reason"
|
||||
FAILED_SVCS+=("spare-sync: $_reason")
|
||||
rc=1
|
||||
fi
|
||||
fi
|
||||
|
||||
DURATION=$(( $(date +%s) - START_TS ))
|
||||
DURATION_STR="$((DURATION/60))m $((DURATION%60))s"
|
||||
|
||||
|
||||
@@ -68,12 +68,16 @@ while [ $# -gt 0 ]; do
|
||||
esac
|
||||
done
|
||||
|
||||
# Bounded so a stalled repo connection (offsite mirror host unreachable
|
||||
# mid-restore, etc.) can't hang forever and block every service behind it in
|
||||
# the batch — 300s is generous for this stack's data sizes; raise it if a
|
||||
# service's dataset genuinely needs longer.
|
||||
k_for() {
|
||||
local dest="$1"; shift
|
||||
local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD"
|
||||
local cfg="${!cfg_var:-}" pw="${!pw_var:-}"
|
||||
[ -n "$cfg" ] || return 1
|
||||
env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
|
||||
timeout 300 env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
|
||||
}
|
||||
|
||||
# ── Discover the latest restorable snapshot per service, across every
|
||||
@@ -172,8 +176,8 @@ for i in "${!SRC_PATH_LIST[@]}"; do
|
||||
mkdir -p "$path"
|
||||
|
||||
if ! k_for "$dest" restore "$snap" "$path"; then
|
||||
err " Restore failed for $svc"
|
||||
FAILED_SVCS+=("$svc: restore failed")
|
||||
err " Restore failed (or timed out) for $svc"
|
||||
FAILED_SVCS+=("$svc: restore failed or timed out")
|
||||
continue
|
||||
fi
|
||||
chown -R "$ACTUAL_USER:$ACTUAL_USER" "$path" 2>/dev/null || true
|
||||
@@ -185,11 +189,15 @@ for i in "${!SRC_PATH_LIST[@]}"; do
|
||||
FAILED_SVCS+=("$svc: no compose file")
|
||||
continue
|
||||
fi
|
||||
if docker compose -f "$compose" up -d 2>/dev/null; then
|
||||
# Bounded so a stuck image pull or a compose file waiting on something
|
||||
# (e.g. an interactive prompt) can't stall the rest of the batch — one
|
||||
# bad service should never cost the others their spot in the 10-minute
|
||||
# budget this script exists for.
|
||||
if timeout 120 docker compose -f "$compose" up -d 2>/dev/null; then
|
||||
ok " Started"
|
||||
else
|
||||
err " docker compose up -d failed for $svc"
|
||||
FAILED_SVCS+=("$svc: compose up failed")
|
||||
err " docker compose up -d failed (or timed out) for $svc"
|
||||
FAILED_SVCS+=("$svc: compose up failed or timed out")
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
@@ -218,9 +226,16 @@ if [ "${#UP_SVCS[@]}" -gt 0 ]; then
|
||||
fi
|
||||
|
||||
if [ "${#FAILED_SVCS[@]}" -gt 0 ]; then
|
||||
echo " Failed:"
|
||||
echo " Failed (skipped — did not stop the rest of the batch):"
|
||||
for s in "${FAILED_SVCS[@]}"; do echo " ✗ $s"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# One bad service is a partial success, not a failed run — the whole point of
|
||||
# this script is getting as much of the stack back up as possible. Only a
|
||||
# fully empty result (nothing came up at all) counts as a failed exit code.
|
||||
if [ "$DRY" = false ] && [ "${#UP_SVCS[@]}" -eq 0 ]; then
|
||||
err "Nothing came up."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
+151
-2
@@ -452,6 +452,40 @@ install_backup() {
|
||||
if [ -n "$NTFY_URL" ]; then
|
||||
prompt_text " ntfy access token (blank if public/no auth):" "" NTFY_TOKEN
|
||||
fi
|
||||
|
||||
# ── Disaster-recovery spare box (optional) ────────────────────────────────
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
echo " DISASTER-RECOVERY SPARE (optional)"
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
echo " If you keep a spare box ready to take over on failure (running"
|
||||
echo " dr_bringup.sh), this can push backup.conf + README.md to it after"
|
||||
echo " every successful backup, so it's always ready without a manual copy."
|
||||
echo ""
|
||||
echo " Requires passwordless SSH (key-based) from THIS box to the spare,"
|
||||
echo " as the account below. Since the backup timer runs as root, that"
|
||||
echo " usually means a key in /root/.ssh authorized on the spare — set that"
|
||||
echo " up first if you haven't (ssh-keygen, then ssh-copy-id to the spare)."
|
||||
echo ""
|
||||
local DR_SYNC_HOST="" DR_SYNC_PATH=""
|
||||
prompt_text " Spare box SSH destination, user@host (blank to skip):" "" DR_SYNC_HOST
|
||||
if [ -n "$DR_SYNC_HOST" ]; then
|
||||
prompt_text " Path for backup.conf/README on the spare:" "~/docker/backup" DR_SYNC_PATH
|
||||
DR_SYNC_PATH="${DR_SYNC_PATH:-~/docker/backup}"
|
||||
|
||||
# Catch a missing/unauthorized key now, not at 2am during the first
|
||||
# scheduled backup. Non-fatal either way — the setting is saved
|
||||
# regardless, since the key may simply not be set up yet.
|
||||
if ssh -o BatchMode=yes -o ConnectTimeout=5 "$DR_SYNC_HOST" true 2>/dev/null; then
|
||||
log_success " SSH to $DR_SYNC_HOST works — spare sync will run after each backup."
|
||||
else
|
||||
log_warning " Couldn't SSH to $DR_SYNC_HOST without a password right now."
|
||||
log_warning " Spare sync is saved but will fail until this works (as root, since"
|
||||
log_warning " the backup timer runs as root): ssh-keygen; ssh-copy-id $DR_SYNC_HOST"
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "$DIR"
|
||||
ensure_docker_dir_ownership "$DIR"
|
||||
|
||||
@@ -531,6 +565,14 @@ install_backup() {
|
||||
echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)."
|
||||
printf "NTFY_URL='%s'\n" "${NTFY_URL:-}"
|
||||
printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}"
|
||||
echo ""
|
||||
echo "# ── Disaster-recovery spare sync ───────────────────────────────────────────"
|
||||
echo "# If set, backup_kopia.sh scp's this backup.conf + README.md to"
|
||||
echo "# DR_SYNC_HOST:DR_SYNC_PATH after every successful backup, so a spare box"
|
||||
echo "# running dr_bringup.sh is always ready with no manual copy step. Requires"
|
||||
echo "# passwordless SSH from this box to the spare (see README.md)."
|
||||
printf "DR_SYNC_HOST='%s'\n" "${DR_SYNC_HOST:-}"
|
||||
printf "DR_SYNC_PATH='%s'\n" "${DR_SYNC_PATH:-~/docker/backup}"
|
||||
} > "$CONF_FILE"
|
||||
chown root:root "$CONF_FILE" 2>/dev/null || true
|
||||
chmod 600 "$CONF_FILE"
|
||||
@@ -676,6 +718,106 @@ SVCEOF
|
||||
AUTORUN="cat /etc/cron.d/${SVC_NAME}"
|
||||
fi
|
||||
|
||||
# ── Write README ─────────────────────────────────────────────────────────
|
||||
# Written before the optional first run below so, if DR sync is enabled,
|
||||
# the very first backup already ships an up-to-date README to the spare.
|
||||
local DEST_LIST_MD=""
|
||||
for dn in "${DEST_NAMES_ARR[@]}"; do
|
||||
DEST_LIST_MD+="- **${dn}**: ${DEST_REPOS[$dn]}"$'\n'
|
||||
done
|
||||
|
||||
local DR_SYNC_MD OFFSITE_MD
|
||||
if [ -n "${DR_SYNC_HOST:-}" ]; then
|
||||
DR_SYNC_MD="Configured: after every successful backup, this box copies backup.conf + this README to \`${DR_SYNC_HOST}:${DR_SYNC_PATH:-~/docker/backup}\` over SSH."
|
||||
else
|
||||
DR_SYNC_MD="Not configured. Re-run this installer to set it up, or copy backup.conf to the spare manually whenever it changes."
|
||||
fi
|
||||
if [ "${REMOTE_TYPE:-none}" != "none" ]; then
|
||||
OFFSITE_MD="Configured: every backup also runs \`kopia repository sync-to ${REMOTE_TYPE}\` to mirror the repo off this box."
|
||||
else
|
||||
OFFSITE_MD="Not configured. Set REMOTE_TYPE/REMOTE_ARGS in backup.conf (see the comment above them) to mirror the repo off this box."
|
||||
fi
|
||||
|
||||
write_readme "$DIR" << MD
|
||||
# Backup — Kopia
|
||||
|
||||
Full recovery for every Docker service under \`$DOCKER_DIR\`: each service's
|
||||
entire directory (compose file, \`.env\`, config, data, databases) is
|
||||
snapshotted with Kopia — deduplicated, compressed (zstd), and encrypted.
|
||||
Databases are captured consistently (container stopped briefly, snapshotted,
|
||||
restarted); Minecraft instead gets a live save-all flush, no downtime.
|
||||
|
||||
## Destinations
|
||||
|
||||
$DEST_LIST_MD
|
||||
## Schedule
|
||||
|
||||
$SCHED_LABEL — keeps the latest $KEEP_LATEST snapshots (plus 7 daily / 4
|
||||
weekly / 3 monthly).
|
||||
|
||||
## Commands
|
||||
|
||||
\`\`\`bash
|
||||
sudo $WORKER # back up now
|
||||
sudo $WORKER snapshots # list all snapshots
|
||||
sudo $WORKER policy # show retention policy
|
||||
\`\`\`
|
||||
|
||||
### Restore — interactive, one service at a time
|
||||
|
||||
\`\`\`bash
|
||||
sudo $RESTORE
|
||||
sudo $RESTORE --list
|
||||
\`\`\`
|
||||
|
||||
### Disaster recovery — unattended, every service, for a cold spare box
|
||||
|
||||
\`\`\`bash
|
||||
sudo $DR_BRINGUP # restore + start everything
|
||||
sudo $DR_BRINGUP --list # list what's restorable
|
||||
sudo $DR_BRINGUP --dry-run # preview, touch nothing
|
||||
sudo $DR_BRINGUP --service NAME # just one service
|
||||
\`\`\`
|
||||
|
||||
A single service failing to restore or start does not stop the rest of the
|
||||
batch — it's logged and skipped so the run maximizes what actually comes
|
||||
back up. The exit code is only non-zero if nothing came up at all.
|
||||
|
||||
**On the spare box**, \`dr_bringup.sh\` needs \`backup.conf\` from this
|
||||
directory to connect to the repo — see the DR spare sync section below.
|
||||
|
||||
## Disaster-recovery spare sync
|
||||
|
||||
$DR_SYNC_MD
|
||||
|
||||
Requires passwordless SSH (key-based) from this box to the spare — since the
|
||||
backup timer runs as root, generate/authorize a key for root:
|
||||
\`ssh-keygen\`, then \`ssh-copy-id\` to the spare.
|
||||
|
||||
## Offsite mirror
|
||||
|
||||
$OFFSITE_MD
|
||||
|
||||
## Backup test — stop / restore / compare / restore-back
|
||||
|
||||
\`\`\`bash
|
||||
sudo $TEST_SCRIPT # test most recent backup, all services
|
||||
sudo $TEST_SCRIPT --list # list testable services
|
||||
sudo $TEST_SCRIPT --service NAME
|
||||
\`\`\`
|
||||
|
||||
## Files
|
||||
|
||||
- \`backup.conf\` — destinations, passwords, retention, DR-sync/offsite settings (chmod 600)
|
||||
- \`backup_kopia.sh\` — the worker the systemd timer runs
|
||||
- \`restore_kopia.sh\` — interactive restore
|
||||
- \`dr_bringup.sh\` — unattended full-stack restore + start
|
||||
- \`test_backup_kopia.sh\` / \`test_backup.sh\` — automated restore tests
|
||||
|
||||
**Save the passwords in \`backup.conf\` somewhere safe** — without them the
|
||||
encrypted repos cannot be restored.
|
||||
MD
|
||||
|
||||
# ── 12. Optional first run ────────────────────────────────────────────────
|
||||
echo ""
|
||||
local _now=""
|
||||
@@ -717,8 +859,15 @@ SVCEOF
|
||||
echo " sudo $DR_BRINGUP restore + start everything"
|
||||
echo " sudo $DR_BRINGUP --list list what's restorable"
|
||||
echo " sudo $DR_BRINGUP --dry-run preview, touch nothing"
|
||||
echo " Copy backup.conf to the spare box first — it holds the repo path(s)"
|
||||
echo " and password(s) this needs to connect."
|
||||
if [ -n "${DR_SYNC_HOST:-}" ]; then
|
||||
echo " backup.conf + README.md sync to $DR_SYNC_HOST after every backup — the"
|
||||
echo " spare stays ready with no manual copy step."
|
||||
else
|
||||
echo " Copy backup.conf to the spare box first — it holds the repo path(s)"
|
||||
echo " and password(s) this needs to connect."
|
||||
fi
|
||||
echo ""
|
||||
echo " Full docs: $DIR/README.md"
|
||||
echo ""
|
||||
echo " Backup test (stop/restore/compare/restore-back):"
|
||||
echo " sudo $TEST_SCRIPT test most recent backup (all services)"
|
||||
|
||||
Reference in New Issue
Block a user