Sync backup.conf/README to a spare box; make DR bring-up fault-tolerant

- dr_bringup.sh: bound every kopia call and docker compose up with a
  timeout so one stuck service can't stall the rest of the batch, and
  only exit non-zero if literally nothing came up — a partial recovery
  is a partial success, not a failed run.
- backup_kopia.sh: optional DR_SYNC_HOST/DR_SYNC_PATH in backup.conf
  scp's backup.conf + README.md to a spare box over SSH after every
  successful backup, so dr_bringup.sh is ready there with no manual
  copy step.
- backup.sh: prompts for the spare's SSH destination, verifies
  connectivity at install time instead of failing silently at 2am, and
  writes ~/docker/backup/README.md (this service never had one) so the
  synced copy documents every command listed above.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NQkdAn3iG5A4WoqU9FHMaN

extras/backup_kopia.sh

@@ -170,6 +170,29 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
     done
 fi
 
+# ── Keep a spare box's copy of backup.conf + README current ─────────────────
+# Runs after the data itself is backed up (and mirrored, if configured) so a
+# sync never ships config pointing at a repo state that isn't actually there
+# yet. dr_bringup.sh on the spare only needs these two small files — the
+# repo data itself already lives wherever REMOTE_TYPE mirrored it (or is
+# local, if the spare IS that target).
+if [ -n "${DR_SYNC_HOST:-}" ]; then
+    _dr_path="${DR_SYNC_PATH:-~/docker/backup}"
+    log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..."
+    _dr_files=("$CONF")
+    [ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md")
+    if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \
+        && scp -o BatchMode=yes -o ConnectTimeout=10 "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \
+        && ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then
+        log "OK spare sync ($DR_SYNC_HOST)"
+    else
+        _reason="$(categorize_error "$(cat "$_ERR")")"
+        log "WARNING: spare sync failed — $_reason"
+        FAILED_SVCS+=("spare-sync: $_reason")
+        rc=1
+    fi
+fi
+
 DURATION=$(( $(date +%s) - START_TS ))
 DURATION_STR="$((DURATION/60))m $((DURATION%60))s"
 

extras/dr_bringup_kopia.sh

@@ -68,12 +68,16 @@ while [ $# -gt 0 ]; do
     esac
 done
 
+# Bounded so a stalled repo connection (offsite mirror host unreachable
+# mid-restore, etc.) can't hang forever and block every service behind it in
+# the batch — 300s is generous for this stack's data sizes; raise it if a
+# service's dataset genuinely needs longer.
 k_for() {
     local dest="$1"; shift
     local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD"
     local cfg="${!cfg_var:-}" pw="${!pw_var:-}"
     [ -n "$cfg" ] || return 1
-    env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
+    timeout 300 env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
 }
 
 # ── Discover the latest restorable snapshot per service, across every
@@ -172,8 +176,8 @@ for i in "${!SRC_PATH_LIST[@]}"; do
     mkdir -p "$path"
 
     if ! k_for "$dest" restore "$snap" "$path"; then
-        err "  Restore failed for $svc"
+        err "  Restore failed (or timed out) for $svc"
-        FAILED_SVCS+=("$svc: restore failed")
+        FAILED_SVCS+=("$svc: restore failed or timed out")
         continue
     fi
     chown -R "$ACTUAL_USER:$ACTUAL_USER" "$path" 2>/dev/null || true
@@ -185,11 +189,15 @@ for i in "${!SRC_PATH_LIST[@]}"; do
             FAILED_SVCS+=("$svc: no compose file")
             continue
         fi
-        if docker compose -f "$compose" up -d 2>/dev/null; then
+        # Bounded so a stuck image pull or a compose file waiting on something
+        # (e.g. an interactive prompt) can't stall the rest of the batch — one
+        # bad service should never cost the others their spot in the 10-minute
+        # budget this script exists for.
+        if timeout 120 docker compose -f "$compose" up -d 2>/dev/null; then
             ok "  Started"
         else
-            err "  docker compose up -d failed for $svc"
+            err "  docker compose up -d failed (or timed out) for $svc"
-            FAILED_SVCS+=("$svc: compose up failed")
+            FAILED_SVCS+=("$svc: compose up failed or timed out")
             continue
         fi
     fi
@@ -218,9 +226,16 @@ if [ "${#UP_SVCS[@]}" -gt 0 ]; then
 fi
 
 if [ "${#FAILED_SVCS[@]}" -gt 0 ]; then
-    echo "  Failed:"
+    echo "  Failed (skipped — did not stop the rest of the batch):"
     for s in "${FAILED_SVCS[@]}"; do echo "    ✗ $s"; done
     echo ""
+fi
+
+# One bad service is a partial success, not a failed run — the whole point of
+# this script is getting as much of the stack back up as possible. Only a
+# fully empty result (nothing came up at all) counts as a failed exit code.
+if [ "$DRY" = false ] && [ "${#UP_SVCS[@]}" -eq 0 ]; then
+    err "Nothing came up."
     exit 1
 fi
 

services/backup.sh

@@ -452,6 +452,40 @@ install_backup() {
     if [ -n "$NTFY_URL" ]; then
         prompt_text "  ntfy access token (blank if public/no auth):" "" NTFY_TOKEN
     fi
+
+    # ── Disaster-recovery spare box (optional) ────────────────────────────────
+    echo ""
+    echo "═══════════════════════════════════════════════════════"
+    echo "  DISASTER-RECOVERY SPARE (optional)"
+    echo "═══════════════════════════════════════════════════════"
+    echo ""
+    echo "  If you keep a spare box ready to take over on failure (running"
+    echo "  dr_bringup.sh), this can push backup.conf + README.md to it after"
+    echo "  every successful backup, so it's always ready without a manual copy."
+    echo ""
+    echo "  Requires passwordless SSH (key-based) from THIS box to the spare,"
+    echo "  as the account below. Since the backup timer runs as root, that"
+    echo "  usually means a key in /root/.ssh authorized on the spare — set that"
+    echo "  up first if you haven't (ssh-keygen, then ssh-copy-id to the spare)."
+    echo ""
+    local DR_SYNC_HOST="" DR_SYNC_PATH=""
+    prompt_text "  Spare box SSH destination, user@host (blank to skip):" "" DR_SYNC_HOST
+    if [ -n "$DR_SYNC_HOST" ]; then
+        prompt_text "  Path for backup.conf/README on the spare:" "~/docker/backup" DR_SYNC_PATH
+        DR_SYNC_PATH="${DR_SYNC_PATH:-~/docker/backup}"
+
+        # Catch a missing/unauthorized key now, not at 2am during the first
+        # scheduled backup. Non-fatal either way — the setting is saved
+        # regardless, since the key may simply not be set up yet.
+        if ssh -o BatchMode=yes -o ConnectTimeout=5 "$DR_SYNC_HOST" true 2>/dev/null; then
+            log_success "  SSH to $DR_SYNC_HOST works — spare sync will run after each backup."
+        else
+            log_warning "  Couldn't SSH to $DR_SYNC_HOST without a password right now."
+            log_warning "  Spare sync is saved but will fail until this works (as root, since"
+            log_warning "  the backup timer runs as root): ssh-keygen; ssh-copy-id $DR_SYNC_HOST"
+        fi
+    fi
+
     mkdir -p "$DIR"
     ensure_docker_dir_ownership "$DIR"
 
@@ -531,6 +565,14 @@ install_backup() {
         echo "# Leave blank to disable. NTFY_TOKEN is optional (for private topics)."
         printf "NTFY_URL='%s'\n" "${NTFY_URL:-}"
         printf "NTFY_TOKEN='%s'\n" "${NTFY_TOKEN:-}"
+        echo ""
+        echo "# ── Disaster-recovery spare sync ───────────────────────────────────────────"
+        echo "# If set, backup_kopia.sh scp's this backup.conf + README.md to"
+        echo "# DR_SYNC_HOST:DR_SYNC_PATH after every successful backup, so a spare box"
+        echo "# running dr_bringup.sh is always ready with no manual copy step. Requires"
+        echo "# passwordless SSH from this box to the spare (see README.md)."
+        printf "DR_SYNC_HOST='%s'\n" "${DR_SYNC_HOST:-}"
+        printf "DR_SYNC_PATH='%s'\n" "${DR_SYNC_PATH:-~/docker/backup}"
     } > "$CONF_FILE"
     chown root:root "$CONF_FILE" 2>/dev/null || true
     chmod 600 "$CONF_FILE"
@@ -676,6 +718,106 @@ SVCEOF
         AUTORUN="cat /etc/cron.d/${SVC_NAME}"
     fi
 
+    # ── Write README ─────────────────────────────────────────────────────────
+    # Written before the optional first run below so, if DR sync is enabled,
+    # the very first backup already ships an up-to-date README to the spare.
+    local DEST_LIST_MD=""
+    for dn in "${DEST_NAMES_ARR[@]}"; do
+        DEST_LIST_MD+="- **${dn}**: ${DEST_REPOS[$dn]}"$'\n'
+    done
+
+    local DR_SYNC_MD OFFSITE_MD
+    if [ -n "${DR_SYNC_HOST:-}" ]; then
+        DR_SYNC_MD="Configured: after every successful backup, this box copies backup.conf + this README to \`${DR_SYNC_HOST}:${DR_SYNC_PATH:-~/docker/backup}\` over SSH."
+    else
+        DR_SYNC_MD="Not configured. Re-run this installer to set it up, or copy backup.conf to the spare manually whenever it changes."
+    fi
+    if [ "${REMOTE_TYPE:-none}" != "none" ]; then
+        OFFSITE_MD="Configured: every backup also runs \`kopia repository sync-to ${REMOTE_TYPE}\` to mirror the repo off this box."
+    else
+        OFFSITE_MD="Not configured. Set REMOTE_TYPE/REMOTE_ARGS in backup.conf (see the comment above them) to mirror the repo off this box."
+    fi
+
+    write_readme "$DIR" << MD
+# Backup — Kopia
+
+Full recovery for every Docker service under \`$DOCKER_DIR\`: each service's
+entire directory (compose file, \`.env\`, config, data, databases) is
+snapshotted with Kopia — deduplicated, compressed (zstd), and encrypted.
+Databases are captured consistently (container stopped briefly, snapshotted,
+restarted); Minecraft instead gets a live save-all flush, no downtime.
+
+## Destinations
+
+$DEST_LIST_MD
+## Schedule
+
+$SCHED_LABEL — keeps the latest $KEEP_LATEST snapshots (plus 7 daily / 4
+weekly / 3 monthly).
+
+## Commands
+
+\`\`\`bash
+sudo $WORKER                     # back up now
+sudo $WORKER snapshots           # list all snapshots
+sudo $WORKER policy              # show retention policy
+\`\`\`
+
+### Restore — interactive, one service at a time
+
+\`\`\`bash
+sudo $RESTORE
+sudo $RESTORE --list
+\`\`\`
+
+### Disaster recovery — unattended, every service, for a cold spare box
+
+\`\`\`bash
+sudo $DR_BRINGUP                 # restore + start everything
+sudo $DR_BRINGUP --list          # list what's restorable
+sudo $DR_BRINGUP --dry-run       # preview, touch nothing
+sudo $DR_BRINGUP --service NAME  # just one service
+\`\`\`
+
+A single service failing to restore or start does not stop the rest of the
+batch — it's logged and skipped so the run maximizes what actually comes
+back up. The exit code is only non-zero if nothing came up at all.
+
+**On the spare box**, \`dr_bringup.sh\` needs \`backup.conf\` from this
+directory to connect to the repo — see the DR spare sync section below.
+
+## Disaster-recovery spare sync
+
+$DR_SYNC_MD
+
+Requires passwordless SSH (key-based) from this box to the spare — since the
+backup timer runs as root, generate/authorize a key for root:
+\`ssh-keygen\`, then \`ssh-copy-id\` to the spare.
+
+## Offsite mirror
+
+$OFFSITE_MD
+
+## Backup test — stop / restore / compare / restore-back
+
+\`\`\`bash
+sudo $TEST_SCRIPT                # test most recent backup, all services
+sudo $TEST_SCRIPT --list         # list testable services
+sudo $TEST_SCRIPT --service NAME
+\`\`\`
+
+## Files
+
+- \`backup.conf\` — destinations, passwords, retention, DR-sync/offsite settings (chmod 600)
+- \`backup_kopia.sh\` — the worker the systemd timer runs
+- \`restore_kopia.sh\` — interactive restore
+- \`dr_bringup.sh\` — unattended full-stack restore + start
+- \`test_backup_kopia.sh\` / \`test_backup.sh\` — automated restore tests
+
+**Save the passwords in \`backup.conf\` somewhere safe** — without them the
+encrypted repos cannot be restored.
+MD
+
     # ── 12. Optional first run ────────────────────────────────────────────────
     echo ""
     local _now=""
@@ -717,8 +859,15 @@ SVCEOF
     echo "    sudo $DR_BRINGUP              restore + start everything"
     echo "    sudo $DR_BRINGUP --list       list what's restorable"
     echo "    sudo $DR_BRINGUP --dry-run    preview, touch nothing"
-    echo "    Copy backup.conf to the spare box first — it holds the repo path(s)"
+    if [ -n "${DR_SYNC_HOST:-}" ]; then
-    echo "    and password(s) this needs to connect."
+        echo "    backup.conf + README.md sync to $DR_SYNC_HOST after every backup — the"
+        echo "    spare stays ready with no manual copy step."
+    else
+        echo "    Copy backup.conf to the spare box first — it holds the repo path(s)"
+        echo "    and password(s) this needs to connect."
+    fi
+    echo ""
+    echo "  Full docs: $DIR/README.md"
     echo ""
     echo "  Backup test (stop/restore/compare/restore-back):"
     echo "    sudo $TEST_SCRIPT                test most recent backup (all services)"