Sync backup.conf/README to a spare box; make DR bring-up fault-tolerant

- dr_bringup.sh: bound every kopia call and docker compose up with a
  timeout so one stuck service can't stall the rest of the batch, and
  only exit non-zero if literally nothing came up — a partial recovery
  is a partial success, not a failed run.
- backup_kopia.sh: optional DR_SYNC_HOST/DR_SYNC_PATH in backup.conf
  scp's backup.conf + README.md to a spare box over SSH after every
  successful backup, so dr_bringup.sh is ready there with no manual
  copy step.
- backup.sh: prompts for the spare's SSH destination, verifies
  connectivity at install time instead of failing silently at 2am, and
  writes ~/docker/backup/README.md (this service never had one) so the
  synced copy documents every command listed above.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NQkdAn3iG5A4WoqU9FHMaN

extras/backup_kopia.sh

@@ -170,6 +170,29 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
     done
 fi
 
+# ── Keep a spare box's copy of backup.conf + README current ─────────────────
+# Runs after the data itself is backed up (and mirrored, if configured) so a
+# sync never ships config pointing at a repo state that isn't actually there
+# yet. dr_bringup.sh on the spare only needs these two small files — the
+# repo data itself already lives wherever REMOTE_TYPE mirrored it (or is
+# local, if the spare IS that target).
+if [ -n "${DR_SYNC_HOST:-}" ]; then
+    _dr_path="${DR_SYNC_PATH:-~/docker/backup}"
+    log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..."
+    _dr_files=("$CONF")
+    [ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md")
+    if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \
+        && scp -o BatchMode=yes -o ConnectTimeout=10 "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \
+        && ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then
+        log "OK spare sync ($DR_SYNC_HOST)"
+    else
+        _reason="$(categorize_error "$(cat "$_ERR")")"
+        log "WARNING: spare sync failed — $_reason"
+        FAILED_SVCS+=("spare-sync: $_reason")
+        rc=1
+    fi
+fi
+
 DURATION=$(( $(date +%s) - START_TS ))
 DURATION_STR="$((DURATION/60))m $((DURATION%60))s"
 

extras/dr_bringup_kopia.sh

@@ -68,12 +68,16 @@ while [ $# -gt 0 ]; do
     esac
 done
 
+# Bounded so a stalled repo connection (offsite mirror host unreachable
+# mid-restore, etc.) can't hang forever and block every service behind it in
+# the batch — 300s is generous for this stack's data sizes; raise it if a
+# service's dataset genuinely needs longer.
 k_for() {
     local dest="$1"; shift
     local cfg_var="DEST_${dest}_CONFIG" pw_var="DEST_${dest}_PASSWORD"
     local cfg="${!cfg_var:-}" pw="${!pw_var:-}"
     [ -n "$cfg" ] || return 1
-    env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
+    timeout 300 env KOPIA_PASSWORD="$pw" "$KOPIA" --config-file="$cfg" "$@"
 }
 
 # ── Discover the latest restorable snapshot per service, across every
@@ -172,8 +176,8 @@ for i in "${!SRC_PATH_LIST[@]}"; do
     mkdir -p "$path"
 
     if ! k_for "$dest" restore "$snap" "$path"; then
-        err "  Restore failed for $svc"
-        FAILED_SVCS+=("$svc: restore failed")
+        err "  Restore failed (or timed out) for $svc"
+        FAILED_SVCS+=("$svc: restore failed or timed out")
         continue
     fi
     chown -R "$ACTUAL_USER:$ACTUAL_USER" "$path" 2>/dev/null || true
@@ -185,11 +189,15 @@ for i in "${!SRC_PATH_LIST[@]}"; do
             FAILED_SVCS+=("$svc: no compose file")
             continue
         fi
-        if docker compose -f "$compose" up -d 2>/dev/null; then
+        # Bounded so a stuck image pull or a compose file waiting on something
+        # (e.g. an interactive prompt) can't stall the rest of the batch — one
+        # bad service should never cost the others their spot in the 10-minute
+        # budget this script exists for.
+        if timeout 120 docker compose -f "$compose" up -d 2>/dev/null; then
             ok "  Started"
         else
-            err "  docker compose up -d failed for $svc"
-            FAILED_SVCS+=("$svc: compose up failed")
+            err "  docker compose up -d failed (or timed out) for $svc"
+            FAILED_SVCS+=("$svc: compose up failed or timed out")
             continue
         fi
     fi
@@ -218,9 +226,16 @@ if [ "${#UP_SVCS[@]}" -gt 0 ]; then
 fi
 
 if [ "${#FAILED_SVCS[@]}" -gt 0 ]; then
-    echo "  Failed:"
+    echo "  Failed (skipped — did not stop the rest of the batch):"
     for s in "${FAILED_SVCS[@]}"; do echo "    ✗ $s"; done
     echo ""
+fi
+
+# One bad service is a partial success, not a failed run — the whole point of
+# this script is getting as much of the stack back up as possible. Only a
+# fully empty result (nothing came up at all) counts as a failed exit code.
+if [ "$DRY" = false ] && [ "${#UP_SVCS[@]}" -eq 0 ]; then
+    err "Nothing came up."
     exit 1
 fi