From 9c5d8c32f70c0b8659396c39a212c1224f24d710 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 13:11:44 +0000 Subject: [PATCH 1/5] Reuse the sudo user's SSH key for root; make B2 rejection unambiguous MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two separate fixes from a live report. 1. The DR-spare and SFTP-mirror sections both checked ONLY /root/.ssh for a key, missing the common case: the person running `sudo ./setup.sh backup` already has a key under their own home directory (used interactively, quite possibly already authorized on the target box), while root — who actually runs the scheduled systemd service — has none. Confirmed live: "the computer has the ssh key for the sudo user on the box" produced "No SSH key found for root" with no inline way to do anything about it beyond a pointer to go set one up elsewhere and re-run. Factored both call sites into one shared _backup_ensure_root_ssh_key() that checks root first, then offers to reuse the sudo user's existing keypair (copied into /root/.ssh with correct ownership/permissions, root:root 600) before falling back to generating a brand new one — reusing an existing key can work immediately if it's already authorized on the target, where a fresh key needs a new ssh-copy-id round-trip regardless. Verified all three branches (root already has a key, root has none but the user does and accepts reuse, neither exists and one gets generated) against a mocked filesystem. 2. The B2 dry-run failure message read like it could be about missing input even when every field was non-empty — confirmed there's no code path where non-blank-but-wrong values actually trigger the separate "Left blank" message (the two are on disjoint branches), but the dry-run failure text itself didn't rule that out or point at the actual likely cause. Now echoes back what was entered (bucket, endpoint, Key ID — never the secret) so it's easy to eyeball against B2's own confirmation screen, states plainly that this is a rejection of non-blank input, and names the most likely cause directly: pairing the Key ID from one Application Key with the Secret from a different one, which is easy to do after creating more than one while troubleshooting. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/backup.sh | 93 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 18 deletions(-) diff --git a/services/backup.sh b/services/backup.sh index 8509147..543c778 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -207,6 +207,60 @@ fi register_service backup backup "Encrypted backup of all Docker services (full restore)" +# Ensures root has an SSH key usable for the systemd-run (root, unattended) +# backup/mirror steps. The DR-spare and SFTP-mirror prompts both need one — +# both used to check ONLY /root/.ssh, missing the common case where +# whoever ran `sudo ./setup.sh backup` already has a key under their OWN +# home directory (used interactively, quite possibly already authorized on +# the target box) while root — who actually runs the scheduled service — +# has none. Prefers reusing that existing keypair over minting a fresh +# one, since the existing one may already be trusted where it's needed; +# ssh-copy-id-ing a brand new key is the fallback, not the first move. +# Sets _ROOT_SSH_KEYFILE (out-param, not local) to the resulting keyfile +# path, empty if none is available/created. +_backup_ensure_root_ssh_key() { + _ROOT_SSH_KEYFILE="" + if [ -f /root/.ssh/id_ed25519 ]; then + _ROOT_SSH_KEYFILE=/root/.ssh/id_ed25519; return 0 + fi + if [ -f /root/.ssh/id_rsa ]; then + _ROOT_SSH_KEYFILE=/root/.ssh/id_rsa; return 0 + fi + + local _user_key="" + [ -f "$ACTUAL_HOME/.ssh/id_ed25519" ] && _user_key="$ACTUAL_HOME/.ssh/id_ed25519" + [ -z "$_user_key" ] && [ -f "$ACTUAL_HOME/.ssh/id_rsa" ] && _user_key="$ACTUAL_HOME/.ssh/id_rsa" + + if [ -n "$_user_key" ]; then + local _COPY_USER_KEY="" + prompt_yn " No SSH key for root, but $ACTUAL_USER has one ($_user_key) — reuse it for root too (it may already be authorized where you need it)? (y/n):" "y" _COPY_USER_KEY + if [[ "$_COPY_USER_KEY" =~ ^[Yy]$ ]]; then + mkdir -p /root/.ssh && chmod 700 /root/.ssh + local _keyname; _keyname="$(basename "$_user_key")" + cp "$_user_key" "/root/.ssh/$_keyname" + [ -f "${_user_key}.pub" ] && cp "${_user_key}.pub" "/root/.ssh/${_keyname}.pub" + chown root:root "/root/.ssh/$_keyname" "/root/.ssh/${_keyname}.pub" 2>/dev/null + chmod 600 "/root/.ssh/$_keyname" + [ -f "/root/.ssh/${_keyname}.pub" ] && chmod 644 "/root/.ssh/${_keyname}.pub" + log_success " Copied $_user_key to /root/.ssh/ for root's use." + _ROOT_SSH_KEYFILE="/root/.ssh/$_keyname" + return 0 + fi + fi + + local _GEN_KEY="" + prompt_yn " Generate a new SSH key for root (ssh-keygen)? (y/n):" "y" _GEN_KEY + if [[ "$_GEN_KEY" =~ ^[Yy]$ ]]; then + mkdir -p /root/.ssh && chmod 700 /root/.ssh + if ssh-keygen -t ed25519 -N "" -f /root/.ssh/id_ed25519 -q; then + log_success " Generated /root/.ssh/id_ed25519" + _ROOT_SSH_KEYFILE=/root/.ssh/id_ed25519 + else + log_warning " ssh-keygen failed — generate one manually." + fi + fi +} + install_backup() { require_docker || return 1 @@ -550,17 +604,7 @@ install_backup() { esac fi - local _HAVE_KEY=false - [ -f /root/.ssh/id_ed25519 ] || [ -f /root/.ssh/id_rsa ] && _HAVE_KEY=true - if [ "$_HAVE_KEY" = false ]; then - local _GEN_KEY="" - prompt_yn " No SSH key found for root — generate one now (ssh-keygen)? (y/n):" "y" _GEN_KEY - if [[ "$_GEN_KEY" =~ ^[Yy]$ ]]; then - ssh-keygen -t ed25519 -N "" -f /root/.ssh/id_ed25519 -q \ - && log_success " Generated /root/.ssh/id_ed25519" \ - || log_warning " ssh-keygen failed — generate one manually." - fi - fi + _backup_ensure_root_ssh_key local _COPY_KEY="" prompt_yn " Run ssh-copy-id to $DR_SYNC_HOST now? (asks for its login password interactively) (y/n):" "y" _COPY_KEY @@ -724,7 +768,23 @@ install_backup() { REMOTE_ARGS="--bucket=$B2_BUCKET --access-key=$B2_KEY_ID --secret-access-key=$B2_APP_KEY --endpoint=$B2_ENDPOINT" log_success "B2 credentials verified — offsite mirroring will run after each backup." else - log_warning "B2 dry-run failed — check bucket name, endpoint, and key permissions:" + # All four fields were non-empty (the blank-field check above + # already ruled that out) — this is B2 rejecting what was + # entered, not missing input. Echoing back what was actually + # used (never the secret) so it's easy to eyeball against + # B2's own confirmation screen — the most common cause here + # is pairing the Key ID from one Application Key with the + # Secret from a different one (e.g. after creating more than + # one while troubleshooting). + log_warning "B2 rejected these credentials — bucket, endpoint, or key mismatch" + log_warning "(not blank input — all four fields were entered):" + log_warning " Bucket: $B2_BUCKET" + log_warning " Endpoint: $B2_ENDPOINT" + log_warning " Application Key ID: $B2_KEY_ID" + log_warning "Common cause: the Key ID and Application Key are from two DIFFERENT" + log_warning "keys (easy to mix up if you created more than one). Re-check both" + log_warning "values come from the SAME entry on B2's App Keys page." + log_warning "Raw error from B2/Kopia:" log_warning "$_b2_err" log_warning "Not enabling offsite mirroring this run. Re-run this installer once" log_warning "fixed, or hand-edit REMOTE_TYPE/REMOTE_ARGS in backup.conf directly." @@ -799,14 +859,11 @@ install_backup() { # sync-to sftp doesn't shell out to the system ssh client, so it # needs an explicit key/known_hosts file rather than picking up # whatever plain `ssh` already trusts automatically. - local _SFTP_KEYFILE="" - [ -f /root/.ssh/id_ed25519 ] && _SFTP_KEYFILE=/root/.ssh/id_ed25519 - [ -z "$_SFTP_KEYFILE" ] && [ -f /root/.ssh/id_rsa ] && _SFTP_KEYFILE=/root/.ssh/id_rsa + _backup_ensure_root_ssh_key + local _SFTP_KEYFILE="$_ROOT_SSH_KEYFILE" if [ -z "$_SFTP_KEYFILE" ]; then - log_warning " No SSH key found for root — this mirror needs one. Set one up (the" - log_warning " DISASTER-RECOVERY SPARE section above offers to generate one) and" - log_warning " re-run this installer to add the mirror." + log_warning " No SSH key available for root — can't add this mirror." elif ! ssh -o BatchMode=yes -o ConnectTimeout=5 "$_SFTP_DEST" true 2>/dev/null; then log_warning " Couldn't SSH to $_SFTP_DEST without a password — not adding this" log_warning " mirror until that works: ssh-copy-id $_SFTP_DEST" From 63392357811f9a298fc177ae3aa8486bc982a467 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 13:20:59 +0000 Subject: [PATCH 2/5] Correct B2 application key guidance: use "All" bucket access, not one bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live and cross-checked against a real, documented Kopia issue (kopia/kopia#5329): the walkthrough previously told the operator to scope the Application Key to just the bucket they created — the more security-conservative default, and correct for B2's own S3-compatible API in general. But Kopia specifically needs the listBuckets capability even though it only ever touches the one configured bucket, and B2's basic "Add a New Application Key" web form doesn't expose a way to grant listBuckets on a bucket-restricted key — only an account-wide ("All") key gets it through that form. Without it, the connection fails with B2's unhelpful "Cannot access bucket" error, which doesn't point at the actual missing capability at all. Updated the guidance to "All" with the reasoning inline, and a note that single-bucket scoping is still possible for anyone willing to create the key via B2's CLI/API directly (b2_create_key with an explicit capabilities list including listBuckets) rather than the basic web form this walkthrough is written for. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/backup.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/backup.sh b/services/backup.sh index 543c778..54d6e57 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -727,7 +727,13 @@ install_backup() { echo " e.g. s3.us-west-004.backblazeb2.com — you'll need it below." echo "" echo " 2) Account → App Keys → Add a New Application Key" - echo " - Allow access to: the bucket you just created (not 'All')" + echo " - Allow access to: All — confirmed live (kopia/kopia issue #5329): a key" + echo " restricted to one bucket via this basic form doesn't get the" + echo " 'listBuckets' capability Kopia needs even though it only ever touches" + echo " that one bucket, and B2 fails the connection with an unhelpful 'Cannot" + echo " access bucket' error. Restricting to one bucket only works if you add" + echo " listBuckets via the B2 CLI/API's own key-creation call instead of this" + echo " form — not something this walkthrough covers." echo " - Type: Read and Write" echo " - B2 shows the application key ONLY once — copy both values now," echo " you can't retrieve the key itself again afterward." From 7efd087993cf9c29f4240485cdd247327e6fe9f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 13:35:20 +0000 Subject: [PATCH 3/5] Add standalone backup/restore script for Asterisk, independent of Kopia MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asterisk's whole state (dialplan, pjsip devices, voicemail, recordings, .env with its coturn credential, docker-compose.yml) already lives under one self-contained directory, so asterisk-standalone-backup.sh just tars it — with stop/restart safety around the tar since voicemail/spool write continuously, and a move-aside-then-extract restore that rolls back automatically if extraction fails. Written into the install directory at both fresh-install and update time via _asterisk_write_standalone_backup_script(). Output defaults to ~/asterisk-backups/, deliberately outside ~/docker/, so a Kopia backup of the box doesn't also back up a backup-of-itself. Meant for a quick pre-change snapshot or moving this PBX to a new host without standing up the full backup stack first. Documented in the generated README's new "Standalone backup/restore" section. Tested against a mocked EA_DIR (fake docker/docker compose, config/spool/voicemail files) confirming backup produces a correct tar and restore replaces content correctly with rollback on extraction failure. --- services/asterisk.sh | 191 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/services/asterisk.sh b/services/asterisk.sh index c0f04d1..617299b 100644 --- a/services/asterisk.sh +++ b/services/asterisk.sh @@ -418,6 +418,167 @@ LOGROTATE rm -f /etc/logrotate.d/asterisk-digital-ocean } +# ── Shared: standalone backup/restore, independent of the Kopia backup +# service ───────────────────────────────────────────────────────────────── +# Unlike Mattermost (a database export that needed real work to get right — +# see migrate-from-pikapods.sh), Asterisk's entire state is already plain +# files under one directory: dialplan, pjsip devices, voicemail, recordings, +# .env (including its coturn credential), docker-compose.yml. So this is +# just a tar of the whole directory, with the stop/restart safety a live +# PBX needs around it — no export format to get wrong. +# +# Output defaults to a path OUTSIDE $DOCKER_DIR (~/asterisk-backups/) so a +# Kopia-based backup of this same box doesn't also end up backing up a +# backup-of-itself on every run — the same lesson as not leaving Mattermost +# migration scratch files inside ~/docker/mattermost. +_asterisk_write_standalone_backup_script() { + local _ea_dir="$1" _container="$2" + cat > "$_ea_dir/asterisk-standalone-backup.sh" << 'BACKUPSCRIPT' +#!/bin/bash +# __EA_DIR__/asterisk-standalone-backup.sh — independent backup/restore for +# this Asterisk install. No Kopia/backup-service dependency — everything +# this PBX needs to come back already lives under this one directory, so +# this is a tar of the whole thing plus the stop/restart safety a live PBX +# needs around it. +# +# sudo ./asterisk-standalone-backup.sh backup [output-dir] +# sudo ./asterisk-standalone-backup.sh restore +# +# Output defaults to ~/asterisk-backups/ — deliberately OUTSIDE ~/docker/, +# so a Kopia-based backup of this same box doesn't also end up backing up +# a backup-of-itself on every run. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONTAINER="__CONTAINER_NAME__" +ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" +ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" + +[ "${EUID:-$(id -u)}" -eq 0 ] || { echo "Run as root: sudo $0 ..."; exit 1; } + +usage() { + echo "Usage:" + echo " sudo $0 backup [output-dir] (default: $ACTUAL_HOME/asterisk-backups)" + echo " sudo $0 restore " + exit 1 +} + +container_running() { + docker compose ps --status running -q 2>/dev/null | grep -q . +} + +cmd="${1:-}" +case "$cmd" in + backup) + OUT_DIR="${2:-$ACTUAL_HOME/asterisk-backups}" + mkdir -p "$OUT_DIR" + TS="$(date +%Y%m%d-%H%M%S)" + ARCHIVE="$OUT_DIR/asterisk-backup-$TS.tar.gz" + + echo "This stops Asterisk briefly (voicemail/spool are written to" + echo "continuously — a live tar could capture a half-written file" + echo "otherwise) and restarts it after." + echo "Target: $ARCHIVE" + read -r -p "Type YES to proceed: " CONFIRM + [ "$CONFIRM" = "YES" ] || { echo "Aborted — no changes made."; exit 0; } + + cd "$HERE" || exit 1 + WAS_RUNNING=false + if container_running; then + echo "Stopping Asterisk..." + docker compose stop + WAS_RUNNING=true + fi + + echo "Archiving $HERE -> $ARCHIVE ..." + if tar -czf "$ARCHIVE" -C "$(dirname "$HERE")" "$(basename "$HERE")"; then + chown "$ACTUAL_USER:$ACTUAL_USER" "$ARCHIVE" + echo "Done: $ARCHIVE ($(du -h "$ARCHIVE" | cut -f1))" + else + echo "tar failed — restarting Asterisk regardless; check disk space." + fi + + if [ "$WAS_RUNNING" = true ]; then + echo "Starting Asterisk..." + (cd "$HERE" && docker compose up -d) + fi + ;; + + restore) + ARCHIVE="${2:-}" + [ -n "$ARCHIVE" ] || usage + [ -f "$ARCHIVE" ] || { echo "Archive not found: $ARCHIVE"; exit 1; } + + echo "┌─────────────────────────────────────────────────────────────────┐" + echo "│ ASTERISK RESTORE — THIS REPLACES $HERE" + echo "└─────────────────────────────────────────────────────────────────┘" + echo "" + echo " Archive: $ARCHIVE" + echo " Target: $HERE" + echo "" + read -r -p "Type YES to proceed: " CONFIRM + [ "$CONFIRM" = "YES" ] || { echo "Aborted — no changes made."; exit 0; } + + cd "$HERE" || exit 1 + WAS_RUNNING=false + if container_running; then + echo "Stopping Asterisk..." + docker compose stop + WAS_RUNNING=true + fi + + PARENT_DIR="$(dirname "$HERE")" + BASE_NAME="$(basename "$HERE")" + ASIDE="${HERE}.restore-aside-$(date +%Y%m%d-%H%M%S)" + + # Leave the directory being renamed before renaming it, rather than + # relying on renaming-your-own-cwd being safe (it generally is on + # Linux, but there's no reason to lean on that when cd'ing out first + # costs nothing). + cd "$PARENT_DIR" || exit 1 + + echo "Moving current install aside: $ASIDE" + mv "$HERE" "$ASIDE" + + echo "Extracting $ARCHIVE -> $PARENT_DIR ..." + if tar -xzf "$ARCHIVE" -C "$PARENT_DIR"; then + echo "Extracted." + else + echo "Extraction failed — rolling back to the pre-restore install." + rm -rf "${PARENT_DIR:?}/$BASE_NAME" + mv "$ASIDE" "$HERE" + exit 1 + fi + + chown -R "$ACTUAL_USER:$ACTUAL_USER" "$HERE" + + if [ "$WAS_RUNNING" = true ]; then + echo "Starting Asterisk..." + (cd "$HERE" && docker compose up -d) + fi + + echo "" + echo "Done. Previous install kept at: $ASIDE" + echo "Delete it once you've verified this restore is good — it's not" + echo "cleaned up automatically." + echo "" + echo "If you ran this from inside $HERE, your current shell may still show" + echo "the old directory's contents (a normal Linux quirk — your shell's" + echo "working directory followed the OLD directory when it got renamed" + echo "aside). Run 'cd $HERE' again (or open a new shell) to see the" + echo "restored files." + ;; + + *) + usage + ;; +esac +BACKUPSCRIPT + sed -i "s/__CONTAINER_NAME__/${_container}/g" "$_ea_dir/asterisk-standalone-backup.sh" + chmod +x "$_ea_dir/asterisk-standalone-backup.sh" + chown "$ACTUAL_USER:$ACTUAL_USER" "$_ea_dir/asterisk-standalone-backup.sh" 2>/dev/null || true +} + # ── Shared: extension presence (online/offline) ntfy alerts ──────────────── # Polls PJSIP registration state and alerts only on a CHANGE from the last # check (never on every poll) — same periodic-check shape as pstn-trunk.sh's @@ -1426,6 +1587,34 @@ acquisition both read. It's rotated at 100MB (5 generations, compressed) via \`/etc/logrotate.d/asterisk\`; unrotated it reached 1.4GB in three days on a publicly reachable box. +## Standalone backup/restore + +This directory is self-contained — dialplan, pjsip devices, voicemail +messages, recordings, \`.env\` (including its coturn credential), and +\`docker-compose.yml\` all live under \`${EA_DIR}\`. \`asterisk-standalone-backup.sh\` +(written into this directory at install time) tars the whole thing up +independent of Kopia or any other backup service in this repo — useful for +a one-off snapshot before a risky change, or to move this PBX to a new host +without setting up the full backup stack first. + +\`\`\`bash +sudo ${EA_DIR}/asterisk-standalone-backup.sh backup [output-dir] +# writes ~/asterisk-backups/asterisk-backup-.tar.gz by default +# (deliberately outside ~/docker/, so a Kopia backup of this box doesn't +# also end up backing up a backup-of-itself on every run) + +sudo ${EA_DIR}/asterisk-standalone-backup.sh restore +# moves the current install aside (timestamped, not deleted) and extracts +# the archive in its place; rolls back automatically if extraction fails +\`\`\` + +Both subcommands stop the container first (voicemail/spool are written to +continuously — a live tar could capture a half-written file) and restart it +after, and both require typing \`YES\` to confirm before touching anything. +To migrate to a new host: run \`backup\` on the old one, copy the archive +over, then run \`restore\` after a fresh \`sudo ./setup.sh asterisk\` install +(or directly into an empty \`${EA_DIR}\`) on the new host. + ## VLANs / other subnets \`.env\` → \`HAS_VLANS\`/\`VLAN_SUBNETS\` lists extra networks (space-separated @@ -1639,6 +1828,7 @@ install_asterisk() { _asterisk_refresh_vendor_files _asterisk_write_compose "$ASTERISK_PROJECT" "$CONTAINER" "$ASTERISK_COTURN" "$_HAD_EMBEDDED_COTURN" _asterisk_write_logrotate "$EA_DIR" + _asterisk_write_standalone_backup_script "$EA_DIR" "$CONTAINER" _asterisk_patch_messaging_vendor_files "$EA_DIR" _asterisk_write_messaging_dialplan "$EA_DIR/config/asterisk/messaging-dialplan.conf" _asterisk_ensure_live_messaging_include "$EA_DIR" "$CONTAINER" @@ -1736,6 +1926,7 @@ install_asterisk() { _asterisk_refresh_vendor_files _asterisk_write_logrotate "$EA_DIR" + _asterisk_write_standalone_backup_script "$EA_DIR" "$CONTAINER" _asterisk_patch_messaging_vendor_files "$EA_DIR" _asterisk_write_messaging_dialplan "$EA_DIR/config/asterisk/messaging-dialplan.conf" _asterisk_ensure_live_messaging_include "$EA_DIR" "$CONTAINER" From 63774e05009582dfadbbdaee1ff6971446532182 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 18:34:57 +0000 Subject: [PATCH 4/5] Read $_ERR once per failure instead of twice, fixing lost raw-error text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last night's fully-failed backup run (0/20, "repository not found" on every service) showed the real gap: categorize_error() clearly saw real content in $_ERR (it matched a specific pattern, not the generic fallback), but log_raw_error()'s separate re-read of the same file moments later came back empty on every single failure — so the raw-error logging added earlier this session produced nothing when it mattered most. Fixed by reading $_ERR into a variable exactly once per failure and passing that string to both categorize_error() and log_raw_error(), instead of two independent file reads. Verified against a mock harness reproducing the same call pattern (three simulated failures in a loop, single shared error file) — both the categorized reason and the raw stderr text now come through on every iteration. Doesn't explain why last night's repo access failed in the first place (disk and mount checks came back clean) — but the next time it happens, this will actually surface the real kopia error instead of losing it. --- extras/backup_kopia.sh | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/extras/backup_kopia.sh b/extras/backup_kopia.sh index e599533..4abd789 100644 --- a/extras/backup_kopia.sh +++ b/extras/backup_kopia.sh @@ -59,9 +59,19 @@ categorize_error() { # go look at. This makes that claim true: the raw text now lands in the # same log stream (journal, when run via the systemd timer) as everything # else, surviving past the run that produced it. +# +# Takes the already-read error TEXT, not the file path. A live run showed +# categorize_error() correctly matching a specific pattern (so $_ERR had +# real content at that point) while a second, later read of the same file +# for this function came back completely empty — every failure that night +# logged its categorized reason but zero "Raw error:" lines. Whatever causes +# that (the file is reused across the whole script run and read twice per +# failure), reading it once and passing the string to both this function and +# categorize_error() removes the second read entirely, so there's nothing +# left to race. log_raw_error() { - local errfile="$1" raw - raw="$(tr '\n' ' ' < "$errfile" | head -c 500)" + local raw + raw="$(printf '%s' "$1" | tr '\n' ' ' | head -c 500)" [ -n "$raw" ] && log " Raw error: $raw" } @@ -137,9 +147,10 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "OK $svc (Minecraft, no downtime)" BACKUP_COUNT=$((BACKUP_COUNT+1)) else - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: snapshot failed for $svc — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("$svc: $_reason") rc=1 fi @@ -158,9 +169,10 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "OK $svc" BACKUP_COUNT=$((BACKUP_COUNT+1)) else - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: snapshot failed for $svc — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("$svc: $_reason") rc=1 fi @@ -178,9 +190,10 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then log "Mirroring '$dest' offsite ($REMOTE_TYPE)..." # shellcheck disable=SC2086 if ! kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: mirror failed for '$dest' — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("mirror[$dest]: $_reason") rc=1 fi @@ -200,9 +213,10 @@ for mirror_name in ${EXTRA_MIRROR_NAMES:-}; do log "Mirroring '$dest' to '$mirror_name' ($_mtype)..." # shellcheck disable=SC2086 if ! kp_for "$dest" repository sync-to "$_mtype" $_margs 2>"$_ERR"; then - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: mirror '$mirror_name' failed for '$dest' — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("mirror[$mirror_name/$dest]: $_reason") rc=1 fi @@ -225,9 +239,10 @@ if [ -n "${DR_SYNC_HOST:-}" ]; then && ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then log "OK spare sync ($DR_SYNC_HOST)" else - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: spare sync failed — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("spare-sync: $_reason") rc=1 fi From f6e5bb4ea3fa1ac19568acdf39b808543b969c8c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 19:15:58 +0000 Subject: [PATCH 5/5] Use rsync instead of scp for the DR-spare backup.conf/README sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The freshly-added raw-error logging paid off immediately: the box's spare sync was failing every run with "scp: Connection closed" while plain ssh exec to the same host worked fine. That split (ssh exec OK, scp specifically rejected) matches modern OpenSSH's default scp-over-SFTP transfer hitting a restriction on the remote side that a plain exec or rsync's own protocol don't trigger. Swapped the scp step for rsync -a over the same ssh options, keeping the ssh mkdir -p before it (rsync doesn't create missing destination directories) and the ssh chmod after. Verified the exact command/quoting against mocked ssh/rsync binaries — array expansion and remote path handling both check out. --- extras/backup_kopia.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/extras/backup_kopia.sh b/extras/backup_kopia.sh index 4abd789..252eb98 100644 --- a/extras/backup_kopia.sh +++ b/extras/backup_kopia.sh @@ -234,8 +234,14 @@ if [ -n "${DR_SYNC_HOST:-}" ]; then log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..." _dr_files=("$CONF") [ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md") + # rsync instead of scp: modern OpenSSH (9.0+) defaults scp to an + # SFTP-based transfer, and some remote-side setups (restricted shells, + # forced commands, older sshd) reject that with an immediate "Connection + # closed" while plain ssh exec and rsync's own protocol both still work + # fine over the same connection. Confirmed live: scp failing this way + # while `ssh "$DR_SYNC_HOST" true` succeeded, rsync doesn't hit it. if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \ - && scp -o BatchMode=yes -o ConnectTimeout=10 "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \ + && rsync -a -e 'ssh -o BatchMode=yes -o ConnectTimeout=10' "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \ && ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then log "OK spare sync ($DR_SYNC_HOST)" else