Read $_ERR once per failure instead of twice, fixing lost raw-error text

Last night's fully-failed backup run (0/20, "repository not found" on
every service) showed the real gap: categorize_error() clearly saw real
content in $_ERR (it matched a specific pattern, not the generic
fallback), but log_raw_error()'s separate re-read of the same file moments
later came back empty on every single failure — so the raw-error logging
added earlier this session produced nothing when it mattered most.

Fixed by reading $_ERR into a variable exactly once per failure and
passing that string to both categorize_error() and log_raw_error(),
instead of two independent file reads. Verified against a mock harness
reproducing the same call pattern (three simulated failures in a loop,
single shared error file) — both the categorized reason and the raw
stderr text now come through on every iteration.

Doesn't explain why last night's repo access failed in the first place
(disk and mount checks came back clean) — but the next time it happens,
this will actually surface the real kopia error instead of losing it.
This commit is contained in:
Claude
2026-08-14 18:34:57 +00:00
parent 7efd087993
commit 63774e0500
+27 -12
View File
@@ -59,9 +59,19 @@ categorize_error() {
# go look at. This makes that claim true: the raw text now lands in the
# same log stream (journal, when run via the systemd timer) as everything
# else, surviving past the run that produced it.
#
# Takes the already-read error TEXT, not the file path. A live run showed
# categorize_error() correctly matching a specific pattern (so $_ERR had
# real content at that point) while a second, later read of the same file
# for this function came back completely empty — every failure that night
# logged its categorized reason but zero "Raw error:" lines. Whatever causes
# that (the file is reused across the whole script run and read twice per
# failure), reading it once and passing the string to both this function and
# categorize_error() removes the second read entirely, so there's nothing
# left to race.
log_raw_error() {
local errfile="$1" raw
raw="$(tr '\n' ' ' < "$errfile" | head -c 500)"
local raw
raw="$(printf '%s' "$1" | tr '\n' ' ' | head -c 500)"
[ -n "$raw" ] && log " Raw error: $raw"
}
@@ -137,9 +147,10 @@ for svc_dir in "$DOCKER_DIR"/*/; do
log "OK $svc (Minecraft, no downtime)"
BACKUP_COUNT=$((BACKUP_COUNT+1))
else
_reason="$(categorize_error "$(cat "$_ERR")")"
_err_text="$(cat "$_ERR" 2>/dev/null)"
_reason="$(categorize_error "$_err_text")"
log "WARNING: snapshot failed for $svc$_reason"
log_raw_error "$_ERR"
log_raw_error "$_err_text"
FAILED_SVCS+=("$svc: $_reason")
rc=1
fi
@@ -158,9 +169,10 @@ for svc_dir in "$DOCKER_DIR"/*/; do
log "OK $svc"
BACKUP_COUNT=$((BACKUP_COUNT+1))
else
_reason="$(categorize_error "$(cat "$_ERR")")"
_err_text="$(cat "$_ERR" 2>/dev/null)"
_reason="$(categorize_error "$_err_text")"
log "WARNING: snapshot failed for $svc$_reason"
log_raw_error "$_ERR"
log_raw_error "$_err_text"
FAILED_SVCS+=("$svc: $_reason")
rc=1
fi
@@ -178,9 +190,10 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then
log "Mirroring '$dest' offsite ($REMOTE_TYPE)..."
# shellcheck disable=SC2086
if ! kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then
_reason="$(categorize_error "$(cat "$_ERR")")"
_err_text="$(cat "$_ERR" 2>/dev/null)"
_reason="$(categorize_error "$_err_text")"
log "WARNING: mirror failed for '$dest' — $_reason"
log_raw_error "$_ERR"
log_raw_error "$_err_text"
FAILED_SVCS+=("mirror[$dest]: $_reason")
rc=1
fi
@@ -200,9 +213,10 @@ for mirror_name in ${EXTRA_MIRROR_NAMES:-}; do
log "Mirroring '$dest' to '$mirror_name' ($_mtype)..."
# shellcheck disable=SC2086
if ! kp_for "$dest" repository sync-to "$_mtype" $_margs 2>"$_ERR"; then
_reason="$(categorize_error "$(cat "$_ERR")")"
_err_text="$(cat "$_ERR" 2>/dev/null)"
_reason="$(categorize_error "$_err_text")"
log "WARNING: mirror '$mirror_name' failed for '$dest' — $_reason"
log_raw_error "$_ERR"
log_raw_error "$_err_text"
FAILED_SVCS+=("mirror[$mirror_name/$dest]: $_reason")
rc=1
fi
@@ -225,9 +239,10 @@ if [ -n "${DR_SYNC_HOST:-}" ]; then
&& ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then
log "OK spare sync ($DR_SYNC_HOST)"
else
_reason="$(categorize_error "$(cat "$_ERR")")"
_err_text="$(cat "$_ERR" 2>/dev/null)"
_reason="$(categorize_error "$_err_text")"
log "WARNING: spare sync failed — $_reason"
log_raw_error "$_ERR"
log_raw_error "$_err_text"
FAILED_SVCS+=("spare-sync: $_reason")
rc=1
fi