diff --git a/extras/backup_kopia.sh b/extras/backup_kopia.sh index e599533..252eb98 100644 --- a/extras/backup_kopia.sh +++ b/extras/backup_kopia.sh @@ -59,9 +59,19 @@ categorize_error() { # go look at. This makes that claim true: the raw text now lands in the # same log stream (journal, when run via the systemd timer) as everything # else, surviving past the run that produced it. +# +# Takes the already-read error TEXT, not the file path. A live run showed +# categorize_error() correctly matching a specific pattern (so $_ERR had +# real content at that point) while a second, later read of the same file +# for this function came back completely empty — every failure that night +# logged its categorized reason but zero "Raw error:" lines. Whatever causes +# that (the file is reused across the whole script run and read twice per +# failure), reading it once and passing the string to both this function and +# categorize_error() removes the second read entirely, so there's nothing +# left to race. log_raw_error() { - local errfile="$1" raw - raw="$(tr '\n' ' ' < "$errfile" | head -c 500)" + local raw + raw="$(printf '%s' "$1" | tr '\n' ' ' | head -c 500)" [ -n "$raw" ] && log " Raw error: $raw" } @@ -137,9 +147,10 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "OK $svc (Minecraft, no downtime)" BACKUP_COUNT=$((BACKUP_COUNT+1)) else - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: snapshot failed for $svc — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("$svc: $_reason") rc=1 fi @@ -158,9 +169,10 @@ for svc_dir in "$DOCKER_DIR"/*/; do log "OK $svc" BACKUP_COUNT=$((BACKUP_COUNT+1)) else - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: snapshot failed for $svc — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("$svc: $_reason") rc=1 fi @@ -178,9 +190,10 @@ if [ "${REMOTE_TYPE:-none}" != "none" ] && [ -n "${REMOTE_TYPE:-}" ]; then log "Mirroring '$dest' offsite ($REMOTE_TYPE)..." # shellcheck disable=SC2086 if ! kp_for "$dest" repository sync-to "$REMOTE_TYPE" $REMOTE_ARGS 2>"$_ERR"; then - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: mirror failed for '$dest' — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("mirror[$dest]: $_reason") rc=1 fi @@ -200,9 +213,10 @@ for mirror_name in ${EXTRA_MIRROR_NAMES:-}; do log "Mirroring '$dest' to '$mirror_name' ($_mtype)..." # shellcheck disable=SC2086 if ! kp_for "$dest" repository sync-to "$_mtype" $_margs 2>"$_ERR"; then - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: mirror '$mirror_name' failed for '$dest' — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("mirror[$mirror_name/$dest]: $_reason") rc=1 fi @@ -220,14 +234,21 @@ if [ -n "${DR_SYNC_HOST:-}" ]; then log "Syncing backup.conf + README to spare ($DR_SYNC_HOST:$_dr_path)..." _dr_files=("$CONF") [ -f "$HERE/README.md" ] && _dr_files+=("$HERE/README.md") + # rsync instead of scp: modern OpenSSH (9.0+) defaults scp to an + # SFTP-based transfer, and some remote-side setups (restricted shells, + # forced commands, older sshd) reject that with an immediate "Connection + # closed" while plain ssh exec and rsync's own protocol both still work + # fine over the same connection. Confirmed live: scp failing this way + # while `ssh "$DR_SYNC_HOST" true` succeeded, rsync doesn't hit it. if ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "mkdir -p '$_dr_path'" 2>"$_ERR" \ - && scp -o BatchMode=yes -o ConnectTimeout=10 "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \ + && rsync -a -e 'ssh -o BatchMode=yes -o ConnectTimeout=10' "${_dr_files[@]}" "$DR_SYNC_HOST:$_dr_path/" 2>>"$_ERR" \ && ssh -o BatchMode=yes -o ConnectTimeout=10 "$DR_SYNC_HOST" "chmod 600 '$_dr_path/backup.conf'" 2>>"$_ERR"; then log "OK spare sync ($DR_SYNC_HOST)" else - _reason="$(categorize_error "$(cat "$_ERR")")" + _err_text="$(cat "$_ERR" 2>/dev/null)" + _reason="$(categorize_error "$_err_text")" log "WARNING: spare sync failed — $_reason" - log_raw_error "$_ERR" + log_raw_error "$_err_text" FAILED_SVCS+=("spare-sync: $_reason") rc=1 fi diff --git a/services/asterisk.sh b/services/asterisk.sh index 3922847..701f83f 100644 --- a/services/asterisk.sh +++ b/services/asterisk.sh @@ -459,6 +459,167 @@ LOGROTATE rm -f /etc/logrotate.d/asterisk-digital-ocean } +# ── Shared: standalone backup/restore, independent of the Kopia backup +# service ───────────────────────────────────────────────────────────────── +# Unlike Mattermost (a database export that needed real work to get right — +# see migrate-from-pikapods.sh), Asterisk's entire state is already plain +# files under one directory: dialplan, pjsip devices, voicemail, recordings, +# .env (including its coturn credential), docker-compose.yml. So this is +# just a tar of the whole directory, with the stop/restart safety a live +# PBX needs around it — no export format to get wrong. +# +# Output defaults to a path OUTSIDE $DOCKER_DIR (~/asterisk-backups/) so a +# Kopia-based backup of this same box doesn't also end up backing up a +# backup-of-itself on every run — the same lesson as not leaving Mattermost +# migration scratch files inside ~/docker/mattermost. +_asterisk_write_standalone_backup_script() { + local _ea_dir="$1" _container="$2" + cat > "$_ea_dir/asterisk-standalone-backup.sh" << 'BACKUPSCRIPT' +#!/bin/bash +# __EA_DIR__/asterisk-standalone-backup.sh — independent backup/restore for +# this Asterisk install. No Kopia/backup-service dependency — everything +# this PBX needs to come back already lives under this one directory, so +# this is a tar of the whole thing plus the stop/restart safety a live PBX +# needs around it. +# +# sudo ./asterisk-standalone-backup.sh backup [output-dir] +# sudo ./asterisk-standalone-backup.sh restore +# +# Output defaults to ~/asterisk-backups/ — deliberately OUTSIDE ~/docker/, +# so a Kopia-based backup of this same box doesn't also end up backing up +# a backup-of-itself on every run. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONTAINER="__CONTAINER_NAME__" +ACTUAL_USER="${SUDO_USER:-${USER:-$(id -un)}}" +ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "/home/$ACTUAL_USER")" + +[ "${EUID:-$(id -u)}" -eq 0 ] || { echo "Run as root: sudo $0 ..."; exit 1; } + +usage() { + echo "Usage:" + echo " sudo $0 backup [output-dir] (default: $ACTUAL_HOME/asterisk-backups)" + echo " sudo $0 restore " + exit 1 +} + +container_running() { + docker compose ps --status running -q 2>/dev/null | grep -q . +} + +cmd="${1:-}" +case "$cmd" in + backup) + OUT_DIR="${2:-$ACTUAL_HOME/asterisk-backups}" + mkdir -p "$OUT_DIR" + TS="$(date +%Y%m%d-%H%M%S)" + ARCHIVE="$OUT_DIR/asterisk-backup-$TS.tar.gz" + + echo "This stops Asterisk briefly (voicemail/spool are written to" + echo "continuously — a live tar could capture a half-written file" + echo "otherwise) and restarts it after." + echo "Target: $ARCHIVE" + read -r -p "Type YES to proceed: " CONFIRM + [ "$CONFIRM" = "YES" ] || { echo "Aborted — no changes made."; exit 0; } + + cd "$HERE" || exit 1 + WAS_RUNNING=false + if container_running; then + echo "Stopping Asterisk..." + docker compose stop + WAS_RUNNING=true + fi + + echo "Archiving $HERE -> $ARCHIVE ..." + if tar -czf "$ARCHIVE" -C "$(dirname "$HERE")" "$(basename "$HERE")"; then + chown "$ACTUAL_USER:$ACTUAL_USER" "$ARCHIVE" + echo "Done: $ARCHIVE ($(du -h "$ARCHIVE" | cut -f1))" + else + echo "tar failed — restarting Asterisk regardless; check disk space." + fi + + if [ "$WAS_RUNNING" = true ]; then + echo "Starting Asterisk..." + (cd "$HERE" && docker compose up -d) + fi + ;; + + restore) + ARCHIVE="${2:-}" + [ -n "$ARCHIVE" ] || usage + [ -f "$ARCHIVE" ] || { echo "Archive not found: $ARCHIVE"; exit 1; } + + echo "┌─────────────────────────────────────────────────────────────────┐" + echo "│ ASTERISK RESTORE — THIS REPLACES $HERE" + echo "└─────────────────────────────────────────────────────────────────┘" + echo "" + echo " Archive: $ARCHIVE" + echo " Target: $HERE" + echo "" + read -r -p "Type YES to proceed: " CONFIRM + [ "$CONFIRM" = "YES" ] || { echo "Aborted — no changes made."; exit 0; } + + cd "$HERE" || exit 1 + WAS_RUNNING=false + if container_running; then + echo "Stopping Asterisk..." + docker compose stop + WAS_RUNNING=true + fi + + PARENT_DIR="$(dirname "$HERE")" + BASE_NAME="$(basename "$HERE")" + ASIDE="${HERE}.restore-aside-$(date +%Y%m%d-%H%M%S)" + + # Leave the directory being renamed before renaming it, rather than + # relying on renaming-your-own-cwd being safe (it generally is on + # Linux, but there's no reason to lean on that when cd'ing out first + # costs nothing). + cd "$PARENT_DIR" || exit 1 + + echo "Moving current install aside: $ASIDE" + mv "$HERE" "$ASIDE" + + echo "Extracting $ARCHIVE -> $PARENT_DIR ..." + if tar -xzf "$ARCHIVE" -C "$PARENT_DIR"; then + echo "Extracted." + else + echo "Extraction failed — rolling back to the pre-restore install." + rm -rf "${PARENT_DIR:?}/$BASE_NAME" + mv "$ASIDE" "$HERE" + exit 1 + fi + + chown -R "$ACTUAL_USER:$ACTUAL_USER" "$HERE" + + if [ "$WAS_RUNNING" = true ]; then + echo "Starting Asterisk..." + (cd "$HERE" && docker compose up -d) + fi + + echo "" + echo "Done. Previous install kept at: $ASIDE" + echo "Delete it once you've verified this restore is good — it's not" + echo "cleaned up automatically." + echo "" + echo "If you ran this from inside $HERE, your current shell may still show" + echo "the old directory's contents (a normal Linux quirk — your shell's" + echo "working directory followed the OLD directory when it got renamed" + echo "aside). Run 'cd $HERE' again (or open a new shell) to see the" + echo "restored files." + ;; + + *) + usage + ;; +esac +BACKUPSCRIPT + sed -i "s/__CONTAINER_NAME__/${_container}/g" "$_ea_dir/asterisk-standalone-backup.sh" + chmod +x "$_ea_dir/asterisk-standalone-backup.sh" + chown "$ACTUAL_USER:$ACTUAL_USER" "$_ea_dir/asterisk-standalone-backup.sh" 2>/dev/null || true +} + # ── Shared: extension presence (online/offline) ntfy alerts ──────────────── # Polls PJSIP registration state and alerts only on a CHANGE from the last # check (never on every poll) — same periodic-check shape as pstn-trunk.sh's @@ -1516,6 +1677,34 @@ acquisition both read. It's rotated at 100MB (5 generations, compressed) via \`/etc/logrotate.d/asterisk\`; unrotated it reached 1.4GB in three days on a publicly reachable box. +## Standalone backup/restore + +This directory is self-contained — dialplan, pjsip devices, voicemail +messages, recordings, \`.env\` (including its coturn credential), and +\`docker-compose.yml\` all live under \`${EA_DIR}\`. \`asterisk-standalone-backup.sh\` +(written into this directory at install time) tars the whole thing up +independent of Kopia or any other backup service in this repo — useful for +a one-off snapshot before a risky change, or to move this PBX to a new host +without setting up the full backup stack first. + +\`\`\`bash +sudo ${EA_DIR}/asterisk-standalone-backup.sh backup [output-dir] +# writes ~/asterisk-backups/asterisk-backup-.tar.gz by default +# (deliberately outside ~/docker/, so a Kopia backup of this box doesn't +# also end up backing up a backup-of-itself on every run) + +sudo ${EA_DIR}/asterisk-standalone-backup.sh restore +# moves the current install aside (timestamped, not deleted) and extracts +# the archive in its place; rolls back automatically if extraction fails +\`\`\` + +Both subcommands stop the container first (voicemail/spool are written to +continuously — a live tar could capture a half-written file) and restart it +after, and both require typing \`YES\` to confirm before touching anything. +To migrate to a new host: run \`backup\` on the old one, copy the archive +over, then run \`restore\` after a fresh \`sudo ./setup.sh asterisk\` install +(or directly into an empty \`${EA_DIR}\`) on the new host. + ## VLANs / other subnets \`.env\` → \`HAS_VLANS\`/\`VLAN_SUBNETS\` lists extra networks (space-separated @@ -1726,6 +1915,7 @@ install_asterisk() { _asterisk_refresh_vendor_files _asterisk_write_compose "$ASTERISK_PROJECT" "$CONTAINER" "$ASTERISK_COTURN" "$_HAD_EMBEDDED_COTURN" _asterisk_write_logrotate "$EA_DIR" + _asterisk_write_standalone_backup_script "$EA_DIR" "$CONTAINER" _asterisk_patch_messaging_vendor_files "$EA_DIR" _asterisk_write_messaging_dialplan "$EA_DIR/config/asterisk/messaging-dialplan.conf" _asterisk_ensure_live_messaging_include "$EA_DIR" "$CONTAINER" @@ -1821,6 +2011,7 @@ install_asterisk() { _asterisk_refresh_vendor_files _asterisk_write_logrotate "$EA_DIR" + _asterisk_write_standalone_backup_script "$EA_DIR" "$CONTAINER" _asterisk_patch_messaging_vendor_files "$EA_DIR" _asterisk_write_messaging_dialplan "$EA_DIR/config/asterisk/messaging-dialplan.conf" _asterisk_ensure_live_messaging_include "$EA_DIR" "$CONTAINER" diff --git a/services/backup.sh b/services/backup.sh index 8509147..54d6e57 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -207,6 +207,60 @@ fi register_service backup backup "Encrypted backup of all Docker services (full restore)" +# Ensures root has an SSH key usable for the systemd-run (root, unattended) +# backup/mirror steps. The DR-spare and SFTP-mirror prompts both need one — +# both used to check ONLY /root/.ssh, missing the common case where +# whoever ran `sudo ./setup.sh backup` already has a key under their OWN +# home directory (used interactively, quite possibly already authorized on +# the target box) while root — who actually runs the scheduled service — +# has none. Prefers reusing that existing keypair over minting a fresh +# one, since the existing one may already be trusted where it's needed; +# ssh-copy-id-ing a brand new key is the fallback, not the first move. +# Sets _ROOT_SSH_KEYFILE (out-param, not local) to the resulting keyfile +# path, empty if none is available/created. +_backup_ensure_root_ssh_key() { + _ROOT_SSH_KEYFILE="" + if [ -f /root/.ssh/id_ed25519 ]; then + _ROOT_SSH_KEYFILE=/root/.ssh/id_ed25519; return 0 + fi + if [ -f /root/.ssh/id_rsa ]; then + _ROOT_SSH_KEYFILE=/root/.ssh/id_rsa; return 0 + fi + + local _user_key="" + [ -f "$ACTUAL_HOME/.ssh/id_ed25519" ] && _user_key="$ACTUAL_HOME/.ssh/id_ed25519" + [ -z "$_user_key" ] && [ -f "$ACTUAL_HOME/.ssh/id_rsa" ] && _user_key="$ACTUAL_HOME/.ssh/id_rsa" + + if [ -n "$_user_key" ]; then + local _COPY_USER_KEY="" + prompt_yn " No SSH key for root, but $ACTUAL_USER has one ($_user_key) — reuse it for root too (it may already be authorized where you need it)? (y/n):" "y" _COPY_USER_KEY + if [[ "$_COPY_USER_KEY" =~ ^[Yy]$ ]]; then + mkdir -p /root/.ssh && chmod 700 /root/.ssh + local _keyname; _keyname="$(basename "$_user_key")" + cp "$_user_key" "/root/.ssh/$_keyname" + [ -f "${_user_key}.pub" ] && cp "${_user_key}.pub" "/root/.ssh/${_keyname}.pub" + chown root:root "/root/.ssh/$_keyname" "/root/.ssh/${_keyname}.pub" 2>/dev/null + chmod 600 "/root/.ssh/$_keyname" + [ -f "/root/.ssh/${_keyname}.pub" ] && chmod 644 "/root/.ssh/${_keyname}.pub" + log_success " Copied $_user_key to /root/.ssh/ for root's use." + _ROOT_SSH_KEYFILE="/root/.ssh/$_keyname" + return 0 + fi + fi + + local _GEN_KEY="" + prompt_yn " Generate a new SSH key for root (ssh-keygen)? (y/n):" "y" _GEN_KEY + if [[ "$_GEN_KEY" =~ ^[Yy]$ ]]; then + mkdir -p /root/.ssh && chmod 700 /root/.ssh + if ssh-keygen -t ed25519 -N "" -f /root/.ssh/id_ed25519 -q; then + log_success " Generated /root/.ssh/id_ed25519" + _ROOT_SSH_KEYFILE=/root/.ssh/id_ed25519 + else + log_warning " ssh-keygen failed — generate one manually." + fi + fi +} + install_backup() { require_docker || return 1 @@ -550,17 +604,7 @@ install_backup() { esac fi - local _HAVE_KEY=false - [ -f /root/.ssh/id_ed25519 ] || [ -f /root/.ssh/id_rsa ] && _HAVE_KEY=true - if [ "$_HAVE_KEY" = false ]; then - local _GEN_KEY="" - prompt_yn " No SSH key found for root — generate one now (ssh-keygen)? (y/n):" "y" _GEN_KEY - if [[ "$_GEN_KEY" =~ ^[Yy]$ ]]; then - ssh-keygen -t ed25519 -N "" -f /root/.ssh/id_ed25519 -q \ - && log_success " Generated /root/.ssh/id_ed25519" \ - || log_warning " ssh-keygen failed — generate one manually." - fi - fi + _backup_ensure_root_ssh_key local _COPY_KEY="" prompt_yn " Run ssh-copy-id to $DR_SYNC_HOST now? (asks for its login password interactively) (y/n):" "y" _COPY_KEY @@ -683,7 +727,13 @@ install_backup() { echo " e.g. s3.us-west-004.backblazeb2.com — you'll need it below." echo "" echo " 2) Account → App Keys → Add a New Application Key" - echo " - Allow access to: the bucket you just created (not 'All')" + echo " - Allow access to: All — confirmed live (kopia/kopia issue #5329): a key" + echo " restricted to one bucket via this basic form doesn't get the" + echo " 'listBuckets' capability Kopia needs even though it only ever touches" + echo " that one bucket, and B2 fails the connection with an unhelpful 'Cannot" + echo " access bucket' error. Restricting to one bucket only works if you add" + echo " listBuckets via the B2 CLI/API's own key-creation call instead of this" + echo " form — not something this walkthrough covers." echo " - Type: Read and Write" echo " - B2 shows the application key ONLY once — copy both values now," echo " you can't retrieve the key itself again afterward." @@ -724,7 +774,23 @@ install_backup() { REMOTE_ARGS="--bucket=$B2_BUCKET --access-key=$B2_KEY_ID --secret-access-key=$B2_APP_KEY --endpoint=$B2_ENDPOINT" log_success "B2 credentials verified — offsite mirroring will run after each backup." else - log_warning "B2 dry-run failed — check bucket name, endpoint, and key permissions:" + # All four fields were non-empty (the blank-field check above + # already ruled that out) — this is B2 rejecting what was + # entered, not missing input. Echoing back what was actually + # used (never the secret) so it's easy to eyeball against + # B2's own confirmation screen — the most common cause here + # is pairing the Key ID from one Application Key with the + # Secret from a different one (e.g. after creating more than + # one while troubleshooting). + log_warning "B2 rejected these credentials — bucket, endpoint, or key mismatch" + log_warning "(not blank input — all four fields were entered):" + log_warning " Bucket: $B2_BUCKET" + log_warning " Endpoint: $B2_ENDPOINT" + log_warning " Application Key ID: $B2_KEY_ID" + log_warning "Common cause: the Key ID and Application Key are from two DIFFERENT" + log_warning "keys (easy to mix up if you created more than one). Re-check both" + log_warning "values come from the SAME entry on B2's App Keys page." + log_warning "Raw error from B2/Kopia:" log_warning "$_b2_err" log_warning "Not enabling offsite mirroring this run. Re-run this installer once" log_warning "fixed, or hand-edit REMOTE_TYPE/REMOTE_ARGS in backup.conf directly." @@ -799,14 +865,11 @@ install_backup() { # sync-to sftp doesn't shell out to the system ssh client, so it # needs an explicit key/known_hosts file rather than picking up # whatever plain `ssh` already trusts automatically. - local _SFTP_KEYFILE="" - [ -f /root/.ssh/id_ed25519 ] && _SFTP_KEYFILE=/root/.ssh/id_ed25519 - [ -z "$_SFTP_KEYFILE" ] && [ -f /root/.ssh/id_rsa ] && _SFTP_KEYFILE=/root/.ssh/id_rsa + _backup_ensure_root_ssh_key + local _SFTP_KEYFILE="$_ROOT_SSH_KEYFILE" if [ -z "$_SFTP_KEYFILE" ]; then - log_warning " No SSH key found for root — this mirror needs one. Set one up (the" - log_warning " DISASTER-RECOVERY SPARE section above offers to generate one) and" - log_warning " re-run this installer to add the mirror." + log_warning " No SSH key available for root — can't add this mirror." elif ! ssh -o BatchMode=yes -o ConnectTimeout=5 "$_SFTP_DEST" true 2>/dev/null; then log_warning " Couldn't SSH to $_SFTP_DEST without a password — not adding this" log_warning " mirror until that works: ssh-copy-id $_SFTP_DEST"