diff --git a/services/backup.sh b/services/backup.sh index c32b388..81969bd 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -520,6 +520,106 @@ install_backup() { done unset -f kp_d + # ── 7b. Offsite mirror (Backblaze B2) ──────────────────────────────────── + # Kopia's dedicated "b2" sync-to provider is marked [DEPRECATED] in + # Kopia's own docs (kopia.io/docs/reference/command-line/common/ + # repository-sync-to-b2/) — confirmed before writing this rather than + # building on a command that's on its way out. B2's S3-compatible + # endpoint plus the actively-maintained `sync-to s3` provider is the + # supported path instead: same B2 application key, just pointed at + # B2's own s3..backblazeb2.com endpoint instead of AWS. + # + # Bucket creation and the application key can't be automated here on + # purpose — Object Lock in particular is a deliberate, one-time choice + # B2 only lets you make at bucket creation, not something safe for a + # script to flip on (or skip) silently on someone's behalf. This walks + # through both console steps, then handles the mechanical part: taking + # the resulting bucket/endpoint/key and writing a verified + # REMOTE_TYPE/REMOTE_ARGS into backup.conf. + # + # Encryption is NOT a separate step here — Kopia already encrypts + # everything client-side (AES-256-GCM) using the repository password + # set above, before any of it leaves this box. B2's own optional + # Server-Side Encryption toggle is redundant on top of that; harmless + # to also enable for defense-in-depth, but nothing here depends on it. + local REMOTE_TYPE="none" REMOTE_ARGS="" + if [ -f "$CONF_FILE" ]; then + # Preserve whatever's already configured if this is a re-run and + # the operator doesn't re-answer the prompt below — re-running this + # installer has no update/fresh distinction, so without this an + # already-working offsite mirror would silently reset to "none". + REMOTE_TYPE="$(grep '^REMOTE_TYPE=' "$CONF_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"')" + REMOTE_ARGS="$(grep '^REMOTE_ARGS=' "$CONF_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"')" + [ -z "$REMOTE_TYPE" ] && REMOTE_TYPE="none" + fi + + echo "" + echo "═══════════════════════════════════════════════════════" + echo " OFFSITE MIRROR (optional)" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Mirrors every local repo above to Backblaze B2 after each backup run —" + echo " the actual '1 copy offsite' piece of a real 3-2-1 backup. Skip this if" + echo " you don't have a B2 account yet, or would rather set REMOTE_TYPE/" + echo " REMOTE_ARGS in backup.conf by hand later." + if [ "$REMOTE_TYPE" != "none" ]; then + echo "" + echo " Offsite mirroring is already configured (REMOTE_TYPE=$REMOTE_TYPE)." + echo " Answering yes below replaces it; answering no leaves it as-is." + fi + echo "" + local _setup_b2="" + prompt_yn " Set up Backblaze B2 offsite mirroring now? (y/N):" "n" _setup_b2 + if [[ "$_setup_b2" =~ ^[Yy]$ ]]; then + echo "" + echo " Two one-time steps in the B2 web console first — this script can't do" + echo " these for you:" + echo "" + echo " 1) Buckets → Create a Bucket" + echo " - Files in Bucket: Private" + echo " - Object Lock: your call. ON means backups in this bucket can't be" + echo " deleted or overwritten for a retention period you choose, even by" + echo " someone holding valid credentials for it — protects the offsite" + echo " copy if this box is ever compromised, at the cost of genuinely not" + echo " being able to delete early yourself either. Can only be set at" + echo " bucket creation, not turned on later." + echo " - Note the endpoint shown on the bucket's details page afterward," + echo " e.g. s3.us-west-004.backblazeb2.com — you'll need it below." + echo "" + echo " 2) Account → App Keys → Add a New Application Key" + echo " - Allow access to: the bucket you just created (not 'All')" + echo " - Type: Read and Write" + echo " - B2 shows the application key ONLY once — copy both values now," + echo " you can't retrieve the key itself again afterward." + echo "" + local B2_BUCKET="" B2_ENDPOINT="" B2_KEY_ID="" B2_APP_KEY="" + prompt_text " Bucket name:" "" B2_BUCKET + prompt_text " Endpoint (e.g. s3.us-west-004.backblazeb2.com):" "" B2_ENDPOINT + prompt_text " Application Key ID:" "" B2_KEY_ID + read -rsp " Application Key (input hidden): " B2_APP_KEY; echo + + if [ -z "$B2_BUCKET" ] || [ -z "$B2_ENDPOINT" ] || [ -z "$B2_KEY_ID" ] || [ -z "$B2_APP_KEY" ]; then + log_warning "One or more fields left blank — skipping B2 setup this run." + else + log_info "Verifying B2 credentials (dry-run sync against the 'default' repo)..." + local _b2_err + if _b2_err="$(env KOPIA_PASSWORD="${DEST_PASSWORDS[default]}" "$KOPIA_BIN" \ + --config-file="${DEST_CONFIGS[default]}" repository sync-to s3 \ + --bucket="$B2_BUCKET" --access-key="$B2_KEY_ID" \ + --secret-access-key="$B2_APP_KEY" --endpoint="$B2_ENDPOINT" \ + --dry-run 2>&1)"; then + REMOTE_TYPE="s3" + REMOTE_ARGS="--bucket=$B2_BUCKET --access-key=$B2_KEY_ID --secret-access-key=$B2_APP_KEY --endpoint=$B2_ENDPOINT" + log_success "B2 credentials verified — offsite mirroring will run after each backup." + else + log_warning "B2 dry-run failed — check bucket name, endpoint, and key permissions:" + log_warning "$_b2_err" + log_warning "Not enabling offsite mirroring this run. Re-run this installer once" + log_warning "fixed, or hand-edit REMOTE_TYPE/REMOTE_ARGS in backup.conf directly." + fi + fi + fi + # ── 8. Write backup.conf ───────────────────────────────────────────────── log_info "Writing $CONF_FILE ..." { @@ -556,9 +656,12 @@ install_backup() { echo "" echo "# ── Optional offsite mirror ─────────────────────────────────────────────────" echo "# Mirror ALL repos offsite after each run (see kopia repository sync-to --help)." + echo "# B2: use the s3 provider against B2's S3-compatible endpoint, not the b2" + echo "# provider — kopia.io marks repository-sync-to-b2 as deprecated. Example:" + echo "# REMOTE_TYPE=s3 REMOTE_ARGS=\"--bucket=NAME --access-key=KEYID --secret-access-key=KEY --endpoint=s3.us-west-004.backblazeb2.com\"" echo "# Example SFTP: REMOTE_TYPE=sftp REMOTE_ARGS=\"--host H --username U --path /srv/...\"" - echo "REMOTE_TYPE=\"none\"" - echo "REMOTE_ARGS=\"\"" + echo "REMOTE_TYPE=\"$REMOTE_TYPE\"" + echo "REMOTE_ARGS=\"$REMOTE_ARGS\"" echo "" echo "# ── Notifications (ntfy) ─────────────────────────────────────────────────────" echo "# Set NTFY_URL to receive backup success/failure alerts." @@ -633,15 +736,36 @@ install_backup() { log_success "test_backup.sh installed" fi - # ── 11b. Weekly backup test timer ──────────────────────────────────────── + # ── 11b. Backup test timer ─────────────────────────────────────────────── + # Every service in this test stops briefly (seconds) while its data gets + # moved aside and restored back — same interruption profile as the main + # backup job itself. Weekly is the most thorough default, but that's a + # standing tradeoff against a weekly blip on every service; offer the + # same Daily/Weekly/Monthly/Custom shape the main backup schedule above + # already gives, rather than hardcoding one choice. local TEST_SVC_NAME="post-install-backup-test" local _add_test="" - prompt_yn " Schedule a weekly automated backup test? (y/N):" "n" _add_test + prompt_yn " Schedule an automated backup restore test? (y/N):" "n" _add_test if [[ "$_add_test" =~ ^[Yy]$ ]]; then + echo "" + echo " 1) Weekly (Saturday 03:00) (recommended)" + echo " 2) Monthly (1st of the month, 03:00)" + echo " 3) Custom (systemd OnCalendar)" + echo "" + local _test_sch="" + prompt_text " How often? [1]:" "1" _test_sch + local TEST_ONCALENDAR TEST_SCHED_LABEL TEST_CRON="" + case "${_test_sch:-1}" in + 2) TEST_ONCALENDAR="*-*-01 03:00:00"; TEST_SCHED_LABEL="monthly (1st, 03:00)"; TEST_CRON="0 3 1 * *" ;; + 3) prompt_text " OnCalendar expression:" "Sat *-*-* 03:00:00" TEST_ONCALENDAR + TEST_SCHED_LABEL="$TEST_ONCALENDAR" ;; + *) TEST_ONCALENDAR="Sat *-*-* 03:00:00"; TEST_SCHED_LABEL="weekly (Saturday 03:00)"; TEST_CRON="0 3 * * 6" ;; + esac + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then tee "/etc/systemd/system/${TEST_SVC_NAME}.service" >/dev/null << SVCEOF [Unit] -Description=Weekly restore test for Kopia backup +Description=Automated restore test for Kopia backup After=docker.service [Service] @@ -651,10 +775,10 @@ SVCEOF tee "/etc/systemd/system/${TEST_SVC_NAME}.timer" >/dev/null << SVCEOF [Unit] -Description=Weekly Kopia backup restore test (Saturday 03:00) +Description=Kopia backup restore test ($TEST_SCHED_LABEL) [Timer] -OnCalendar=Sat *-*-* 03:00:00 +OnCalendar=$TEST_ONCALENDAR Persistent=true RandomizedDelaySec=600 @@ -664,11 +788,24 @@ SVCEOF systemctl daemon-reload systemctl enable --now "${TEST_SVC_NAME}.timer" - log_success "Weekly test timer enabled (Saturday 03:00)" + log_success "Backup test timer enabled ($TEST_SCHED_LABEL)" else - echo "0 3 * * 6 root /bin/bash $TEST_SCRIPT >> /var/log/${TEST_SVC_NAME}.log 2>&1" \ + if [ -z "$TEST_CRON" ]; then + log_warning "Custom OnCalendar schedules aren't auto-translated to cron — installing" + log_warning "a weekly placeholder; edit /etc/cron.d/${TEST_SVC_NAME} to adjust the timing." + TEST_CRON="0 3 * * 6" + fi + echo "$TEST_CRON root /bin/bash $TEST_SCRIPT >> /var/log/${TEST_SVC_NAME}.log 2>&1" \ > "/etc/cron.d/${TEST_SVC_NAME}" - log_success "Weekly test cron installed (Saturday 03:00)" + log_success "Backup test cron installed ($TEST_SCHED_LABEL)" + fi + + echo "" + local _run_now="" + prompt_yn " Run the first test now, instead of waiting for the schedule? (y/N):" "n" _run_now + if [[ "$_run_now" =~ ^[Yy]$ ]]; then + log_info "Running initial backup restore test..." + bash "$TEST_SCRIPT" || log_warning "Initial test reported failures — see the output above and /var/log/post-install-backup-test.log." fi fi diff --git a/services/mattermost.sh b/services/mattermost.sh index 2c0833f..958559b 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -283,19 +283,12 @@ install_mattermost() { fi fi - # Free-port scan — runs unconditionally, not just when adding an explicit - # additional instance, so a plain first install also can't collide with - # an unrelated service that already claimed these default ports. Same - # pattern services/asterisk.sh uses for its web admin port. WEB_PORT is - # also set as Mattermost's own internal ListenAddress below (not just the - # host publish side), so configure_caddy_for_service's single upstream - # "name:port" string works unmodified in both local and remote-Caddy mode — - # it assumes host-published-port == container-internal-port, true for - # every other service in this repo and made true here too rather than - # special-casing the shared helper for one caller. See CLAUDE.md's "Port - # collision avoidance" section. - find_free_port WEB_PORT "$WEB_PORT" - find_free_port CALLS_UDP_PORT "$CALLS_UDP_PORT" udp + # WEB_PORT/CALLS_UDP_PORT are resolved further down, after we know + # whether this is an update (read the existing port back, never rescan + # — see the comment there) or a fresh/new install (scan from these + # defaults). Scanning here, before that's known, used to mean an + # update run could see this instance's OWN currently-published port as + # "in use" and silently move it — see git history on this block. log_info "Installing Mattermost${INSTANCE_SUFFIX:+ ($INSTANCE_SUFFIX)}..." @@ -340,6 +333,41 @@ install_mattermost() { esac fi + # ── Web/Calls ports: read back on update, scan fresh on a new/full install ── + # Mirrors services/asterisk.sh's WEB_ADMIN_PORT handling: update must + # never rescan here — nothing has stopped the currently-running + # container yet at this point (fresh/"Full reinstall" already did, + # above), so find_free_port would see this instance's OWN + # already-published port as "in use" and silently move it to a + # different one on every single update run. Falls back to parsing the + # existing MM_SERVICESETTINGS_LISTENADDRESS / docker-compose.yml port + # mapping for installs from before WEB_PORT/CALLS_UDP_PORT were written + # to .env directly — so this doesn't regress anyone already running. + if [ "$MODE" = "update" ] && [ -f "$DIR/.env" ]; then + local _EXISTING_WEB_PORT _EXISTING_CALLS_PORT + _EXISTING_WEB_PORT="$(grep '^WEB_PORT=' "$DIR/.env" 2>/dev/null | cut -d= -f2-)" + [ -z "$_EXISTING_WEB_PORT" ] && _EXISTING_WEB_PORT="$(grep '^MM_SERVICESETTINGS_LISTENADDRESS=:' "$DIR/.env" 2>/dev/null | sed 's/.*://')" + [ -n "$_EXISTING_WEB_PORT" ] && WEB_PORT="$_EXISTING_WEB_PORT" + + _EXISTING_CALLS_PORT="$(grep '^CALLS_UDP_PORT=' "$DIR/.env" 2>/dev/null | cut -d= -f2-)" + [ -z "$_EXISTING_CALLS_PORT" ] && _EXISTING_CALLS_PORT="$(grep -oE '"[0-9]+:8443/udp"' "$DIR/docker-compose.yml" 2>/dev/null | head -1 | cut -d: -f1 | tr -d '"')" + [ -n "$_EXISTING_CALLS_PORT" ] && CALLS_UDP_PORT="$_EXISTING_CALLS_PORT" + else + # Fresh/new install — runs unconditionally, not just when adding an + # explicit additional instance, so a plain first install also can't + # collide with an unrelated service that already claimed these + # default ports. WEB_PORT is also set as Mattermost's own internal + # ListenAddress below (not just the host publish side), so + # configure_caddy_for_service's single upstream "name:port" string + # works unmodified in both local and remote-Caddy mode — it assumes + # host-published-port == container-internal-port, true for every + # other service in this repo and made true here too rather than + # special-casing the shared helper for one caller. See CLAUDE.md's + # "Port collision avoidance" section. + find_free_port WEB_PORT "$WEB_PORT" + find_free_port CALLS_UDP_PORT "$CALLS_UDP_PORT" udp + fi + mkdir -p "$DIR" ensure_docker_dir_ownership "$DIR" cd "$DIR" || return 1 @@ -514,6 +542,13 @@ EOF TZ=$TZ_VAL CADDY_NET=$SITE_CADDY_NET +# Host-published ports this install is actually using — read back on every +# Update run (see the WEB_PORT/CALLS_UDP_PORT resolution above +# install_mattermost's mkdir) instead of rescanning while this instance's +# own container is still up. +WEB_PORT=$WEB_PORT +CALLS_UDP_PORT=$CALLS_UDP_PORT + # PostgreSQL POSTGRES_DB=mattermost POSTGRES_USER=mattermost