From 158df0d54527dc1fbf6161cdecfecd3395071729 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 03:40:32 +0000 Subject: [PATCH 1/6] Fix embedded-coturn relay-port overlap between Asterisk and Mattermost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live on a box that retired the shared coturn service in favor of each service running its own dedicated/embedded coturn permanently: Mattermost's embedded-coturn fallback used relay range 49153-49352, which overlaps Asterisk's embedded coturn range (49152-49252) by ~100 UDP ports. Both run network_mode: host, so with shared coturn out of the picture this is the exact same collision CLAUDE.md documents as the original, already-fixed-once bug that the shared coturn service was built to solve in the first place — reintroduced here because Mattermost's embedded-coturn fallback path apparently never got checked against Asterisk's numbers when it was written. Moved Mattermost's embedded relay range to 49253-49452 (same 200-port width, now contiguous with and non-overlapping Asterisk's 49152-49252). Updated the docker-compose command flags, the matching UFW rule, and added a comment explaining the offset so it doesn't drift back into collision — and noting the known residual gap this doesn't cover: two Mattermost instances *both* falling back to embedded coturn at once would still collide with each other on these same fixed numbers. Not fixed here since it requires more than one Mattermost instance to be running without shared coturn at the same time, which isn't this box's situation; flagged in-code for whoever hits it. Also made asterisk.sh's generated README port table stop unconditionally claiming a TURN relay range it isn't actually publishing when the shared coturn service (not this install's own container) is fronting TURN instead — it now branches on USE_EMBEDDED_COTURN, which the function already receives as a parameter. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/asterisk.sh | 6 ++++-- services/mattermost.sh | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/services/asterisk.sh b/services/asterisk.sh index 0cfa883..c0f04d1 100644 --- a/services/asterisk.sh +++ b/services/asterisk.sh @@ -1447,9 +1447,11 @@ docker exec -it ${CONTAINER} easy-asterisk | 5061 | TCP | SIP over TLS | | ${WEB_ADMIN_PORT_VAL} | TCP | Easy Asterisk web admin (auto-picked — see \`.env\`) | | 8088/8089 | TCP | Asterisk HTTP/WS (ARI/AMI) | -| 3478 | UDP/TCP | TURN/STUN (coturn) | | 10000–20000 | UDP | RTP media streams | -| 49152–49252 | UDP | TURN relay media ports | +$( [[ "$USE_EMBEDDED_COTURN" == true ]] \ + && echo "| 3478 | UDP/TCP | TURN/STUN (this install's own dedicated coturn) | +| 49152–49252 | UDP | TURN relay media ports (dedicated coturn) |" \ + || echo "| See \`~/docker/coturn/.env\` | UDP/TCP | TURN/STUN — shared coturn service, not opened by this install |" ) ## Data directories (all inside ${EA_DIR}/, included in backup) diff --git a/services/mattermost.sh b/services/mattermost.sh index 7ccb3a6..063ab63 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -415,6 +415,18 @@ networks: fi [ -n "$MM_SECRET" ] || MM_SECRET=$(generate_password 48) + # Listening port (3479, vs. Asterisk's embedded coturn on 3478) and relay + # range (49253-49452, vs. Asterisk's 49152-49252) are both deliberately + # offset from services/asterisk.sh's embedded coturn defaults. Confirmed + # live: an earlier version of this range (49153-49352) overlapped + # Asterisk's by ~100 ports — the exact bug the shared coturn service + # (services/coturn.sh) exists to avoid, reintroduced here because this + # is the no-shared-coturn fallback path. If more than one Mattermost + # instance ever falls back to embedded coturn at the same time, they'll + # collide with EACH OTHER on these same fixed numbers — not handled here + # (single-instance-without-shared-coturn is the case this fallback is + # actually for; install services/coturn.sh instead if you need more than + # one consumer without hand-managing per-instance port math). local _COTURN_SERVICE="" if [ "$USE_EMBEDDED_COTURN" = true ]; then _COTURN_SERVICE=" @@ -431,8 +443,8 @@ networks: - --use-auth-secret - --static-auth-secret=\${COTURN_SECRET} - --realm=\${MM_REALM:-localhost} - - --min-port=49153 - - --max-port=49352 + - --min-port=49253 + - --max-port=49452 - --no-tls - --no-dtls - --no-cli @@ -532,7 +544,7 @@ EOF ufw allow "${CALLS_UDP_PORT}/udp" comment "Mattermost Calls RTC${INSTANCE_SUFFIX:+ ($INSTANCE_SUFFIX)}" if [ "$USE_EMBEDDED_COTURN" = true ]; then ufw allow 3479/udp; ufw allow 3479/tcp - ufw allow 49153:49352/udp comment "Mattermost coturn relay" + ufw allow 49253:49452/udp comment "Mattermost coturn relay" fi # Shared coturn opens its own ports once, at its own install time. fi From 4a09d3d1def3f6ee4e754b0a487a0725b5f3bef8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 03:45:28 +0000 Subject: [PATCH 2/6] Give each Mattermost instance's embedded coturn its own port slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the Asterisk/Mattermost relay-range overlap fix: that fix only handled the two-service collision, and left a documented gap for what happens when a second (or third...) Mattermost instance also falls back to embedded coturn — they'd have collided with each other on the same fixed 3479/49253-49452 numbers, same bug, different pair. find_free_port-style scanning doesn't work for the relay range itself — it's a scan for a single free port, not a free contiguous 200-port block — so this follows the same fixed-offset-per-instance approach CLAUDE.md documents for traccar.sh's large port range instead. Each instance gets an integer slot (control port = 3479 + slot, relay range = 49253 + slot*200 through +199) computed once as the smallest slot number not already claimed by another mattermost*/.env on the box, then cached in that instance's own .env as EMBEDDED_COTURN_SLOT so it reads back the same value on every later update or full reinstall instead of potentially landing on a different slot (which would silently move an already-configured instance's TURN port out from under it — the same "never touch what's already the box's answer" rule everything else in update mode already follows). Verified the allocation logic against a mock: first instance gets slot 0, a second gets slot 1 without stepping on the first, both instances keep their own slot across a simulated re-run, and a third new instance correctly lands on the next free slot (2) rather than reusing either. Threaded the computed port/range through every place that used to hardcode 3479/49253/49452: the coturn compose block, the UFW rule (now also labeled with the instance suffix, matching this file's other UFW comments), and the Calls-plugin TURN config text in the generated README/System-Console instructions. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/mattermost.sh | 68 ++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/services/mattermost.sh b/services/mattermost.sh index 063ab63..828a427 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -415,18 +415,42 @@ networks: fi [ -n "$MM_SECRET" ] || MM_SECRET=$(generate_password 48) - # Listening port (3479, vs. Asterisk's embedded coturn on 3478) and relay - # range (49253-49452, vs. Asterisk's 49152-49252) are both deliberately - # offset from services/asterisk.sh's embedded coturn defaults. Confirmed - # live: an earlier version of this range (49153-49352) overlapped - # Asterisk's by ~100 ports — the exact bug the shared coturn service - # (services/coturn.sh) exists to avoid, reintroduced here because this - # is the no-shared-coturn fallback path. If more than one Mattermost - # instance ever falls back to embedded coturn at the same time, they'll - # collide with EACH OTHER on these same fixed numbers — not handled here - # (single-instance-without-shared-coturn is the case this fallback is - # actually for; install services/coturn.sh instead if you need more than - # one consumer without hand-managing per-instance port math). + # Each embedded-coturn instance (this Mattermost falls back to its own + # dedicated coturn when the shared one isn't available) needs its own + # listening port and relay range, or two instances both on embedded + # coturn collide on identical fixed numbers — confirmed live for the + # Asterisk-vs-Mattermost case this same offset scheme now also fixes + # (see the git history on this block). A relay range can't be found by + # scanning port-by-port like find_free_port does for a single port + # (CLAUDE.md's port-collision-avoidance section is explicit about this + # for large ranges) — so instead each instance gets an integer "slot" + # or the next one already used elsewhere, and a wide-enough width + # (200, matching this range's existing size) keeps slots from + # overlapping each other. base 3479/49253 is right after Asterisk's own + # embedded-coturn numbers (3478/49152-49252) so slot 0 doesn't collide + # with Asterisk either. + # + # The slot is assigned once (the smallest integer not already claimed + # by another mattermost*/.env on this box) and cached in THIS + # instance's own .env as EMBEDDED_COTURN_SLOT, so re-running this same + # instance's installer (update or full reinstall) always reads the + # same slot back instead of potentially reassigning it — reassignment + # would silently move an already-configured instance's TURN port out + # from under it. + local EMBEDDED_COTURN_SLOT="" + [ -f "$DIR/.env" ] && EMBEDDED_COTURN_SLOT="$(grep '^EMBEDDED_COTURN_SLOT=' "$DIR/.env" 2>/dev/null | cut -d= -f2-)" + if [ -z "$EMBEDDED_COTURN_SLOT" ]; then + local _used_slots + _used_slots="$(grep -h '^EMBEDDED_COTURN_SLOT=' "$DOCKER_DIR"/mattermost*/.env 2>/dev/null | cut -d= -f2-)" + EMBEDDED_COTURN_SLOT=0 + while echo "$_used_slots" | grep -qx "$EMBEDDED_COTURN_SLOT"; do + EMBEDDED_COTURN_SLOT=$((EMBEDDED_COTURN_SLOT + 1)) + done + fi + local _MM_COTURN_PORT=$((3479 + EMBEDDED_COTURN_SLOT)) + local _MM_COTURN_MIN=$((49253 + EMBEDDED_COTURN_SLOT * 200)) + local _MM_COTURN_MAX=$((_MM_COTURN_MIN + 199)) + local _COTURN_SERVICE="" if [ "$USE_EMBEDDED_COTURN" = true ]; then _COTURN_SERVICE=" @@ -437,14 +461,14 @@ networks: user: root command: - -n - - --listening-port=3479 + - --listening-port=${_MM_COTURN_PORT} - --listening-ip=0.0.0.0 - --fingerprint - --use-auth-secret - --static-auth-secret=\${COTURN_SECRET} - --realm=\${MM_REALM:-localhost} - - --min-port=49253 - - --max-port=49452 + - --min-port=${_MM_COTURN_MIN} + - --max-port=${_MM_COTURN_MAX} - --no-tls - --no-dtls - --no-cli @@ -521,6 +545,12 @@ TURN_HOST=$TURN_HOST_VAL TURN_PORT=$TURN_PORT_VAL TURN_USERNAME=$TURN_USERNAME_VAL TURN_PASSWORD=$TURN_PASSWORD_VAL +# This instance's embedded-coturn port slot (see the comment above the +# EMBEDDED_COTURN_SLOT assignment in mattermost.sh) — read back on every +# re-run so it never gets reassigned out from under an already-running +# instance. Reserved even when USE_EMBEDDED_COTURN is currently false, in +# case this instance ever falls back to its own coturn later. +EMBEDDED_COTURN_SLOT=$EMBEDDED_COTURN_SLOT EOF chmod 600 .env @@ -543,8 +573,8 @@ EOF ufw allow "${WEB_PORT}/tcp" comment "Mattermost${INSTANCE_SUFFIX:+ ($INSTANCE_SUFFIX)}" ufw allow "${CALLS_UDP_PORT}/udp" comment "Mattermost Calls RTC${INSTANCE_SUFFIX:+ ($INSTANCE_SUFFIX)}" if [ "$USE_EMBEDDED_COTURN" = true ]; then - ufw allow 3479/udp; ufw allow 3479/tcp - ufw allow 49253:49452/udp comment "Mattermost coturn relay" + ufw allow "${_MM_COTURN_PORT}/udp"; ufw allow "${_MM_COTURN_PORT}/tcp" + ufw allow "${_MM_COTURN_MIN}:${_MM_COTURN_MAX}/udp" comment "Mattermost coturn relay${INSTANCE_SUFFIX:+ ($INSTANCE_SUFFIX)}" fi # Shared coturn opens its own ports once, at its own install time. fi @@ -562,9 +592,9 @@ EOF # cannot run at the same time as --lt-cred-mech on one instance). local _ICE_JSON _turn_config_md if [ "$USE_EMBEDDED_COTURN" = true ]; then - _ICE_JSON="[{\"urls\":[\"turn:${SITE_DOMAIN:-YOUR_IP}:3479?transport=udp\"],\"username\":\"static\",\"credential\":\"see COTURN_SECRET below — this dedicated coturn uses use-auth-secret/HMAC, not a fixed credential\"}]" + _ICE_JSON="[{\"urls\":[\"turn:${SITE_DOMAIN:-YOUR_IP}:${_MM_COTURN_PORT}?transport=udp\"],\"username\":\"static\",\"credential\":\"see COTURN_SECRET below — this dedicated coturn uses use-auth-secret/HMAC, not a fixed credential\"}]" _turn_config_md="This instance runs its own dedicated coturn (HMAC/REST-API auth): -- TURN Server URI: \`turn:${SITE_DOMAIN:-YOUR_IP}:3479?transport=udp\` +- TURN Server URI: \`turn:${SITE_DOMAIN:-YOUR_IP}:${_MM_COTURN_PORT}?transport=udp\` - System Console → Plugins → Calls → **TURN Static Auth Secret**: value of \`COTURN_SECRET\` in \`.env\`" else _ICE_JSON="[{\"urls\":[\"turn:${TURN_HOST_VAL}:${TURN_PORT_VAL}?transport=udp\"],\"username\":\"${TURN_USERNAME_VAL}\",\"credential\":\"${TURN_PASSWORD_VAL}\"}]" From cce8147059b1653281813aa674e9a34a0d4cf6fb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 15:04:51 +0000 Subject: [PATCH 3/6] Live-verify a newly assigned Mattermost coturn slot isn't already bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requested check: the slot-allocation scheme added in the previous commit only checked against OTHER mattermost*/.env files on the box, not against what's actually listening. A slot whose numbers happen to be free by that bookkeeping could still be squatted by something this script doesn't track (a manually-run process, an unrelated service) — this box already learned that lesson once, from Asterisk and Mattermost's embedded coturn ranges overlapping without either side knowing. Only a NEWLY assigned slot gets the live check — an already-cached slot (read back from this instance's own .env) is trusted as-is, since a live conflict on an already-configured, already-running instance's own port is a real problem to report, not something to silently route around by moving that instance's TURN port out from under it. Can't scan the full 200-port relay range port-by-port (large ranges use the offset scheme instead of scanning per CLAUDE.md's port-collision section) — checks the control port plus both relay-range boundaries as the practical middle ground. Verified against a mock: a candidate slot whose control port is already bound gets skipped in favor of the next free one. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/mattermost.sh | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/services/mattermost.sh b/services/mattermost.sh index 828a427..acfacfd 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -440,12 +440,33 @@ networks: local EMBEDDED_COTURN_SLOT="" [ -f "$DIR/.env" ] && EMBEDDED_COTURN_SLOT="$(grep '^EMBEDDED_COTURN_SLOT=' "$DIR/.env" 2>/dev/null | cut -d= -f2-)" if [ -z "$EMBEDDED_COTURN_SLOT" ]; then - local _used_slots + # Assigning a NEW slot — also live-verify the candidate control port + # and relay-range boundaries aren't already bound by something this + # box's own .env files don't know about (a manually-run process, an + # unrelated service). An already-cached slot (the branch above) is + # trusted as-is and never re-verified — that's what "stable across + # re-runs" means; a live process squatting on an already-assigned + # slot's port is a conflict to report, not silently route around by + # moving an already-configured instance. Can't scan the full + # 200-port relay range port-by-port (CLAUDE.md's + # port-collision-avoidance section covers why large ranges use an + # offset instead of scanning) — checking the control port plus the + # relay range's own two boundary ports is the practical middle + # ground between "no live check at all" and a full range scan. + local _used_slots _cand _p _min _max _used_slots="$(grep -h '^EMBEDDED_COTURN_SLOT=' "$DOCKER_DIR"/mattermost*/.env 2>/dev/null | cut -d= -f2-)" - EMBEDDED_COTURN_SLOT=0 - while echo "$_used_slots" | grep -qx "$EMBEDDED_COTURN_SLOT"; do - EMBEDDED_COTURN_SLOT=$((EMBEDDED_COTURN_SLOT + 1)) + _cand=0 + while true; do + _p=$((3479 + _cand)); _min=$((49253 + _cand * 200)); _max=$((_min + 199)) + if echo "$_used_slots" | grep -qx "$_cand" \ + || port_in_use "$_p" || port_in_use "$_p" udp \ + || port_in_use "$_min" udp || port_in_use "$_max" udp; then + _cand=$((_cand + 1)) + continue + fi + break done + EMBEDDED_COTURN_SLOT="$_cand" fi local _MM_COTURN_PORT=$((3479 + EMBEDDED_COTURN_SLOT)) local _MM_COTURN_MIN=$((49253 + EMBEDDED_COTURN_SLOT * 200)) From 624ae3d2f3e1136a4c7057bc05532a41f9f54955 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 15:12:05 +0000 Subject: [PATCH 4/6] Stop Mattermost's WEB_PORT/CALLS_UDP_PORT rescanning on every update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while adding a live scanner to the coturn-slot code: WEB_PORT and CALLS_UDP_PORT were scanned unconditionally, before the reinstall-mode prompt even ran and before anything stopped the currently-running container. On an "Update" run that meant find_free_port would see this instance's OWN already-published port as occupied and silently shift it to the next free one — every plain update could have moved the service's port out from under already-configured Caddy routes, bookmarks, and the Calls plugin's client config, without the operator asking for that. services/asterisk.sh already gets this right for WEB_ADMIN_PORT: update reads the existing port back from .env (no rescan), fresh scans from the plain default only after stopping the old container. Brought Mattermost in line with the same shape — the port resolution moved from before the reinstall-mode block to after it, so MODE is known and, for a fresh install/"Full reinstall", the old containers are already stopped by the time it scans. WEB_PORT/CALLS_UDP_PORT are now also written to .env directly (they weren't before), with a fallback to parse them from the existing MM_SERVICESETTINGS_LISTENADDRESS / docker-compose.yml port mapping for installs made before this change — so an update on an already-running instance doesn't regress just because its .env predates the new variables. Verified the explicit-var, fallback-parse, and priority-order (explicit wins over fallback) cases against a mock before shipping. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/mattermost.sh | 61 +++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/services/mattermost.sh b/services/mattermost.sh index acfacfd..e98614f 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -270,19 +270,12 @@ install_mattermost() { fi fi - # Free-port scan — runs unconditionally, not just when adding an explicit - # additional instance, so a plain first install also can't collide with - # an unrelated service that already claimed these default ports. Same - # pattern services/asterisk.sh uses for its web admin port. WEB_PORT is - # also set as Mattermost's own internal ListenAddress below (not just the - # host publish side), so configure_caddy_for_service's single upstream - # "name:port" string works unmodified in both local and remote-Caddy mode — - # it assumes host-published-port == container-internal-port, true for - # every other service in this repo and made true here too rather than - # special-casing the shared helper for one caller. See CLAUDE.md's "Port - # collision avoidance" section. - find_free_port WEB_PORT "$WEB_PORT" - find_free_port CALLS_UDP_PORT "$CALLS_UDP_PORT" udp + # WEB_PORT/CALLS_UDP_PORT are resolved further down, after we know + # whether this is an update (read the existing port back, never rescan + # — see the comment there) or a fresh/new install (scan from these + # defaults). Scanning here, before that's known, used to mean an + # update run could see this instance's OWN currently-published port as + # "in use" and silently move it — see git history on this block. log_info "Installing Mattermost${INSTANCE_SUFFIX:+ ($INSTANCE_SUFFIX)}..." @@ -332,6 +325,41 @@ install_mattermost() { esac fi + # ── Web/Calls ports: read back on update, scan fresh on a new/full install ── + # Mirrors services/asterisk.sh's WEB_ADMIN_PORT handling: update must + # never rescan here — nothing has stopped the currently-running + # container yet at this point (fresh/"Full reinstall" already did, + # above), so find_free_port would see this instance's OWN + # already-published port as "in use" and silently move it to a + # different one on every single update run. Falls back to parsing the + # existing MM_SERVICESETTINGS_LISTENADDRESS / docker-compose.yml port + # mapping for installs from before WEB_PORT/CALLS_UDP_PORT were written + # to .env directly — so this doesn't regress anyone already running. + if [ "$MODE" = "update" ] && [ -f "$DIR/.env" ]; then + local _EXISTING_WEB_PORT _EXISTING_CALLS_PORT + _EXISTING_WEB_PORT="$(grep '^WEB_PORT=' "$DIR/.env" 2>/dev/null | cut -d= -f2-)" + [ -z "$_EXISTING_WEB_PORT" ] && _EXISTING_WEB_PORT="$(grep '^MM_SERVICESETTINGS_LISTENADDRESS=:' "$DIR/.env" 2>/dev/null | sed 's/.*://')" + [ -n "$_EXISTING_WEB_PORT" ] && WEB_PORT="$_EXISTING_WEB_PORT" + + _EXISTING_CALLS_PORT="$(grep '^CALLS_UDP_PORT=' "$DIR/.env" 2>/dev/null | cut -d= -f2-)" + [ -z "$_EXISTING_CALLS_PORT" ] && _EXISTING_CALLS_PORT="$(grep -oE '"[0-9]+:8443/udp"' "$DIR/docker-compose.yml" 2>/dev/null | head -1 | cut -d: -f1 | tr -d '"')" + [ -n "$_EXISTING_CALLS_PORT" ] && CALLS_UDP_PORT="$_EXISTING_CALLS_PORT" + else + # Fresh/new install — runs unconditionally, not just when adding an + # explicit additional instance, so a plain first install also can't + # collide with an unrelated service that already claimed these + # default ports. WEB_PORT is also set as Mattermost's own internal + # ListenAddress below (not just the host publish side), so + # configure_caddy_for_service's single upstream "name:port" string + # works unmodified in both local and remote-Caddy mode — it assumes + # host-published-port == container-internal-port, true for every + # other service in this repo and made true here too rather than + # special-casing the shared helper for one caller. See CLAUDE.md's + # "Port collision avoidance" section. + find_free_port WEB_PORT "$WEB_PORT" + find_free_port CALLS_UDP_PORT "$CALLS_UDP_PORT" udp + fi + mkdir -p "$DIR" ensure_docker_dir_ownership "$DIR" cd "$DIR" || return 1 @@ -541,6 +569,13 @@ EOF TZ=$TZ_VAL CADDY_NET=$SITE_CADDY_NET +# Host-published ports this install is actually using — read back on every +# Update run (see the WEB_PORT/CALLS_UDP_PORT resolution above +# install_mattermost's mkdir) instead of rescanning while this instance's +# own container is still up. +WEB_PORT=$WEB_PORT +CALLS_UDP_PORT=$CALLS_UDP_PORT + # PostgreSQL POSTGRES_DB=mattermost POSTGRES_USER=mattermost From 93d5459a67d85b8e8e513bfc082ec7d54624faa3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 19:22:52 +0000 Subject: [PATCH 5/6] Make the backup restore-test schedule configurable, add a run-now option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers a direct ask: the automated restore-verify test (extras/test_backup_kopia.sh — verifies the latest snapshot, restores it over a moved-aside copy, compares, rolls back, reports PASS/FAIL, sends an ntfy notification) was already fully non-interactive and already wired to a systemd timer/cron fallback by install_backup() — it just had no schedule choice at all, hardcoded to weekly (Saturday 03:00). Every service in this test stops briefly while its data gets moved aside and restored back, same interruption profile as the main backup job — so the schedule is a real tradeoff (more frequent verification vs. more frequent blips), not a free "always pick the most frequent" choice. Gave it the same Weekly/Monthly/Custom shape the main backup schedule prompt above it already offers, instead of a single hardcoded option. Also added an explicit "run the first test now?" prompt right after scheduling it — otherwise choosing Monthly means waiting up to a month before finding out whether the test even works, rather than getting that initial confirmation immediately and then settling into the chosen cadence. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/backup.sh | 50 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/services/backup.sh b/services/backup.sh index c32b388..35ffab5 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -633,15 +633,36 @@ install_backup() { log_success "test_backup.sh installed" fi - # ── 11b. Weekly backup test timer ──────────────────────────────────────── + # ── 11b. Backup test timer ─────────────────────────────────────────────── + # Every service in this test stops briefly (seconds) while its data gets + # moved aside and restored back — same interruption profile as the main + # backup job itself. Weekly is the most thorough default, but that's a + # standing tradeoff against a weekly blip on every service; offer the + # same Daily/Weekly/Monthly/Custom shape the main backup schedule above + # already gives, rather than hardcoding one choice. local TEST_SVC_NAME="post-install-backup-test" local _add_test="" - prompt_yn " Schedule a weekly automated backup test? (y/N):" "n" _add_test + prompt_yn " Schedule an automated backup restore test? (y/N):" "n" _add_test if [[ "$_add_test" =~ ^[Yy]$ ]]; then + echo "" + echo " 1) Weekly (Saturday 03:00) (recommended)" + echo " 2) Monthly (1st of the month, 03:00)" + echo " 3) Custom (systemd OnCalendar)" + echo "" + local _test_sch="" + prompt_text " How often? [1]:" "1" _test_sch + local TEST_ONCALENDAR TEST_SCHED_LABEL TEST_CRON="" + case "${_test_sch:-1}" in + 2) TEST_ONCALENDAR="*-*-01 03:00:00"; TEST_SCHED_LABEL="monthly (1st, 03:00)"; TEST_CRON="0 3 1 * *" ;; + 3) prompt_text " OnCalendar expression:" "Sat *-*-* 03:00:00" TEST_ONCALENDAR + TEST_SCHED_LABEL="$TEST_ONCALENDAR" ;; + *) TEST_ONCALENDAR="Sat *-*-* 03:00:00"; TEST_SCHED_LABEL="weekly (Saturday 03:00)"; TEST_CRON="0 3 * * 6" ;; + esac + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then tee "/etc/systemd/system/${TEST_SVC_NAME}.service" >/dev/null << SVCEOF [Unit] -Description=Weekly restore test for Kopia backup +Description=Automated restore test for Kopia backup After=docker.service [Service] @@ -651,10 +672,10 @@ SVCEOF tee "/etc/systemd/system/${TEST_SVC_NAME}.timer" >/dev/null << SVCEOF [Unit] -Description=Weekly Kopia backup restore test (Saturday 03:00) +Description=Kopia backup restore test ($TEST_SCHED_LABEL) [Timer] -OnCalendar=Sat *-*-* 03:00:00 +OnCalendar=$TEST_ONCALENDAR Persistent=true RandomizedDelaySec=600 @@ -664,11 +685,24 @@ SVCEOF systemctl daemon-reload systemctl enable --now "${TEST_SVC_NAME}.timer" - log_success "Weekly test timer enabled (Saturday 03:00)" + log_success "Backup test timer enabled ($TEST_SCHED_LABEL)" else - echo "0 3 * * 6 root /bin/bash $TEST_SCRIPT >> /var/log/${TEST_SVC_NAME}.log 2>&1" \ + if [ -z "$TEST_CRON" ]; then + log_warning "Custom OnCalendar schedules aren't auto-translated to cron — installing" + log_warning "a weekly placeholder; edit /etc/cron.d/${TEST_SVC_NAME} to adjust the timing." + TEST_CRON="0 3 * * 6" + fi + echo "$TEST_CRON root /bin/bash $TEST_SCRIPT >> /var/log/${TEST_SVC_NAME}.log 2>&1" \ > "/etc/cron.d/${TEST_SVC_NAME}" - log_success "Weekly test cron installed (Saturday 03:00)" + log_success "Backup test cron installed ($TEST_SCHED_LABEL)" + fi + + echo "" + local _run_now="" + prompt_yn " Run the first test now, instead of waiting for the schedule? (y/N):" "n" _run_now + if [[ "$_run_now" =~ ^[Yy]$ ]]; then + log_info "Running initial backup restore test..." + bash "$TEST_SCRIPT" || log_warning "Initial test reported failures — see the output above and /var/log/post-install-backup-test.log." fi fi From c48ed039e089ffb88dc07deac23caa29f86c2696 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 19:28:00 +0000 Subject: [PATCH 6/6] Guide + automate Backblaze B2 offsite mirror setup in backup.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers a direct ask: offsite mirroring existed only as a REMOTE_TYPE/ REMOTE_ARGS placeholder in backup.conf with a comment pointing at `kopia repository sync-to --help` — no interactive setup at all, B2 or otherwise. Checked before building anything: Kopia's dedicated `sync-to b2` provider is marked [DEPRECATED] on kopia.io's own command reference. B2 also offers an S3-compatible endpoint (s3..backblazeb2.com, same application key works as the access/secret key pair), and Kopia's `sync-to s3` provider isn't deprecated — so this targets that path instead of building on a command on its way out. What's now automated vs. guided, deliberately split: - Bucket creation and the application key are walked through as console steps, not automated. Object Lock specifically is a one-time, bucket-creation-only decision with a real tradeoff (undeletable-by- design vs. genuinely can't delete early) that shouldn't be silently flipped either way by a script on someone's behalf. - Once the operator has a bucket + endpoint + scoped application key (B2 requires a key scoped to one bucket, not the account master key — noted in the walkthrough), this becomes mechanical: run a `sync-to s3 --dry-run` against the just-created 'default' repo to verify the credentials actually work, and only then write REMOTE_TYPE=s3 / REMOTE_ARGS into backup.conf. A bad bucket name or key leaves REMOTE_TYPE at "none" with a clear error instead of saving a broken config that fails silently at 2am. - Encryption isn't a separate step — Kopia already encrypts client-side with the repository password set earlier in this same flow; called that out explicitly since it was asked about as if it needed its own setup step. Also fixed a regression the new prompt would otherwise have caused: backup.sh has no update/fresh distinction and re-asks everything on every run, so an already-configured offsite mirror is now read back from the existing backup.conf and preserved by default — answering "no" on a re-run no longer silently resets REMOTE_TYPE to "none". Verified the control flow (not just bash -n) against a mock kopia binary and stubbed prompts: good credentials wire up REMOTE_TYPE/ REMOTE_ARGS correctly, a rejected credential leaves REMOTE_TYPE at "none" rather than saving something broken, an existing configured value survives a "no" answer on re-run, and blank fields skip cleanly without attempting a dry-run at all. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H4k6J1qXXyYxhGEgnJaMvn --- services/backup.sh | 107 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/services/backup.sh b/services/backup.sh index 35ffab5..81969bd 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -520,6 +520,106 @@ install_backup() { done unset -f kp_d + # ── 7b. Offsite mirror (Backblaze B2) ──────────────────────────────────── + # Kopia's dedicated "b2" sync-to provider is marked [DEPRECATED] in + # Kopia's own docs (kopia.io/docs/reference/command-line/common/ + # repository-sync-to-b2/) — confirmed before writing this rather than + # building on a command that's on its way out. B2's S3-compatible + # endpoint plus the actively-maintained `sync-to s3` provider is the + # supported path instead: same B2 application key, just pointed at + # B2's own s3..backblazeb2.com endpoint instead of AWS. + # + # Bucket creation and the application key can't be automated here on + # purpose — Object Lock in particular is a deliberate, one-time choice + # B2 only lets you make at bucket creation, not something safe for a + # script to flip on (or skip) silently on someone's behalf. This walks + # through both console steps, then handles the mechanical part: taking + # the resulting bucket/endpoint/key and writing a verified + # REMOTE_TYPE/REMOTE_ARGS into backup.conf. + # + # Encryption is NOT a separate step here — Kopia already encrypts + # everything client-side (AES-256-GCM) using the repository password + # set above, before any of it leaves this box. B2's own optional + # Server-Side Encryption toggle is redundant on top of that; harmless + # to also enable for defense-in-depth, but nothing here depends on it. + local REMOTE_TYPE="none" REMOTE_ARGS="" + if [ -f "$CONF_FILE" ]; then + # Preserve whatever's already configured if this is a re-run and + # the operator doesn't re-answer the prompt below — re-running this + # installer has no update/fresh distinction, so without this an + # already-working offsite mirror would silently reset to "none". + REMOTE_TYPE="$(grep '^REMOTE_TYPE=' "$CONF_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"')" + REMOTE_ARGS="$(grep '^REMOTE_ARGS=' "$CONF_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"')" + [ -z "$REMOTE_TYPE" ] && REMOTE_TYPE="none" + fi + + echo "" + echo "═══════════════════════════════════════════════════════" + echo " OFFSITE MIRROR (optional)" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Mirrors every local repo above to Backblaze B2 after each backup run —" + echo " the actual '1 copy offsite' piece of a real 3-2-1 backup. Skip this if" + echo " you don't have a B2 account yet, or would rather set REMOTE_TYPE/" + echo " REMOTE_ARGS in backup.conf by hand later." + if [ "$REMOTE_TYPE" != "none" ]; then + echo "" + echo " Offsite mirroring is already configured (REMOTE_TYPE=$REMOTE_TYPE)." + echo " Answering yes below replaces it; answering no leaves it as-is." + fi + echo "" + local _setup_b2="" + prompt_yn " Set up Backblaze B2 offsite mirroring now? (y/N):" "n" _setup_b2 + if [[ "$_setup_b2" =~ ^[Yy]$ ]]; then + echo "" + echo " Two one-time steps in the B2 web console first — this script can't do" + echo " these for you:" + echo "" + echo " 1) Buckets → Create a Bucket" + echo " - Files in Bucket: Private" + echo " - Object Lock: your call. ON means backups in this bucket can't be" + echo " deleted or overwritten for a retention period you choose, even by" + echo " someone holding valid credentials for it — protects the offsite" + echo " copy if this box is ever compromised, at the cost of genuinely not" + echo " being able to delete early yourself either. Can only be set at" + echo " bucket creation, not turned on later." + echo " - Note the endpoint shown on the bucket's details page afterward," + echo " e.g. s3.us-west-004.backblazeb2.com — you'll need it below." + echo "" + echo " 2) Account → App Keys → Add a New Application Key" + echo " - Allow access to: the bucket you just created (not 'All')" + echo " - Type: Read and Write" + echo " - B2 shows the application key ONLY once — copy both values now," + echo " you can't retrieve the key itself again afterward." + echo "" + local B2_BUCKET="" B2_ENDPOINT="" B2_KEY_ID="" B2_APP_KEY="" + prompt_text " Bucket name:" "" B2_BUCKET + prompt_text " Endpoint (e.g. s3.us-west-004.backblazeb2.com):" "" B2_ENDPOINT + prompt_text " Application Key ID:" "" B2_KEY_ID + read -rsp " Application Key (input hidden): " B2_APP_KEY; echo + + if [ -z "$B2_BUCKET" ] || [ -z "$B2_ENDPOINT" ] || [ -z "$B2_KEY_ID" ] || [ -z "$B2_APP_KEY" ]; then + log_warning "One or more fields left blank — skipping B2 setup this run." + else + log_info "Verifying B2 credentials (dry-run sync against the 'default' repo)..." + local _b2_err + if _b2_err="$(env KOPIA_PASSWORD="${DEST_PASSWORDS[default]}" "$KOPIA_BIN" \ + --config-file="${DEST_CONFIGS[default]}" repository sync-to s3 \ + --bucket="$B2_BUCKET" --access-key="$B2_KEY_ID" \ + --secret-access-key="$B2_APP_KEY" --endpoint="$B2_ENDPOINT" \ + --dry-run 2>&1)"; then + REMOTE_TYPE="s3" + REMOTE_ARGS="--bucket=$B2_BUCKET --access-key=$B2_KEY_ID --secret-access-key=$B2_APP_KEY --endpoint=$B2_ENDPOINT" + log_success "B2 credentials verified — offsite mirroring will run after each backup." + else + log_warning "B2 dry-run failed — check bucket name, endpoint, and key permissions:" + log_warning "$_b2_err" + log_warning "Not enabling offsite mirroring this run. Re-run this installer once" + log_warning "fixed, or hand-edit REMOTE_TYPE/REMOTE_ARGS in backup.conf directly." + fi + fi + fi + # ── 8. Write backup.conf ───────────────────────────────────────────────── log_info "Writing $CONF_FILE ..." { @@ -556,9 +656,12 @@ install_backup() { echo "" echo "# ── Optional offsite mirror ─────────────────────────────────────────────────" echo "# Mirror ALL repos offsite after each run (see kopia repository sync-to --help)." + echo "# B2: use the s3 provider against B2's S3-compatible endpoint, not the b2" + echo "# provider — kopia.io marks repository-sync-to-b2 as deprecated. Example:" + echo "# REMOTE_TYPE=s3 REMOTE_ARGS=\"--bucket=NAME --access-key=KEYID --secret-access-key=KEY --endpoint=s3.us-west-004.backblazeb2.com\"" echo "# Example SFTP: REMOTE_TYPE=sftp REMOTE_ARGS=\"--host H --username U --path /srv/...\"" - echo "REMOTE_TYPE=\"none\"" - echo "REMOTE_ARGS=\"\"" + echo "REMOTE_TYPE=\"$REMOTE_TYPE\"" + echo "REMOTE_ARGS=\"$REMOTE_ARGS\"" echo "" echo "# ── Notifications (ntfy) ─────────────────────────────────────────────────────" echo "# Set NTFY_URL to receive backup success/failure alerts."