From d2568ecb091839ac9c20a315d414745663215eb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 22:51:37 +0000 Subject: [PATCH 1/6] Read back existing destinations, ntfy, schedule, and B2 fields on rerun MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requested after a rerun silently reset DR_SYNC_PATH (fixed separately) — auditing the rest of install_backup() turned up the same class of bug in several other places, one of them worse than the one that prompted this: - Default destination repo path defaulted to $ACTUAL_HOME/backups/... even when the real configured repo was somewhere else entirely (this user's actual path is /root/backups/kopia-backup) — accepting the shown default on a rerun would have pointed the installer at the wrong location. - Extra (non-"default") destinations weren't preserved AT ALL on a rerun — skipping "Add more destinations?" silently dropped every extra destination, and anything mapped to it, from the rewritten backup.conf. - The per-service destination-assignment prompt always showed "[default]" regardless of the service's actual existing mapping. - ntfy URL/token always started blank, silently disabling notifications on any rerun where they weren't retyped. - The schedule prompt always defaulted to option 1 (daily 02:00) instead of reading back whatever OnCalendar was actually already running. - B2's four sub-fields (bucket/endpoint/key ID/secret) always started blank even when reconfiguring an already-working REMOTE_TYPE=s3 setup — a mispaste on any one of the four meant retyping all four blind, since there was nothing to fall back to per-field (the existing REMOTE_ARGS was already preserved as a whole on a blank/failed attempt, just not offered back as individual editable defaults). All six read the same way: pull the existing value from backup.conf (or, for the schedule, from the live systemd timer unit — schedule isn't stored in backup.conf) and use it as the prompt default, so accepting the default keeps what's already there instead of silently reverting it. Verified all six against a mock backup.conf + timer fixture with pre-existing values for every field this touches. Known remaining gap: KEEP_LATEST (retention count) still isn't read back — doing so correctly needs the repo already connected, which happens later in this same function's flow. Flagging rather than rushing a reorder here. --- services/backup.sh | 100 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/services/backup.sh b/services/backup.sh index b3a18b7..47a0669 100644 --- a/services/backup.sh +++ b/services/backup.sh @@ -421,6 +421,11 @@ install_backup() { echo "" local DEFAULT_DEST="$ACTUAL_HOME/backups/kopia-backup" + if [ -f "$CONF_FILE" ]; then + local _existing_default_repo + _existing_default_repo="$(grep '^DEST_default_REPO=' "$CONF_FILE" 2>/dev/null | sed -E 's/^DEST_default_REPO="(.*)"$/\1/')" + [ -n "$_existing_default_repo" ] && DEFAULT_DEST="$_existing_default_repo" + fi local _repo="" prompt_text " Default repository path [${DEFAULT_DEST}]:" "$DEFAULT_DEST" _repo _repo="${_repo/#\~/$ACTUAL_HOME}"; _repo="${_repo%/}" @@ -430,6 +435,28 @@ install_backup() { DEST_REPOS["default"]="$_repo" DEST_CONFIGS["default"]="/etc/kopia-backup/default.config" + # Preserve any extra (non-"default") destinations already configured. + # This script has no update/fresh distinction, so without this, + # skipping the "Add more destinations?" prompt below on a rerun would + # silently drop every extra destination — and anything mapped to it — + # from the rewritten backup.conf, rather than just leaving it as-is. + if [ -f "$CONF_FILE" ]; then + local _existing_dest_names _en _existing_repo _existing_cfg + _existing_dest_names="$(grep '^DEST_NAMES=' "$CONF_FILE" 2>/dev/null | sed -E 's/^DEST_NAMES="(.*)"$/\1/')" + for _en in $_existing_dest_names; do + [ "$_en" = "default" ] && continue + _existing_repo="$(grep "^DEST_${_en}_REPO=" "$CONF_FILE" 2>/dev/null | sed -E "s/^DEST_${_en}_REPO=\"(.*)\"\$/\1/")" + [ -z "$_existing_repo" ] && continue + _existing_cfg="$(grep "^DEST_${_en}_CONFIG=" "$CONF_FILE" 2>/dev/null | sed -E "s/^DEST_${_en}_CONFIG=\"(.*)\"\$/\1/")" + DEST_REPOS["$_en"]="$_existing_repo" + DEST_CONFIGS["$_en"]="${_existing_cfg:-/etc/kopia-backup/${_en}.config}" + DEST_NAMES_ARR+=("$_en") + done + if [ "${#DEST_NAMES_ARR[@]}" -gt 1 ]; then + log_info " Keeping already-configured destination(s): ${DEST_NAMES_ARR[*]:1}" + fi + fi + local _extra="" prompt_yn " Add more destinations (for services on different drives)? (y/N):" "n" _extra if [[ "$_extra" =~ ^[Yy]$ ]]; then @@ -440,12 +467,16 @@ install_backup() { [ -z "$_dn" ] && break _dn="${_dn//[^a-zA-Z0-9_]/_}" [ "$_dn" = "default" ] && { log_warning " 'default' is reserved — use another name."; continue; } - prompt_text " Path for '$_dn' repository:" "" _dr + # Typing an already-known name reconfigures its path rather than + # duplicating it in DEST_NAMES_ARR. + prompt_text " Path for '$_dn' repository:" "${DEST_REPOS[$_dn]:-}" _dr [ -z "$_dr" ] && continue _dr="${_dr/#\~/$ACTUAL_HOME}"; _dr="${_dr%/}" DEST_REPOS["$_dn"]="$_dr" DEST_CONFIGS["$_dn"]="/etc/kopia-backup/${_dn}.config" - DEST_NAMES_ARR+=("$_dn") + if [[ " ${DEST_NAMES_ARR[*]} " != *" $_dn "* ]]; then + DEST_NAMES_ARR+=("$_dn") + fi log_success " Destination '$_dn' → $_dr" done fi @@ -464,11 +495,17 @@ install_backup() { printf " %-16s %s\n" "$dn" "${DEST_REPOS[$dn]}" done echo "" - echo " Press Enter to accept the default for each service." + echo " Press Enter to accept the shown default for each service." echo "" - local _d + local _d _svc_var _existing_svc_dest for svc in "${ALL_SVCS[@]}"; do - prompt_text " $svc [default]:" "default" _d + _svc_var="${svc//-/_}" + _existing_svc_dest="default" + if [ -f "$CONF_FILE" ]; then + _existing_svc_dest="$(grep -E "^#?SVC_${_svc_var}=" "$CONF_FILE" 2>/dev/null | tail -1 | sed -E 's/^#?SVC_[A-Za-z0-9_]+="(.*)"$/\1/')" + [ -z "$_existing_svc_dest" ] && _existing_svc_dest="default" + fi + prompt_text " $svc [$_existing_svc_dest]:" "$_existing_svc_dest" _d if [ -n "$_d" ] && [ "$_d" != "default" ] && [ -n "${DEST_REPOS[$_d]:-}" ]; then SVC_DEST_MAP["$svc"]="$_d" fi @@ -518,13 +555,27 @@ install_backup() { echo " 3) Weekly (Sunday 02:00)" echo " 4) Custom (systemd OnCalendar)" echo "" - local _sch="" - prompt_text " How often? [1]:" "1" _sch + # Preselect whatever's already scheduled, read back from the live timer + # unit rather than backup.conf (the schedule isn't stored there — it's + # baked directly into the .timer file). Without this, re-running the + # installer and just hitting Enter through this prompt would silently + # revert a customized schedule back to "1) Daily at 02:00" every time. + local _sch="1" _existing_oncal="" + if [ -f "/etc/systemd/system/${SVC_NAME}.timer" ]; then + _existing_oncal="$(grep '^OnCalendar=' "/etc/systemd/system/${SVC_NAME}.timer" 2>/dev/null | cut -d= -f2-)" + case "$_existing_oncal" in + "*-*-* 02,14:00:00") _sch="2" ;; + "Sun *-*-* 02:00:00") _sch="3" ;; + "*-*-* 02:00:00"|"") _sch="1" ;; + *) _sch="4" ;; + esac + fi + prompt_text " How often? [$_sch]:" "$_sch" _sch local ONCALENDAR SCHED_LABEL case "${_sch:-1}" in 2) ONCALENDAR="*-*-* 02,14:00:00"; SCHED_LABEL="every 12 hours" ;; 3) ONCALENDAR="Sun *-*-* 02:00:00"; SCHED_LABEL="weekly Sunday 02:00" ;; - 4) prompt_text " OnCalendar expression:" "*-*-* 02:00:00" ONCALENDAR; SCHED_LABEL="$ONCALENDAR" ;; + 4) prompt_text " OnCalendar expression:" "${_existing_oncal:-*-*-* 02:00:00}" ONCALENDAR; SCHED_LABEL="$ONCALENDAR" ;; *) ONCALENDAR="*-*-* 02:00:00"; SCHED_LABEL="daily at 02:00" ;; esac local KEEP_LATEST="" @@ -542,9 +593,13 @@ install_backup() { echo " Example URL: https://ntfy.sh/my-backup-alerts" echo "" local NTFY_URL="" NTFY_TOKEN="" - prompt_text " ntfy topic URL (blank to skip):" "" NTFY_URL + if [ -f "$CONF_FILE" ]; then + NTFY_URL="$(grep "^NTFY_URL=" "$CONF_FILE" 2>/dev/null | sed -E "s/^NTFY_URL='(.*)'\$/\1/")" + NTFY_TOKEN="$(grep "^NTFY_TOKEN=" "$CONF_FILE" 2>/dev/null | sed -E "s/^NTFY_TOKEN='(.*)'\$/\1/")" + fi + prompt_text " ntfy topic URL (blank to skip)${NTFY_URL:+ — already set to $NTFY_URL}:" "$NTFY_URL" NTFY_URL if [ -n "$NTFY_URL" ]; then - prompt_text " ntfy access token (blank if public/no auth):" "" NTFY_TOKEN + prompt_text " ntfy access token (blank if public/no auth${NTFY_TOKEN:+ — one is already set, Enter keeps it}):" "$NTFY_TOKEN" NTFY_TOKEN fi # ── Disaster-recovery spare box (optional) ──────────────────────────────── @@ -774,14 +829,29 @@ install_backup() { # live: a paste into the hidden Application Key field can silently # capture nothing depending on the terminal/SSH client, with no # other symptom until this point. - local B2_BUCKET="" B2_ENDPOINT="" B2_KEY_ID="" B2_APP_KEY="" - prompt_text " Bucket name:" "" B2_BUCKET + # Pre-fill from whatever's already configured (only meaningful if the + # existing REMOTE_TYPE really is s3/B2 — a REMOTE_ARGS left over from + # a different provider, e.g. sftp, wouldn't parse into anything + # useful here and is harmlessly skipped). Otherwise reconfiguring + # just to rotate one field means blindly retyping all four, and a + # mispaste on any one of them loses the other three that were + # already typed correctly this run. + local _existing_b2_bucket="" _existing_b2_endpoint="" _existing_b2_keyid="" _existing_b2_appkey="" + if [ "$REMOTE_TYPE" = "s3" ]; then + _existing_b2_bucket="$(echo "$REMOTE_ARGS" | grep -oE -- '--bucket=[^ ]*' | cut -d= -f2-)" + _existing_b2_endpoint="$(echo "$REMOTE_ARGS" | grep -oE -- '--endpoint=[^ ]*' | cut -d= -f2-)" + _existing_b2_keyid="$(echo "$REMOTE_ARGS" | grep -oE -- '--access-key=[^ ]*' | cut -d= -f2-)" + _existing_b2_appkey="$(echo "$REMOTE_ARGS" | grep -oE -- '--secret-access-key=[^ ]*' | cut -d= -f2-)" + fi + local B2_BUCKET="$_existing_b2_bucket" B2_ENDPOINT="$_existing_b2_endpoint" B2_KEY_ID="$_existing_b2_keyid" B2_APP_KEY="" + prompt_text " Bucket name:" "$B2_BUCKET" B2_BUCKET echo " (${#B2_BUCKET} characters entered)" - prompt_text " Endpoint (e.g. s3.us-west-004.backblazeb2.com):" "" B2_ENDPOINT + prompt_text " Endpoint (e.g. s3.us-west-004.backblazeb2.com):" "$B2_ENDPOINT" B2_ENDPOINT echo " (${#B2_ENDPOINT} characters entered)" - prompt_text " Application Key ID:" "" B2_KEY_ID + prompt_text " Application Key ID:" "$B2_KEY_ID" B2_KEY_ID echo " (${#B2_KEY_ID} characters entered)" - read -rsp " Application Key (input hidden): " B2_APP_KEY; echo + read -rsp " Application Key (input hidden${_existing_b2_appkey:+ — leave blank to keep the existing one}): " B2_APP_KEY; echo + [ -z "$B2_APP_KEY" ] && B2_APP_KEY="$_existing_b2_appkey" echo " (${#B2_APP_KEY} characters entered)" local _B2_MISSING="" From 0f9ce0a82b589a6412cf44512e7fdc44ae77b886 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 23:35:44 +0000 Subject: [PATCH 2/6] Add a flag to permanently retire shared coturn without auto-reinstalling it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensure_coturn_user() auto-installs shared coturn (services/coturn.sh) any time $DOCKER_DIR/coturn doesn't exist — correct behavior for "first service that needs TURN", wrong behavior for "an operator deliberately decided every consumer should run its own dedicated coturn instead and removed the shared one on purpose". The function had no way to tell those two states apart, so deleting ~/docker/coturn didn't actually retire it — the next service to call this function (a Mattermost reinstall, a fresh Asterisk install) would silently bring it right back. touch ~/docker/.coturn-retired now short-circuits the function straight to the existing "no TURN available, caller degrades gracefully" return path, before it ever looks at install_coturn. Every existing consumer (Asterisk, Mattermost) already handles that path correctly today — it's exactly what happens if shared coturn simply fails to install — so this needed no changes on the consumer side, only closing the gap in the shared function. Verified with a mock: with the flag present, install_coturn is never invoked and the function returns empty COTURN_HOST/rc=1 as expected. --- lib/common.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/common.sh b/lib/common.sh index 016aaf5..477491b 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1031,6 +1031,21 @@ ensure_coturn_user() { local _consumer="$1" COTURN_HOST="" COTURN_PORT="" COTURN_USERNAME="" COTURN_PASSWORD="" + # An operator can permanently retire shared coturn (e.g. deciding every + # consumer should run its own dedicated instance instead) by creating + # this flag file. Without checking it here first, retiring shared coturn + # by just deleting $DOCKER_DIR/coturn wouldn't actually stick — the very + # next service that calls this function (a Mattermost reinstall, a fresh + # Asterisk install, ...) would silently reinstall it right back, since + # the only signal this function otherwise has is "does the directory + # exist yet", which looks identical to "never installed" and "removed on + # purpose". touch/rm $DOCKER_DIR/.coturn-retired to toggle this. + if [ -f "$DOCKER_DIR/.coturn-retired" ]; then + log_info "Shared coturn is retired on this box ($DOCKER_DIR/.coturn-retired exists) —" + log_info "$_consumer will run its own dedicated coturn instead." + return 1 + fi + if [ ! -d "$DOCKER_DIR/coturn" ]; then if declare -F install_coturn >/dev/null 2>&1; then log_info "No shared coturn (TURN/STUN) server yet — setting one up for $_consumer..." From 10b985dc9551877c9e16be97f9633eadd8ca00d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 02:22:16 +0000 Subject: [PATCH 3/6] Add standalone Pi-hole service; move wg-easy's default port off Netbird's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent, requested changes: - services/pihole.sh: new standalone service, Pi-hole v6 (the image moved entirely to a TOML-based /etc/pihole config — the old WEBPASSWORD env var and separate /etc/dnsmasq.d volume are both gone; uses FTLCONF_webserver_api_password and FTLCONF_dns_listeningMode=ALL instead). Deliberately not wired into wg-easy or any other VPN — a device has to be pointed at it manually (per-device or via router DHCP). DNS itself (53/tcp+udp) is never scanned/moved since shifting it off the standard port would defeat the point; a port_in_use check warns instead of blocking, since the common case (systemd-resolved on 127.0.0.53 only) doesn't actually collide with Pi-hole binding the host's real interfaces. Web admin UI is Caddy-fronted like everything else in this repo. Added to the README services table. - services/wg-easy.sh: default VPN/web ports moved from 51820/51821 to 51830/51831. Netbird's own WireGuard listener also defaults to exactly 51820 — installing both on one box means wg-easy's existing scan-and-move logic would silently shift its port every time, which is harder to predict/document than just not starting on the collision in the first place. The scan itself is unchanged and still moves both ports further if even the new default is taken. Tested pihole.sh's full standalone install flow (no-Caddy and Caddy-present-locally cases) against a mocked environment, validating both generated docker-compose.yml files with `docker compose config`, and confirmed the reinstall-mode gate correctly no-ops on a second run in unattended mode. --- README.md | 2 +- services/pihole.sh | 449 ++++++++++++++++++++++++++++++++++++++++++++ services/wg-easy.sh | 20 +- 3 files changed, 465 insertions(+), 6 deletions(-) create mode 100644 services/pihole.sh diff --git a/README.md b/README.md index 7911739..1391c29 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ a ready-to-copy Caddy config snippet to `~/docker/caddy-snippets/`. |-------|---------| | `base` | `net-tools`, `ncdu`, `git`, `curl`, `wget`, `htop`, `tree`, `zip`/`unzip`, `ca-certificates`, `gnupg`, `jq`, `rsync`; `glow` (terminal markdown reader, Charm apt repo); Docker CE + Compose plugin; `openssh-server` with GitHub/Launchpad SSH key import, optional password-auth lockdown, and SSH Host aliases; optional NetBird overlay network | | `homelab` | `caddy`, `crowdsec`, `authelia`, `coturn` (shared TURN/STUN relay — Asterisk, Mattermost Calls, and future WebRTC-capable services all register a dedicated credential against one instance instead of each running its own), `homeassistant`, `asterisk`, `pstn-trunk`, `sms-inbound`, `security-dashboard`, `sunshine`, `vpn-data-mount` (mount existing SMB shares from a NetBird-connected home box — SSH trust bootstrap, then read-only discovery of shares already configured there; never writes to the home box's Samba config; repeatable, pick from any number of a home box's shares in one pass; optional per-share [gocryptfs decrypt layer](#client-side-encryption-for-vpn-data-mount) so the VPS only ever handles ciphertext) | -| `utilities` | `actualbudget`, `ai-gpu`, `ai-stack`, `archivebox`, `beszel` (lightweight server + Docker monitoring — CPU/RAM/disk/network, auto-discovers running containers via the Docker socket; complements Gatus rather than replacing it — Gatus is a black-box HTTP check, Beszel is white-box host/process monitoring), `beszel-agent` (agent-only Beszel install for a remote/homelab box reporting to a hub elsewhere — connects outbound over HTTPS, no VPN/port-forwarding/FQDN needed on that box), `changedetection`, `ddclient`, `filebrowser`, `fmd`, `gatus`, `homebox`, `iopaint`, `joplin`, `koha`, `magicmirror`, `mail-archiver`, `mattermost`, `mealie`, `meshcentral`, `n8n`, `nextcloud`, `ntfy`, `onlyoffice`, `paintplus`, `portainer`, `rustdesk`, `stirling-pdf`, `syncthing`, `traccar`, `unifi`, `uptimekuma`, `vaultwarden`, `watchyourlan`, `watchtower`, `wg-easy`, `wordpress` (multi-site, dedicated MariaDB per site — blogs, business sites, e-commerce via WooCommerce) | +| `utilities` | `actualbudget`, `ai-gpu`, `ai-stack`, `archivebox`, `beszel` (lightweight server + Docker monitoring — CPU/RAM/disk/network, auto-discovers running containers via the Docker socket; complements Gatus rather than replacing it — Gatus is a black-box HTTP check, Beszel is white-box host/process monitoring), `beszel-agent` (agent-only Beszel install for a remote/homelab box reporting to a hub elsewhere — connects outbound over HTTPS, no VPN/port-forwarding/FQDN needed on that box), `changedetection`, `ddclient`, `filebrowser`, `fmd`, `gatus`, `homebox`, `iopaint`, `joplin`, `koha`, `magicmirror`, `mail-archiver`, `mattermost`, `mealie`, `meshcentral`, `n8n`, `nextcloud`, `ntfy`, `onlyoffice`, `paintplus`, `pihole` (standalone DNS ad/tracker blocking — not wired into any VPN's DNS push), `portainer`, `rustdesk`, `stirling-pdf`, `syncthing`, `traccar`, `unifi`, `uptimekuma`, `vaultwarden`, `watchyourlan`, `watchtower`, `wg-easy`, `wordpress` (multi-site, dedicated MariaDB per site — blogs, business sites, e-commerce via WooCommerce) | | `media` | `arm`, `audiobookshelf`, `calibre-web`, `emby`, `immich`, `jellyfin`, `lyrion` | | `cameras` | `frigate`, `frigate-audio`, `frigate-notify`, `sky-cam` | | `gaming` | `drum-rhythm-game`, `js99er`, `kyber-launcher`, `kyber-server`, `minecraft`, `wolf`, `wolf-pair` | diff --git a/services/pihole.sh b/services/pihole.sh new file mode 100644 index 0000000..3673363 --- /dev/null +++ b/services/pihole.sh @@ -0,0 +1,449 @@ +#!/bin/bash +# services/pihole.sh — Pi-hole network-wide DNS ad/tracker blocking. +# Part of the modular post-install system (sourced by setup.sh). +# +# Can also be run standalone on any machine: +# sudo bash pihole.sh +# (Docker must already be installed when run standalone) +# +# Deliberately standalone — not wired into wg-easy or any other VPN/DNS +# push here. Point a device's DNS settings at this box's IP manually to use +# it. If you want it pushed automatically to VPN clients, that's a +# wg-easy-side change (WG_CONFIG's DNS setting), not something this +# installer does on its own. + +# ── Standalone bootstrap ────────────────────────────────────────────────────── +# Detected when the script is executed directly rather than sourced by setup.sh. +# Sets up helpers and globals, then defers execution until after the function +# definition at the bottom of this file. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + [[ "$(id -u)" == "0" ]] || { echo "Run with sudo: sudo bash $0"; exit 1; } + + _SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _COMMON="$_SELF_DIR/../lib/common.sh" + + if [[ -f "$_COMMON" ]]; then + # Full repo present — use the real helpers (picks up ~/docker/.config too) + # shellcheck source=../lib/common.sh + source "$_COMMON" + else + # One-off copy — inline minimal stubs so the script works without the repo + log_info() { echo -e "\033[0;34m[INFO]\033[0m $*"; } + log_success() { echo -e "\033[0;32m[OK]\033[0m $*"; } + log_warning() { echo -e "\033[1;33m[WARN]\033[0m $*"; } + log_error() { echo -e "\033[0;31m[ERROR]\033[0m $*" >&2; } + + require_docker() { + command -v docker &>/dev/null || { + log_error "Docker not found. Install it first:" + log_error " curl -fsSL https://get.docker.com | sudo sh" + return 1 + } + docker compose version &>/dev/null || { + log_error "Docker Compose plugin missing:" + log_error " sudo apt-get install -y docker-compose-plugin" + return 1 + } + } + + ensure_docker_dir_ownership() { + chown -R "$ACTUAL_USER:$ACTUAL_USER" "$@" 2>/dev/null || true + } + + generate_password() { + local _len="${1:-32}" + tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c "$_len" + } + + port_in_use() { + local _port="$1" _proto="${2:-tcp}" + local _flag="-tlnH" + [ "$_proto" = "udp" ] && _flag="-ulnH" + ss "$_flag" "sport = :${_port}" 2>/dev/null | grep -q . + } + + find_free_port() { + local _varname="$1" _port="$2" _proto="${3:-tcp}" + while port_in_use "$_port" "$_proto"; do + _port=$((_port + 1)) + done + eval "$_varname='$_port'" + } + + # Match common.sh's eval-based pattern so local vars in install_* are set correctly + prompt_text() { + local _q="$1" _def="$2" _var="$3" _r + [[ "${UNATTENDED:-false}" == "true" ]] && { eval "$_var='$_def'"; return; } + read -r -p " $_q " _r + eval "$_var='${_r:-$_def}'" + } + + prompt_yn() { + local _q="$1" _def="$2" _var="$3" _r + [[ "${UNATTENDED:-false}" == "true" ]] && { eval "$_var='$_def'"; return; } + read -r -p " $_q " _r + eval "$_var='${_r:-$_def}'" + } + + prompt_reinstall_mode() { + local _var="$1" _r + [[ "${UNATTENDED:-false}" == "true" ]] && { eval "$_var='cancel'"; return; } + echo "" + echo " 1) Update — refresh the image only, leave config/data as-is" + echo " 2) Full reinstall — wipe and reconfigure from scratch" + echo " 3) Cancel — leave the existing install untouched" + read -r -p " Choice [3]: " _r + case "$_r" in + 1) eval "$_var='update'" ;; + 2) eval "$_var='fresh'" ;; + *) eval "$_var='cancel'" ;; + esac + } + + configure_caddy_for_service() { + local _name="$1" _upstream="$2" _subdomain="$3" _extra="${4:-}" + local _caddy_dir="$DOCKER_DIR/caddy" + local _caddyfile="$_caddy_dir/Caddyfile" + local _display_port="${_upstream##*:}" + + # Determine mode: local Caddy, remote Caddy, or none + local _mode="none" + [[ -d "$_caddy_dir" ]] && _mode="local" + [[ -n "${CADDY_REMOTE_HOST:-}" ]] && [[ "$_mode" != "local" ]] && _mode="remote" + [[ "$_mode" == "none" ]] && { + log_info "Access $_name directly on port $_display_port." + return 0 + } + + echo "" + local _do_caddy="" + if [[ "$_mode" == "remote" ]]; then + log_info "Remote Caddy configured (${CADDY_REMOTE_HOST})." + log_info "A snippet file will be saved to ~/docker/caddy-snippets/." + fi + read -r -p " Configure Caddy reverse proxy for $_name? [y/N]: " _do_caddy + [[ "${_do_caddy,,}" == "y" ]] || { + log_info "Skipping — access at: http://localhost:$_display_port" + return 0 + } + + # Domain prompt — pre-fill from SITE_DOMAIN when available + local _default_domain="" + if [[ -n "${SITE_DOMAIN:-}" ]] && [[ "$SITE_DOMAIN" != "example.com" ]]; then + _default_domain="${_subdomain}.${SITE_DOMAIN}" + log_info "Default: $_default_domain" + fi + local _domain="" + read -r -p " Domain [${_default_domain:-required}]: " _domain + _domain="${_domain:-$_default_domain}" + [[ -n "$_domain" ]] || { log_warning "No domain entered — skipping Caddy."; return 0; } + + # Build upstream — remote Caddy uses host IP:port, not container name + local _block_upstream="$_upstream" + if [[ "$_mode" == "remote" ]]; then + _block_upstream="${CADDY_REMOTE_HOST}:${_display_port}" + fi + + local _site_block + _site_block="$(cat << CBLOCK + +# $_name +${_domain} { + reverse_proxy ${_block_upstream} + + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "SAMEORIGIN" + Referrer-Policy "strict-origin-when-cross-origin" + } + + log { + output file /var/log/caddy/${_domain}.log + format json + } +${_extra} +} +CBLOCK +)" + + if [[ "$_mode" == "local" ]]; then + if [[ -f "$_caddyfile" ]]; then + local _bk="$_caddy_dir/Caddyfile.backup.$(date +%Y%m%d-%H%M%S)" + cp "$_caddyfile" "$_bk" + log_info "Backed up Caddyfile to $(basename "$_bk")" + else + touch "$_caddyfile" + fi + + if grep -q "^${_domain}" "$_caddyfile" 2>/dev/null; then + log_warning "$_domain already in Caddyfile" + local _ow="" + read -r -p " Overwrite? [y/N]: " _ow + [[ "${_ow,,}" == "y" ]] || { log_info "Keeping existing entry."; return 0; } + sed -i "/^${_domain}/,/^}/d" "$_caddyfile" + fi + + printf '%s\n' "$_site_block" >> "$_caddyfile" + log_success "Added $_domain to Caddyfile" + docker exec caddy caddy fmt --overwrite /etc/caddy/Caddyfile 2>/dev/null || true + if docker exec caddy caddy reload --config /etc/caddy/Caddyfile 2>/dev/null; then + log_success "$_name accessible at: https://$_domain" + else + log_warning "Reload failed — check: docker logs caddy" + log_info "Manual reload: docker exec caddy caddy reload --config /etc/caddy/Caddyfile" + fi + else + local _snippet_dir="$DOCKER_DIR/caddy-snippets" + local _snippet_file="$_snippet_dir/${_subdomain}.caddy" + mkdir -p "$_snippet_dir" + printf '%s\n' "$_site_block" > "$_snippet_file" + chown "$ACTUAL_USER:$ACTUAL_USER" "$_snippet_file" 2>/dev/null || true + log_success "Snippet saved: $_snippet_file" + log_info "Copy to Caddy machine:" + log_info " scp $_snippet_file caddy-host:~/caddy-snippets/" + log_info " rsync -av $_snippet_dir/ caddy-host:~/caddy-snippets/ (all at once)" + fi + } + write_readme() { + local _dir="$1"; shift + mkdir -p "$_dir" + cat > "$_dir/README.md" + } + fi + + # Globals — ACTUAL_USER/ACTUAL_HOME must come before DOCKER_DIR + # ($HOME under sudo is /root, not the real user's home) + ACTUAL_USER="${ACTUAL_USER:-${SUDO_USER:-$USER}}" + ACTUAL_HOME="$(getent passwd "$ACTUAL_USER" 2>/dev/null | cut -d: -f6 || echo "${HOME:-/root}")" + DOCKER_DIR="${DOCKER_DIR:-$ACTUAL_HOME/docker}" + DRY_RUN="${DRY_RUN:-false}" + UNATTENDED="${UNATTENDED:-false}" + SITE_TZ="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" + SITE_DOMAIN="${SITE_DOMAIN:-example.com}" + SITE_CADDY_NET="${SITE_CADDY_NET:-caddy_net}" + + register_service() { :; } # no-op — no wizard to register into + _RUN_STANDALONE=1 +fi +# ───────────────────────────────────────────────────────────────────────────── + +register_service pihole utilities "Network-wide DNS ad/tracker blocking (Pi-hole) — standalone, not wired into any VPN" 80 + +install_pihole() { + require_docker || return 1 + + local DIR="$DOCKER_DIR/pihole" + local WEB_PORT="8081" + + if [ "$DRY_RUN" = true ]; then + echo "[DRY-RUN] Would create $DIR with docker-compose.yml + .env" + echo "[DRY-RUN] Would warn (not block) if port 53/tcp or 53/udp is already in use" + echo "[DRY-RUN] Would auto-scan for a free host port for the web admin UI" + echo "[DRY-RUN] Would generate a random admin password" + echo "[DRY-RUN] Would offer a Caddy reverse proxy for the admin UI only (never DNS)" + return 0 + fi + + if [[ -f "$DIR/docker-compose.yml" && -f "$DIR/.env" ]]; then + local MODE="" + prompt_reinstall_mode MODE + case "$MODE" in + update) + log_info "Refreshing the Pi-hole image only — existing blocklists, config, and" + log_info "port are left as-is." + ( cd "$DIR" && docker compose pull && docker compose up -d ) \ + && log_success "Pi-hole image refreshed" \ + || log_warning "Refresh failed — check: docker compose -f $DIR/docker-compose.yml logs" + return 0 + ;; + cancel) + log_info "Leaving the existing install as-is." + return 0 + ;; + fresh) ;; # fall through to the full install flow below + esac + fi + + # DNS itself is never scanned/moved — shifting Pi-hole off port 53 would + # defeat the point, since every device on the network expects to find DNS + # there by convention (unlike a web UI port, nothing lets a client be told + # "try a different port instead"). Warn instead: the common case on Ubuntu + # is systemd-resolved bound only to 127.0.0.53:53 (loopback), which does + # NOT collide with Pi-hole's container publishing 53 on the host's real + # interfaces — but if something else really is bound to 0.0.0.0:53, this + # says so up front instead of failing silently at `docker compose up`. + if port_in_use 53 tcp || port_in_use 53 udp; then + log_warning "Something is already listening on port 53 (DNS) — check with:" + log_warning " ss -tulnp | grep ':53 '" + log_warning "If that's systemd-resolved bound to 127.0.0.53 only, this is fine —" + log_warning "Pi-hole binds the host's real interfaces, not the loopback stub. If" + log_warning "it's something else bound to 0.0.0.0:53, Pi-hole's container won't" + log_warning "be able to start until that's freed or reconfigured." + fi + + find_free_port WEB_PORT "$WEB_PORT" + + mkdir -p "$DIR" + ensure_docker_dir_ownership "$DIR" + cd "$DIR" || return 1 + + # Mirrors configure_caddy_for_service's own mode resolution (lib/common.sh): + # explicit CADDY_MODE from the site config wins, then a local ~/docker/caddy, + # then the legacy CADDY_REMOTE_HOST var. Only "local" joins caddy_net — a + # remote Caddy box can't resolve container names on this host's bridge + # network anyway; it reaches this service via the host's published port. + # This only ever applies to the WEB admin UI — DNS itself is never behind + # Caddy (Caddy only speaks HTTP). + local _CADDY_MODE="${CADDY_MODE:-none}" + [ "$_CADDY_MODE" = "none" ] && [ -d "$DOCKER_DIR/caddy" ] && _CADDY_MODE="local" + [ "$_CADDY_MODE" = "none" ] && [ -n "${CADDY_REMOTE_HOST:-}" ] && _CADDY_MODE="remote" + + local _CADDY_NET_BLOCK="" + local _CADDY_NET_SECTION="" + if [ "$_CADDY_MODE" = "local" ]; then + _CADDY_NET_BLOCK=" networks: + - caddy_net +" + _CADDY_NET_SECTION=" +networks: + caddy_net: + external: true + name: ${SITE_CADDY_NET:-caddy_net} +" + fi + + local WEBPASSWORD + WEBPASSWORD="$(generate_password 24)" + + # v6 image: config lives entirely under /etc/pihole (TOML-based + # pihole.toml) — the old v5 split with a separate /etc/dnsmasq.d volume + # and WEBPASSWORD env var are both gone. FTLCONF_webserver_api_password + # replaces WEBPASSWORD; FTLCONF_dns_listeningMode=ALL is required under + # Docker's default bridge networking or Pi-hole ignores queries from + # anything but localhost. cap_add matches Pi-hole's own official compose + # example — NET_ADMIN specifically is only needed if this instance is + # ever used as a DHCP server too, which it isn't here, but the other two + # (SYS_TIME, SYS_NICE) are part of that same documented baseline. + cat > docker-compose.yml << PIHOLE_COMPOSE +name: pihole + +services: + pihole: + image: pihole/pihole:latest + container_name: pihole + hostname: pihole + restart: unless-stopped + environment: + TZ: \${TZ} + FTLCONF_webserver_api_password: \${WEBPASSWORD} + FTLCONF_dns_listeningMode: ALL + volumes: + - ./etc-pihole:/etc/pihole + ports: + - "53:53/tcp" + - "53:53/udp" + - "${WEB_PORT}:80/tcp" + cap_add: + - NET_ADMIN + - SYS_TIME + - SYS_NICE +${_CADDY_NET_BLOCK}${_CADDY_NET_SECTION} +PIHOLE_COMPOSE + + cat > .env << PIHOLE_ENV +TZ=${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)} +# Admin web UI password (System Console / login screen). +WEBPASSWORD='${WEBPASSWORD}' +CADDY_NET=$SITE_CADDY_NET +PIHOLE_ENV + chmod 600 .env + + mkdir -p etc-pihole + chown -R "$ACTUAL_USER:$ACTUAL_USER" "$DIR" + + echo "" + log_success "Pi-hole configured at $DIR (DNS on 53, admin UI on port $WEB_PORT)" + + configure_caddy_for_service "Pi-hole" "pihole:80" "pihole" + local _caddy_configured=false _caddy_mode="" + if declare -p CADDY_SERVICE_CONFIGURED >/dev/null 2>&1; then + _caddy_configured="$CADDY_SERVICE_CONFIGURED" + _caddy_mode="$CADDY_SERVICE_MODE" + fi + + # ── Firewall ───────────────────────────────────────────────────────────── + # DNS (53) is opened unconditionally — this is meant to be queried + # directly by devices on the network, never fronted by Caddy. The web UI + # port only needs opening if Caddy ISN'T fronting it locally (matches the + # pattern documented in CLAUDE.md for every other service in this repo). + if command -v ufw &>/dev/null; then + ufw allow 53/tcp comment "Pi-hole DNS" + ufw allow 53/udp comment "Pi-hole DNS" + if [ "$_caddy_configured" != "true" ] || [ "$_caddy_mode" = "remote" ]; then + ufw allow "${WEB_PORT}/tcp" comment "Pi-hole admin UI" + fi + fi + + write_readme "$DIR" << MD +# Pi-hole + +Network-wide DNS-based ad/tracker blocking. Standalone install — **not** +wired into wg-easy, Netbird, or any other VPN here. To actually use it, +point a device's DNS settings at this box's IP on port 53, either by hand +per-device or via your router's DHCP DNS setting. + +## Access +- Admin UI: http://localhost:${WEB_PORT} (or via Caddy if configured above) +- Admin password: see \`.env\` → \`WEBPASSWORD\` +- DNS: this box's IP, port 53 (standard DNS port — not configurable per-instance) + +## Using it +Nothing points at this automatically. Options, in order of how much you +want blocked by default: +- **Per-device**: change that device's DNS server setting to this box's IP. +- **Whole LAN**: change your router's DHCP-assigned DNS server to this box's IP + (every device on that network picks it up automatically going forward). +- **Over the VPN too**: this needs a manual edit on the wg-easy side — set + wg-easy's DNS setting to this box's IP so it's pushed to VPN peers. Not + done automatically by this installer, on purpose (you said standalone). + +## Port 53 already in use? +The installer warns but doesn't block if something's already listening on +53 at install time. The common, harmless case on Ubuntu is systemd-resolved +bound only to \`127.0.0.53:53\` (loopback stub) — Pi-hole's container +publishes on the host's real interfaces, not that loopback address, so the +two normally coexist fine. If something else genuinely holds \`0.0.0.0:53\`, +free it first or Pi-hole's container won't start: +\`\`\` +ss -tulnp | grep ':53 ' +\`\`\` + +## Manage +\`\`\`bash +cd $DIR +docker compose up -d # start +docker compose down # stop +docker compose logs -f # logs +docker exec -it pihole pihole -g # force a blocklist update (gravity) +\`\`\` +MD + + local START="" + prompt_yn "Start Pi-hole now? (y/n):" "y" START + if [ "$START" = "y" ] || [ "$START" = "Y" ]; then + docker compose up -d \ + && log_success "Pi-hole started" \ + || log_warning "Start failed — check: docker compose logs" + fi + + echo "" + echo " Admin UI: http://localhost:${WEB_PORT}" + echo " Password: see $DIR/.env" + echo " Nothing points at this DNS server yet — see the README for how to" + echo " actually route devices to it." + echo "" +} + +# Run immediately when executed directly (deferred until after function definition) +[[ "${_RUN_STANDALONE:-0}" == 1 ]] && install_pihole diff --git a/services/wg-easy.sh b/services/wg-easy.sh index edf0fc6..846ff7a 100644 --- a/services/wg-easy.sh +++ b/services/wg-easy.sh @@ -9,7 +9,17 @@ # Ported from ubuntu-post-install-24.04-crowdsec.sh (# ---- WG-EASY ----). # Own ~/docker/wg-easy/ with a standalone docker-compose.yml + .env. # Requires cap_add: NET_ADMIN + SYS_MODULE and ip_forward sysctl. -# Forward UDP 51820 on your router to this server for external VPN access. +# Forward UDP 51830 on your router to this server for external VPN access +# (default — scanned/moved at install time if already taken; see below). +# +# Default port deliberately isn't WireGuard's conventional 51820: Netbird's +# own WireGuard listener also defaults to exactly 51820, and this is the +# one service in this repo where two completely independent tools (this +# repo's own wg-easy and a separately-installed Netbird) are both likely to +# reach for the same hardcoded upstream default with no scanning of their +# own on Netbird's side. Starting one port family away avoids that +# collision in the common case; the scan below still moves both further if +# even the new default is somehow already taken. # ── Standalone bootstrap ────────────────────────────────────────────────────── # Detected when the script is executed directly rather than sourced by setup.sh. @@ -207,13 +217,13 @@ CBLOCK fi # ───────────────────────────────────────────────────────────────────────────── -register_service wg-easy utilities "WireGuard VPN with web management UI (wg-easy); peers mesh through this hub automatically, with a script to sync SSH aliases" 51821 +register_service wg-easy utilities "WireGuard VPN with web management UI (wg-easy); peers mesh through this hub automatically, with a script to sync SSH aliases" 51831 install_wg-easy() { require_docker || return 1 local WGEASY_DIR="$DOCKER_DIR/wg-easy" - local WEB_PORT="51821" VPN_PORT="51820" + local WEB_PORT="51831" VPN_PORT="51830" if [ "$DRY_RUN" = true ]; then echo "[DRY-RUN] wg-easy would:" @@ -221,7 +231,7 @@ install_wg-easy() { echo " - Auto-detect public IP for WG_HOST" echo " - Generate a random web UI password" echo " - Pin WG_DEFAULT_ADDRESS=10.8.0.x (subnet 10.8.0.0/24)" - echo " - Expose port 51821 (web UI) + 51820/udp (VPN), both auto-scanned if occupied" + echo " - Expose port 51831 (web UI) + 51830/udp (VPN), both auto-scanned if occupied" echo " - Require router port-forward: UDP → this server" echo " - Offer a Caddy reverse proxy and to start the container" echo " - Offer to also allow SSH from the VPN subnet (additive, doesn't remove public SSH)" @@ -232,7 +242,7 @@ install_wg-easy() { # Scan for free host ports, moving both together — a plain install # shouldn't silently claim a port another already-running service holds. # Whatever VPN_PORT ends up as is what needs forwarding on the router - # (the messaging below reflects the final value, not the 51820 default). + # (the messaging below reflects the final value, not the 51830 default). # See CLAUDE.md's "Port collision avoidance" section. while port_in_use "$WEB_PORT" || port_in_use "$VPN_PORT" udp; do WEB_PORT=$((WEB_PORT + 1)) From 767479d1131ddbfd20b68f4a6471faf33c9315e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 02:34:40 +0000 Subject: [PATCH 4/6] Self-heal Mattermost data/logs/config/plugins ownership on every start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root-caused a live "stream closed" image-upload failure to data/20260814/.../mkdir: permission denied — the volumes weren't owned by the fixed UID 2000 mattermost/mattermost-team-edition runs as, most likely left that way by the PikaPods data import. The install script already chown -R 2000:2000's these on every run (fresh or update), so re-running the installer would have fixed it — but that still means remembering to re-run it every time ownership drifts for any reason, including causes this repo doesn't control (a future migration, a manual restore, anything that copies files in as a different UID). Added a small mattermost-fix-perms init container (busybox, chown, exit) that the mattermost service now depends on via condition: service_completed_successfully. Runs on every `docker compose up` — including a plain host reboot, since restart: unless-stopped brings the stack back on its own — so this self-heals permanently instead of needing a human to notice and fix it by hand again. Verified the generated compose file (with representative variable values) against real `docker compose config`: valid YAML, and the dependency graph correctly shows mattermost waiting on both db (service_healthy) and mattermost-fix-perms (service_completed_successfully). --- services/mattermost.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/services/mattermost.sh b/services/mattermost.sh index e98614f..309d3e6 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -545,6 +545,28 @@ ${_CADDY_NET_BLOCK} healthcheck: timeout: 5s retries: 5 + # Fixes ownership on the bind-mounted volumes below to the fixed UID/GID + # (2000) mattermost/mattermost-team-edition runs as, before the mattermost + # service starts — every time, not just at install time. Confirmed live: + # importing data from another host (e.g. a PikaPods migration) can leave + # these owned by whatever UID did the copy instead of 2000, and the + # container doesn't fix this itself on start the way postgres's official + # image does — it just fails every file write with "permission denied" + # until someone notices and runs chown by hand. This removes the "by + # hand" part permanently: runs on every `docker compose up`, including a + # plain host reboot, so ownership drift from any future cause self-heals + # without needing this installer re-run again. + mattermost-fix-perms: + image: busybox:latest + container_name: ${MM_CONTAINER}-fix-perms + command: sh -c "chown -R 2000:2000 /data /logs /config /plugins" + volumes: + - ./data:/data + - ./logs:/logs + - ./config:/config + - ./plugins:/plugins + restart: "no" + mattermost: image: mattermost/mattermost-team-edition:latest container_name: ${MM_CONTAINER} @@ -554,6 +576,8 @@ ${_CADDY_NET_BLOCK} healthcheck: depends_on: db: condition: service_healthy + mattermost-fix-perms: + condition: service_completed_successfully volumes: - ./data:/mattermost/data - ./logs:/mattermost/logs From a7d3dc5b5d4a43bcbd7ce6c879f35d3ce20c4abd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 02:58:59 +0000 Subject: [PATCH 5/6] Fix imported file ownership in the generated PikaPods migration script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generated migrate-from-pikapods.sh (services/mattermost.sh's existing "Migrating from an existing Mattermost instance?" prompt on fresh installs) already correctly parameterizes PROJECT_DIR/MM_CONTAINER/DB_CONTAINER per instance — no bug there. What it missed: after rsync/cp-ing files in from the export, it never touched ownership, so the imported ./data landed owned by whoever ran the script instead of the fixed UID 2000 mattermost/mattermost-team-edition runs as. Every file write then failed with permission denied — confirmed live as the actual cause of a client-side "stream closed" error on image/file uploads after a real migration. Adds chown -R 2000:2000 ./data right after the copy step, and a root check up front since chowning to an arbitrary UID needs it (docker/psql access already implied running as root in practice, just never enforced explicitly). Usage lines updated to say `sudo` to match. Verified by reconstructing the exact generated script from the real source heredocs (head + variable substitution + body, the same three pieces the actual cat/cat>> sequence produces) and syntax-checking the result — root check and chown both land in the right place, and PROJECT_DIR/MM_CONTAINER/DB_CONTAINER still resolve correctly per instance. --- services/mattermost.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/services/mattermost.sh b/services/mattermost.sh index 309d3e6..1d4c1c3 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -788,7 +788,7 @@ MD # before running. # # Usage: -# ./migrate-from-pikapods.sh +# sudo ./migrate-from-pikapods.sh ################################################################################ MIGRATE_HEAD @@ -803,13 +803,18 @@ MIGRATE_VARS cat >> "$DIR/migrate-from-pikapods.sh" << 'MIGRATE_BODY' set -uo pipefail + +# Needed for the chown to UID 2000 near the end (an ordinary user generally +# can't chown files to an arbitrary UID that isn't their own). +[ "${EUID:-$(id -u)}" -eq 0 ] || { echo "Run as root: sudo $0 ..."; exit 1; } + cd "$PROJECT_DIR" || exit 1 SQL_DUMP="${1:-}" FILES_DIR="${2:-}" if [ -z "$SQL_DUMP" ] || [ -z "$FILES_DIR" ]; then - echo "Usage: $0 " + echo "Usage: sudo $0 " exit 1 fi [ -f "$SQL_DUMP" ] || { echo "SQL dump not found: $SQL_DUMP"; exit 1; } @@ -856,6 +861,14 @@ echo "Copying files into ./data..." mkdir -p ./data rsync -a "$FILES_DIR"/ ./data/ 2>/dev/null || cp -a "$FILES_DIR"/. ./data/ +# mattermost/mattermost-team-edition runs as fixed UID/GID 2000 — an SFTP'd +# copy from elsewhere lands owned by whoever ran this script instead, and +# every file write then fails with "permission denied" until this is +# fixed. Confirmed live: this is what broke image/file uploads with a +# client-side "stream closed" error after a migration. +echo "Fixing ownership on imported files (mattermost image expects UID/GID 2000)..." +chown -R 2000:2000 ./data + echo "Starting Mattermost..." docker compose up -d From 29db2da5fccf42fd255bded58417e41d3cc40688 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 03:09:57 +0000 Subject: [PATCH 6/6] Add fix_pikapods_dump.py to the repo; cross-reference it from the migration script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extras/fix_pikapods_dump.py patches two confirmed Adminer PostgreSQL-export bugs that otherwise make a PikaPods Mattermost migration fail outright: unquoted enum-label DEFAULT values (Postgres reads the bare label as a column reference and rejects the CREATE TABLE) and boolean columns serialized as bare 0/1 instead of true/false (Postgres doesn't implicitly cast integers to boolean). Boolean columns are discovered by actually parsing each CREATE TABLE in the dump rather than working from a hand-curated list — Postgres only reports the first bad column per failed row, so a list built from error output alone would likely be incomplete. Already verified earlier this session against a real local Postgres 16 instance; reviewed now for anything needing redaction before committing — it's a generic text-processing tool with no hostnames, credentials, file paths, or personal data in it, so nothing needed changing. Cross-referenced from the generated migrate-from-pikapods.sh's header comment (services/mattermost.sh) so anyone hitting a CREATE TYPE/CREATE TABLE or boolean-column import error is pointed at the fix instead of having to rediscover it. --- extras/fix_pikapods_dump.py | 302 ++++++++++++++++++++++++++++++++++++ services/mattermost.sh | 9 ++ 2 files changed, 311 insertions(+) create mode 100755 extras/fix_pikapods_dump.py diff --git a/extras/fix_pikapods_dump.py b/extras/fix_pikapods_dump.py new file mode 100755 index 0000000..6c90409 --- /dev/null +++ b/extras/fix_pikapods_dump.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +""" +fix_pikapods_dump.py — Patches two confirmed Adminer PostgreSQL-export bugs +in a Mattermost SQL dump, before importing it via migrate-from-pikapods.sh. + +Bug 1: Adminer omits quotes around enum-label DEFAULT values, e.g. + DEFAULT link instead of DEFAULT 'link' + DEFAULT client_credentials instead of DEFAULT 'client_credentials' + This makes Postgres treat the label as a column reference, which is + illegal in a DEFAULT expression, so the whole CREATE TABLE fails. + +Bug 2: Adminer serializes PostgreSQL boolean columns as bare integer + literals (0 / 1) in INSERT statements instead of true/false. + Postgres does not implicitly cast integer literals to boolean, so + every row touching one of those columns is rejected. + +Bug 2's fix parses each CREATE TABLE in the dump to find every column +declared as `boolean` (not a hand-curated list from partial error +messages — Postgres only reports the FIRST bad column per row, so a +list built from error output alone would likely be incomplete). It then +rewrites only the VALUES-tuple positions that correspond to those +specific boolean columns, leaving every other value in the row +(including other literal 0/1 integers) untouched. + +Usage: + python3 fix_pikapods_dump.py input.sql output.sql + +Always writes to a NEW file — never modifies the input in place — so the +original export is preserved if something looks wrong afterward. +""" +import re +import sys + + +def split_top_level(s, sep=","): + """Split s on sep, but only outside single-quoted strings and outside + nested parens. '' inside a quoted string is the SQL escape for a + literal quote and does not end the string.""" + parts = [] + buf = [] + depth = 0 + in_quote = False + i = 0 + n = len(s) + while i < n: + c = s[i] + if in_quote: + if c == "'": + # doubled quote = escaped literal quote, stays in_quote + if i + 1 < n and s[i + 1] == "'": + buf.append("''") + i += 2 + continue + in_quote = False + buf.append(c) + i += 1 + continue + buf.append(c) + i += 1 + continue + if c == "'": + in_quote = True + buf.append(c) + i += 1 + continue + if c == "(": + depth += 1 + buf.append(c) + i += 1 + continue + if c == ")": + depth -= 1 + buf.append(c) + i += 1 + continue + if c == sep and depth == 0: + parts.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(c) + i += 1 + parts.append("".join(buf)) + return parts + + +def split_statements(sql_text): + """Split a whole dump into individual statements at semicolons that + are not inside a quoted string. Returns list of (statement_text, + trailing_terminator) so the exact original text can be reassembled + byte-for-byte from the pieces.""" + stmts = [] + buf = [] + in_quote = False + i = 0 + n = len(sql_text) + while i < n: + c = sql_text[i] + if in_quote: + buf.append(c) + if c == "'": + if i + 1 < n and sql_text[i + 1] == "'": + buf.append(sql_text[i + 1]) + i += 2 + continue + in_quote = False + i += 1 + continue + if c == "'": + in_quote = True + buf.append(c) + i += 1 + continue + if c == ";": + buf.append(c) + stmts.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(c) + i += 1 + if buf: + stmts.append("".join(buf)) + return stmts + + +CREATE_TABLE_NAME_RE = re.compile( + r'CREATE TABLE\s+(?:"public"\.)?"([^"]+)"\s*\(', + re.IGNORECASE, +) +INSERT_RE = re.compile( + r'^(INSERT INTO\s+(?:"public"\.)?"([^"]+)"\s*\()([^)]*)\)\s*VALUES\s*(.*);\s*$', + re.IGNORECASE | re.DOTALL, +) + + +def find_matching_paren(s, open_idx): + """Return the index of the ')' matching the '(' at open_idx, skipping + over quoted strings so a paren inside a quoted default value (e.g. a + function call in a DEFAULT expression) doesn't miscount depth.""" + depth = 0 + in_quote = False + i = open_idx + n = len(s) + while i < n: + c = s[i] + if in_quote: + if c == "'": + if i + 1 < n and s[i + 1] == "'": + i += 2 + continue + in_quote = False + i += 1 + continue + if c == "'": + in_quote = True + i += 1 + continue + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth == 0: + return i + i += 1 + return -1 + + +def parse_boolean_columns(create_table_stmt): + """Given a full CREATE TABLE statement (trailing syntax after the + closing paren — WITHOUT OIDS, TABLESPACE, etc — is fine, not assumed + absent), return the set of column names declared as boolean.""" + name_m = CREATE_TABLE_NAME_RE.search(create_table_stmt) + if not name_m: + return set() + open_idx = name_m.end() - 1 # the '(' the regex matched + close_idx = find_matching_paren(create_table_stmt, open_idx) + if close_idx == -1: + return set() + body = create_table_stmt[open_idx + 1 : close_idx] + cols = set() + for coldef in split_top_level(body): + coldef = coldef.strip() + cm = re.match(r'^"([^"]+)"\s+([A-Za-z_][A-Za-z0-9_]*)', coldef) + if cm and cm.group(2).lower() == "boolean": + cols.add(cm.group(1)) + return cols + + +def fix_default_quoting(stmt): + """Bug 1: unquoted enum-label DEFAULTs. Only touches DEFAULT clauses + that name one of the two enum types confirmed broken in this export + (channel_bookmark_type, outgoingoauthconnections_granttype) — narrow + and conservative rather than a blanket 'quote anything after DEFAULT' + rule that could misfire on legitimate unquoted defaults elsewhere + (numbers, now(), etc).""" + stmt, n1 = re.subn( + r"(channel_bookmark_type\s+DEFAULT\s+)([A-Za-z_][A-Za-z0-9_]*)(?=[,\)])", + r"\1'\2'", + stmt, + ) + stmt, n2 = re.subn( + r"(outgoingoauthconnections_granttype\s+DEFAULT\s+)([A-Za-z_][A-Za-z0-9_]*)(?=[,\)])", + r"\1'\2'", + stmt, + ) + return stmt, n1 + n2 + + +def patch_insert_booleans(stmt, table, col_list_raw, values_raw, bool_cols): + """Rewrite bare 0/1 literals to false/true at the positions in + col_list_raw that correspond to bool_cols. Returns (new_values_text, + count_of_values_changed).""" + col_names = [c.strip().strip('"') for c in split_top_level(col_list_raw)] + bool_positions = {i for i, c in enumerate(col_names) if c in bool_cols} + if not bool_positions: + return values_raw, 0 + + tuples = split_top_level(values_raw) + changed = 0 + new_tuples = [] + for tup in tuples: + tup_stripped = tup.strip() + if not (tup_stripped.startswith("(") and tup_stripped.endswith(")")): + new_tuples.append(tup) + continue + inner = tup_stripped[1:-1] + vals = split_top_level(inner) + for pos in bool_positions: + if pos >= len(vals): + continue + v = vals[pos].strip() + if v == "0": + vals[pos] = "false" + changed += 1 + elif v == "1": + vals[pos] = "true" + changed += 1 + prefix = tup[: len(tup) - len(tup.lstrip())] + suffix = tup[len(tup.rstrip()):] + new_tuples.append(prefix + "(" + ",".join(vals) + ")" + suffix) + return ",".join(new_tuples), changed + + +def process(input_path, output_path): + with open(input_path, "r", encoding="utf-8", errors="surrogateescape") as f: + text = f.read() + + statements = split_statements(text) + bool_cols_by_table = {} + out = [] + default_fixes = 0 + total_value_fixes = 0 + tables_patched = set() + + for stmt in statements: + stripped = stmt.strip() + upper = stripped.upper() + + if upper.startswith("CREATE TABLE"): + fixed_stmt, n = fix_default_quoting(stmt) + default_fixes += n + name_m = CREATE_TABLE_NAME_RE.search(fixed_stmt) + if name_m: + bool_cols_by_table[name_m.group(1)] = parse_boolean_columns(fixed_stmt) + out.append(fixed_stmt) + continue + + if upper.startswith("INSERT INTO"): + m = INSERT_RE.match(stripped) + if m: + prefix, table, col_list_raw, values_raw = m.groups() + bool_cols = bool_cols_by_table.get(table, set()) + if bool_cols: + new_values, n = patch_insert_booleans( + stripped, table, col_list_raw, values_raw, bool_cols + ) + if n: + total_value_fixes += n + tables_patched.add(table) + rebuilt = f'{prefix}{col_list_raw}) VALUES {new_values};' + out.append(rebuilt) + continue + out.append(stmt) + continue + + out.append(stmt) + + with open(output_path, "w", encoding="utf-8", errors="surrogateescape") as f: + f.write("".join(out)) + + print(f"DEFAULT-clause quoting fixes: {default_fixes}") + print(f"Boolean literal fixes: {total_value_fixes} across {len(tables_patched)} table(s)") + if tables_patched: + print("Tables patched: " + ", ".join(sorted(tables_patched))) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} input.sql output.sql", file=sys.stderr) + sys.exit(1) + process(sys.argv[1], sys.argv[2]) diff --git a/services/mattermost.sh b/services/mattermost.sh index 1d4c1c3..4a5a3bf 100644 --- a/services/mattermost.sh +++ b/services/mattermost.sh @@ -781,6 +781,15 @@ MD # default). If you have a custom-format pg_dump instead, use `pg_restore` # in place of the `psql < dump` step below. # +# Adminer's plain-text export has two confirmed bugs of its own — unquoted +# enum-label DEFAULTs, and boolean columns serialized as bare 0/1 instead +# of true/false — either of which makes the import below fail outright. +# extras/fix_pikapods_dump.py patches both; run it on the dump BEFORE this +# script if psql reports errors on CREATE TYPE/CREATE TABLE or boolean +# columns: +# python3 fix_pikapods_dump.py input.sql output.sql +# + # IMPORTANT: point the files argument at the SUBDIRECTORY that holds # Mattermost's own file storage inside whatever you downloaded via SFTP # (commonly named `data`), not the whole SFTP root — PikaPods' exact