From cd59549e3955d5458cae5a472d1d2f2cb1304d39 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 19:24:23 +0000 Subject: [PATCH 01/12] chore: add versioned snapshot setup_v0.9.5.sh, reset VERSION to 0.9.5 Introduces the versioned-snapshot naming convention: each release creates a new setup_v.sh file alongside the live setup.sh; old snapshots are never removed. Resets VERSION from 1.0.0 to 0.9.5. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- CHANGELOG.md | 14 +++- VERSION | 2 +- setup_v0.9.5.sh | 209 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 3 deletions(-) create mode 100755 setup_v0.9.5.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index ed01653..408e1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ # Changelog All notable changes to this project. Versions follow `MAJOR.MINOR.PATCH`. -The project is pre-1.0 while the modular system reaches parity with the -monolithic `ubuntu-post-install-*.sh` scripts. + +## [0.9.5] - 2026-06-03 + +### Changed +- VERSION reset from 1.0.0 to 0.9.5 — versioning now tracks `setup_v.sh` + snapshot files. Each release creates a new numbered file (old files stay). The + current `setup.sh` is always the live version; `setup_v0.9.5.sh` is the first + named snapshot. + +### Added +- `setup_v0.9.5.sh` — first versioned snapshot of `setup.sh`. Future changes + produce `setup_v0.9.6.sh`, etc. Previous snapshots are never removed. ## [1.0.0] - 2026-06-03 diff --git a/VERSION b/VERSION index 3eefcb9..b0bb878 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +0.9.5 diff --git a/setup_v0.9.5.sh b/setup_v0.9.5.sh new file mode 100755 index 0000000..ed6fa9b --- /dev/null +++ b/setup_v0.9.5.sh @@ -0,0 +1,209 @@ +#!/bin/bash +# setup.sh — modular post-install dispatcher. +# +# One source of truth, multiple ways to run it: +# sudo ./setup.sh guided install: required packages, then a +# category menu you loop through +# sudo ./setup.sh ... install one or more services directly +# ./setup.sh --list list available services (grouped) +# ./setup.sh --version print version +# +# Flags: +# --dry-run preview actions without making changes +# --unattended use defaults, no prompts (pair with explicit service names) +# +# Every service lives in services/.sh, registers itself with +# register_service, and defines install_. Adding a service = adding one +# file; it appears in the menu automatically. Nothing is generated. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Category display order (groups not listed here are appended alphabetically). +CATEGORY_ORDER=(base homelab utilities media cameras gaming extras backup) +# Service ordering hint within a category (lower = earlier). Default 50. +declare -A SERVICE_PRIORITY=( [caddy]=1 [crowdsec]=2 [authelia]=3 ) + +# ── Parse flags / collect service names ────────────────────────────────────── +DRY_RUN=false; UNATTENDED=false; DO_LIST=false +REQUESTED=() +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --unattended) UNATTENDED=true ;; + --list|-l) DO_LIST=true ;; + --version|-V) cat "$HERE/VERSION" 2>/dev/null || echo "unknown"; exit 0 ;; + -h|--help) sed -n '2,18p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + -*) echo "Unknown flag: $arg" >&2; exit 1 ;; + *) REQUESTED+=("$arg") ;; + esac +done +export DRY_RUN UNATTENDED + +# ── Load helpers + all service modules (they self-register) ────────────────── +# shellcheck source=lib/common.sh +source "$HERE/lib/common.sh" +shopt -s nullglob +for _mod in "$HERE"/services/*.sh; do source "$_mod"; done +shopt -u nullglob + +# ── Helpers over the registry ──────────────────────────────────────────────── +# Groups present, in CATEGORY_ORDER first, then any extras alphabetically. +groups_present() { + local g present=() seen=" " + for name in "${SERVICE_ORDER[@]}"; do + g="${SERVICE_GROUP[$name]}" + case "$seen" in *" $g "*) : ;; *) present+=("$g"); seen="$seen$g " ;; esac + done + local out=() + for g in "${CATEGORY_ORDER[@]}"; do + printf '%s\n' "${present[@]}" | grep -qx "$g" && out+=("$g") + done + for g in "${present[@]}"; do + printf '%s\n' "${CATEGORY_ORDER[@]}" | grep -qx "$g" || out+=("$g") + done + printf '%s\n' "${out[@]}" +} + +# Services in a group, ordered by SERVICE_PRIORITY then name. +services_in_group() { + local group="$1" name + for name in "${SERVICE_ORDER[@]}"; do + [ "${SERVICE_GROUP[$name]}" = "$group" ] && echo "${SERVICE_PRIORITY[$name]:-50} $name" + done | sort -n -k1 | awk '{print $2}' +} + +# Best-effort "is it already installed?" for the [installed] marker. +is_installed() { + case "$1" in + base) command -v ncdu >/dev/null 2>&1 ;; + glow) command -v glow >/dev/null 2>&1 ;; + crowdsec) command -v cscli >/dev/null 2>&1 ;; + silent-send) [ -d "$ACTUAL_HOME/silent-send/.git" ] ;; + linux-to-sync) [ -d "$ACTUAL_HOME/linux-to-sync/.git" ] ;; + *) [ -e "$DOCKER_DIR/$1" ] ;; + esac +} + +run_service() { + local name="$1" + if [ -z "${SERVICE_GROUP[$name]:-}" ]; then log_error "Unknown service: $name (try --list)"; return 1; fi + declare -F "install_${name}" >/dev/null || { log_error "Service '$name' has no install_${name}"; return 1; } + log_info "=== ${name} (${SERVICE_DESC[$name]}) ===" + "install_${name}" +} + +list_services() { + local g name + while IFS= read -r g; do + echo ""; echo "── ${g^^} ──" + while IFS= read -r name; do + printf " %-16s %s\n" "$name" "${SERVICE_DESC[$name]}" + done < <(services_in_group "$g") + done < <(groups_present) + echo "" +} + +# ── --list ─────────────────────────────────────────────────────────────────── +if [ "$DO_LIST" = true ]; then list_services; exit 0; fi + +# ── Direct install: ./setup.sh caddy homeassistant ────────────────────────── +if [ "${#REQUESTED[@]}" -gt 0 ]; then + require_root + rc=0; for name in "${REQUESTED[@]}"; do run_service "$name" || rc=1; done + exit "$rc" +fi + +# ── Guided interactive flow ────────────────────────────────────────────────── +require_root + +# 1) Show the REQUIRED set and let the user cancel before anything happens. +echo "" +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ Ubuntu Post-Install · v$(cat "$HERE/VERSION" 2>/dev/null || echo '?')" +echo "╚══════════════════════════════════════════════════════════════╝" +echo "" +echo "REQUIRED (installed/verified first):" +echo " • Essential CLI packages: net-tools, git, curl, wget, htop, tree," +echo " ncdu, zip/unzip, jq, rsync, and glow (markdown reader)" +echo " • Docker presence check (needed by all containerized services)" +echo "" +echo "Then you'll get a category menu to pick optional services." +echo "" +PROCEED="" +prompt_yn "Proceed with the required setup? (y/n):" "y" PROCEED +if [ "$PROCEED" != "y" ] && [ "$PROCEED" != "Y" ]; then + echo "Cancelled. Nothing was changed." + exit 0 +fi + +# 2) Run required. +run_service base +if ! command -v docker >/dev/null 2>&1; then + log_warning "Docker is not installed. Containerized services need it." + echo " Install with: curl -fsSL https://get.docker.com | sh" +fi + +# 3) Offer Caddy first (most services proxy through it). +if [ -n "${SERVICE_GROUP[caddy]:-}" ] && ! is_installed caddy; then + echo "" + OFFER_CADDY="" + prompt_yn "Install Caddy now? It's the reverse proxy most services use. (y/n):" "y" OFFER_CADDY + [ "$OFFER_CADDY" = "y" ] || [ "$OFFER_CADDY" = "Y" ] && run_service caddy +fi + +# 4) Category menu loop: pick a category → checklist → install → back to menu. +have_whiptail=false +command -v whiptail >/dev/null 2>&1 && have_whiptail=true + +while true; do + mapfile -t CATS < <(groups_present) + + if [ "$have_whiptail" = true ]; then + cat_items=() + for g in "${CATS[@]}"; do + n=$(services_in_group "$g" | wc -l) + cat_items+=("$g" "$n service(s)") + done + cat_items+=("DONE" "Finish and exit") + CHOSEN_CAT=$(whiptail --title "Service Categories" --menu \ + "Pick a category (services you install come back here):" 22 70 14 \ + "${cat_items[@]}" 3>&1 1>&2 2>&3) || break + else + echo ""; echo "Categories:"; i=1 + for g in "${CATS[@]}"; do echo " $i) $g"; i=$((i+1)); done + echo " d) Done" + read -rp "Pick a category [d]: " pick + [ "$pick" = "d" ] || [ -z "$pick" ] && break + CHOSEN_CAT="${CATS[$((pick-1))]:-}" + [ -z "$CHOSEN_CAT" ] && { echo "Invalid."; continue; } + fi + [ "$CHOSEN_CAT" = "DONE" ] && break + + mapfile -t SVCS < <(services_in_group "$CHOSEN_CAT") + SELECTED=() + if [ "$have_whiptail" = true ]; then + svc_items=() + for name in "${SVCS[@]}"; do + tag="${SERVICE_DESC[$name]}" + is_installed "$name" && tag="$tag [installed]" + svc_items+=("$name" "$tag" "OFF") + done + CHOICE=$(whiptail --title "${CHOSEN_CAT^^}" --checklist \ + "Space to select, Enter to install. Already-installed are marked:" 22 78 14 \ + "${svc_items[@]}" 3>&1 1>&2 2>&3) || continue + eval "SELECTED=($CHOICE)" + else + echo ""; echo "${CHOSEN_CAT^^}:" + for name in "${SVCS[@]}"; do + m=""; is_installed "$name" && m=" [installed]" + printf " %-16s %s%s\n" "$name" "${SERVICE_DESC[$name]}" "$m" + done + read -rp "Enter service names to install (space-separated, blank to go back): " -a SELECTED + fi + + for name in "${SELECTED[@]}"; do run_service "$name"; done +done + +echo "" +log_success "Done. Re-run 'sudo ./setup.sh' any time to add more." From a63f72ef17c8b8f702edbadf12b68d526f3e72dc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 21:32:39 +0000 Subject: [PATCH 02/12] feat(minecraft): port whitelist import + VT share links + pause from standalone Three improvements ported from the standalone setupminecraft.sh: - Whitelist: detect existing whitelist.json on re-run, offer to import players by number (0=all, comma list, Enter=skip), no UUID re-lookup for already-resolved entries; new gamertags still looked up via Mojang API - Vanilla Tweaks: add vanillatweaks.net pre-configured share links at the top of the download instructions section (datapacks + crafting tweaks) - Vanilla Tweaks: pause with "Press Enter when datapacks are in datapacks-download/" so user can SCP the ZIP before the build starts https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- services/minecraft.sh | 160 +++++++++++++++++++++++++++++++++--------- 1 file changed, 126 insertions(+), 34 deletions(-) diff --git a/services/minecraft.sh b/services/minecraft.sh index 815f9b4..ebd90b8 100644 --- a/services/minecraft.sh +++ b/services/minecraft.sh @@ -145,11 +145,68 @@ print(snaps[0] if snaps else '') [[ $WHITELIST =~ ^[Yy]$ ]] && WHITELIST_ENABLED=true || WHITELIST_ENABLED=false local WHITELIST_PLAYERS=() + declare -A WHITELIST_PRELOADED # name → uuid, already resolved from existing file if [ "$WHITELIST_ENABLED" = true ] && [ "$UNATTENDED" != true ]; then echo "" log_info "Whitelist Players" - echo " Enter player gamertags to pre-populate the whitelist." - echo " UUIDs are looked up automatically. Press Enter alone when done." + + # Import from existing whitelist.json when re-running against an existing instance + local _EXISTING_WL="$MC_DIR/data/whitelist.json" + if [ -f "$_EXISTING_WL" ]; then + local -a _EX_NAMES _EX_UUIDS + mapfile -t _EX_NAMES < <(python3 -c " +import json, sys +try: + data = json.load(open(sys.argv[1])) + for p in data: + if p.get('name'): print(p['name']) +except: pass +" "$_EXISTING_WL" 2>/dev/null) + mapfile -t _EX_UUIDS < <(python3 -c " +import json, sys +try: + data = json.load(open(sys.argv[1])) + for p in data: + if p.get('uuid'): print(p['uuid']) +except: pass +" "$_EXISTING_WL" 2>/dev/null) + if [ ${#_EX_NAMES[@]} -gt 0 ]; then + echo "" + echo " Existing whitelist found (${#_EX_NAMES[@]} player(s)):" + local _i + for _i in "${!_EX_NAMES[@]}"; do + printf " %d) %s\n" "$((_i+1))" "${_EX_NAMES[$_i]}" + done + echo "" + echo " Import from existing? Enter numbers (e.g. 1,3,4), 0=all, Enter=skip:" + local _WL_IMPORT="" + read -p " Selection: " _WL_IMPORT + if [ -n "$_WL_IMPORT" ]; then + if [ "$_WL_IMPORT" = "0" ]; then + for _i in "${!_EX_NAMES[@]}"; do + WHITELIST_PRELOADED["${_EX_NAMES[$_i]}"]="${_EX_UUIDS[$_i]}" + done + log_success " Imported all ${#_EX_NAMES[@]} existing player(s)" + else + local -a _SEL_NUMS + IFS=',' read -ra _SEL_NUMS <<< "$_WL_IMPORT" + local _n + for _n in "${_SEL_NUMS[@]}"; do + _n="${_n// /}" + if [[ "$_n" =~ ^[0-9]+$ ]] && [ "$_n" -ge 1 ] && \ + [ "$_n" -le "${#_EX_NAMES[@]}" ]; then + WHITELIST_PRELOADED["${_EX_NAMES[$((_n-1))]}"]="${_EX_UUIDS[$((_n-1))]}" + log_info " Imported: ${_EX_NAMES[$((_n-1))]}" + fi + done + fi + fi + fi + fi + + echo "" + echo " Enter additional gamertags to add. UUIDs looked up automatically." + echo " Press Enter alone when done." echo "" while true; do local _GT="" @@ -158,8 +215,9 @@ print(snaps[0] if snaps else '') WHITELIST_PLAYERS+=("$_GT") log_info " Added: $_GT" done - if [ ${#WHITELIST_PLAYERS[@]} -gt 0 ]; then - log_success " ${#WHITELIST_PLAYERS[@]} player(s) queued for whitelist" + local _WL_TOTAL=$(( ${#WHITELIST_PLAYERS[@]} + ${#WHITELIST_PRELOADED[@]} )) + if [ "$_WL_TOTAL" -gt 0 ]; then + log_success " $_WL_TOTAL player(s) queued for whitelist" else log_info " No players entered — whitelist will be empty until you add players manually" fi @@ -918,38 +976,61 @@ print(snaps[0] if snaps else '') cd "$MC_DIR" || return 1 # ── Whitelist pre-population ──────────────────────────────────────────────── - if [ "$WHITELIST_ENABLED" = true ] && [ ${#WHITELIST_PLAYERS[@]} -gt 0 ]; then - log_info "Looking up UUIDs for whitelist players..." + local _WL_NEED_WRITE=false + [ "$WHITELIST_ENABLED" = true ] && \ + [ $(( ${#WHITELIST_PLAYERS[@]} + ${#WHITELIST_PRELOADED[@]} )) -gt 0 ] && \ + _WL_NEED_WRITE=true + + if [ "$_WL_NEED_WRITE" = true ]; then + log_info "Building whitelist.json..." local _WL_JSON="[" local _WL_FIRST=true local _WL_COUNT=0 - local _player _resp _uuid _name - for _player in "${WHITELIST_PLAYERS[@]}"; do - _resp=$(curl -sf --max-time 10 \ - "https://api.mojang.com/users/profiles/minecraft/${_player}" 2>/dev/null || echo "") - if [ -z "$_resp" ]; then - log_warning " '$_player' not found — skipping (account may not exist)" - continue - fi - _uuid=$(echo "$_resp" | python3 -c " + + # Preloaded entries — UUIDs already known, no API call needed + local _wl_name _uuid + for _wl_name in "${!WHITELIST_PRELOADED[@]}"; do + _uuid="${WHITELIST_PRELOADED[$_wl_name]}" + log_success " $_wl_name → $_uuid (from existing whitelist)" + [ "$_WL_FIRST" = true ] || _WL_JSON+="," + _WL_FIRST=false + _WL_COUNT=$((_WL_COUNT + 1)) + _WL_JSON+=" + {\"uuid\": \"$_uuid\", \"name\": \"$_wl_name\"}" + done + + # New gamertags — look up via Mojang API + if [ ${#WHITELIST_PLAYERS[@]} -gt 0 ]; then + log_info " Looking up UUIDs via Mojang API..." + local _player _resp _name + for _player in "${WHITELIST_PLAYERS[@]}"; do + _resp=$(curl -sf --max-time 10 \ + "https://api.mojang.com/users/profiles/minecraft/${_player}" 2>/dev/null || echo "") + if [ -z "$_resp" ]; then + log_warning " '$_player' not found — skipping (account may not exist)" + continue + fi + _uuid=$(echo "$_resp" | python3 -c " import sys, json d = json.load(sys.stdin) uid = d['id'] print(f'{uid[:8]}-{uid[8:12]}-{uid[12:16]}-{uid[16:20]}-{uid[20:]}') " 2>/dev/null || echo "") - _name=$(echo "$_resp" | python3 -c " + _name=$(echo "$_resp" | python3 -c " import sys, json; d=json.load(sys.stdin); print(d.get('name',''))" 2>/dev/null || echo "$_player") - if [ -z "$_uuid" ]; then - log_warning " Could not parse UUID for '$_player' — skipping" - continue - fi - log_success " $_name → $_uuid" - [ "$_WL_FIRST" = true ] || _WL_JSON+="," - _WL_FIRST=false - _WL_COUNT=$((_WL_COUNT + 1)) - _WL_JSON+=" + if [ -z "$_uuid" ]; then + log_warning " Could not parse UUID for '$_player' — skipping" + continue + fi + log_success " $_name → $_uuid" + [ "$_WL_FIRST" = true ] || _WL_JSON+="," + _WL_FIRST=false + _WL_COUNT=$((_WL_COUNT + 1)) + _WL_JSON+=" {\"uuid\": \"$_uuid\", \"name\": \"$_name\"}" - done + done + fi + _WL_JSON+=" ]" echo "$_WL_JSON" > "$MC_DIR/data/whitelist.json" @@ -1012,9 +1093,13 @@ for v in versions: echo "" log_info "Vanilla Tweaks — download your selected packs manually:" echo "" - echo " 1. Go to: https://vanillatweaks.net/picker/datapacks/" - echo " 2. Select Minecraft version ${VT_VERSION} in the version dropdown" - echo " 3. Enable these packs (your selections from the toggle menu):" + echo " ┌─ Quick start: pre-configured share links (opens VT pre-selected) ─┐" + echo " │ Datapacks: https://vanillatweaks.net/share#B3QqSd │" + echo " │ Crafting tweaks: https://vanillatweaks.net/share#SqzGkO │" + echo " └───────────────────────────────────────────────────────────────────-┘" + echo "" + echo " Or pick manually — go to https://vanillatweaks.net/picker/datapacks/" + echo " and select version ${VT_VERSION}, then enable your chosen packs:" echo "" local _LAST_CAT="" _cat dp for dp in "${DPACK_ORDER[@]}"; do @@ -1027,13 +1112,20 @@ for v in versions: echo " • ${DPACKS[$dp]}" done echo "" - echo " 4. Click Download and save the .zip file" - echo " 5. Place the .zip in: ${MC_DIR}/datapacks-download/" - echo " 6. Rebuild: cd ${MC_DIR} && docker compose build" - echo " 7. Restart: cd ${MC_DIR} && docker compose up -d" - echo "" + echo " ── How to install ────────────────────────────────────────────────────" + echo " 1. Download the ZIP from vanillatweaks.net (use share link or pick)" + echo " 2. SCP it to this server (run on your local machine):" + echo " scp ~/Downloads/VanillaTweaks*.zip $(whoami)@$(hostname -I | awk '{print $1}'):${MC_DIR}/datapacks-download/" + echo " 3. On this server:" + echo " cd ${MC_DIR}/datapacks-download" + echo " unzip 'VanillaTweaks*.zip' && rm VanillaTweaks*.zip" + echo " 4. Rebuild: cd ${MC_DIR} && docker compose build" + echo " 5. Restart: cd ${MC_DIR} && docker compose up -d" + echo " ──────────────────────────────────────────────────────────────────────" echo " The itzg image extracts .zip files from /datapacks/ on startup." echo " Datapacks land in ${MC_NAME}/data/datapacks/ and persist across restarts." + echo "" + read -p " Press Enter when datapacks are in datapacks-download/ (or Enter to skip): " fi # ── LuckPerms bootstrap script ────────────────────────────────────────────── From 1d4b38674dfcf63c205cbcc2490ae3929d4d2fb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 21:36:22 +0000 Subject: [PATCH 03/12] =?UTF-8?q?feat(extras):=20add=20sync-cc=20service?= =?UTF-8?q?=20=E2=80=94=20Whisper/ffsubsync=20subtitle=20tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds sync_cc as an extras service module: - extras/sync_cc.py: the Python tool (3196 lines) — 8 modes: SYNC, GENERATE, BATCH, RENAME (TMDB), EXTRACT, REMUX, EMBED, BURNSUBS - services/sync-cc.sh: installs system deps (python3, ffmpeg, mkvtoolnix, ccextractor), pip installs openai-whisper + ffsubsync, copies the script to ~/sync-cc/, prompts for TMDB API key → .env, creates /usr/local/bin/sync-cc wrapper so users run it from any directory containing video/SRT files Heavy optional deps (easyocr, pgsreader) are installed on first use by the script itself. GPU (CUDA/MPS) is used automatically if detected. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- MODULAR.md | 2 +- extras/sync_cc.py | 3195 +++++++++++++++++++++++++++++++++++++++++++ services/sync-cc.sh | 130 ++ setup.sh | 1 + 4 files changed, 3327 insertions(+), 1 deletion(-) create mode 100644 extras/sync_cc.py create mode 100644 services/sync-cc.sh diff --git a/MODULAR.md b/MODULAR.md index c36b58f..5c3c920 100644 --- a/MODULAR.md +++ b/MODULAR.md @@ -97,5 +97,5 @@ is retained as a frozen evolution record. | `media` | `arm`, `audiobookshelf`, `emby`, `immich`, `jellyfin`, `lyrion` | | `cameras` | `frigate`, `frigate-notify` | | `gaming` | `js99er`, `minecraft`, `wolf`, `wolf-pair` | -| `extras` | `linux-to-sync`, `silent-send` | +| `extras` | `linux-to-sync`, `silent-send`, `sync-cc` | | `backup` | `backup` | diff --git a/extras/sync_cc.py b/extras/sync_cc.py new file mode 100644 index 0000000..5b56f62 --- /dev/null +++ b/extras/sync_cc.py @@ -0,0 +1,3195 @@ +#!/usr/bin/env python3 +""" +SRT subtitle tool - three modes: + + 1. GENERATE - Whisper transcribes the video and creates a perfectly-synced SRT. + 2. SYNC - ffsubsync syncs an existing SRT, Whisper cross-checks the result. + 3. BATCH - sync all video+SRT pairs in the current directory. + +GPU is used automatically if CUDA (NVIDIA) or MPS (Apple Silicon) is detected. +openai-whisper and ffsubsync are installed automatically if missing. + +Flags: + --translate Mode 2 outputs English regardless of source language + --lang CODE Source language hint (e.g. fr, id, es) — speeds up detection + --lang-auto Auto-detect language (default when --translate is used) + --extract-all FILE Non-interactive: extract all subtitle tracks and exit + +Requirements: + Python 3, ffmpeg in PATH. +""" +import os, sys, re, subprocess, struct, difflib, urllib.request, urllib.parse, json, glob +from statistics import median + +# ---------- .env loader ------------------------------------------------------- + +def _load_env(): + """Parse KEY=value lines from .env in cwd or script directory.""" + import pathlib + for candidate in [pathlib.Path('.env'), + pathlib.Path(__file__).resolve().parent / '.env']: + try: + for line in candidate.read_text().splitlines(): + line = line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + k, _, v = line.partition('=') + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and k not in os.environ: + os.environ[k] = v + except FileNotFoundError: + pass + +_load_env() + +# ============================================================================= +# TMDB API key — set here OR put TMDB_API_KEY=your_key in a .env file +# Get a free key at https://www.themoviedb.org/settings/api +TMDB_API_KEY = os.environ.get('TMDB_API_KEY', '') +# ============================================================================= + +# ---------- Path setup ------------------------------------------------------- + +def _extend_path(): + import site, pathlib + candidates = [] + try: + candidates.append(site.getusersitepackages()) + except Exception: + pass + home = str(pathlib.Path.home()) + candidates += glob.glob( + os.path.join(home, '.local', 'lib', 'python*', 'site-packages') + ) + for p in candidates: + if p and os.path.isdir(p) and p not in sys.path: + sys.path.insert(0, p) + +_extend_path() + +# ---------- ffsubsync finder ------------------------------------------------- + +def _find_ffsubsync(): + import shutil, pathlib + found = shutil.which('ffsubsync') + if found: + return found + local_bin = os.path.join(str(pathlib.Path.home()), '.local', 'bin', 'ffsubsync') + if os.path.isfile(local_bin): + return local_bin + return None + +# ---------- Optional dependency detection ------------------------------------ + +try: + import whisper as _whisper + WHISPER_AVAILABLE = True +except ImportError: + _whisper = None + WHISPER_AVAILABLE = False + +FFSUBSYNC_AVAILABLE = _find_ffsubsync() is not None + +TS_RE = re.compile( + r'(\d{1,2}:\d{2}:\d{2}[,\.]\d{1,3})\s*-->\s*(\d{1,2}:\d{2}:\d{2}[,\.]\d{1,3})' +) +VIDEO_EXTS = ('.mp4','.mkv','.mov','.avi','.ts','.m2ts','.webm','.flv','.wmv','.mpg','.mpeg') + +# ---------- Startup diagnostic ----------------------------------------------- + +def _check_deps(): + print("--- dependency check ---") + try: + import whisper as _w, inspect + print(f" whisper : found at {os.path.dirname(inspect.getfile(_w))}") + except ImportError: + print(" whisper : NOT found") + exe = _find_ffsubsync() + print(f" ffsubsync : {'found at ' + exe if exe else 'NOT found'}") + try: + r = subprocess.run(['ffmpeg', '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + line = r.stdout.decode(errors='ignore').splitlines()[0] + print(f" ffmpeg : {line}") + except FileNotFoundError: + print(" ffmpeg : NOT found - required!") + local_paths = [p for p in sys.path if 'local' in p or 'site' in p] + if local_paths: + print(" sys.path (local/site entries):") + for p in local_paths: + print(f" {p}") + print("------------------------") + +_check_deps() + +# ---------- Tunable constants ------------------------------------------------ + +WHISPER_MODEL = "large-v3-turbo" +WHISPER_LANGUAGE = "en" +WHISPER_TASK = "transcribe" # or "translate" (→ English output) + +WHISPER_MODELS = { + "1": ("tiny", "~39 MB - very fast, low accuracy"), + "2": ("base", "~74 MB - fast, basic accuracy"), + "3": ("small", "~244 MB - good for simple audio"), + "4": ("medium", "~769 MB - better accuracy, slower"), + "5": ("large-v3-turbo", "~809 MB - best speed/accuracy balance (recommended)"), + "6": ("large-v3", "~1.5 GB - highest accuracy, slowest"), +} + +WHISPER_MODEL_SIZES = { + "tiny": "39 MB", "base": "74 MB", "small": "244 MB", + "medium": "769 MB", "large-v3-turbo": "809 MB", "large-v3": "1.5 GB", +} + +WHISPER_PROMPT = ( + "Transcript with proper punctuation, capitalization, and grammar. " + "Mark all sung lyrics and songs with ♪ symbols at the start and end. " + "Use italics tags for off-screen or narrator dialogue." +) + +START_SKIP_S = 0 +ANALYZE_S = 600 # 10 minutes of audio for alignment +MIN_WORD_LEN = 4 +OFFSET_AGREE_THRESHOLD = 1.5 # seconds - warn if ffsubsync and Whisper differ more than this + +STOP_WORDS = { + 'the','and','you','that','was','for','are','with','his','they','this', + 'have','from','not','but','had','her','she','him','been','has','its', + 'who','did','get','may','now','can','our','out','all','yes','no', + 'what','just','will','your','when','them','than','then','some','into', + 'said','more','also','very','here','well','like','even','back','much', +} + +MAX_OFFSET_S = 90.0 +RESOLUTION_S = 0.1 +RESAMPLE_HZ = 100 +SPEECH_LO = 300 +SPEECH_HI = 3400 +CHUNK_SIZE = max(1, int(RESAMPLE_HZ * RESOLUTION_S)) + +_NOISE_RE = re.compile( + r'\b(720p|1080p|2160p|4k|uhd|webrip|web|bluray|bdrip|dvdrip|hdtv|dl' + r'|x264|x265|hevc|avc|h264|h265|aac|dts|ac3|nf|amzn|hulu|dsnp|atvp' + r'|hmax|pcok|repack|proper|extended|theatrical|directors?cut|remux' + r'|episode|episodes?)\b', + re.IGNORECASE +) +_SXXEXX_RE = re.compile(r'\bS(\d{1,2})E(\d{1,2})\b', re.IGNORECASE) +_SEASON_DIR_RE = re.compile(r'^[Ss]eason[\s._-]*\d+$') +_BRACKET_RE = re.compile(r'^\s*\[[^\]]*\]\s*') # leading [SubGroup] tags + +_TEXT_SUB_CODECS = {'subrip', 'srt', 'ass', 'ssa', 'mov_text', + 'webvtt', 'microdvd', 'text', 'dvb_teletext'} +_IMAGE_SUB_CODECS = {'dvd_subtitle', 'hdmv_pgs_subtitle', + 'dvb_subtitle', 'dvbsub', 'pgssub', 'xsub'} + +# ---------- Auto-install helpers --------------------------------------------- + +def _find_pip(): + for cmd in (['pip3'], ['pip']): + try: + if subprocess.run(cmd + ['--version'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0: + return cmd + except FileNotFoundError: + pass + for py in [sys.executable, 'python3', 'python']: + try: + if subprocess.run([py, '-m', 'pip', '--version'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0: + return [py, '-m', 'pip'] + except FileNotFoundError: + pass + # Try bootstrapping pip via ensurepip + try: + if subprocess.run([sys.executable, '-m', 'ensurepip', '--upgrade'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0: + if subprocess.run([sys.executable, '-m', 'pip', '--version'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0: + return [sys.executable, '-m', 'pip'] + except Exception: + pass + # Last resort: apt-get + print(" pip not found - attempting: sudo apt-get install python3-pip ...") + try: + if subprocess.run(['sudo', 'apt-get', 'install', '-y', 'python3-pip'], + timeout=120).returncode == 0: + for cmd in (['pip3'], [sys.executable, '-m', 'pip']): + try: + if subprocess.run(cmd + ['--version'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0: + return cmd + except FileNotFoundError: + pass + except Exception: + pass + return None + +def _pip_install(package): + pip = _find_pip() + if pip is None: + print(f" Cannot find pip. Try manually: pip3 install {package}") + return False + for flags in [[], ['--user']]: + if subprocess.run(pip + ['install'] + flags + [package]).returncode == 0: + _extend_path() + return True + print(" Standard and --user installs failed.") + if input(" Try --break-system-packages? [y/N]: ").strip().lower() == 'y': + if subprocess.run(pip + ['install', '--break-system-packages', + package]).returncode == 0: + _extend_path() + return True + return False + +def ensure_whisper(): + global _whisper, WHISPER_AVAILABLE + if WHISPER_AVAILABLE: + return True + print("\nopenai-whisper is not installed.") + if input("Install it now? [y/N]: ").strip().lower() != 'y': + print("Skipping - will fall back to audio energy method.") + return False + print("Installing openai-whisper...") + if not _pip_install('openai-whisper'): + print("Installation failed.") + return False + import importlib + importlib.invalidate_caches() + try: + import whisper as _w + _whisper = _w + WHISPER_AVAILABLE = True + print("Installed successfully.\n") + return True + except ImportError: + print("Installed but import failed - try restarting the script.") + return False + +def ensure_ffsubsync(): + global FFSUBSYNC_AVAILABLE + if FFSUBSYNC_AVAILABLE: + return True + print("\nffsubsync is not installed (recommended for syncing existing SRTs).") + if input("Install it now? [y/N]: ").strip().lower() != 'y': + return False + print("Installing ffsubsync...") + if not _pip_install('ffsubsync'): + print("Installation failed.") + return False + import importlib + importlib.invalidate_caches() + if _find_ffsubsync(): + FFSUBSYNC_AVAILABLE = True + print("ffsubsync installed successfully.") + return True + print("Installed but ffsubsync not found - try restarting the script.") + return False + +def ensure_easyocr(): + try: + import easyocr # noqa: F401 + return True + except ImportError: + pass + print("\neasyocr not installed (needed to scan video frames for a title card).") + if input("Install it now? (~200 MB package, ~170 MB model download on first use) [y/N]: ").strip().lower() != 'y': + return False + print("Installing easyocr...") + if not _pip_install('easyocr'): + print("Installation failed.") + return False + import importlib + importlib.invalidate_caches() + try: + import easyocr # noqa: F401 + return True + except ImportError: + print("Installed but import failed - try restarting the script.") + return False + +def ensure_ccextractor(): + """Return ccextractor command, or None if unavailable.""" + for cmd in ['ccextractor', 'ccextractorwin', 'ccx']: + try: + if subprocess.run([cmd, '--version'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode in (0, 1): + return cmd + except FileNotFoundError: + pass + print("\nccextractor not found (needed for CC and some DVD subtitles).") + if input("Try to install via apt-get? [y/N]: ").strip().lower() != 'y': + print(" Install manually: https://ccextractor.org") + return None + try: + if subprocess.run(['sudo', 'apt-get', 'install', '-y', 'ccextractor'], + timeout=120).returncode == 0: + return 'ccextractor' + except Exception: + pass + print(" apt-get failed. Install manually: https://ccextractor.org") + return None + +def ensure_pgsreader(): + try: + import pgsreader # noqa: F401 + return True + except ImportError: + pass + print("\npgsreader not installed (needed for Blu-ray PGS subtitles).") + if input("Install it now? [y/N]: ").strip().lower() != 'y': + return False + if not _pip_install('pgsreader'): + return False + import importlib + importlib.invalidate_caches() + try: + import pgsreader # noqa: F401 + return True + except ImportError: + print("Installed but import failed - try restarting the script.") + return False + + +def ensure_mkvtoolnix(): + """Return True if mkvmerge is available, offering to install if not.""" + import shutil, platform + if shutil.which('mkvmerge'): + return True + print("\nmkvmerge not found — needed to embed subtitles into MKV files.") + system = platform.system() + if system == 'Darwin': + if input(" Try to install via brew? [y/N]: ").strip().lower() == 'y': + try: + if subprocess.run(['brew', 'install', 'mkvtoolnix'], + timeout=300).returncode == 0: + return bool(shutil.which('mkvmerge')) + except Exception: + pass + print(" Install manually: brew install mkvtoolnix") + else: + if input(" Try to install via apt-get? [y/N]: ").strip().lower() == 'y': + try: + if subprocess.run(['sudo', 'apt-get', 'install', '-y', 'mkvtoolnix'], + timeout=120).returncode == 0: + return bool(shutil.which('mkvmerge')) + except Exception: + pass + print(" Install manually:") + print(" Debian/Ubuntu : sudo apt install mkvtoolnix") + print(" Arch : sudo pacman -S mkvtoolnix-cli") + print(" Other : https://mkvtoolnix.download/") + return False + + +def ensure_vobsub2srt(): + """Return True if vobsub2srt is available, offering to install if not.""" + import shutil + if shutil.which('vobsub2srt'): + return True + print("\nvobsub2srt not found — needed for DVD VOB subtitle OCR to SRT.") + if input(" Try to install via apt-get? [y/N]: ").strip().lower() == 'y': + try: + if subprocess.run(['sudo', 'apt-get', 'install', '-y', 'vobsub2srt'], + timeout=120).returncode == 0: + return bool(shutil.which('vobsub2srt')) + except Exception: + pass + print(" Install manually: sudo apt install vobsub2srt") + print(" Alternative GUI : https://github.com/SubtitleEdit/subtitleedit") + return False + +# ---------- GPU detection ---------------------------------------------------- + +def get_device(): + try: + import torch + if torch.cuda.is_available(): + print(f" GPU detected: {torch.cuda.get_device_name(0)} (CUDA)") + return "cuda" + if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + print(" GPU detected: Apple Silicon (MPS)") + return "mps" + except Exception: + pass + print(" No GPU detected - running on CPU.") + return "cpu" + +def load_whisper_model(model_name): + device = get_device() + size = WHISPER_MODEL_SIZES.get(model_name, '?') + print(f" Loading Whisper '{model_name}' model " + f"(first run downloads ~{size} to ~/.cache/whisper)...") + try: + return _whisper.load_model(model_name, device=device), device + except Exception as e: + if 'out of memory' in str(e).lower() and device != 'cpu': + print(" GPU out of memory - clearing cache and retrying on CPU...") + try: + import torch + torch.cuda.empty_cache() + torch.cuda.synchronize() + except Exception: + pass + return _whisper.load_model(model_name, device='cpu'), 'cpu' + raise + +# ---------- File listing / selection ----------------------------------------- + +def list_files(exts, label): + exts = (exts,) if isinstance(exts, str) else exts + files = [f for f in sorted(os.listdir('.')) if f.lower().endswith(exts)] + if not files: + print(f"No {label} files found in current directory.") + else: + for i, f in enumerate(files, 1): + print(f"{i}: {f}") + return files + +def pick_file(files, prompt, allow_skip=False): + skip_hint = " or Enter to skip" if allow_skip else "" + while True: + choice = input(prompt + skip_hint + " (0 to cancel): ").strip() + if choice == '0': + return None + if choice == "" and allow_skip: + return "" + if choice == "": + for i, f in enumerate(files, 1): + print(f"{i}: {f}") + continue + if choice.isdigit(): + idx = int(choice) + if 1 <= idx <= len(files): + return files[idx - 1] + print("Invalid number.") + continue + if os.path.isfile(choice): + return choice + print("File not found.") + +# ---------- SRT parsing / writing -------------------------------------------- + +def srt_to_seconds(t): + h, m, rest = t.split(':') + s, ms = rest.split(',') + return int(h)*3600 + int(m)*60 + int(s) + int(ms)/1000.0 + +def seconds_to_srt(t): + t = max(0.0, t) + h = int(t) // 3600 + m = (int(t) // 60) % 60 + s = int(t) % 60 + ms = int(round((t - int(t)) * 1000)) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + +def _read_srt_text(path): + """Read an SRT file, auto-detecting encoding and stripping BOM.""" + for enc in ('utf-8-sig', 'utf-16', 'cp1252', 'latin-1'): + try: + text = open(path, encoding=enc).read() + # utf-16 files decoded correctly won't have lone surrogates + return text + except (UnicodeDecodeError, UnicodeError): + continue + return open(path, encoding='utf-8', errors='replace').read() + + +def _normalise_srt_ts(text): + """Accept HH:MM:SS.mmm or H:MM:SS,mm etc. — normalise to HH:MM:SS,mmm.""" + def _fix(m): + ts = m.group(0) + ts = ts.replace('.', ',') + ms_part = ts.rsplit(',', 1)[1] + ts = ts.rsplit(',', 1)[0] + ',' + ms_part.ljust(3, '0')[:3] + return ts + return re.sub(r'\d{1,2}:\d{2}:\d{2}[,\.]\d{1,3}', _fix, text) + + +_HI_LINE_RE = re.compile(r'^\s*[\(\[].+[\)\]]\s*$') # lines that are ONLY a bracketed description + +def _is_hi_subtitle(path): + """Return True if >25% of text lines look like HI sound descriptions.""" + entries = parse_srt_full(path, limit=80) + if not entries: + return False + total = hi = 0 + for _, _, text in entries: + for line in text.splitlines(): + line = line.strip() + if not line: + continue + total += 1 + if _HI_LINE_RE.match(line): + hi += 1 + return total > 0 and (hi / total) > 0.25 + + +def _strip_hi_for_sync(src_path, dst_path): + """Write a copy of src_path with description-only entries removed. + Entries that mix dialogue with descriptions are kept (stripped to dialogue only). + Returns True if any entries were removed/modified.""" + text = _normalise_srt_ts(_read_srt_text(src_path)) + blocks = re.split(r'\n\s*\n', text.strip()) + out = [] + changed = False + for block in blocks: + lines = block.strip().splitlines() + ts_idx = next((i for i, l in enumerate(lines) if TS_RE.search(l)), None) + if ts_idx is None: + out.append(block) + continue + text_lines = [l for l in lines[ts_idx + 1:] if l.strip()] + dialogue = [l for l in text_lines if not _HI_LINE_RE.match(l)] + if not text_lines: + out.append(block) + elif not dialogue: + # entry is entirely sound descriptions — drop it + changed = True + else: + if len(dialogue) < len(text_lines): + changed = True + out.append('\n'.join(lines[:ts_idx + 1] + dialogue)) + with open(dst_path, 'w', encoding='utf-8') as f: + f.write('\n\n'.join(out)) + return changed + +def parse_srt_full(path, limit=9999): + entries = [] + try: + text = _normalise_srt_ts(_read_srt_text(path)) + except Exception: + return entries + for block in re.split(r'\n\s*\n', text.strip()): + lines = block.strip().splitlines() + for i, line in enumerate(lines): + m = TS_RE.search(line) + if m: + start = srt_to_seconds(m.group(1)) + end = srt_to_seconds(m.group(2)) + body = re.sub(r'<[^>]+>', '', ' '.join(lines[i+1:]).strip()) + entries.append((start, end, body)) + break + if len(entries) >= limit: + break + return entries + +def normalize_word(w): + return re.sub(r"[^a-z0-9']", '', w.lower()) + +def srt_to_word_times(entries): + result = [] + for start, _end, text in entries: + for raw in text.split(): + w = normalize_word(raw) + if len(w) >= MIN_WORD_LEN and w not in STOP_WORDS: + result.append((w, start)) + return result + +def shift_srt(inpath, outpath, offset): + text = _normalise_srt_ts(_read_srt_text(inpath)) + with open(outpath, 'w', encoding='utf-8') as fout, \ + __import__('io').StringIO(text) as fin: + for line in fin: + m = TS_RE.search(line) + if m: + s = srt_to_seconds(m.group(1)) + offset + e = srt_to_seconds(m.group(2)) + offset + fout.write(f"{seconds_to_srt(s)} --> {seconds_to_srt(e)}\n") + else: + fout.write(line) + +def parse_offset(s): + try: + return float(s) + except Exception: + return None + +# ---------- Filename / show info parsing ------------------------------------- + +def extract_show_info(filepath, extra_paths=None): + """ + Extract (show_name, SxxExx) by checking, in order: + 1. The video filename + 2. Any extra_paths (e.g. matching SRT filename) + 3. Directory path components (handles SxxExx in a folder name) + 4. Plex-style layout: .../Show Name/Season NN/file + 5. Immediate parent directory name as a last resort + """ + show = '' + episode = '' + + def _parse_name(path): + base = os.path.splitext(os.path.basename(path))[0] + base = _BRACKET_RE.sub('', base) # strip leading [SubGroup] + base = re.sub(r'[._]', ' ', base) + m = _SXXEXX_RE.search(base) + if m: + s = _NOISE_RE.sub('', base[:m.start()]).strip() + return re.sub(r'\s+', ' ', s).strip(), m.group(0).upper() + s = _NOISE_RE.sub('', base).strip() + return re.sub(r'\s+', ' ', s).strip(), '' + + for path in [filepath] + (extra_paths or []): + s, e = _parse_name(path) + if not show and s: + show = s + if not episode and e: + episode = e + if show and episode: + break + + if not show or not episode: + parts = os.path.normpath(os.path.abspath(filepath)).split(os.sep) + for part in reversed(parts[:-1]): + part_clean = re.sub(r'[._]', ' ', part) + m = _SXXEXX_RE.search(part_clean) + if m: + if not episode: + episode = m.group(0).upper() + if not show: + s = _NOISE_RE.sub('', part_clean[:m.start()]).strip() + show = re.sub(r'\s+', ' ', s).strip() + + if not show: + for i, part in enumerate(parts): + if _SEASON_DIR_RE.match(part) and i > 0: + show = re.sub(r'[._]', ' ', parts[i - 1]).strip() + show = re.sub(r'\s+', ' ', show).strip() + break + + if not show: + parent = os.path.basename(os.path.dirname(os.path.abspath(filepath))) + if parent not in ('', '.') and not _SEASON_DIR_RE.match(parent): + show = re.sub(r'[._]', ' ', parent).strip() + show = re.sub(r'\s+', ' ', show).strip() + + return show, episode + +# ---------- Text post-processing --------------------------------------------- + +def postprocess_text(text): + text = text.strip() + if not text: + return text + # OCR misreads \u266a as $. Strip $ embedded inside words; replace remaining + # $ (not before a digit) with \u266a so music-note lines are handled correctly. + text = re.sub(r'(?<=[A-Za-z])\$(?=[A-Za-z])', '', text) + text = re.sub(r'\$(?!\d)', '\u266a', text) + music_rx = re.compile( + r'\[\s*(music|singing|song|humming|instrumental|melody)\s*\]', + re.IGNORECASE + ) + has_music = bool(music_rx.search(text)) or '\u266a' in text + text = music_rx.sub('\u266a', text) + text = re.sub(r'\[[^\]]{1,40}\]', '', text).strip() + text = re.sub(r' +', ' ', text).strip() + if has_music: + core = re.sub(r'[\u266a]+', '', text).strip() + text = f'\u266a {core} \u266a' if core else '\u266a' + if text.startswith('\u266a'): + after = text[1:].lstrip() + if after and after[0].islower(): + text = '\u266a ' + after[0].upper() + after[1:] + elif text and text[0].islower(): + text = text[0].upper() + text[1:] + return text + +# ---------- SRT vocabulary extraction ---------------------------------------- + +def extract_srt_vocab(srt_path, max_words=60): + entries = parse_srt_full(srt_path) + proper = {} + for _, _, text in entries: + words = text.split() + for i, raw in enumerate(words): + w = re.sub(r"[^a-zA-Z']", '', raw) + if not w: + continue + if i > 0 and w[0].isupper() and w.lower() not in STOP_WORDS: + proper[w] = proper.get(w, 0) + 1 + return sorted(proper, key=lambda w: -proper[w])[:max_words] + +def build_prompt(video_path, srt_path=None): + show, episode = extract_show_info(video_path) + prompt = WHISPER_PROMPT + if show: + prompt += f" This is '{show}'" + prompt += f", {episode}." if episode else "." + if srt_path and os.path.isfile(srt_path): + vocab = extract_srt_vocab(srt_path) + if vocab: + prompt += f" Vocabulary: {', '.join(vocab)}." + return prompt + +# ---------- Mode 2: Generate SRT from scratch -------------------------------- + +def _detect_language(video_path, model): + """Sample 30 s of audio and return (code, confidence, display_name).""" + import numpy as np + raw = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-t', '30', + '-vn', '-ac', '1', '-ar', '16000', '-f', 'f32le', 'pipe:1' + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60).stdout + n = len(raw) // 4 + if n == 0: + return None, None, None + audio = np.frombuffer(raw, dtype=np.float32).copy() + audio = _whisper.pad_or_trim(audio) + n_mels = getattr(getattr(model, 'dims', None), 'n_mels', 80) + mel = _whisper.log_mel_spectrogram(audio, n_mels=n_mels).to(model.device) + _, probs = model.detect_language(mel) + code = max(probs, key=probs.get) + conf = probs[code] + names = getattr(_whisper.tokenizer, 'LANGUAGES', {}) + name = names.get(code, code).title() + return code, conf, name + + +def generate_srt(video_path, output_path, model_name, srt_path=None, + task='transcribe', language=None, _model=None): + if _model is None: + _model, _ = load_whisper_model(model_name) + show, episode = extract_show_info(video_path) + if show: + print(f" Detected show: '{show}'" + (f" Episode: {episode}" if episode else "")) + if srt_path: + print(f" Vocabulary seeded from: {os.path.basename(srt_path)}") + if task == 'translate': + hint = f" (source: {language})" if language else " (auto-detect source)" + print(f" Translating to English{hint} - lines will appear as recognised...") + else: + print(" Transcribing - lines will appear as they are recognised...") + result = _model.transcribe(video_path, + initial_prompt=build_prompt(video_path, srt_path), + language=language, + task=task, + verbose=True) + segs = result.get('segments', []) + idx = 0 + with open(output_path, 'w', encoding='utf-8') as f: + for seg in segs: + txt = postprocess_text(seg['text']) + if not txt: + continue + idx += 1 + f.write(f"{idx}\n") + f.write(f"{seconds_to_srt(seg['start'])} --> {seconds_to_srt(seg['end'])}\n") + f.write(f"{txt}\n\n") + return idx, output_path + + +def scan_title_card(video_path, start=20, duration=160, interval=5): + """ + Extract frames from the video and OCR them to find on-screen episode title cards. + Returns list of (text, frame_count, timestamp_seconds) sorted by frame count. + """ + try: + import easyocr + except ImportError: + print(" easyocr not available.") + return [] + + import tempfile, glob + + end = start + duration + print(f" Extracting frames ({start}s – {end}s, one every {interval}s)...") + seen = {} # lower-normalised key -> (original_case, count, first_timestamp) + + with tempfile.TemporaryDirectory() as tmpdir: + frame_pattern = os.path.join(tmpdir, 'frame_%04d.png') + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-ss', str(start), '-i', video_path, + '-t', str(duration), + '-vf', f'fps=1/{interval},scale=1280:-1', + frame_pattern + ], timeout=120) + frames = sorted(glob.glob(os.path.join(tmpdir, 'frame_*.png'))) + if not frames: + print(" No frames extracted.") + return [] + + print(f" Running OCR on {len(frames)} frames" + f" (first run downloads ~170 MB model)...") + reader = easyocr.Reader(['en'], verbose=False) + + for frame_idx, frame_path in enumerate(frames): + ts = start + frame_idx * interval + try: + results = reader.readtext(frame_path, detail=1, paragraph=False) + frame_seen = set() + for (_, text, conf) in results: + text = text.strip() + if conf < 0.4: + continue + words = text.split() + if not (2 <= len(words) <= 8) or not (4 <= len(text) <= 60): + continue + if re.search(r'[©®@]|\d{2}:\d{2}|www\.', text): + continue + key = re.sub(r'\s+', ' ', text).lower() + if key not in frame_seen: + frame_seen.add(key) + if key in seen: + seen[key] = (seen[key][0], seen[key][1] + 1, seen[key][2]) + else: + seen[key] = (text, 1, ts) + except Exception: + continue + + return sorted(seen.values(), key=lambda x: -x[1]) + + +def _timed_input(prompt, timeout=15): + """Print prompt and wait for Enter; auto-continues after timeout seconds.""" + import select as _sel + print(prompt, end='', flush=True) + ready, _, _ = _sel.select([sys.stdin], [], [], timeout) + if ready: + sys.stdin.readline() + else: + print(f" (timed out after {timeout}s)") + + +def _preview_frame(video_path, timestamp): + """Extract the frame at timestamp and open it in the system image viewer.""" + import tempfile + fd, png = tempfile.mkstemp(suffix='.png', prefix='cc_preview_') + os.close(fd) + try: + subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-ss', str(timestamp), '-i', video_path, + '-frames:v', '1', '-y', png + ], timeout=30, check=True) + viewer = 'open' if sys.platform == 'darwin' else 'xdg-open' + subprocess.Popen([viewer, png]) + _timed_input(" (Press Enter to continue, auto-closes in 15s...)", timeout=15) + except Exception as e: + print(f" Preview failed: {e}") + finally: + try: + os.unlink(png) + except Exception: + pass + + +def _sync_pass(video_path, whisper_out, final_out, ffsubsync_ok): + """Run ffsubsync on whisper_out → final_out. Returns path of best result.""" + if not ffsubsync_ok: + print(" ffsubsync not available, skipping timing pass.") + return whisper_out + ok, offset = sync_with_ffsubsync(video_path, whisper_out, final_out) + if ok: + if offset is not None: + print(f" Timing adjusted by {offset:+.3f} s") + return final_out + print(" ffsubsync timing pass failed - using Whisper output as-is.") + return whisper_out + + +def generate_and_sync(video_path, model_name, srt_path=None, ffsubsync_ok=False): + """Load model once, detect language, ask user, then transcribe/translate/both.""" + global WHISPER_TASK, WHISPER_LANGUAGE + + base = os.path.splitext(video_path)[0] + model, _ = load_whisper_model(model_name) + + # --- Language detection --- + print("\n Detecting language from first 30 seconds...") + lang_code, conf, lang_name = _detect_language(video_path, model) + if lang_code: + print(f" Detected: {lang_name} ({lang_code}) {conf*100:.0f}% confidence") + else: + print(" Language detection failed — defaulting to current setting.") + lang_code = WHISPER_LANGUAGE + + is_english = lang_code in ('en', None) + + # --- Skip choice if --translate was passed explicitly --- + if WHISPER_TASK == 'translate' and not is_english: + task = 'translate' + src_lang = lang_code + do_orig = False + do_en = True + elif is_english: + task = 'transcribe' + src_lang = lang_code + do_orig = True + do_en = False + else: + # Non-English detected — ask what to generate + print(f"\n Source language: {lang_name}. What would you like?") + print(f" 1: {lang_name} SRT - transcribe in original language") + print( " 2: English SRT - translate to English") + print(f" 3: Both - {lang_name} + English SRT") + print( " 0: Cancel") + while True: + ch = input(" Choose [2]: ").strip() or '2' + if ch in ('0', '1', '2', '3'): + break + print(" Enter 0-3.") + if ch == '0': + return None + do_orig = ch in ('1', '3') + do_en = ch in ('2', '3') + src_lang = lang_code + + outputs = [] + + # --- Original language pass --- + if do_orig: + suffix = f'-whisper-{src_lang}' if src_lang and src_lang != 'en' else '-whisper' + w_out = f"{base}{suffix}.srt" + f_out = f"{base}{suffix}-synced.srt" + print(f"\nWhisper transcription → {os.path.basename(w_out)}") + n, _ = generate_srt(video_path, w_out, model_name, + srt_path=srt_path, task='transcribe', + language=src_lang, _model=model) + print(f" {n} segments written.") + print(f"\nffsubsync timing pass → {os.path.basename(f_out)}") + outputs.append(_sync_pass(video_path, w_out, f_out, ffsubsync_ok)) + + # --- English translation pass --- + if do_en: + w_out = f"{base}-whisper-en.srt" + f_out = f"{base}-whisper-en-synced.srt" + print(f"\nWhisper translation → English → {os.path.basename(w_out)}") + n, _ = generate_srt(video_path, w_out, model_name, + srt_path=srt_path, task='translate', + language=src_lang, _model=model) + print(f" {n} segments written.") + print(f"\nffsubsync timing pass → {os.path.basename(f_out)}") + outputs.append(_sync_pass(video_path, w_out, f_out, ffsubsync_ok)) + + return outputs[-1] if outputs else None + +# ---------- ffsubsync -------------------------------------------------------- + +def sync_with_ffsubsync(video_path, srt_path, output_path): + exe = _find_ffsubsync() + if not exe: + return False, None + + import tempfile + + # HI subtitles (hearing impaired) have many [sound] descriptions that + # don't correspond to speech, wrecking VAD-based cross-correlation. + # Sync on a dialogue-only copy; apply the resulting offset to the original. + hi = _is_hi_subtitle(srt_path) + if hi: + print(" Detected HI (hearing-impaired) subtitle — stripping sound " + "descriptions for sync pass, will reapply to original.") + fd, stripped_path = tempfile.mkstemp(suffix='.srt') + os.close(fd) + _strip_hi_for_sync(srt_path, stripped_path) + sync_src = stripped_path + else: + stripped_path = None + sync_src = srt_path + + print(" Running ffsubsync (WebRTC VAD + FFT) - usually 20-30 seconds...") + result = subprocess.run( + [exe, video_path, '-i', sync_src, '-o', output_path], + capture_output=True, text=True + ) + + if stripped_path: + try: + os.remove(stripped_path) + except OSError: + pass + + combined = result.stdout + result.stderr + + if result.returncode != 0 or not os.path.isfile(output_path): + return False, None + + # Parse scale factor; if significant, apply it to correct framerate drift. + # A plain offset fixes a constant gap; scaling fixes drift that grows over + # time when the SRT was authored for a different framerate than the video. + scale_m = re.search(r'framerate scale factor[:\s]+([\d.]+)', combined) + if scale_m: + scale = float(scale_m.group(1)) + if not 0.98 <= scale <= 1.02: + src_fps = 'NTSC 23.976' if scale < 1.0 else 'PAL 25' + vid_fps = 'PAL 25' if scale < 1.0 else 'NTSC 23.976' + drift = abs(1.0 - scale) * 100 + print(f" Framerate mismatch: SRT={src_fps}fps, video={vid_fps}fps " + f"(scale {scale:.4f}, ~{drift:.1f}% drift) — applying correction.") + scaled = parse_srt_full(output_path) + with open(output_path, 'w', encoding='utf-8') as _f: + for _i, (_s, _e, _t) in enumerate(scaled, 1): + _f.write(f"{_i}\n{seconds_to_srt(_s * scale)} --> " + f"{seconds_to_srt(_e * scale)}\n{_t}\n\n") + + # If HI, we got a synced version of the stripped file; now shift the + # original (with all descriptions) by the same offset instead. + def first_ts(path): + try: + for line in _read_srt_text(path).splitlines(): + m = TS_RE.search(line) + if m: + return srt_to_seconds(m.group(1)) + except Exception: + pass + return None + + t_orig = first_ts(srt_path) + t_synced = first_ts(output_path) + offset = (t_synced - t_orig) if (t_orig is not None and t_synced is not None) else None + + if hi and offset is not None: + # Replace ffsubsync's output (stripped) with shifted original (full HI) + shift_srt(srt_path, output_path, offset) + + return True, offset + +# ---------- Whisper word alignment ------------------------------------------- + +def whisper_word_times(video_path, model_name, srt_path=None): + import numpy as np + model, device = load_whisper_model(model_name) + print(f" Extracting audio (first {ANALYZE_S//60} min)...") + raw = subprocess.run([ + 'ffmpeg', '-hide_banner', + '-i', video_path, + '-t', str(ANALYZE_S), + '-vn', '-ac', '1', '-ar', '16000', + '-f', 'f32le', 'pipe:1' + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300).stdout + n = len(raw) // 4 + if n == 0: + raise RuntimeError("ffmpeg returned no audio.") + audio = np.frombuffer(raw, dtype=np.float32).copy() + print(" Transcribing...") + result = model.transcribe(audio, + initial_prompt=build_prompt(video_path, srt_path), + language=WHISPER_LANGUAGE, + word_timestamps=True, verbose=False) + words = [] + for seg in result.get('segments', []): + for wd in seg.get('words', []): + w = normalize_word(wd.get('word', '')) + if len(w) >= MIN_WORD_LEN and w not in STOP_WORDS: + words.append((w, wd['start'])) + return words + +def compute_offset_whisper(srt_path, video_path, model_name): + entries = parse_srt_full(srt_path) + if not entries: + return None, 0, 0, "No entries found in SRT." + window_entries = [(s, e, t) for s, e, t in entries + if s <= ANALYZE_S + MAX_OFFSET_S] + if not window_entries: + print(" Warning: no SRT entries in analysis window - using first 100.") + window_entries = entries[:100] + srt_wt = srt_to_word_times(window_entries) + if not srt_wt: + return None, 0, 0, "No usable words in SRT window." + print(f" Analysis window: 0-{ANALYZE_S}s | {len(window_entries)} SRT cues") + try: + whi_wt = whisper_word_times(video_path, model_name, srt_path) + except Exception as e: + return None, 0, 0, f"Whisper failed: {e}" + if not whi_wt: + return None, 0, 0, "Whisper produced no output." + print(f" SRT: {len(srt_wt)} words | Whisper: {len(whi_wt)} words") + print(" Aligning word sequences...") + matcher = difflib.SequenceMatcher( + None, [w for w, _ in srt_wt], [w for w, _ in whi_wt], autojunk=False + ) + raw_offsets = [] + for i, j, n in matcher.get_matching_blocks(): + for k in range(n): + raw_offsets.append(whi_wt[j+k][1] - srt_wt[i+k][1]) + if len(raw_offsets) < 5: + return None, len(raw_offsets), 0, ( + f"Only {len(raw_offsets)} word matches. Is this SRT for this video?" + ) + rough = median(raw_offsets) + cleaned = [o for o in raw_offsets if abs(o - rough) <= 2.0] + if len(cleaned) < 5: + cleaned = raw_offsets + off = median(cleaned) + spread = max(cleaned) - min(cleaned) + print(f" Matches after outlier filter: {len(cleaned)}/{len(raw_offsets)}") + return off, len(cleaned), spread, None + +# ---------- Whisper cross-check of ffsubsync result -------------------------- + +def whisper_verify(srt_path, video_path, model_name, ffsubsync_offset): + print(" Verifying with Whisper word alignment...") + w_offset, n_matches, spread, err = compute_offset_whisper( + srt_path, video_path, model_name + ) + if err: + return None, None, 0, 0, err + agree = abs(w_offset - ffsubsync_offset) <= OFFSET_AGREE_THRESHOLD + return agree, w_offset, n_matches, spread, None + +# ---------- Fallback: speech-band energy cross-correlation ------------------ + +def extract_speech_energy(video_path): + total = int(ANALYZE_S / RESOLUTION_S) + 1 + cmd = [ + 'ffmpeg', '-hide_banner', + '-i', video_path, + '-t', str(ANALYZE_S), '-vn', '-ac', '1', + '-af', f'highpass=f={SPEECH_LO},lowpass=f={SPEECH_HI}', + '-ar', str(RESAMPLE_HZ), '-f', 'f32le', 'pipe:1' + ] + try: + r = subprocess.run(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=300) + raw = r.stdout + n = len(raw) // 4 + if n == 0: + return [] + samples = struct.unpack(f'<{n}f', raw) + energy = [0.0] * total + for i in range(0, n, CHUNK_SIZE): + seg = samples[i:i+CHUNK_SIZE] + rms = (sum(x*x for x in seg) / len(seg)) ** 0.5 + bi = i // CHUNK_SIZE + if bi < total: + energy[bi] = rms + return energy + except Exception as e: + print(f" Audio extraction error: {e}") + return [] + +def compute_onsets(energy, lookback=2): + onsets = [0.0] * len(energy) + for i in range(lookback, len(energy)): + d = energy[i] - energy[i - lookback] + if d > 0: + onsets[i] = d + nz = sorted(o for o in onsets if o > 0) + if nz: + thr = nz[len(nz) // 2] + onsets = [o if o >= thr else 0.0 for o in onsets] + return onsets + +def crosscorr_offset(entries, energy): + n_bins = len(energy) + max_lag = int(MAX_OFFSET_S / RESOLUTION_S) + onsets = compute_onsets(energy) + seen, starts = set(), [] + for s, _e, _t in entries: + si = max(0, int(s / RESOLUTION_S)) + if si not in seen: + starts.append(si) + seen.add(si) + if not starts or not any(onsets): + return 0.0, 0.0 + scores = [ + sum(onsets[i+lag] for i in starts if 0 <= i+lag < n_bins) + for lag in range(-max_lag, max_lag + 1) + ] + best = max(range(len(scores)), key=lambda i: scores[i]) + mean = sum(scores) / len(scores) + return (best - max_lag) * RESOLUTION_S, scores[best] / max(1e-9, mean) + +def compute_offset_fallback(srt_path, video_path): + entries = parse_srt_full(srt_path) + if not entries: + return None, None, "No entries in SRT." + print(" Extracting speech-band audio energy...") + energy = extract_speech_energy(video_path) + if not energy or not any(energy): + return None, None, "Could not extract audio from video." + print(" Running onset cross-correlation...") + offset, conf = crosscorr_offset(entries, energy) + return offset, conf, None + +# ---------- Core sync logic (used by Mode 2 and batch) ---------------------- + +def sync_single(video, src, out, ffsubsync_ok, whisper_ok, interactive=True): + """ + Sync src SRT to video, writing result to out. + ffsubsync runs first; Whisper independently verifies the offset. + interactive=True prompts user on disagreement; False just warns and keeps ffsubsync. + Returns True on success. + """ + synced = False + final_offset = None + + # Primary: ffsubsync + if ffsubsync_ok: + ok, fs_offset = sync_with_ffsubsync(video, src, out) + if ok: + if fs_offset is not None: + print(f" ffsubsync offset : {fs_offset:+.3f} s") + + # Cross-check with Whisper + if whisper_ok and fs_offset is not None: + agree, w_offset, n_matches, spread, err = whisper_verify( + src, video, WHISPER_MODEL, fs_offset + ) + if err: + print(f" Whisper verify skipped: {err}") + else: + quality = ("good" if spread < 2.0 else + "moderate" if spread < 5.0 else "low") + diff = abs(w_offset - fs_offset) + print(f" Whisper offset : {w_offset:+.3f} s " + f"({n_matches} words, spread {spread:.1f}s, {quality})") + if agree: + print(f" Agreement : YES (differ by {diff:.2f}s) " + f"- using ffsubsync result.") + else: + print(f" Agreement : NO (differ by {diff:.2f}s, " + f"threshold {OFFSET_AGREE_THRESHOLD}s)") + if interactive: + print(f" [f] Use ffsubsync ({fs_offset:+.3f}s)") + print(f" [w] Use Whisper ({w_offset:+.3f}s)") + print(f" [e] Enter offset manually") + while True: + choice = input(" Choose [f/w/e]: ").strip().lower() + if choice == 'f': + print(" Using ffsubsync offset.") + break + elif choice == 'w': + print(" Re-applying Whisper offset...") + shift_srt(src, out, w_offset) + final_offset = w_offset + break + elif choice == 'e': + while True: + resp = input(" Enter offset in seconds: ").strip() + manual = parse_offset(resp) + if manual is not None: + shift_srt(src, out, manual) + final_offset = manual + break + print(" Invalid number.") + break + else: + print(f" WARNING: methods disagree by {diff:.2f}s. " + f"Keeping ffsubsync - review manually.") + + synced = True + final_offset = final_offset or fs_offset + else: + print(" ffsubsync failed - falling back to Whisper...") + + # Fallback 1: Whisper word alignment + if not synced and whisper_ok: + offset, n_matches, spread, err = compute_offset_whisper(src, video, WHISPER_MODEL) + if not err: + quality = ("good" if spread < 2.0 else + "moderate" if spread < 5.0 else "low") + print(f" Whisper offset: {offset:+.3f} s " + f"({n_matches} matches, spread {spread:.1f}s, {quality})") + shift_srt(src, out, offset) + final_offset = offset + synced = True + else: + print(f" Whisper failed: {err}") + print(" Trying audio energy cross-correlation...") + + # Fallback 2: energy cross-correlation + if not synced: + offset, conf, err = compute_offset_fallback(src, video) + if not err: + q = "LOW" if conf < 1.5 else "moderate" if conf < 2.5 else "good" + print(f" Energy offset: {offset:+.3f} s (confidence {conf:.2f}x, {q})") + shift_srt(src, out, offset) + final_offset = offset + synced = True + else: + print(f" All methods failed: {err}") + + if synced and final_offset is not None: + print(f" Final offset: {final_offset:+.3f} s") + + return synced + +# ---------- Batch helpers ---------------------------------------------------- + +def find_srt_for_video(video_path, srt_files): + _, ep = extract_show_info(video_path) + ep_lower = ep.lower() if ep else None + base = os.path.splitext(os.path.basename(video_path))[0] + + candidates = [f for f in srt_files + if not f.lower().endswith('-synced.srt') + and not f.lower().endswith('-whisper.srt')] + + if ep_lower: + ep_matches = [f for f in candidates if ep_lower in f.lower()] + if ep_matches: + return sorted(ep_matches, key=len)[0] + + exact = base + '.srt' + if exact in candidates: + return exact + return None + +def batch_sync(ffsubsync_ok, whisper_ok): + video_files = [f for f in sorted(os.listdir('.')) + if f.lower().endswith(VIDEO_EXTS)] + srt_files = [f for f in sorted(os.listdir('.')) + if f.lower().endswith('.srt')] + + if not video_files: + print("No video files found.") + return + if not srt_files: + print("No SRT files found.") + return + + pairs, unmatched = [], [] + for vf in video_files: + sf = find_srt_for_video(vf, srt_files) + if sf: + out = os.path.splitext(sf)[0] + '-synced.srt' + if os.path.isfile(out): + print(f" Skipping {vf} - {os.path.basename(out)} already exists.") + else: + pairs.append((vf, sf, out)) + else: + unmatched.append(vf) + + if not pairs: + print("No unprocessed pairs found.") + if unmatched: + print("Videos with no matching SRT:") + for v in unmatched: + print(f" {v}") + return + + print(f"\nFound {len(pairs)} pair(s) to process:") + for vf, sf, out in pairs: + print(f" {vf} + {sf} -> {os.path.basename(out)}") + if unmatched: + print(f"\n{len(unmatched)} video(s) with no matching SRT (skipped):") + for v in unmatched: + print(f" {v}") + + if input("\nProceed? [Y/n]: ").strip().lower() not in ('', 'y'): + print("Cancelled.") + return + + ok_count, fail_count, failed = 0, 0, [] + for vf, sf, out in pairs: + print(f"\n{'='*60}") + print(f" Video : {vf}") + print(f" SRT : {sf}") + print(f" Output: {os.path.basename(out)}") + if sync_single(vf, sf, out, ffsubsync_ok, whisper_ok, interactive=False): + ok_count += 1 + else: + fail_count += 1 + failed.append(vf) + + print(f"\n{'='*60}") + print(f"Batch complete: {ok_count} synced, {fail_count} failed.") + if failed: + print("Run Mode 2 manually on these:") + for v in failed: + print(f" {v}") + +# ---------- TMDB episode lookup + rename ------------------------------------- + +def tmdb_get(path, params, api_key): + params = dict(params) # don't mutate caller's dict + params['api_key'] = api_key + url = f"https://api.themoviedb.org/3{path}?{urllib.parse.urlencode(params)}" + req = urllib.request.Request(url, headers={'Accept-Encoding': 'gzip, deflate'}) + try: + with urllib.request.urlopen(req, timeout=10) as r: + raw = r.read() + if raw[:2] == b'\x1f\x8b': + import gzip + raw = gzip.decompress(raw) + return json.loads(raw.decode('utf-8')) + except Exception as e: + print(f" TMDB error: {e}") + return None + +def _get_tmdb_key(): + key = TMDB_API_KEY.strip() + if not key: + print(" Get a free key at https://www.themoviedb.org/settings/api") + key = input(" Enter TMDB API key: ").strip() + if not key: + print(" No key - skipping.") + return None + return key + + +def tmdb_pick_show(show_name, key): + """Search TMDB for show_name and let the user pick. Returns (show_id, canonical) or (None, None).""" + data = tmdb_get('/search/tv', {'query': show_name, 'page': 1}, key) + if not data or not data.get('results'): + print(" No results found.") + return None, None + results = data['results'][:6] + if len(results) > 1: + print(" Multiple results:") + for i, r in enumerate(results, 1): + year = r.get('first_air_date', '')[:4] + print(f" {i}: {r['name']} ({year})") + choice = input(" Choose [1]: ").strip() + idx = (int(choice)-1) if choice.isdigit() and 1 <= int(choice) <= len(results) else 0 + else: + idx = 0 + return results[idx]['id'], results[idx]['name'] + + +def tmdb_find_episode_by_title(show_id, ep_title, key): + """ + Scan every season of show_id on TMDB looking for an episode whose title + matches ep_title (case-insensitive). Returns (season, episode_number) or (None, None). + """ + show_data = tmdb_get(f'/tv/{show_id}', {}, key) + if not show_data: + return None, None + n_seasons = show_data.get('number_of_seasons', 0) + target = ep_title.strip().lower() + for s in range(1, n_seasons + 1): + season_data = tmdb_get(f'/tv/{show_id}/season/{s}', {}, key) + if not season_data: + continue + for ep in season_data.get('episodes', []): + if ep.get('name', '').strip().lower() == target: + return s, ep['episode_number'] + return None, None + + +def safe_filename(s): + return re.sub(r'[<>:"/\\|?*]', '', s).strip() + +def find_matching_srt(video_path): + base = os.path.splitext(video_path)[0] + dirpath = os.path.dirname(video_path) or '.' + for suffix in ('', '-synced', '-offset', '-whisper'): + c = base + suffix + '.srt' + if os.path.isfile(c): + return c + _, ep_code = extract_show_info(video_path) + if ep_code: + for f in os.listdir(dirpath): + if f.lower().endswith('.srt') and ep_code.lower() in f.lower(): + return os.path.join(dirpath, f) + return None + +def do_rename(filepath, new_base): + ext = os.path.splitext(filepath)[1] + dirpath = os.path.dirname(filepath) or '.' + new_path = os.path.join(dirpath, new_base + ext) + if os.path.abspath(filepath) == os.path.abspath(new_path): + print(" Already named correctly.") + return new_path + try: + os.rename(filepath, new_path) + print(f" -> {os.path.basename(new_path)}") + return new_path + except Exception as e: + print(f" Rename failed: {e}") + return filepath + +def _tmdb_rename(video_path, show_name, episode_code, srt_path): + """Core TMDB lookup + rename. show_name / episode_code may be empty strings.""" + if not episode_code: + print(" No SxxExx found in filename, SRT, or directory path - skipping.") + return + m = re.match(r'S(\d+)E(\d+)', episode_code, re.IGNORECASE) + if not m: + return + season, episode = int(m.group(1)), int(m.group(2)) + if not show_name: + show_name = input(" Could not detect show name. Enter show name: ").strip() + if not show_name: + print(" No show name - skipping.") + return + key = _get_tmdb_key() + if not key: + return + print(f" Searching TMDB for '{show_name}'...") + show_id, canonical = tmdb_pick_show(show_name, key) + if not show_id: + return + ep_data = tmdb_get(f'/tv/{show_id}/season/{season}/episode/{episode}', {}, key) + if ep_data and 'name' in ep_data: + new_base = (f"{safe_filename(canonical)} - " + f"S{season:02d}E{episode:02d} - {safe_filename(ep_data['name'])}") + else: + print(" Episode title not found - using show name + SxxExx only.") + new_base = f"{safe_filename(canonical)} - S{season:02d}E{episode:02d}" + _confirm_and_rename(video_path, new_base, srt_path) + + +def _confirm_and_rename(video_path, new_base, srt_path): + ext = os.path.splitext(video_path)[1] + print(f"\n New name: {new_base}{ext}") + if srt_path: + print(f" SRT : {new_base}.srt") + if input(" Rename? [Y/n]: ").strip().lower() not in ('', 'y'): + print(" Skipped.") + return + do_rename(video_path, new_base) + if srt_path: + do_rename(srt_path, new_base) + + +def offer_rename(video_path): + if input("\nLook up episode title on TMDB and rename files? [y/N]: ").strip().lower() != 'y': + return + srt_path = find_matching_srt(video_path) + extra = [srt_path] if srt_path else [] + show_name, ep_code = extract_show_info(video_path, extra_paths=extra) + print(f" Show : {show_name or '(not detected)'}") + print(f" Episode: {ep_code or '(not detected)'}") + _tmdb_rename(video_path, show_name, ep_code, srt_path) + + +def rename_mode(video_path, ocr_ok, _method=None): + srt_path = find_matching_srt(video_path) + extra = [srt_path] if srt_path else [] + show_name, ep_code = extract_show_info(video_path, extra_paths=extra) + + print(f"\n Show : {show_name or '(not detected)'}") + print(f" Episode: {ep_code or '(not detected)'}") + + if _method is None: + print("\n How to find the episode title?") + print(" 1: TMDB lookup - search by show name + SxxExx [default]") + if ocr_ok: + print(" 2: Scan video - OCR the first 3 min for a title card") + choice = input(" Choose [1]: ").strip() or '1' + else: + choice = _method + + if choice == '2' and ocr_ok: + candidates = scan_title_card(video_path) + if not candidates: + print(" No title candidates found - falling back to TMDB.") + else: + top = candidates[:20] + print(f"\n Candidates (sorted by how many frames they appeared in):") + for i, (text, count, _ts) in enumerate(top, 1): + print(f" {i}: {text} ({count} frame{'s' if count != 1 else ''})") + print("\n Enter a number to select, p to preview that frame, " + "or Enter to fall back to TMDB.") + ep_title = None + while True: + sel = input(" > ").strip() + if not sel: + break + pm = re.match(r'^[pP](\d+)$', sel) + if pm: + pidx = int(pm.group(1)) + if 1 <= pidx <= len(top): + _preview_frame(video_path, top[pidx - 1][2]) + else: + print(f" Choose 1–{len(top)}.") + continue + if sel.isdigit() and 1 <= int(sel) <= len(top): + ep_title = top[int(sel) - 1][0] + break + print(f" Enter a number (1–{len(top)}), p to preview, or Enter to skip.") + + if ep_title: + if not show_name: + show_name = input(" Enter show name: ").strip() + if not show_name: + print(" No show name - skipping.") + return + key = _get_tmdb_key() + if not key: + return + print(f" Searching TMDB for '{show_name}' / episode '{ep_title}'...") + show_id, canonical = tmdb_pick_show(show_name, key) + if show_id: + season, ep_num = tmdb_find_episode_by_title(show_id, ep_title, key) + if season and ep_num: + new_base = (f"{safe_filename(canonical)} - " + f"S{season:02d}E{ep_num:02d} - {safe_filename(ep_title)}") + _confirm_and_rename(video_path, new_base, srt_path) + return + print(" Episode title not found on TMDB.") + if ep_code: + m = re.match(r'S(\d+)E(\d+)', ep_code, re.IGNORECASE) + if m: + season, ep_num = int(m.group(1)), int(m.group(2)) + new_base = (f"{safe_filename(canonical or show_name)} - " + f"S{season:02d}E{ep_num:02d} - {safe_filename(ep_title)}") + _confirm_and_rename(video_path, new_base, srt_path) + return + print(" No episode code available either - skipping.") + return + + # Default: TMDB lookup + _tmdb_rename(video_path, show_name, ep_code, srt_path) + + +def _rename_one_batch(video_path, srt_path, show_id, canonical, key, auto): + """Rename one file within a batch. Returns True if renamed/confirmed, False if skipped.""" + extra = [srt_path] if srt_path else [] + _, ep_code = extract_show_info(video_path, extra_paths=extra) + if not ep_code: + print(f" {os.path.basename(video_path)}: no SxxExx found - skipping.") + return False + m = re.match(r'S(\d+)E(\d+)', ep_code, re.IGNORECASE) + if not m: + print(f" {os.path.basename(video_path)}: cannot parse {ep_code} - skipping.") + return False + season, ep_num = int(m.group(1)), int(m.group(2)) + ep_data = tmdb_get(f'/tv/{show_id}/season/{season}/episode/{ep_num}', {}, key) + if ep_data and 'name' in ep_data: + new_base = (f"{safe_filename(canonical)} - " + f"S{season:02d}E{ep_num:02d} - {safe_filename(ep_data['name'])}") + else: + print(f" {os.path.basename(video_path)}: episode title not found - using SxxExx only.") + new_base = f"{safe_filename(canonical)} - S{season:02d}E{ep_num:02d}" + ext = os.path.splitext(video_path)[1] + print(f" {os.path.basename(video_path)}") + print(f" -> {new_base}{ext}") + if auto: + do_rename(video_path, new_base) + if srt_path: + do_rename(srt_path, new_base) + else: + if input(" Rename? [Y/n]: ").strip().lower() in ('', 'y'): + do_rename(video_path, new_base) + if srt_path: + do_rename(srt_path, new_base) + else: + print(" Skipped.") + return True + + +def _next_file_prompt(vid_files, idx, allow_auto=False): + """ + After processing vid_files[idx], ask what to do next. + Returns (next_index, go_auto). next_index is None to stop. + Enter = next file, 0 = stop, a = auto rest (if allow_auto), N = jump. + """ + next_idx = idx + 1 + if next_idx >= len(vid_files): + print(" No more files.") + return None, False + print(f"\n Next: {os.path.basename(vid_files[next_idx])}") + auto_hint = " [a] auto rest | " if allow_auto else " " + print(f"{auto_hint}[Enter] continue | [0] stop | [1-{len(vid_files)}] jump to file") + ans = input(" > ").strip().lower() + if ans == '0': + return None, False + if ans == 'a' and allow_auto: + return next_idx, True + if ans == '': + return next_idx, False + if ans.isdigit() and 1 <= int(ans) <= len(vid_files): + return int(ans) - 1, False + return next_idx, False + + +def rename_tmdb_loop(): + print("\n Video files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if not vid_files: + print(" No video files found.") + return + + video = pick_file(vid_files, " Choose starting file") + if not video or not os.path.isfile(video): + print(" No valid video selected.") + return + + srt0 = find_matching_srt(video) + extra0 = [srt0] if srt0 else [] + show_name, _ = extract_show_info(video, extra_paths=extra0) + if not show_name: + show_name = input(" Could not detect show name. Enter show name: ").strip() + if not show_name: + return + + key = _get_tmdb_key() + if not key: + return + + print(f" Searching TMDB for '{show_name}'...") + show_id, canonical = tmdb_pick_show(show_name, key) + if not show_id: + return + print(f" Show: {canonical}\n") + + idx = vid_files.index(video) if video in vid_files else 0 + auto = False + while True: + vf = vid_files[idx] + srt = find_matching_srt(vf) + _rename_one_batch(vf, srt, show_id, canonical, key, auto=auto) + if auto: + idx += 1 + if idx >= len(vid_files): + print(" No more files.") + break + else: + idx, auto = _next_file_prompt(vid_files, idx, allow_auto=True) + if idx is None: + break + + +def rename_scan_loop(ocr_ok): + print("\n Video files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if not vid_files: + print(" No video files found.") + return + + video = pick_file(vid_files, " Choose starting file") + if not video or not os.path.isfile(video): + print(" No valid video selected.") + return + + idx = vid_files.index(video) if video in vid_files else 0 + while True: + rename_mode(vid_files[idx], ocr_ok, _method='2') + idx, _ = _next_file_prompt(vid_files, idx) + if idx is None: + break + + +def rename_menu(ocr_ok): + while True: + print("\n RENAME") + print(" 1: TMDB lookup [default]") + if ocr_ok: + print(" 2: Scan video for title card") + print(" 0: Back to main menu") + valid = ('0', '1', '2') if ocr_ok else ('0', '1') + while True: + choice = input(" Choose [1]: ").strip() or '1' + if choice in valid: + break + print(f" Please enter {'0, 1 or 2' if ocr_ok else '0 or 1'}.") + + if choice == '0': + break + elif choice == '1': + rename_tmdb_loop() + elif choice == '2': + rename_scan_loop(ocr_ok) + +# ---------- Subtitle extraction ---------------------------------------------- + +def probe_subtitle_streams(video_path): + """Return list of subtitle stream dicts from ffprobe.""" + try: + r = subprocess.run([ + 'ffprobe', '-v', 'quiet', '-print_format', 'json', + '-show_streams', '-select_streams', 's', video_path + ], capture_output=True, text=True, timeout=30) + return json.loads(r.stdout).get('streams', []) + except Exception: + return [] + + +def _sub_out_path(video_path, lang=''): + base = os.path.splitext(video_path)[0] + return f"{base}.{lang}.srt" if lang else f"{base}.srt" + + +def _extract_text_track(video_path, stream_index, out_path): + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-map', f'0:{stream_index}', + '-c:s', 'srt', '-y', out_path + ], timeout=300) + return r.returncode == 0 and os.path.isfile(out_path) + + +def _extract_cc(video_path, out_path, cce_cmd): + print(" Running ccextractor...") + r = subprocess.run([cce_cmd, video_path, '-o', out_path], timeout=600) + return r.returncode == 0 and os.path.isfile(out_path) + + +def _extract_pgs(video_path, stream_index, out_path): + """Extract Blu-ray PGS subtitle track → SRT via pgsreader + easyocr.""" + try: + import easyocr + from pgsreader import PGSReader + import numpy as np + from PIL import Image as _PILImage + except ImportError as e: + print(f" Missing dependency: {e}") + return False + + import tempfile + fd, sup_path = tempfile.mkstemp(suffix='.sup') + os.close(fd) + try: + print(" Extracting PGS stream...") + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-map', f'0:{stream_index}', + '-c:s', 'copy', '-y', sup_path + ], timeout=300) + if r.returncode != 0: + print(" ffmpeg extraction failed.") + return False + + print(" Reading PGS display sets...") + pgs = PGSReader(sup_path) + reader = easyocr.Reader(['en'], verbose=False) + entries = [] + pending = None + + for ds in pgs.displaySets: + ts_s = ds.pcs.presentation_timestamp / 90000.0 + if ds.has_image: + img = ds.to_image().convert('RGB') + results = reader.readtext(np.array(img), detail=0, paragraph=True) + text = ' '.join(results).strip() + if pending: + entries.append(pending) + pending = [ts_s, None, text] if text else None + else: + if pending: + pending[1] = ts_s + entries.append(pending) + pending = None + + if pending: + pending[1] = pending[0] + 3.0 + entries.append(pending) + + print(f" Writing {len(entries)} subtitle entries...") + with open(out_path, 'w', encoding='utf-8') as f: + for i, (start, end, text) in enumerate(entries, 1): + f.write(f"{i}\n") + f.write(f"{seconds_to_srt(start)} --> {seconds_to_srt(end)}\n") + f.write(f"{text}\n\n") + return True + finally: + try: + os.unlink(sup_path) + except Exception: + pass + + +def _extract_vobsub(video_path, stream_index, out_path): + """Extract DVD VOB subtitle track → SRT via vobsub2srt.""" + if not ensure_vobsub2srt(): + return False + import tempfile, shutil + with tempfile.TemporaryDirectory() as tmpdir: + sub_base = os.path.join(tmpdir, 'subs') + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-map', f'0:{stream_index}', + '-c:s', 'copy', '-y', sub_base + '.sub' + ], timeout=300) + if r.returncode != 0: + print(" ffmpeg extraction failed.") + return False + r2 = subprocess.run(['vobsub2srt', sub_base], timeout=300) + if r2.returncode == 0 and os.path.isfile(sub_base + '.srt'): + shutil.copy(sub_base + '.srt', out_path) + return True + print(" vobsub2srt conversion failed.") + return False + + +def extract_subs_mode(): + print("\n Video files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if vid_files: + video = pick_file(vid_files, " Choose video by number or filename") + else: + video = input(" Enter path to video file (0 to cancel): ").strip() + if video == '0': + return + if not video or not os.path.isfile(video): + print(" No valid video selected.") + return + + video = _offer_mp4_remux(video) + streams = probe_subtitle_streams(video) + + # Build menu: numbered subtitle tracks + CC option + options = [] + if streams: + print("\n Subtitle tracks found:") + for s in streams: + codec = s.get('codec_name', 'unknown') + idx = s.get('index', '?') + lang = s.get('tags', {}).get('language', '') + title = s.get('tags', {}).get('title', '') + label = codec + if lang: label += f" [{lang}]" + if title: label += f" — {title}" + if codec in _TEXT_SUB_CODECS: + label += " (text, instant)" + elif codec in _IMAGE_SUB_CODECS: + label += " (image, needs OCR)" + print(f" {len(options)+1}: {label}") + options.append(('track', s)) + else: + print("\n No subtitle tracks found in file.") + + print(f" {len(options)+1}: Closed captions from video stream (ccextractor)") + options.append(('cc', None)) + print(" 0: Cancel") + + while True: + sel = input(" Choose: ").strip() + if sel == '0': + return + if sel.isdigit() and 1 <= int(sel) <= len(options): + break + print(f" Enter 1-{len(options)} or 0.") + + kind, stream = options[int(sel) - 1] + base = os.path.splitext(video)[0] + + if kind == 'cc': + cce = ensure_ccextractor() + if not cce: + return + out = _sub_out_path(video, 'cc') + if _extract_cc(video, out, cce): + print(f" Done: {os.path.basename(out)}") + else: + print(" ccextractor found no CC in this file.") + return + + codec = stream.get('codec_name', '') + stream_idx = stream.get('index') + lang = stream.get('tags', {}).get('language', '') + out = _sub_out_path(video, lang) + + if codec in _TEXT_SUB_CODECS: + print(f" Extracting text track {stream_idx} → {os.path.basename(out)} ...") + if _extract_text_track(video, stream_idx, out): + print(f" Done: {os.path.basename(out)}") + else: + print(" Extraction failed.") + + elif codec in _IMAGE_SUB_CODECS: + print(f"\n '{codec}' is an image-based subtitle format.") + print(" 1: Native format - extract as .sup / .sub (perfect quality, instant) [default]") + print(" 2: OCR to SRT - read text via OCR (editable, some quality loss)") + fmt = input(" Choose [1]: ").strip() or '1' + + if fmt != '2': + # Native extraction — no OCR, perfect quality + if codec in {'hdmv_pgs_subtitle', 'pgssub'}: + native_out = base + (f'.{lang}' if lang else '') + '.sup' + else: + native_out = base + (f'.{lang}' if lang else '') + '.sub' + print(f" Extracting → {os.path.basename(native_out)} ...") + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video, '-map', f'0:{stream_idx}', + '-c:s', 'copy', '-y', native_out + ], timeout=300) + if r.returncode == 0 and os.path.isfile(native_out): + print(f" Done: {os.path.basename(native_out)}") + else: + print(" Extraction failed.") + elif codec in {'hdmv_pgs_subtitle', 'pgssub'}: + if not ensure_pgsreader() or not ensure_easyocr(): + return + print(f" Extracting PGS → {os.path.basename(out)} (OCR, may take a while)...") + if _extract_pgs(video, stream_idx, out): + print(f" Done: {os.path.basename(out)}") + else: + print(" PGS extraction failed.") + else: + print(f" Extracting DVD/DVB subtitle → {os.path.basename(out)} ...") + if not _extract_vobsub(video, stream_idx, out): + print(" Could not extract automatically.") + + else: + print(f" Codec '{codec}' not yet supported for direct extraction.") + print(" Try: ffmpeg -i video -map 0:s:N -c:s srt output.srt") + +# ---------- MP4 → MKV remux -------------------------------------------------- + +_LANG_ISO1_TO_639_2 = { + 'en': 'eng', 'fr': 'fre', 'de': 'ger', 'es': 'spa', 'it': 'ita', + 'pt': 'por', 'nl': 'dut', 'ru': 'rus', 'ja': 'jpn', 'zh': 'chi', + 'ko': 'kor', 'ar': 'ara', 'pl': 'pol', 'sv': 'swe', 'no': 'nor', + 'da': 'dan', 'fi': 'fin', 'cs': 'cze', 'tr': 'tur', 'hu': 'hun', +} +_SUB_EXTS = ('.srt', '.ass', '.ssa', '.vtt', '.sup', '.sub') + + +def _do_remux(video_path, out_path): + """Stream-copy video_path → out_path (MKV). Returns True on success.""" + import shutil + use_mkvmerge = bool(shutil.which('mkvmerge')) + if use_mkvmerge: + print(f" mkvmerge: {os.path.basename(video_path)} → {os.path.basename(out_path)}") + cmd = ['mkvmerge', '-o', out_path, video_path] + else: + print(f" ffmpeg stream copy (mkvmerge not found): {os.path.basename(video_path)} → {os.path.basename(out_path)}") + cmd = ['ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-c', 'copy', '-y', out_path] + try: + r = subprocess.run(cmd, timeout=600) + except subprocess.TimeoutExpired: + print(" Timed out.") + return False + if r.returncode == 0 and os.path.isfile(out_path): + return True + print(" Remux failed.") + if os.path.exists(out_path): + os.remove(out_path) + return False + + +def _offer_mp4_remux(video_path): + """If video_path is an MP4, offer (default yes) to remux to MKV first. + Returns the path to use going forward (MKV on success, original otherwise).""" + if not video_path.lower().endswith('.mp4'): + return video_path + base = os.path.splitext(video_path)[0] + mkv_out = base + '.mkv' + print(f"\n '{os.path.basename(video_path)}' is an MP4.") + print(" MKV handles all subtitle types; MP4 only supports mov_text (SRT).") + if os.path.exists(mkv_out): + print(f" MKV already exists: {os.path.basename(mkv_out)}") + resp = input(" Use existing MKV? [Y/n]: ").strip().lower() + if resp != 'n': + return mkv_out + return video_path + resp = input(" Convert to MKV now (lossless)? [Y/n]: ").strip().lower() + if resp == 'n': + return video_path + if _do_remux(video_path, mkv_out): + in_mb = os.path.getsize(video_path) / 1_048_576 + out_mb = os.path.getsize(mkv_out) / 1_048_576 + print(f" Done: {os.path.basename(mkv_out)} ({in_mb:.0f} MB → {out_mb:.0f} MB)") + resp = input(" Delete original MP4? [y/N]: ").strip().lower() + if resp == 'y': + os.remove(video_path) + print(f" Deleted: {os.path.basename(video_path)}") + return mkv_out + return video_path + + +def _detect_lang_tag(sub_path): + """Guess ISO 639-2 language tag from filename stem (e.g. video.en.srt → eng).""" + stem = os.path.splitext(os.path.basename(sub_path))[0] + parts = stem.rsplit('.', 1) + if len(parts) == 2: + code = parts[1].lower() + if code in _LANG_ISO1_TO_639_2: + return _LANG_ISO1_TO_639_2[code] + if len(code) == 3 and code.isalpha(): + return code + return '' + + +def remux_mp4_to_mkv(): + """Mode 6: remux MP4 (or any container) to MKV — stream copy, no re-encode.""" + print("\n MP4 → MKV") + all_vid = list_files(VIDEO_EXTS, "video") + mp4_files = [f for f in all_vid if f.lower().endswith('.mp4')] + + if mp4_files: + candidates = mp4_files + else: + print(" (no .mp4 found — showing all video files)") + candidates = all_vid + + if candidates: + video = pick_file(candidates, " Choose file to remux (0 to cancel)") + else: + video = input(" Enter path to video file (0 to cancel): ").strip() + if video == '0': + return + if not video: + return + if not os.path.isfile(video): + print(" File not found.") + return + + base = os.path.splitext(video)[0] + out = base + '.mkv' + if os.path.exists(out): + print(f" Output already exists: {os.path.basename(out)}") + resp = input(" Overwrite? [y/N]: ").strip().lower() + if resp != 'y': + print(" Cancelled.") + return + + if _do_remux(video, out): + in_mb = os.path.getsize(video) / 1_048_576 + out_mb = os.path.getsize(out) / 1_048_576 + print(f" Done: {os.path.basename(out)} ({in_mb:.0f} MB → {out_mb:.0f} MB)") + resp = input(" Delete original? [y/N]: ").strip().lower() + if resp == 'y': + os.remove(video) + print(f" Deleted: {os.path.basename(video)}") + + +def embed_subs_mode(): + """Mode 7: soft-mux a subtitle file into a video using mkvmerge.""" + if not ensure_mkvtoolnix(): + return + + # --- pick video --- + print("\n EMBED: Video files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if vid_files: + video = pick_file(vid_files, " Choose video (0 to cancel)") + else: + video = input(" Enter path to video file (0 to cancel): ").strip() + if video == '0': + return + if not video or not os.path.isfile(video): + print(" No valid video selected.") + return + + # offer MP4 → MKV before anything else + video = _offer_mp4_remux(video) + + # --- pick subtitle file --- + sub_files = sorted( + f for f in os.listdir('.') + if f.lower().endswith(_SUB_EXTS) and not f.endswith('.idx') + ) + if sub_files: + print("\n Subtitle files in current directory:") + for i, f in enumerate(sub_files, 1): + print(f" {i}: {f}") + sub = pick_file(sub_files, " Choose subtitle file (0 to cancel)") + else: + sub = input(" Enter path to subtitle file (0 to cancel): ").strip() + if sub == '0': + return + if not sub or not os.path.isfile(sub): + print(" No valid subtitle file selected.") + return + + # --- language tag --- + detected = _detect_lang_tag(sub) + if detected: + print(f" Detected language tag: {detected}") + resp = input(f" Use '{detected}'? [Y/n]: ").strip().lower() + lang = detected if resp != 'n' else '' + else: + lang = '' + if not lang: + lang = input(" Enter ISO 639-2 language tag (e.g. eng, fre) or Enter to skip: ").strip().lower() + + # --- build mkvmerge command --- + base = os.path.splitext(video)[0] + tmp_out = base + '._embed_tmp.mkv' + + cmd = ['mkvmerge', '-o', tmp_out, video] + if lang: + cmd += ['--language', f'0:{lang}'] + cmd.append(sub) + + print(f"\n Embedding {os.path.basename(sub)} → {os.path.basename(video)} ...") + try: + r = subprocess.run(cmd, timeout=600) + except subprocess.TimeoutExpired: + print(" Timed out.") + return + + if r.returncode not in (0, 1) or not os.path.isfile(tmp_out): + # mkvmerge returns 1 for warnings (still produces output) + print(" mkvmerge failed.") + if os.path.exists(tmp_out): + os.remove(tmp_out) + return + + # replace original with muxed file + os.replace(tmp_out, video) + print(f" Done: subtitle embedded into {os.path.basename(video)}") + + resp = input(" Delete separate subtitle file? [y/N]: ").strip().lower() + if resp == 'y': + os.remove(sub) + # also remove .idx if present alongside .sub + idx = os.path.splitext(sub)[0] + '.idx' + if os.path.exists(idx): + os.remove(idx) + print(f" Deleted: {os.path.basename(sub)}") + + +def _extract_all_noninteractive(video_path): + """--extract-all: dump every subtitle track + CC without prompting.""" + if not os.path.isfile(video_path): + print(f"File not found: {video_path}", file=sys.stderr) + sys.exit(1) + + print(f"Extracting all subtitles from: {video_path}") + streams = probe_subtitle_streams(video_path) + + extracted = 0 + for s in streams: + codec = s.get('codec_name', '') + stream_idx = s.get('index') + lang = s.get('tags', {}).get('language', '') + out = _sub_out_path(video_path, lang or str(stream_idx)) + + if codec in _TEXT_SUB_CODECS: + if _extract_text_track(video_path, stream_idx, out): + print(f" Extracted text track {stream_idx} → {os.path.basename(out)}") + extracted += 1 + elif codec in {'hdmv_pgs_subtitle', 'pgssub'}: + native_out = os.path.splitext(video_path)[0] + (f'.{lang}' if lang else f'.{stream_idx}') + '.sup' + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-map', f'0:{stream_idx}', '-c:s', 'copy', '-y', native_out + ], timeout=300) + if r.returncode == 0 and os.path.isfile(native_out): + print(f" Extracted PGS track {stream_idx} → {os.path.basename(native_out)}") + extracted += 1 + elif codec in _IMAGE_SUB_CODECS: + native_out = os.path.splitext(video_path)[0] + (f'.{lang}' if lang else f'.{stream_idx}') + '.sub' + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, '-map', f'0:{stream_idx}', '-c:s', 'copy', '-y', native_out + ], timeout=300) + if r.returncode == 0 and os.path.isfile(native_out): + print(f" Extracted VOB SUB track {stream_idx} → {os.path.basename(native_out)}") + extracted += 1 + + # try ccextractor for broadcast CC + import shutil as _sh + cce = _sh.which('ccextractor') or _sh.which('ccextractorwin') + if cce: + cc_out = _sub_out_path(video_path, 'cc') + r = subprocess.run([cce, video_path, '-o', cc_out], timeout=600, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if r.returncode == 0 and os.path.isfile(cc_out): + print(f" Extracted CC → {os.path.basename(cc_out)}") + extracted += 1 + + print(f"Done. {extracted} track(s) extracted.") + sys.exit(0) + +# ---------- Mode 8: Burnt-in subtitle OCR and removal ----------------------- + +def _probe_video_size(video_path): + """Return (width, height) of the first video stream.""" + try: + r = subprocess.run([ + 'ffprobe', '-v', 'quiet', '-print_format', 'json', + '-show_streams', '-select_streams', 'v:0', video_path + ], capture_output=True, text=True, timeout=15) + s = json.loads(r.stdout)['streams'][0] + return int(s['width']), int(s['height']) + except Exception: + return 1920, 1080 + + +def _video_duration(video_path): + try: + r = subprocess.run([ + 'ffprobe', '-v', 'quiet', '-show_entries', 'format=duration', + '-print_format', 'json', video_path + ], capture_output=True, text=True, timeout=15) + return float(json.loads(r.stdout)['format']['duration']) + except Exception: + return 0.0 + + +def scan_burnt_in_subs(video_path, fps=1, crop_fraction=0.28): + """ + OCR burnt-in subtitles from the bottom crop_fraction of each frame at fps. + Returns (entries, region): + entries = [(start_sec, end_sec, text), ...] + region = (x, y, w, h) estimated black-box in full-frame pixels, or None + """ + if not ensure_easyocr(): + return [], None + import easyocr + + width, height = _probe_video_size(video_path) + crop_y = int(height * (1.0 - crop_fraction)) + crop_h = height - crop_y + duration = _video_duration(video_path) + est = int(duration * fps) if duration else '?' + + print(f" Extracting frames at {fps}fps (~{est} frames, bottom {int(crop_fraction*100)}%)...") + + import tempfile + with tempfile.TemporaryDirectory() as tmpdir: + frame_pat = os.path.join(tmpdir, 'f_%06d.png') + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', '-i', video_path, + '-vf', f'crop={width}:{crop_h}:0:{crop_y},fps={fps}', + frame_pat + ], timeout=7200) + if r.returncode != 0: + print(" Frame extraction failed.") + return [], None + + frames = sorted(glob.glob(os.path.join(tmpdir, 'f_*.png'))) + if not frames: + print(" No frames extracted.") + return [], None + + print(f" OCR on {len(frames)} frames (first run downloads ~170 MB model)...") + reader = easyocr.Reader(['en'], verbose=False) + + entries = [] + current_text = None + start_time = None + all_bboxes = [] # (x1,y1,x2,y2) in full-frame pixels + + for i, fp in enumerate(frames): + ts = i / fps + try: + results = reader.readtext(fp, detail=1, paragraph=False) + except Exception: + results = [] + + texts = [] + for (bbox, text, conf) in results: + if conf < 0.35 or not text.strip(): + continue + texts.append(text.strip()) + bx1 = int(min(p[0] for p in bbox)) + by1 = int(min(p[1] for p in bbox)) + crop_y + bx2 = int(max(p[0] for p in bbox)) + by2 = int(max(p[1] for p in bbox)) + crop_y + all_bboxes.append((bx1, by1, bx2, by2)) + + line = postprocess_text(' '.join(texts)) if texts else '' + + if line: + if line != current_text: + if current_text is not None: + entries.append((start_time, ts, current_text)) + current_text = line + start_time = ts + else: + if current_text is not None: + entries.append((start_time, ts, current_text)) + current_text = None + + if current_text is not None and start_time is not None: + entries.append((start_time, len(frames) / fps, current_text)) + + entries = [(s, e, t) for s, e, t in entries if e - s >= 0.4] + + region = None + if all_bboxes: + x1 = max(0, min(b[0] for b in all_bboxes) - 20) + y1 = max(0, min(b[1] for b in all_bboxes) - 15) + x2 = min(width, max(b[2] for b in all_bboxes) + 20) + y2 = min(height,max(b[3] for b in all_bboxes) + 15) + region = (x1, y1, x2 - x1, y2 - y1) + + return entries, region + + +def remove_burnt_in_region(video_path, x, y, w, h, output_path): + """ + Remove a rectangular region using ffmpeg delogo filter. + Re-encodes video; audio and subtitle tracks are stream-copied. + + Limitation: pixels under the box are gone — delogo blends from + surrounding pixels. Simple/static backgrounds look good; busy action + scenes will show visible blending artifacts. + """ + print(f" Applying delogo: x={x} y={y} w={w} h={h}") + print(" Re-encoding video (libx264 CRF 18) — this will take a while...") + cmd = [ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video_path, + '-vf', f'delogo=x={x}:y={y}:w={w}:h={h}:show=0', + '-c:v', 'libx264', '-crf', '18', '-preset', 'medium', + '-c:a', 'copy', '-c:s', 'copy', + '-y', output_path + ] + try: + r = subprocess.run(cmd, timeout=7200) + return r.returncode == 0 and os.path.isfile(output_path) + except subprocess.TimeoutExpired: + print(" Timed out.") + return False + + +def _burnt_in_two_file_sync(): + """ + Two-file workflow: OCR burnt-in subs from a CC copy, then sync the + resulting SRT against a clean (no burnt-in subs) copy of the same video. + Useful when you have both the CC broadcast version and a clean retail copy. + """ + print("\n TWO-FILE SYNC") + print(" Step 1 of 2 — pick the video WITH burnt-in subtitles (the CC copy):") + vid_files = list_files(VIDEO_EXTS, "video") + if vid_files: + cc_video = pick_file(vid_files, " Choose CC video (0 to cancel)") + else: + cc_video = input(" Path to CC video (0 to cancel): ").strip() + if cc_video == '0': + return + if not cc_video or not os.path.isfile(cc_video): + print(" No valid file selected.") + return + + print("\n Step 2 of 2 — pick the CLEAN video (no burnt-in subs):") + if vid_files: + remaining = [f for f in vid_files if f != cc_video] + if remaining: + for i, f in enumerate(remaining, 1): + print(f" {i}: {f}") + clean_video = pick_file(remaining, " Choose clean video (0 to cancel)") + else: + clean_video = input(" Path to clean video (0 to cancel): ").strip() + if clean_video == '0': + return + else: + clean_video = input(" Path to clean video (0 to cancel): ").strip() + if clean_video == '0': + return + if not clean_video or not os.path.isfile(clean_video): + print(" No valid file selected.") + return + + print("\n Scan rate (affects timing accuracy and speed):") + print(" 1: 1 fps - ±1s accuracy, fast [default]") + print(" 2: 2 fps - ±0.5s accuracy, slower") + fps = 2 if (input(" Choose [1]: ").strip() == '2') else 1 + + # ── Step A: OCR the CC video ────────────────────────────────────────────── + print(f"\n Scanning '{os.path.basename(cc_video)}' for burnt-in subtitles...") + entries, _region = scan_burnt_in_subs(cc_video, fps=fps) + + if not entries: + print(" No subtitles detected in the CC video. Aborting.") + return + + print(f" Detected {len(entries)} subtitle entries.") + + import tempfile + fd, raw_srt = tempfile.mkstemp(suffix='-burntocr-raw.srt') + os.close(fd) + with open(raw_srt, 'w', encoding='utf-8') as f: + for i, (s, e, t) in enumerate(entries, 1): + f.write(f"{i}\n{seconds_to_srt(s)} --> {seconds_to_srt(e)}\n{t}\n\n") + + # ── Step B: sync the raw SRT against the clean video ───────────────────── + base = os.path.splitext(clean_video)[0] + out_srt = f"{base}-burntocr-synced.srt" + + print(f"\n Syncing OCR'd SRT against '{os.path.basename(clean_video)}'...") + if not ensure_ffsubsync(): + # No ffsubsync — just write the raw SRT alongside the clean video + import shutil + shutil.copy(raw_srt, out_srt) + os.remove(raw_srt) + print(f" ffsubsync not available — wrote unsynced SRT: {os.path.basename(out_srt)}") + print(" You can sync it later with Mode 1 (SYNC).") + return + + ok, _offset = sync_with_ffsubsync(clean_video, raw_srt, out_srt) + os.remove(raw_srt) + + if ok and os.path.isfile(out_srt): + kb = os.path.getsize(out_srt) / 1024 + print(f"\n Done: {os.path.basename(out_srt)} ({kb:.0f} KB, {len(entries)} entries)") + print(" This SRT is timed to the clean video and ready to use.") + # Offer manual fine-tune: OCR timing is at best ±0.5s so a nudge may help + print("\n Fine-tune timing?") + print(" Subtitle text appears BEFORE you hear it → use a POSITIVE number (+0.5)") + print(" You hear the sound BEFORE the text appears → use a NEGATIVE number (-0.5)") + print(" Enter to skip.") + while True: + resp = input(" Offset seconds [Enter to skip]: ").strip() + if resp == '' or resp == '0': + break + offset = parse_offset(resp) + if offset is None: + print(" Invalid — enter a number like 0.5 or -1.2.") + continue + import shutil + tmp = out_srt + '.bak' + shutil.copy(out_srt, tmp) + shift_srt(tmp, out_srt, offset) + os.remove(tmp) + print(f" Applied {offset:+.3f}s offset to {os.path.basename(out_srt)}") + again = input(" Try another offset? [y/N]: ").strip().lower() + if again != 'y': + break + # Load from the current (already-shifted) file each time — offsets stack + else: + print(" Sync failed. The raw OCR SRT has been discarded.") + print(" Tip: re-run with '1: Transcribe only' on the CC video and sync manually.") + + +def burnt_in_subs_mode(): + """Mode 8: OCR burnt-in subtitles → SRT and/or remove them from video.""" + print("\n BURNSUBS — what would you like to do?") + print(" 1: Transcribe only - OCR burnt-in subs → SRT") + print(" 2: Remove only - erase subtitle band from video (re-encodes)") + print(" 3: Both - transcribe then remove [default]") + print(" 4: Two-file sync - OCR subs from CC copy, sync SRT to clean copy") + print(" 0: Cancel") + while True: + ch = input(" Choose [3]: ").strip() or '3' + if ch in ('0', '1', '2', '3', '4'): + break + print(" Enter 0-4.") + if ch == '0': + return + + # ── Option 4: two-file workflow ────────────────────────────────────────── + if ch == '4': + _burnt_in_two_file_sync() + return + + # ── Options 1-3: single-file workflow ──────────────────────────────────── + print("\n BURNSUBS: Video files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if vid_files: + video = pick_file(vid_files, " Choose video (0 to cancel)") + else: + video = input(" Enter path to video file (0 to cancel): ").strip() + if video == '0': + return + if not video or not os.path.isfile(video): + print(" No valid video selected.") + return + + video = _offer_mp4_remux(video) + + do_ocr = ch in ('1', '3') + do_remove = ch in ('2', '3') + + print("\n Scan rate (affects timing accuracy and speed):") + print(" 1: 1 fps - ±1s accuracy, fast [default]") + print(" 2: 2 fps - ±0.5s accuracy, slower") + fps = 2 if (input(" Choose [1]: ").strip() == '2') else 1 + + region = None + srt_path = None + + if do_ocr: + print(f"\n Scanning for burnt-in subtitles...") + entries, region = scan_burnt_in_subs(video, fps=fps) + + if not entries: + print(" No subtitles detected.") + if do_remove and region is None: + print(" Cannot auto-detect removal region. Run transcribe pass first, or enter region manually.") + do_remove = True # fall through to manual entry below + else: + print(f" Detected {len(entries)} subtitle entries.") + base = os.path.splitext(video)[0] + srt_path = f"{base}-burntocr.srt" + with open(srt_path, 'w', encoding='utf-8') as f: + for i, (s, e, t) in enumerate(entries, 1): + f.write(f"{i}\n{seconds_to_srt(s)} --> {seconds_to_srt(e)}\n{t}\n\n") + print(f" SRT: {os.path.basename(srt_path)}") + + if do_remove: + if region: + x, y, w, h = region + print(f"\n Auto-detected subtitle region: x={x} y={y} w={w} h={h}") + print(" Note: pixels under the black box cannot be recovered.") + print(" delogo blends from surrounding pixels — looks good on") + print(" simple backgrounds, may show artifacts on busy scenes.") + if input(" Adjust region? [y/N]: ").strip().lower() == 'y': + region = None + + if region is None: + vw, vh = _probe_video_size(video) + print(f"\n Enter subtitle region (video is {vw}x{vh}).") + print(" Format: x y width height — e.g. for full-width bottom band: 0 920 1920 100") + while True: + raw = input(" Region (0 to cancel): ").strip() + if raw == '0': + return + try: + x, y, w, h = map(int, raw.split()) + region = (x, y, w, h) + break + except ValueError: + print(" Enter four integers.") + + x, y, w, h = region + base = os.path.splitext(video)[0] + ext = os.path.splitext(video)[1] + out = f"{base}-clean{ext}" + + if input(f"\n Write to {os.path.basename(out)} — proceed? [Y/n]: ").strip().lower() == 'n': + return + + if remove_burnt_in_region(video, x, y, w, h, out): + mb = os.path.getsize(out) / 1_048_576 + print(f" Done: {os.path.basename(out)} ({mb:.0f} MB)") + if input(" Delete original? [y/N]: ").strip().lower() == 'y': + os.remove(video) + print(f" Deleted: {os.path.basename(video)}") + else: + print(" Removal failed.") + + +# ---------- Mode 1: Sync (with language detection + transcribe/translate) ---- + +_LANG_NAMES = { + 'id': 'Indonesian', 'ms': 'Malay', 'fr': 'French', 'es': 'Spanish', + 'de': 'German', 'it': 'Italian', 'pt': 'Portuguese', 'nl': 'Dutch', + 'ru': 'Russian', 'zh-cn': 'Chinese', 'zh-tw': 'Chinese (Traditional)', + 'ja': 'Japanese', 'ko': 'Korean', 'ar': 'Arabic', 'th': 'Thai', + 'vi': 'Vietnamese', 'pl': 'Polish', 'sv': 'Swedish', 'no': 'Norwegian', + 'da': 'Danish', 'fi': 'Finnish', 'tr': 'Turkish', 'cs': 'Czech', + 'hu': 'Hungarian', 'ro': 'Romanian', 'uk': 'Ukrainian', 'tl': 'Filipino', +} + + +def ensure_langdetect(): + try: + import langdetect # noqa: F401 + return True + except ImportError: + pass + print("\nlangdetect not installed (used for subtitle language detection).") + if input(" Install it now? [Y/n]: ").strip().lower() == 'n': + return False + if not _pip_install('langdetect'): + return False + import importlib + importlib.invalidate_caches() + try: + import langdetect # noqa: F401 + return True + except ImportError: + return False + + +def _srt_detect_language(srt_path): + """Detect the language of an SRT file. + Returns (lang_code, lang_name) or (None, None) if detection fails. + Uses langdetect for Latin-script languages (Indonesian, Malay, French, etc.) + and falls back to Unicode character analysis for non-Latin scripts. + """ + entries = parse_srt_full(srt_path, limit=60) + if not entries: + return None, None + + all_text = ' '.join(t for _, _, t in entries) + letters = [c for c in all_text if c.isalpha()] + if not letters: + return None, None + + # Fast path: non-Latin scripts (CJK, Arabic, Cyrillic, etc.) + non_ascii = sum(1 for c in letters if ord(c) > 127) + if (non_ascii / len(letters)) > 0.15: + # Try langdetect for the name, fall back to 'unknown' + try: + if ensure_langdetect(): + from langdetect import detect + code = detect(all_text[:2000]) + return code, _LANG_NAMES.get(code, code.upper()) + except Exception: + pass + return 'xx', 'non-Latin script' + + # Latin-script: needs langdetect to distinguish Indonesian/Malay/English/etc. + if not ensure_langdetect(): + return None, None + try: + from langdetect import detect, DetectorFactory + DetectorFactory.seed = 0 # make results deterministic + code = detect(all_text[:2000]) + if code == 'en': + return 'en', 'English' + return code, _LANG_NAMES.get(code, code.upper()) + except Exception: + return None, None + + +def split_sync_intro_show(video): + """ + Two-pass sync for series episodes with a recurring intro. + + Pass 1: sync intro.srt against the video audio → correct timing for the + intro; the synced intro entries are used directly in the output. + Pass 2: extract show audio from where the intro ends, sync the show SRT + (which is treated as show-only content, starting near 00:00:00) + against that clip → offset_B, then shift timestamps to absolute + video time by adding intro_end_video. + + The episode SRT should cover only the show content; it does not need + intro subtitles — those come from intro.srt. + """ + if not ensure_ffsubsync(): + print(" ffsubsync is required for split sync.") + return + + # --- Locate intro.srt --- + intro_srt = 'intro.srt' + if not os.path.isfile(intro_srt): + vid_dir = os.path.dirname(os.path.abspath(video)) + intro_srt = os.path.join(vid_dir, 'intro.srt') + if os.path.isfile(intro_srt): + ans = input(f" Found {os.path.basename(intro_srt)} — use it as intro reference? [y/N]: ").strip().lower() + if ans != 'y': + intro_srt = '' + if not intro_srt or not os.path.isfile(intro_srt): + print(" SRT files in current directory:") + srt_candidates = list_files('.srt', 'SRT') + if not srt_candidates: + print(" No SRT files found — cannot run split sync.") + return + intro_srt = pick_file(srt_candidates, " Choose intro SRT (0 to cancel)") + if not intro_srt: + return + print(f" Intro reference: {os.path.basename(intro_srt)}") + + # --- Pick show SRT (show content only, need not contain intro lines) --- + print("\n Show SRT files (show content only — intro comes from intro.srt):") + srt_files = [f for f in list_files('.srt', 'SRT') if f != os.path.basename(intro_srt)] + if srt_files: + episode_srt = pick_file(srt_files, " Choose show SRT (0 to cancel)") + else: + episode_srt = input(" Path to show SRT (0 to cancel): ").strip() + if episode_srt == '0': + return + if not episode_srt or not os.path.isfile(episode_srt): + print(" No valid SRT selected.") + return + + import tempfile + + # ── Pass 1: sync intro against the full video ───────────────────────────── + print(f"\n Pass 1 of 2 — syncing {os.path.basename(intro_srt)} against {os.path.basename(video)}...") + fd, intro_synced_tmp = tempfile.mkstemp(suffix='.srt') + os.close(fd) + + ok1, offset_A = sync_with_ffsubsync(video, intro_srt, intro_synced_tmp) + if not ok1 or offset_A is None: + print(" Intro sync failed — cannot determine split point.") + try: os.remove(intro_synced_tmp) + except OSError: pass + return + + print(f" Intro offset: {offset_A:+.3f}s") + + # The synced intro entries already have correct absolute timestamps. + intro_synced_entries = parse_srt_full(intro_synced_tmp) + try: os.remove(intro_synced_tmp) + except OSError: pass + + if not intro_synced_entries: + print(" Could not read synced intro SRT — aborting.") + return + + intro_end_video = max(e for _, e, _ in intro_synced_entries) + print(f" Intro ends at {seconds_to_srt(intro_end_video)} in video") + print(f" Intro: {len(intro_synced_entries)} entries ready") + + # Write a preview file so the user can open it and check before deciding + intro_preview = os.path.splitext(intro_srt)[0] + '-synced-preview.srt' + with open(intro_preview, 'w', encoding='utf-8') as f: + for i, (s, e, t) in enumerate(intro_synced_entries, 1): + f.write(f"{i}\n{seconds_to_srt(s)} --> {seconds_to_srt(e)}\n{t}\n\n") + print(f" Preview written: {os.path.basename(intro_preview)}") + print(" Framerate correction (if needed) was applied automatically.") + print(" Open the preview in a text editor or subtitle viewer to check timing.") + input(" Press Enter when ready to continue...") + + # Optional manual nudge on the intro before combining + print("\n Intro timing fine-tune (or Enter to skip):") + print(" Subtitle text appears BEFORE you hear it → positive number (+3.0)") + print(" You hear the sound BEFORE the text appears → negative number (-3.0)") + while True: + resp = input(" Intro offset seconds [Enter to skip]: ").strip() + if resp == '' or resp == '0': + break + extra = parse_offset(resp) + if extra is None: + print(" Invalid — enter a number like 3.0 or -1.5.") + continue + intro_synced_entries = [ + (max(0.0, s + extra), max(0.0, e + extra), t) + for s, e, t in intro_synced_entries + ] + intro_end_video = max(e for _, e, _ in intro_synced_entries) + with open(intro_preview, 'w', encoding='utf-8') as f: + for i, (s, e, t) in enumerate(intro_synced_entries, 1): + f.write(f"{i}\n{seconds_to_srt(s)} --> {seconds_to_srt(e)}\n{t}\n\n") + print(f" Applied {extra:+.3f}s — intro now ends at {seconds_to_srt(intro_end_video)}") + print(f" Preview updated: {os.path.basename(intro_preview)}") + again = input(" Try another offset? [y/N]: ").strip().lower() + if again != 'y': + break + + # ── Extract show audio from intro_end onwards ───────────────────────────── + print(f"\n Extracting show audio from {seconds_to_srt(intro_end_video)}...") + fd2, show_wav = tempfile.mkstemp(suffix='.wav') + os.close(fd2) + r = subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', + '-i', video, '-ss', str(intro_end_video), + '-vn', '-ac', '1', '-ar', '16000', '-y', show_wav + ], timeout=600) + if r.returncode != 0: + print(" Failed to extract show audio — aborting.") + try: os.remove(show_wav) + except OSError: pass + return + + # ── Pass 2: sync show SRT against the show audio clip ──────────────────── + # ffsubsync finds the best alignment regardless of what offset the show SRT + # currently has; output timestamps are relative to the clip start (i.e. + # relative to intro_end_video). + show_ep = parse_srt_full(episode_srt) + print(f" Pass 2 of 2 — syncing {len(show_ep)} show entries against show audio...") + fd3, show_synced_tmp = tempfile.mkstemp(suffix='.srt') + os.close(fd3) + + ok2, offset_B = sync_with_ffsubsync(show_wav, episode_srt, show_synced_tmp) + try: os.remove(show_wav) + except OSError: pass + + if ok2 and offset_B is not None: + print(f" Show offset: {offset_B:+.3f}s (relative to intro end)") + show_synced_entries = parse_srt_full(show_synced_tmp) + else: + print(" Show sync failed — writing show entries unsynced as fallback.") + show_synced_entries = show_ep + try: os.remove(show_synced_tmp) + except OSError: pass + + # ── Merge: intro (absolute) + show (relative → absolute) ───────────────── + base = os.path.splitext(episode_srt)[0] + out = f"{base}-splitsync.srt" + + with open(out, 'w', encoding='utf-8') as f: + idx = 1 + # Intro: timestamps already correct from pass 1 + for s, e, t in intro_synced_entries: + f.write(f"{idx}\n{seconds_to_srt(s)} --> {seconds_to_srt(e)}\n{t}\n\n") + idx += 1 + # Show: add intro_end_video to convert clip-relative → absolute video time + for s, e, t in show_synced_entries: + ws = s + intro_end_video + we = max(ws + 0.1, e + intro_end_video) + f.write(f"{idx}\n{seconds_to_srt(ws)} --> {seconds_to_srt(we)}\n{t}\n\n") + idx += 1 + + kb = os.path.getsize(out) / 1024 + print(f"\n Done: {os.path.basename(out)} ({kb:.0f} KB, {idx-1} entries)") + print(f" Intro: {len(intro_synced_entries)} entries (offset {offset_A:+.3f}s)") + if ok2 and offset_B is not None: + print(f" Show: {len(show_synced_entries)} entries (offset {offset_B:+.3f}s from intro end)") + + +def sync_mode(): + global WHISPER_MODEL, WHISPER_TASK, WHISPER_LANGUAGE + + # --- Pick video --- + print("\n SYNC: Video files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if vid_files: + video = pick_file(vid_files, " Choose video by number or filename") + else: + video = input(" Enter path to video file (0 to cancel): ").strip() + if video == '0': + return + if not video or not os.path.isfile(video): + print(" No valid video selected.") + return + + ffsubsync_ok = ensure_ffsubsync() + whisper_ok = WHISPER_AVAILABLE # don't install just to show the menu + + # --- Sync method --- + print("\n Sync method:") + print(" f: ffsubsync only (fast, recommended) [default]") + print(" w: Whisper only (speech recognition)") + print(" b: Both - ffsubsync + Whisper cross-check") + print(" m: Manual offset (enter seconds yourself)") + print(" s: Split sync (intro + show have different offsets, uses intro.srt)") + print(" 0: Cancel") + while True: + ch = input(" Choose or Enter for default: ").strip().lower() + if ch == '': + ch = 'f' + break + if ch in ('f', 'w', 'b', 'm', 's', '0'): + break + print(" Enter f, w, b, m, s or 0.") + if ch == '0': + return + + if ch == 's': + split_sync_intro_show(video) + return + + if ch in ('w', 'b'): + whisper_ok = ensure_whisper() + if not whisper_ok: + print(" Whisper required for this method.") + return + print("\n Whisper model:") + for k, (name, desc) in WHISPER_MODELS.items(): + marker = " <-- default" if name == WHISPER_MODEL else "" + print(f" {k}: {name:20s} {desc}{marker}") + choice = input(" Choose model [Enter for default]: ").strip() + if choice in WHISPER_MODELS: + WHISPER_MODEL = WHISPER_MODELS[choice][0] + print(f" Using: {WHISPER_MODEL}\n") + + # --- Pick SRT --- + print("\n SRT files in current directory:") + srt_files = list_files('.srt', "SRT") + if srt_files: + src = pick_file(srt_files, " Choose SRT by number or filename") + else: + src = input(" Enter path to .srt file (0 to cancel): ").strip() + if src == '0': + return + if not src or not os.path.isfile(src): + print(" No valid SRT selected.") + return + + # Detect SRT language from the actual subtitle text — offer English if non-English + also_english = False + lang_code, lang_name = _srt_detect_language(src) + if lang_code and lang_code != 'en': + print(f"\n Detected language: {lang_name}.") + if ensure_whisper(): + whisper_ok = True + also_english = input( + f" Also generate an English SRT via Whisper translate after sync? [Y/n]: " + ).strip().lower() != 'n' + + # --- Sync --- + base = os.path.splitext(src)[0] + out = f"{base}-synced.srt" + print(f"\n Syncing -> {os.path.basename(out)}") + + synced_ok = False + if ch == 'f': + ok, offset = sync_with_ffsubsync(video, src, out) + if ok: + if offset is not None: + print(f" Offset applied: {offset:+.3f} s") + print(f" Done: {os.path.basename(out)}") + synced_ok = True + else: + print(" ffsubsync failed.") + while True: + resp = input(" Enter offset manually (seconds, e.g. -3.5) or Enter to skip: ").strip() + if resp == '': + break + offset = parse_offset(resp) + if offset is None: + print(" Invalid.") + else: + shift_srt(src, out, offset) + print(f" Written -> {os.path.basename(out)}") + synced_ok = True + break + + elif ch == 'w': + offset, n_matches, spread, err = compute_offset_whisper(src, video, WHISPER_MODEL) + if not err: + quality = "good" if spread < 2.0 else "moderate" if spread < 5.0 else "low" + print(f" Whisper offset: {offset:+.3f} s ({n_matches} matches, spread {spread:.1f}s, {quality})") + shift_srt(src, out, offset) + print(f" Done: {os.path.basename(out)}") + synced_ok = True + else: + print(f" Whisper failed: {err}") + + elif ch == 'm': + print(" Subtitle text appears BEFORE you hear it → use a POSITIVE number (+0.52)") + print(" You hear the sound BEFORE the text appears → use a NEGATIVE number (-0.52)") + print(" Enter 0 or blank to cancel.") + last_offset = 0.0 + while True: + hint = f" Offset seconds [last: {last_offset:+.3f}]: " + resp = input(hint).strip() + if resp in ('0', ''): + break + offset = parse_offset(resp) + if offset is None: + print(" Invalid — enter a number like 1.5 or -0.52.") + continue + last_offset = offset + shift_srt(src, out, offset) + print(f" Written -> {os.path.basename(out)}") + synced_ok = True + again = input(" Try another offset? [y/N]: ").strip().lower() + if again != 'y': + break + # re-apply to original each time so offsets don't stack + print(" (applying to original each time — offsets do not stack)") + + else: # b + if sync_single(video, src, out, ffsubsync_ok, whisper_ok, interactive=True): + print(f" Done: {os.path.basename(out)}") + synced_ok = True + else: + while True: + resp = input("\n All methods failed. Enter offset manually or Enter to skip: ").strip() + if resp == '': + break + offset = parse_offset(resp) + if offset is None: + print(" Invalid.") + else: + shift_srt(src, out, offset) + print(f" Written -> {os.path.basename(out)}") + synced_ok = True + break + + # --- Also generate English SRT? --- + if also_english and whisper_ok: + print("\n Generating English SRT via Whisper translate...") + WHISPER_TASK = 'translate' + WHISPER_LANGUAGE = None + final = generate_and_sync(video, WHISPER_MODEL, ffsubsync_ok=ffsubsync_ok) + if final: + print(f" English SRT: {os.path.basename(final)}") + + if synced_ok: + offer_rename(video) + + +# ---------- Main ------------------------------------------------------------- + +def main(): + global WHISPER_MODEL, WHISPER_LANGUAGE, WHISPER_TASK + + if '--translate' in sys.argv: + WHISPER_TASK = 'translate' + WHISPER_LANGUAGE = None # auto-detect source; --lang overrides below + print("Translate mode: Whisper will output English regardless of source language.") + + if '--extract-all' in sys.argv: + idx = sys.argv.index('--extract-all') + if idx + 1 < len(sys.argv): + _extract_all_noninteractive(sys.argv[idx + 1]) + else: + print("--extract-all requires a file path.", file=sys.stderr) + sys.exit(1) + + if '--lang' in sys.argv: + idx = sys.argv.index('--lang') + if idx + 1 < len(sys.argv): + WHISPER_LANGUAGE = sys.argv[idx + 1] + print(f"Language override: {WHISPER_LANGUAGE}") + else: + print("--lang requires a language code (e.g. --lang fr). Using default.") + elif '--lang-auto' in sys.argv: + WHISPER_LANGUAGE = None + print("Language: auto-detect") + + while True: + print("\nWhat would you like to do?") + print(" 1: SYNC - sync an existing SRT to the video") + gen_label = "translate foreign audio → English SRT" if WHISPER_TASK == 'translate' \ + else "create a new SRT by transcribing with Whisper" + print(f" 2: GENERATE - {gen_label}") + print(" 3: BATCH - sync all video+SRT pairs in this directory") + print(" 4: RENAME - rename video + SRT to Plex format") + print(" 5: EXTRACT - extract embedded subtitles / CC to SRT") + print(" 6: REMUX - convert MP4 → MKV (stream copy, no re-encode)") + print(" 7: EMBED - soft-mux subtitle file into video (mkvmerge)") + print(" 8: BURNSUBS - OCR burnt-in subs → SRT and/or erase from video") + print(" 0: Exit") + while True: + mode = input("Choose: ").strip() + if mode in ('0', '1', '2', '3', '4', '5', '6', '7', '8'): + break + print("Please enter 0-8.") + + if mode == '0': + print("Goodbye.") + break + + # ---- Mode 4: Rename (has its own sub-menu loop) --------------------- + if mode == '4': + ocr_ok = ensure_easyocr() + rename_menu(ocr_ok) + continue + + # ---- Mode 5: Extract subtitles -------------------------------------- + if mode == '5': + extract_subs_mode() + continue + + # ---- Mode 6: Remux MP4 → MKV ---------------------------------------- + if mode == '6': + remux_mp4_to_mkv() + continue + + # ---- Mode 7: Embed subtitle into video ------------------------------ + if mode == '7': + embed_subs_mode() + continue + + # ---- Mode 8: Burnt-in subtitle OCR / removal ------------------------ + if mode == '8': + burnt_in_subs_mode() + continue + + # ---- Mode 1: Sync / transcribe / translate -------------------------- + if mode == '1': + sync_mode() + continue + + # ---- Mode 2: Generate SRT ------------------------------------------- + whisper_ok = ensure_whisper() + ffsubsync_ok = ensure_ffsubsync() + + if not whisper_ok: + print("Whisper is required to generate an SRT.") + continue + + if whisper_ok: + print("\nWhisper model (larger = more accurate, more RAM, slower first load):") + for k, (name, desc) in WHISPER_MODELS.items(): + marker = " <-- default" if name == WHISPER_MODEL else "" + print(f" {k}: {name:20s} {desc}{marker}") + choice = input("Choose model [Enter for default]: ").strip() + if choice in WHISPER_MODELS: + WHISPER_MODEL = WHISPER_MODELS[choice][0] + print(f" Using: {WHISPER_MODEL}\n") + + # ---- Mode 3: Batch sync --------------------------------------------- + if mode == '3': + batch_sync(ffsubsync_ok, whisper_ok) + continue + + print("\nVideo files in current directory:") + vid_files = list_files(VIDEO_EXTS, "video") + if vid_files: + video = pick_file(vid_files, "Choose video by number or filename") + else: + video = input("Enter path to video file (0 to cancel): ").strip() + if video == '0': + continue + if not video or not os.path.isfile(video): + print("No valid video selected.") + continue + + final = generate_and_sync(video, WHISPER_MODEL, ffsubsync_ok=ffsubsync_ok) + if final: + print(f"\nDone - final SRT: {final}") + offer_rename(video) + + +if __name__ == '__main__': + main() diff --git a/services/sync-cc.sh b/services/sync-cc.sh new file mode 100644 index 0000000..7117be7 --- /dev/null +++ b/services/sync-cc.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# services/sync-cc.sh — Subtitle sync & generation tool (sync_cc). +# Part of the modular post-install system (sourced by setup.sh). +# +# NON-DOCKER module. sync_cc is a Python CLI tool that: +# - GENERATE: Whisper AI transcribes video audio → SRT +# - SYNC: ffsubsync aligns an existing SRT to the video +# - BATCH: process all video+SRT pairs in a directory +# - RENAME: look up episode titles on TMDB, rename to Plex format +# - EXTRACT: pull embedded subtitle / CC tracks out of MKV/MP4/TS +# - REMUX: MP4 → MKV stream-copy (no re-encode) +# - EMBED: soft-mux an SRT into a container via mkvmerge +# - BURNSUBS: OCR burnt-in subs → SRT (and optionally erase from video) +# +# GPU is used automatically when CUDA or MPS is detected. +# Heavy deps (easyocr, pgsreader) are installed on first use by the script +# itself. This module installs the always-needed system + pip packages. +# +# Source script: extras/sync_cc.py in this repo. + +register_service sync-cc extras "Subtitle sync/generate tool — Whisper + ffsubsync (sync_cc)" + +install_sync-cc() { + local SYNCCC_DIR="$ACTUAL_HOME/sync-cc" + + if [ "$DRY_RUN" = true ]; then + echo "[DRY-RUN] sync-cc would:" + echo " - Install: python3-pip ffmpeg mkvtoolnix ccextractor" + echo " - pip install: openai-whisper ffsubsync" + echo " - Copy extras/sync_cc.py → $SYNCCC_DIR/sync_cc.py" + echo " - Write $SYNCCC_DIR/.env with TMDB_API_KEY" + echo " - Create /usr/local/bin/sync-cc wrapper" + return 0 + fi + + echo "" + echo "╔═══════════════════════════════════════════════════════╗" + echo "║ Subtitle Sync & Generation — sync_cc ║" + echo "║ Whisper AI · ffsubsync · TMDB rename · OCR subs ║" + echo "╚═══════════════════════════════════════════════════════╝" + echo "" + + # ── System packages ────────────────────────────────────────────────────── + log_info "Installing system dependencies..." + run_cmd apt-get update -qq + run_cmd apt-get install -y --no-install-recommends \ + python3 python3-pip ffmpeg mkvtoolnix ccextractor + log_success "System packages installed" + + # ── pip packages ───────────────────────────────────────────────────────── + # Install as the actual (non-root) user so packages land in ~/.local + log_info "Installing Python packages (openai-whisper, ffsubsync)..." + local PIP_CMD="pip3 install --user --quiet openai-whisper ffsubsync" + if sudo -u "$ACTUAL_USER" $PIP_CMD; then + log_success "Python packages installed" + else + log_warning "pip install reported errors — the tool may still work if packages were partially installed" + fi + + # ── Install script ─────────────────────────────────────────────────────── + mkdir -p "$SYNCCC_DIR" + cp "$HERE/extras/sync_cc.py" "$SYNCCC_DIR/sync_cc.py" + chmod +x "$SYNCCC_DIR/sync_cc.py" + chown -R "$ACTUAL_USER:$ACTUAL_USER" "$SYNCCC_DIR" + log_success "sync_cc.py installed to $SYNCCC_DIR/" + + # ── TMDB API key ───────────────────────────────────────────────────────── + echo "" + log_info "TMDB API Key (optional — needed for episode rename mode)" + echo " The rename feature looks up episode titles via The Movie Database." + echo " Get a free key at https://www.themoviedb.org/settings/api" + echo " (Leave blank to skip — you can add it later to $SYNCCC_DIR/.env)" + echo "" + local TMDB_KEY="" + if [ "$UNATTENDED" != true ]; then + read -p " TMDB API key [Enter to skip]: " TMDB_KEY + fi + + # Write .env (creates or replaces) + { + echo "# sync_cc configuration" + echo "# Get a free TMDB key at https://www.themoviedb.org/settings/api" + if [ -n "$TMDB_KEY" ]; then + echo "TMDB_API_KEY=${TMDB_KEY}" + else + echo "# TMDB_API_KEY=your_key_here" + fi + } > "$SYNCCC_DIR/.env" + chown "$ACTUAL_USER:$ACTUAL_USER" "$SYNCCC_DIR/.env" + chmod 600 "$SYNCCC_DIR/.env" + log_success ".env written to $SYNCCC_DIR/.env" + + # ── Wrapper in PATH ─────────────────────────────────────────────────────── + # cd into the user's current dir first so .env from cwd is preferred; + # falls back to the one next to sync_cc.py. + cat > /usr/local/bin/sync-cc << WRAPEOF +#!/bin/bash +exec python3 "$SYNCCC_DIR/sync_cc.py" "\$@" +WRAPEOF + chmod +x /usr/local/bin/sync-cc + log_success "wrapper created: /usr/local/bin/sync-cc" + + # ── Summary ─────────────────────────────────────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " sync_cc installed" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Run from any directory containing video / SRT files:" + echo " sync-cc" + echo "" + echo " Modes:" + echo " 1 SYNC — align an existing SRT to the video" + echo " 2 GENERATE — Whisper AI transcribes video → SRT" + echo " 3 BATCH — sync all video+SRT pairs in directory" + echo " 4 RENAME — TMDB episode lookup + rename to Plex format" + echo " 5 EXTRACT — pull embedded subtitle tracks from MKV/MP4/TS" + echo " 6 REMUX — MP4 → MKV stream copy (no re-encode)" + echo " 7 EMBED — soft-mux an SRT into a container" + echo " 8 BURNSUBS — OCR burnt-in subs → SRT" + echo "" + echo " Config: $SYNCCC_DIR/.env" + if [ -z "$TMDB_KEY" ]; then + echo " → Set TMDB_API_KEY in .env to enable episode rename mode" + fi + echo "" + echo " Whisper models download automatically on first use." + echo " First run may take a few minutes while the model downloads." + echo "" +} diff --git a/setup.sh b/setup.sh index ed6fa9b..d359670 100755 --- a/setup.sh +++ b/setup.sh @@ -81,6 +81,7 @@ is_installed() { crowdsec) command -v cscli >/dev/null 2>&1 ;; silent-send) [ -d "$ACTUAL_HOME/silent-send/.git" ] ;; linux-to-sync) [ -d "$ACTUAL_HOME/linux-to-sync/.git" ] ;; + sync-cc) [ -f "$ACTUAL_HOME/sync-cc/sync_cc.py" ] ;; *) [ -e "$DOCKER_DIR/$1" ] ;; esac } From c3bb2cf18cbe9c0d300346dec6fb947c14d01a60 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 21:40:49 +0000 Subject: [PATCH 04/12] feat(cameras): add sky-cam and frigate-audio service modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sky-cam (cameras/non-docker): Clones outis1one/sky-cam via bootstrap.sh to ~/sky-cam. Prompts for latitude, longitude, timezone, camera names, BASE_DIR, and optional Mattermost webhook. Patches sky-cam.conf and installs systemd user timers via the repo's install.sh. Produces sunrise clips, Four Seasons timelapse, moon-track, and monthly moon-phase images. frigate-audio (cameras/docker): Full stack from outis1one/frigate_w_audio: Frigate 0.17 NVR + Mosquitto MQTT broker + frigate-notify → ntfy push alerts. Audio-ready config template with face recognition and LPR pre-configured. Prompts for camera credentials, media storage path (supports drive detection), MQTT password (auto-generated), and ntfy server. Bootstraps the Mosquitto passwd file. Detector choice: CPU / USB Coral / PCIe Coral. Hardcoded media path from upstream replaced with a configurable prompt. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- MODULAR.md | 2 +- services/frigate-audio.sh | 557 ++++++++++++++++++++++++++++++++++++++ services/sky-cam.sh | 184 +++++++++++++ setup.sh | 1 + 4 files changed, 743 insertions(+), 1 deletion(-) create mode 100644 services/frigate-audio.sh create mode 100644 services/sky-cam.sh diff --git a/MODULAR.md b/MODULAR.md index 5c3c920..c1aa1c6 100644 --- a/MODULAR.md +++ b/MODULAR.md @@ -95,7 +95,7 @@ is retained as a frozen evolution record. | `homelab` | `caddy`, `crowdsec`, `authelia`, `homeassistant` | | `utilities` | `actualbudget`, `ddclient`, `filebrowser`, `fmd`, `magicmirror`, `mealie`, `meshcentral`, `ntfy`, `portainer`, `traccar`, `uptimekuma`, `watchtower`, `wg-easy` | | `media` | `arm`, `audiobookshelf`, `emby`, `immich`, `jellyfin`, `lyrion` | -| `cameras` | `frigate`, `frigate-notify` | +| `cameras` | `frigate`, `frigate-audio`, `frigate-notify`, `sky-cam` | | `gaming` | `js99er`, `minecraft`, `wolf`, `wolf-pair` | | `extras` | `linux-to-sync`, `silent-send`, `sync-cc` | | `backup` | `backup` | diff --git a/services/frigate-audio.sh b/services/frigate-audio.sh new file mode 100644 index 0000000..9406975 --- /dev/null +++ b/services/frigate-audio.sh @@ -0,0 +1,557 @@ +#!/bin/bash +# services/frigate-audio.sh — Frigate NVR + Mosquitto MQTT + frigate-notify +# full-stack with audio support and push notifications via ntfy. +# Part of the modular post-install system (sourced by setup.sh). +# +# Based on outis1one/frigate_w_audio. This is the full stack: +# Frigate 0.17 NVR, face recognition, LPR, motion detection +# Mosquitto MQTT broker (events bus between Frigate and notify) +# frigate-notify Event consumer — sends ntfy push notifications +# +# Audio is OFF by default in the Frigate config (audio.enabled: false). +# To enable it you need at least one camera with a working microphone — +# see the HOW TO ADD A CAMERA WITH A MIC section in the generated config.yml. +# +# Hardware acceleration and Coral TPU are opt-in during setup; the +# default falls back to CPU detection so the stack runs everywhere. +# +# Differs from services/frigate.sh (simpler, standalone Frigate only): +# • includes Mosquitto + frigate-notify +# • audio-ready camera config template +# • Frigate 0.17 schema with face recognition + LPR pre-configured + +register_service frigate-audio cameras "Frigate NVR + MQTT + push notifications (audio-ready stack)" 8971 + +install_frigate-audio() { + require_docker || return 1 + + local DIR="$DOCKER_DIR/frigate-audio" + + if [ "$DRY_RUN" = true ]; then + echo "[DRY-RUN] frigate-audio would:" + echo " Create $DIR with Frigate + Mosquitto + frigate-notify stack" + echo " Prompt for camera RTSP credentials, IPs, MQTT password, ntfy server" + echo " Generate docker-compose.yml, frigate config, mosquitto config, .env" + echo " Bootstrap the Mosquitto password file" + return 0 + fi + + echo "" + echo "╔═══════════════════════════════════════════════════════════════╗" + echo "║ Frigate + Mosquitto + frigate-notify (audio-ready stack) ║" + echo "║ Face recognition · LPR · ntfy push alerts ║" + echo "╚═══════════════════════════════════════════════════════════════╝" + echo "" + + # ── Media storage path ───────────────────────────────────────────────────── + log_info "Frigate media storage (recordings, snapshots)" + echo " Recordings can fill tens of GB quickly — a dedicated drive is recommended." + echo "" + local FRIGATE_MEDIA_DIR="" + local DEFAULT_MEDIA="$DOCKER_DIR/frigate-audio/media" + if declare -f select_storage_path &>/dev/null; then + select_storage_path "Frigate recordings" FRIGATE_MEDIA_DIR + [ -z "$FRIGATE_MEDIA_DIR" ] && FRIGATE_MEDIA_DIR="$DEFAULT_MEDIA" + else + prompt_text "Frigate media path [$DEFAULT_MEDIA]:" "$DEFAULT_MEDIA" FRIGATE_MEDIA_DIR + fi + log_info "Media path: $FRIGATE_MEDIA_DIR" + + # ── Camera credentials ───────────────────────────────────────────────────── + echo "" + log_info "Camera 1 — Front Door (required)" + local CAM1_USER="" CAM1_PASS="" CAM1_IP="" + prompt_text " RTSP username [admin]:" "admin" CAM1_USER + prompt_text " RTSP password:" "" CAM1_PASS + prompt_text " Camera IP [192.168.1.100]:" "192.168.1.100" CAM1_IP + + echo "" + log_info "Camera 2 — Back Door (optional, press Enter to skip IP)" + local CAM2_USER="" CAM2_PASS="" CAM2_IP="" + prompt_text " RTSP username [admin]:" "admin" CAM2_USER + prompt_text " RTSP password [changeme]:" "changeme" CAM2_PASS + prompt_text " Camera IP (Enter to disable):" "" CAM2_IP + + echo "" + log_info "Camera 3 — Third camera (optional)" + local CAM3_USER="" CAM3_PASS="" CAM3_IP="" + prompt_text " RTSP username [admin]:" "admin" CAM3_USER + prompt_text " RTSP password [changeme]:" "changeme" CAM3_PASS + prompt_text " Camera IP (Enter to disable):" "" CAM3_IP + + # ── MQTT password ────────────────────────────────────────────────────────── + echo "" + log_info "MQTT credentials (Frigate ↔ Mosquitto ↔ frigate-notify)" + local MQTT_PASS="" + prompt_text " MQTT username [frigate]:" "frigate" MQTT_USER + if [ -z "${MQTT_USER:-}" ]; then MQTT_USER="frigate"; fi + MQTT_PASS=$(generate_password 24) + log_info " Generated MQTT password: $MQTT_PASS" + + # ── ntfy server ───────────────────────────────────────────────────────── + echo "" + log_info "ntfy push notifications" + echo " frigate-notify sends alerts via ntfy. Set to your ntfy server URL." + local NTFY_SERVER="" NTFY_TOPIC="frigate" + prompt_text " ntfy server URL [https://ntfy.yourdomain.com]:" "https://ntfy.yourdomain.com" NTFY_SERVER + prompt_text " ntfy topic [frigate]:" "frigate" NTFY_TOPIC + + # ── Frigate public URL ───────────────────────────────────────────────────── + echo "" + local BASE_DOMAIN="" + [ -f "$DOCKER_DIR/.config" ] && BASE_DOMAIN=$(grep '^BASE_DOMAIN=' "$DOCKER_DIR/.config" 2>/dev/null | cut -d= -f2-) + local FRIGATE_PUBLIC_URL="" + if [ -n "$BASE_DOMAIN" ]; then + local _PFX="" + prompt_text " Subdomain prefix for Frigate [cam].${BASE_DOMAIN}:" "cam" _PFX + FRIGATE_PUBLIC_URL="https://${_PFX:-cam}.${BASE_DOMAIN}" + else + prompt_text " Frigate public URL [https://cam.yourdomain.com]:" "https://cam.yourdomain.com" FRIGATE_PUBLIC_URL + fi + + # ── Detector choice ──────────────────────────────────────────────────────── + echo "" + log_info "Object detector" + echo " 1) CPU (works everywhere, higher CPU usage)" + echo " 2) USB Coral TPU (faster detection, lower CPU — requires USB Coral stick)" + echo " 3) PCIe Coral TPU" + local DET_CHOICE="" + prompt_text "Detector [1]:" "1" DET_CHOICE + local DETECTOR_BLOCK HWA_COMMENT + case "${DET_CHOICE:-1}" in + 2) DETECTOR_BLOCK="detectors:\n coral:\n type: edgetpu\n device: usb" + HWA_COMMENT=" devices:\n - /dev/bus/usb:/dev/bus/usb # USB Coral" ;; + 3) DETECTOR_BLOCK="detectors:\n coral:\n type: edgetpu\n device: pci" + HWA_COMMENT=" devices:\n - /dev/apex_0:/dev/apex_0 # PCIe Coral" ;; + *) DETECTOR_BLOCK="detectors:\n cpu:\n type: cpu\n num_threads: 3" + HWA_COMMENT="" ;; + esac + + # ── Hardware acceleration for re-encoding ────────────────────────────────── + echo "" + local HWA="" + prompt_yn " Enable hardware video decode (Intel/AMD /dev/dri/renderD128)? (y/n) [n]:" "n" HWA + local DRI_LINE="" + [[ ${HWA:-n} =~ ^[Yy]$ ]] && DRI_LINE=" - /dev/dri/renderD128 # Intel/AMD hwaccel" + + # ── Create directory structure ───────────────────────────────────────────── + mkdir -p "$DIR"/{frigate_config,mosquitto/config,mosquitto/data,mosquitto/log,"frigate-notify"} + mkdir -p "$FRIGATE_MEDIA_DIR" + ensure_docker_dir_ownership "$DIR" + cd "$DIR" || return 1 + + # ── .env ────────────────────────────────────────────────────────────────── + log_info "Writing .env..." + cat > "$DIR/.env" << ENVEOF +# Frigate audio stack — generated by setup.sh +# DO NOT commit this file — it contains credentials. + +# ---- Camera 1: Front Door ---- +FRIGATE_RTSP_USER=${CAM1_USER:-admin} +FRIGATE_RTSP_PASSWORD=${CAM1_PASS:-changeme} +FRIGATE_FRONT_DOOR_IP=${CAM1_IP:-192.168.1.100} + +# ---- Camera 2: Back Door ---- +FRIGATE_RTSP_USER1=${CAM2_USER:-admin} +FRIGATE_RTSP_PASSWORD1=${CAM2_PASS:-changeme} +FRIGATE_BACK_DOOR_IP=${CAM2_IP:-192.168.1.101} + +# ---- Camera 3 (optional) ---- +FRIGATE_RTSP_USER2=${CAM3_USER:-admin} +FRIGATE_RTSP_PASSWORD2=${CAM3_PASS:-changeme} +FRIGATE_SQUIRREL_IP=${CAM3_IP:-192.168.1.102} + +# ---- MQTT ---- +FRIGATE_MQTT_USER=${MQTT_USER:-frigate} +FRIGATE_MQTT_PASSWORD=${MQTT_PASS} + +# ---- frigate-notify ---- +FN_FRIGATE__MQTT__PASSWORD=${MQTT_PASS} +FN_FRIGATE__SERVER=http://frigate:5000 +FN_FRIGATE__PUBLIC_URL=${FRIGATE_PUBLIC_URL} +FN_ALERTS__NTFY__SERVER=${NTFY_SERVER} +ENVEOF + chmod 600 "$DIR/.env" + log_success ".env written" + + # ── docker-compose.yml ───────────────────────────────────────────────────── + log_info "Writing docker-compose.yml..." + + local DEVICES_BLOCK="" + [ -n "$HWA_COMMENT" ] && DEVICES_BLOCK=" devices:\n${HWA_COMMENT}" + [ -n "$DRI_LINE" ] && DEVICES_BLOCK="${DEVICES_BLOCK}\n ${DRI_LINE}" + if [ -n "$DET_CHOICE" ] && [ "$DET_CHOICE" = "2" ]; then + DEVICES_BLOCK=" devices:\n${HWA_COMMENT}" + [ -n "$DRI_LINE" ] && DEVICES_BLOCK="${DEVICES_BLOCK}\n ${DRI_LINE}" + fi + + cat > "$DIR/docker-compose.yml" << 'COMPOSEEOF' +# Frigate NVR + Mosquitto MQTT + frigate-notify +# Generated by ubuntu-post-install setup.sh +name: frigate-audio + +services: + + frigate: + container_name: frigate-audio + image: ghcr.io/blakeblackshear/frigate:0.17.1 + restart: unless-stopped + stop_grace_period: 30s + privileged: true + shm_size: "512mb" + env_file: .env + depends_on: + - mosquitto +COMPOSEEOF + + # Inject devices block if hardware acceleration chosen + if [ -n "$HWA_COMMENT" ] || [ -n "$DRI_LINE" ]; then + echo " devices:" >> "$DIR/docker-compose.yml" + [ -n "$HWA_COMMENT" ] && printf " %s\n" "$HWA_COMMENT" | sed 's|^ *||' >> "$DIR/docker-compose.yml" + [ -n "$DRI_LINE" ] && echo " $DRI_LINE" >> "$DIR/docker-compose.yml" + fi + + cat >> "$DIR/docker-compose.yml" << COMPOSEEOF + volumes: + - /etc/localtime:/etc/localtime:ro + - ./frigate_config:/config + - ${FRIGATE_MEDIA_DIR}:/media/frigate + - type: tmpfs + target: /tmp/cache + tmpfs: + size: 1000000000 + ports: + - "8971:8971" + - "5001:5000" + - "8554:8554" + - "8555:8555/tcp" + - "8555:8555/udp" + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:5000/api/version"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 60s + + mosquitto: + container_name: frigate-audio-mqtt + hostname: mosquitto + image: eclipse-mosquitto:2 + restart: unless-stopped + ports: + - "1883:1883" + volumes: + - ./mosquitto/config:/mosquitto/config + - ./mosquitto/data:/mosquitto/data + - ./mosquitto/log:/mosquitto/log + + frigate-notify: + container_name: frigate-audio-notify + hostname: frigate-notify + image: ghcr.io/0x2142/frigate-notify:latest + restart: unless-stopped + env_file: .env + depends_on: + mosquitto: + condition: service_started + frigate: + condition: service_healthy + volumes: + - ./frigate-notify/config.yml:/app/config.yml:ro +COMPOSEEOF + log_success "docker-compose.yml written" + + # ── Mosquitto config ─────────────────────────────────────────────────────── + log_info "Writing Mosquitto config..." + cat > "$DIR/mosquitto/config/mosquitto.conf" << 'MQTTEOF' +listener 1883 0.0.0.0 +protocol mqtt + +persistence true +persistence_location /mosquitto/data/ + +log_dest stdout +log_dest file /mosquitto/log/mosquitto.log + +allow_anonymous false +password_file /mosquitto/config/passwd +MQTTEOF + + # Bootstrap the Mosquitto password file + log_info "Bootstrapping Mosquitto password file..." + if docker run --rm -i eclipse-mosquitto:2 \ + mosquitto_passwd -b -c /dev/stdout "$MQTT_USER" "$MQTT_PASS" \ + > "$DIR/mosquitto/config/passwd" 2>/dev/null; then + log_success "Mosquitto passwd file created" + else + log_warning "Could not bootstrap Mosquitto passwd — do it manually:" + log_warning " docker run --rm eclipse-mosquitto:2 mosquitto_passwd -b -c /passwd ${MQTT_USER} '${MQTT_PASS}'" + log_warning " Then copy the output to ${DIR}/mosquitto/config/passwd" + fi + + # ── Frigate config.yml ───────────────────────────────────────────────────── + log_info "Writing Frigate config..." + local CAM2_ENABLED="false"; [ -n "$CAM2_IP" ] && CAM2_ENABLED="true" + local CAM3_ENABLED="false"; [ -n "$CAM3_IP" ] && CAM3_ENABLED="true" + + local DETECTOR_YAML + case "${DET_CHOICE:-1}" in + 2) DETECTOR_YAML="detectors:\n coral:\n type: edgetpu\n device: usb" ;; + 3) DETECTOR_YAML="detectors:\n coral:\n type: edgetpu\n device: pci" ;; + *) DETECTOR_YAML="detectors:\n cpu:\n type: cpu\n num_threads: 3" ;; + esac + + cat > "$DIR/frigate_config/config.yml" << FRIGCFGEOF +version: 0.17-0 + +mqtt: + enabled: true + host: mosquitto + port: 1883 + user: "{FRIGATE_MQTT_USER}" + password: "{FRIGATE_MQTT_PASSWORD}" + topic_prefix: frigate + client_id: frigate + stats_interval: 60 + +tls: + enabled: false + +# Audio detection — set true when you have a camera with a working mic. +# See the HOW TO ADD A CAMERA WITH A MIC section at the bottom of this file. +audio: + enabled: false + +$(printf "$DETECTOR_YAML") + +birdseye: + mode: continuous + +semantic_search: + enabled: false + model_size: small + +face_recognition: + enabled: true + model_size: small + +lpr: + enabled: true + model_size: small + +objects: + track: + - person + +record: + enabled: true + continuous: + days: 0 + motion: + days: 10 + +go2rtc: + streams: + front_door: + - rtsp://{FRIGATE_RTSP_USER}:{FRIGATE_RTSP_PASSWORD}@{FRIGATE_FRONT_DOOR_IP}:554/Streaming/Channels/101 + back_door: + - rtsp://{FRIGATE_RTSP_USER1}:{FRIGATE_RTSP_PASSWORD1}@{FRIGATE_BACK_DOOR_IP}:554/Streaming/Channels/101 + squirrel: + - rtsp://{FRIGATE_RTSP_USER2}:{FRIGATE_RTSP_PASSWORD2}@{FRIGATE_SQUIRREL_IP}:554/Streaming/Channels/101 + +cameras: + front_door: + enabled: true + ffmpeg: + inputs: + - path: rtsp://127.0.0.1:8554/front_door + input_args: preset-rtsp-restream + roles: + - detect + - record + detect: + enabled: true + width: 2688 + height: 1520 + fps: 5 + + back_door: + enabled: ${CAM2_ENABLED} + ffmpeg: + inputs: + - path: rtsp://127.0.0.1:8554/back_door + input_args: preset-rtsp-restream + roles: + - detect + - record + detect: + enabled: true + width: 2688 + height: 1520 + fps: 5 + + squirrel: + enabled: ${CAM3_ENABLED} + ffmpeg: + inputs: + - path: rtsp://127.0.0.1:8554/squirrel + input_args: preset-rtsp-restream + roles: + - detect + - record + detect: + enabled: true + width: 2688 + height: 1520 + fps: 5 + +############################################################################## +# HOW TO ADD A CAMERA WITH A MIC +# +# 1. Set audio.enabled: true at the top of this file. +# +# 2. In go2rtc.streams, add the audio transcode line: +# your_cam: +# - rtsp://{FRIGATE_RTSP_USER3}:{FRIGATE_RTSP_PASSWORD3}@{IP}:554/path#backchannel=0 +# - "ffmpeg:your_cam#audio=aac#audio=opus" +# +# 3. In cameras, add the 'audio' role and audio-aware record preset: +# your_cam: +# enabled: true +# ffmpeg: +# output_args: +# record: preset-record-generic-audio-aac +# inputs: +# - path: rtsp://127.0.0.1:8554/your_cam +# input_args: preset-rtsp-restream +# roles: +# - detect +# - record +# - audio +# +# 4. Add credentials to .env: +# FRIGATE_RTSP_USER3=admin +# FRIGATE_RTSP_PASSWORD3=yourpass +# +# 5. RTSP paths by vendor: +# Hikvision / Hikvision OEM: /Streaming/Channels/101 (main), /102 (sub) +# Dahua / Dahua OEM: /cam/realmonitor?channel=1&subtype=0 (main) +############################################################################## +FRIGCFGEOF + log_success "Frigate config.yml written" + + # ── frigate-notify config.yml ───────────────────────────────────────────── + log_info "Writing frigate-notify config..." + cat > "$DIR/frigate-notify/config.yml" << FNEOF +## frigate-notify config +## Docs: https://frigate-notify.0x2142.com +## Secrets come from .env via FN_* environment variables. + +frigate: + server: # FN_FRIGATE__SERVER + ignoressl: true + public_url: # FN_FRIGATE__PUBLIC_URL + + startup_check: + attempts: 5 + interval: 30 + + mqtt: + enabled: true + server: mosquitto + port: 1883 + clientid: frigate-notify + username: ${MQTT_USER:-frigate} + password: # FN_FRIGATE__MQTT__PASSWORD + topic_prefix: frigate + +alerts: + general: + title: 'Frigate - {{ if .SubLabel }}{{ .SubLabel }}{{ else }}{{ .Label }}{{ end }} at {{ .Camera }}' + nosnap: allow + recheck_delay: 10 + + ntfy: + enabled: true + server: # FN_ALERTS__NTFY__SERVER + topic: "${NTFY_TOPIC:-frigate}" + ignoressl: false + headers: + - X-Priority: '{{ if .SubLabel }}3{{ else }}4{{ end }}' + - X-Tags: '{{ if .SubLabel }}wave{{ else }}rotating_light{{ end }}' + template: | + {{ if .SubLabel -}}{{ .SubLabel }}{{ else }}{{ .Label }}{{ end }} at {{ .Camera }} + {{- if gt (len .CurrentZones) 0 }} + Zone: {{ range \$i, \$z := .CurrentZones }}{{ if \$i }}, {{ end }}{{ \$z }}{{ end }}{{ end }} + Score: {{ printf "%.0f" (mul .TopScore 100) }}% + Time: {{ .StartTime.Format "Mon 3:04 PM" }} + +monitor: + enabled: false + + discord: + enabled: false + gotify: + enabled: false + smtp: + enabled: false + telegram: + enabled: false + pushover: + enabled: false + webhook: + enabled: false +FNEOF + log_success "frigate-notify config.yml written" + + # ── Caddy snippet ────────────────────────────────────────────────────────── + if [ -n "$FRIGATE_PUBLIC_URL" ] && [ "$FRIGATE_PUBLIC_URL" != "https://cam.yourdomain.com" ]; then + local _DOM="${FRIGATE_PUBLIC_URL#https://}" + configure_caddy_for_service "Frigate" "8971" "frigate-audio" || true + fi + + ensure_docker_dir_ownership "$DIR" + + # ── Summary ──────────────────────────────────────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " Frigate Audio Stack — Setup Complete" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Directory: $DIR" + echo " Media: $FRIGATE_MEDIA_DIR" + echo " Public URL: $FRIGATE_PUBLIC_URL" + echo " MQTT user: ${MQTT_USER:-frigate}" + echo " ntfy server: $NTFY_SERVER topic: ${NTFY_TOPIC:-frigate}" + echo "" + echo " Before starting:" + echo " 1. Edit frigate_config/config.yml — adjust RTSP paths for your cameras" + echo " (paths vary by vendor; check your camera's manual)" + echo " 2. Edit frigate_config/config.yml — remove/adjust motion masks" + echo " (the masks are blanks — add yours via the Frigate UI after first run)" + echo " 3. Verify .env credentials are correct" + echo "" + echo " Start:" + echo " cd $DIR && docker compose up -d" + echo "" + echo " Face recognition training (after Frigate is running):" + echo " — Go to Frigate UI → Faces → add face photos for household members" + echo " → In frigate-notify/config.yml, add names to alerts.sublabels.block" + echo " to silence push notifications for recognized family members." + echo "" + + local START_NOW="" + prompt_yn "Start the stack now? (y/n) [n]:" "n" START_NOW + if [[ ${START_NOW:-n} =~ ^[Yy]$ ]]; then + log_info "Starting frigate-audio stack..." + if ( cd "$DIR" && docker compose up -d ); then + log_success "Stack started — Frigate UI: http://localhost:8971" + else + log_warning "Start failed — check: cd $DIR && docker compose logs" + fi + else + echo "" + log_info "When ready: cd $DIR && docker compose up -d" + fi + echo "" +} diff --git a/services/sky-cam.sh b/services/sky-cam.sh new file mode 100644 index 0000000..ef34630 --- /dev/null +++ b/services/sky-cam.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# services/sky-cam.sh — Automated sky / timelapse camera scripts. +# Part of the modular post-install system (sourced by setup.sh). +# +# NON-DOCKER module. sky-cam produces: +# • Daily sunrise clip — speed-adjusted video, uploaded to Mattermost +# • Four Seasons timelapse — daily clips sized to Vivaldi movements' music +# • Full-day timelapse — fixed-fps timelapse of every captured image +# • Moon-track timelapse — moon tracked & cropped each visible night +# • Monthly moon-phase close-ups — NASA Dial-a-Moon images posted to MM +# +# Source: https://github.com/outis1one/sky-cam (cloned via bootstrap.sh) +# Installs systemd user timers via sky-cam's install.sh. + +register_service sky-cam cameras "Automated sky / timelapse camera scripts (sky-cam)" + +install_sky-cam() { + local SKYCAM_DIR="$ACTUAL_HOME/sky-cam" + + if [ "$DRY_RUN" = true ]; then + echo "[DRY-RUN] sky-cam would:" + echo " - Install: ffmpeg bc fonts-dejavu curl python3-pip" + echo " - pip install: suntime pytz requests skyfield Pillow numpy scipy" + echo " - Clone sky-cam to $SKYCAM_DIR via bootstrap.sh" + echo " - Edit sky-cam.conf with your location and camera names" + echo " - Copy .env.example → .env and set Mattermost credentials" + echo " - Run ./install.sh to register systemd user timers" + return 0 + fi + + echo "" + echo "╔═══════════════════════════════════════════════════════╗" + echo "║ sky-cam — Automated Sky & Timelapse Camera System ║" + echo "╚═══════════════════════════════════════════════════════╝" + echo "" + + # ── System packages ────────────────────────────────────────────────────── + log_info "Installing system dependencies..." + run_cmd apt-get update -qq + run_cmd apt-get install -y --no-install-recommends \ + ffmpeg bc fonts-dejavu curl python3-pip git + log_success "System packages installed" + + # ── Python packages ────────────────────────────────────────────────────── + log_info "Installing Python packages..." + local PIP="pip3 install --user --quiet" + sudo -u "$ACTUAL_USER" $PIP suntime pytz requests skyfield Pillow numpy scipy \ + || log_warning "Some pip packages may have failed — check output above" + log_success "Python packages installed" + + # ── Clone sky-cam ──────────────────────────────────────────────────────── + if [ -d "$SKYCAM_DIR/.git" ]; then + log_info "sky-cam already cloned at $SKYCAM_DIR — pulling latest..." + sudo -u "$ACTUAL_USER" git -C "$SKYCAM_DIR" pull --ff-only \ + || log_warning "git pull failed — continuing with existing version" + else + log_info "Cloning sky-cam from GitHub..." + sudo -u "$ACTUAL_USER" bash -c " + curl -fsSL https://raw.githubusercontent.com/outis1one/sky-cam/main/bootstrap.sh \ + | bash -s -- '$SKYCAM_DIR' + " || { log_error "Failed to clone sky-cam — check internet connection"; return 1; } + log_success "sky-cam cloned to $SKYCAM_DIR" + fi + + # ── Essential configuration ────────────────────────────────────────────── + echo "" + log_info "Location and camera configuration" + echo " sky-cam needs your GPS coordinates and timezone to calculate" + echo " sunrise times accurately. Use decimal degrees (e.g. 40.7128, -74.0060)." + echo "" + + local LATITUDE="" LONGITUDE="" TIMEZONE="" BASE_DIR="" CAMERAS_LIST="" SUNRISE_CAM="" + prompt_text " Latitude (decimal degrees) [0.0000]:" "0.0000" LATITUDE + prompt_text " Longitude (decimal degrees) [0.0000]:" "0.0000" LONGITUDE + prompt_text " Timezone [America/New_York]:" "America/New_York" TIMEZONE + + echo "" + echo " Camera names are short identifiers, e.g.: east north south west" + echo " These names must match the directories where your camera images are saved." + echo "" + prompt_text " Camera names (space-separated) [east]:" "east" CAMERAS_LIST + prompt_text " Sunrise camera (faces east) [east]:" "east" SUNRISE_CAM + + echo "" + echo " BASE_DIR is where your camera images live." + echo " Each camera should have a sub-folder: BASE_DIR//" + local DEFAULT_BASE="$ACTUAL_HOME/sky-cam/data" + prompt_text " Image base directory [$DEFAULT_BASE]:" "$DEFAULT_BASE" BASE_DIR + [ -z "$BASE_DIR" ] && BASE_DIR="$DEFAULT_BASE" + + # Prompt for Mattermost credentials + echo "" + log_info "Mattermost webhook (for automated uploads)" + echo " sky-cam posts sunrise clips and moon photos to a Mattermost channel." + echo " Create an incoming webhook in Mattermost: Settings → Integrations → Webhooks" + echo " (Leave blank to skip — add to $SKYCAM_DIR/.env later)" + echo "" + local MM_WEBHOOK="" MM_CHANNEL="" + if [ "$UNATTENDED" != true ]; then + read -p " Mattermost webhook URL [Enter to skip]: " MM_WEBHOOK + if [ -n "$MM_WEBHOOK" ]; then + prompt_text " Mattermost channel name [sky-cam]:" "sky-cam" MM_CHANNEL + fi + fi + + # ── Write .env ─────────────────────────────────────────────────────────── + log_info "Writing sky-cam.conf overrides to $SKYCAM_DIR/.env..." + { + echo "# sky-cam site configuration — generated by ubuntu-post-install" + echo "# Edit sky-cam.conf for full settings." + echo "" + echo "LATITUDE=${LATITUDE:-0.0000}" + echo "LONGITUDE=${LONGITUDE:-0.0000}" + echo "TIMEZONE=${TIMEZONE:-America/New_York}" + echo "BASE_DIR=${BASE_DIR}" + if [ -n "$MM_WEBHOOK" ]; then + echo "MM_WEBHOOK_URL=${MM_WEBHOOK}" + echo "MM_CHANNEL=${MM_CHANNEL:-sky-cam}" + else + echo "# MM_WEBHOOK_URL=https://mattermost.yourdomain.com/hooks/your-webhook-id" + echo "# MM_CHANNEL=sky-cam" + fi + } > "$SKYCAM_DIR/.env" + chmod 600 "$SKYCAM_DIR/.env" + + # ── Patch sky-cam.conf with cameras and basic settings ─────────────────── + local CONF="$SKYCAM_DIR/sky-cam.conf" + if [ -f "$CONF" ]; then + log_info "Patching sky-cam.conf with location and camera names..." + # Build CAMERAS=(...) line + local CAM_ARRAY="(${CAMERAS_LIST})" + sed -i "s|^CAMERAS=.*|CAMERAS=${CAM_ARRAY}|" "$CONF" + sed -i "s|^SUNRISE_CAM=.*|SUNRISE_CAM=${SUNRISE_CAM:-east}|" "$CONF" + log_success "sky-cam.conf updated" + else + log_warning "sky-cam.conf not found — check $SKYCAM_DIR" + fi + + # ── Create image directories ───────────────────────────────────────────── + mkdir -p "$BASE_DIR" + for _cam in $CAMERAS_LIST; do + mkdir -p "$BASE_DIR/$_cam" + done + chown -R "$ACTUAL_USER:$ACTUAL_USER" "$SKYCAM_DIR" "$BASE_DIR" 2>/dev/null || true + log_success "Image directories created under $BASE_DIR" + + # ── Install systemd timers ──────────────────────────────────────────────── + if [ -f "$SKYCAM_DIR/install.sh" ]; then + log_info "Installing systemd user timers via install.sh..." + ( cd "$SKYCAM_DIR" && sudo -u "$ACTUAL_USER" bash install.sh ) \ + && log_success "Systemd timers installed" \ + || log_warning "install.sh failed — run manually: cd $SKYCAM_DIR && ./install.sh" + else + log_warning "install.sh not found in $SKYCAM_DIR — run it manually after review" + fi + + # ── Summary ─────────────────────────────────────────────────────────────── + echo "" + echo "═══════════════════════════════════════════════════════" + echo " sky-cam installed" + echo "═══════════════════════════════════════════════════════" + echo "" + echo " Location: $SKYCAM_DIR" + echo " Data: $BASE_DIR" + echo " Cameras: $CAMERAS_LIST" + echo " Timezone: ${TIMEZONE:-America/New_York}" + echo "" + echo " Next steps:" + echo " 1. Review $SKYCAM_DIR/sky-cam.conf" + echo " — SCRIPT_DIR, MUSIC_DIR, schedules, encoding settings" + echo " 2. Put your Vivaldi Four Seasons audio files in:" + echo " $SKYCAM_DIR/music/" + echo " (filenames and expected MUSIC_DIR path are in sky-cam.conf)" + echo " 3. Verify systemd timers:" + echo " systemctl --user list-timers 'sky-cam-*'" + echo " 4. Credentials → $SKYCAM_DIR/.env" + echo "" + echo " To test the sunrise script manually:" + echo " cd $SKYCAM_DIR && ./daily_sunrise_video.sh" + echo "" + echo " Logs:" + echo " journalctl --user -u sky-cam-sunrise.service -f" + echo "" +} diff --git a/setup.sh b/setup.sh index d359670..b57c78e 100755 --- a/setup.sh +++ b/setup.sh @@ -82,6 +82,7 @@ is_installed() { silent-send) [ -d "$ACTUAL_HOME/silent-send/.git" ] ;; linux-to-sync) [ -d "$ACTUAL_HOME/linux-to-sync/.git" ] ;; sync-cc) [ -f "$ACTUAL_HOME/sync-cc/sync_cc.py" ] ;; + sky-cam) [ -d "$ACTUAL_HOME/sky-cam/.git" ] ;; *) [ -e "$DOCKER_DIR/$1" ] ;; esac } From cfad0ebd20ac37120f95a3e4794706158aa66125 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 21:41:54 +0000 Subject: [PATCH 05/12] fix(linux-to-sync): run git clone as actual user, handle re-run, show errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs fixed: 1. git clone ran as root — SSH key lookup failed because user's keys are in ~user/.ssh/, not /root/.ssh/. Both SSH and PAT clones now run via sudo -u ACTUAL_USER so git uses the right key and the clone is owned correctly without a separate chown step. 2. 2>/dev/null on git clone swallowed all error output — errors are now shown so authentication failures are diagnosable. 3. No re-run handling — trying to clone into an existing dir silently failed. Now detects .git, offers git pull instead. Also: checks for SSH key existence before attempting SSH clone, and prints actionable guidance (ssh-keygen, ssh-add, ssh -T git@github.com) on failure. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- services/linux-to-sync.sh | 70 +++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/services/linux-to-sync.sh b/services/linux-to-sync.sh index 7b8cf9e..9879964 100644 --- a/services/linux-to-sync.sh +++ b/services/linux-to-sync.sh @@ -18,6 +18,21 @@ install_linux-to-sync() { return 0 fi + # ── Re-run: already cloned → offer pull ────────────────────────────────── + if [ -d "$SYNC_DIR/.git" ]; then + log_info "linux-to-sync already cloned at $SYNC_DIR" + local DO_PULL="" + prompt_yn "Pull latest changes? (y/n) [y]:" "y" DO_PULL + if [[ ${DO_PULL:-y} =~ ^[Yy]$ ]]; then + if sudo -u "$ACTUAL_USER" git -C "$SYNC_DIR" pull; then + log_success "linux-to-sync updated" + else + log_warning "git pull failed — check connectivity and credentials" + fi + fi + return 0 + fi + echo "" echo " Requires access to github.com/outis1one/linux-to-sync" echo " Authenticate with ONE of:" @@ -41,31 +56,58 @@ install_linux-to-sync() { return 0 fi - if git clone "https://$GH_TOKEN@github.com/outis1one/linux-to-sync.git" "$SYNC_DIR" 2>/dev/null; then - cd "$SYNC_DIR" || return 1 + log_info "Cloning via HTTPS + PAT..." + if sudo -u "$ACTUAL_USER" \ + git clone "https://$GH_TOKEN@github.com/outis1one/linux-to-sync.git" "$SYNC_DIR"; then # Remove token from remote URL so it isn't stored in plain text - git remote set-url origin "https://github.com/outis1one/linux-to-sync.git" - chown -R "$ACTUAL_USER:$ACTUAL_USER" "$SYNC_DIR" + sudo -u "$ACTUAL_USER" git -C "$SYNC_DIR" remote set-url origin \ + "https://github.com/outis1one/linux-to-sync.git" log_success "linux-to-sync cloned to $SYNC_DIR" - echo " Note: re-enter your token for future push/pull, or:" - echo " git config credential.helper store" + echo " Token stripped from remote URL. For future pulls use:" + echo " git -C $SYNC_DIR pull (will prompt for credentials)" + echo " Or set up a credential helper:" + echo " git config --global credential.helper store" else - log_error "Clone failed — check your token and try again." + log_error "Clone failed — check your PAT and network, then retry." return 1 fi else + # SSH auth — git must run as the actual user to use their SSH keys. echo "" - echo " Attempting SSH clone (your SSH key must be added to GitHub)..." - if git clone git@github.com:outis1one/linux-to-sync.git "$SYNC_DIR" 2>/dev/null; then - chown -R "$ACTUAL_USER:$ACTUAL_USER" "$SYNC_DIR" + echo " Checking for SSH key in $ACTUAL_HOME/.ssh/ ..." + local SSH_KEY_FOUND=false + for _k in id_ed25519 id_rsa id_ecdsa; do + if [ -f "$ACTUAL_HOME/.ssh/$_k" ]; then + log_info " Found: $ACTUAL_HOME/.ssh/$_k" + SSH_KEY_FOUND=true + break + fi + done + if [ "$SSH_KEY_FOUND" = false ]; then + log_warning "No SSH key found in $ACTUAL_HOME/.ssh/" + echo "" + echo " To generate one:" + echo " ssh-keygen -t ed25519 -C 'your@email.com'" + echo " cat $ACTUAL_HOME/.ssh/id_ed25519.pub" + echo " → Add the public key at: github.com/settings/keys" + echo "" + local CONTINUE="" + prompt_yn "Continue anyway (will fail if no key on GitHub)? (y/n) [n]:" "n" CONTINUE + [[ ${CONTINUE:-n} =~ ^[Yy]$ ]] || return 0 + fi + + log_info "Cloning via SSH (running as $ACTUAL_USER)..." + if sudo -u "$ACTUAL_USER" \ + git clone git@github.com:outis1one/linux-to-sync.git "$SYNC_DIR"; then log_success "linux-to-sync cloned to $SYNC_DIR" else log_error "SSH clone failed." echo "" - echo " To add your SSH key to GitHub:" - echo " 1. cat ~/.ssh/id_rsa.pub (or id_ed25519.pub)" - echo " 2. github.com/settings/keys → New SSH key → paste" - echo " Then retry: sudo ./setup.sh linux-to-sync" + echo " Common causes:" + echo " • SSH key not added to GitHub — go to github.com/settings/keys" + echo " • Key not accepted by ssh-agent — try: ssh-add $ACTUAL_HOME/.ssh/id_ed25519" + echo " • Test with: sudo -u $ACTUAL_USER ssh -T git@github.com" + echo " Then retry: sudo ./setup.sh linux-to-sync" return 1 fi fi From 56d2f9e85bc40dcaea2daa71c2fdbed76315ff8e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 22:13:59 +0000 Subject: [PATCH 06/12] Add site-wide defaults: timezone, domain, Caddy network Introduces a one-time configuration wizard (sudo ./setup.sh configure) that stores SITE_TZ, SITE_DOMAIN, and SITE_CADDY_NET in ~/docker/.config. Every service now uses these as prompt defaults so the user types common values once instead of re-answering the same questions for each service. - lib/common.sh: load_site_config / save_site_config; auto-loads on source; backward-compat BASE_DOMAIN alias kept for old .config files - setup.sh: run_site_configure wizard; first-run offer after base install; `sudo ./setup.sh configure` command to update defaults at any time - 14 services: TZ_VAL now honours SITE_TZ, falling back to /etc/timezone - 3 inline-heredoc services (filebrowser, homeassistant, ntfy): same fix - authelia: SITE_TZ/SITE_DOMAIN as prompt defaults; SITE_CADDY_NET replaces hardcoded caddy_net throughout (env, compose patch, network creation) - minecraft, frigate-audio: simplify BASE_DOMAIN read to use SITE_DOMAIN - sky-cam: SITE_TZ as default for timezone prompt https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- lib/common.sh | 49 ++++++++++++++++++++++++++++++++++++++ services/actualbudget.sh | 2 +- services/arm.sh | 2 +- services/audiobookshelf.sh | 2 +- services/authelia.sh | 18 +++++++------- services/ddclient.sh | 2 +- services/emby.sh | 2 +- services/filebrowser.sh | 2 +- services/frigate-audio.sh | 3 +-- services/frigate.sh | 2 +- services/homeassistant.sh | 2 +- services/immich.sh | 2 +- services/jellyfin.sh | 2 +- services/lyrion.sh | 2 +- services/magicmirror.sh | 2 +- services/mealie.sh | 2 +- services/minecraft.sh | 3 +-- services/ntfy.sh | 2 +- services/sky-cam.sh | 2 +- setup.sh | 47 ++++++++++++++++++++++++++++++++++-- 20 files changed, 121 insertions(+), 29 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index 9f24c6c..a80ad63 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -47,6 +47,55 @@ register_service() { SERVICE_ORDER+=("$name") } +# ── Site-wide defaults ──────────────────────────────────────────────────────── +# Stored in $DOCKER_DIR/.config (key=value, one per line). +# Service modules read these as prompt defaults so the user only types +# timezone, domain, and Caddy network once. Run: sudo ./setup.sh configure +SITE_TZ="" +SITE_DOMAIN="" +SITE_CADDY_NET="caddy_net" +SITE_PUID="" +SITE_PGID="" + +load_site_config() { + local cfg="$DOCKER_DIR/.config" + [ -f "$cfg" ] || return 0 + local key val + while IFS='=' read -r key val; do + [[ "$key" =~ ^[[:space:]]*# ]] && continue + [[ -z "${key// }" ]] && continue + case "$key" in + SITE_TZ) SITE_TZ="$val" ;; + SITE_DOMAIN) SITE_DOMAIN="$val" ;; + SITE_CADDY_NET) SITE_CADDY_NET="$val" ;; + SITE_PUID) SITE_PUID="$val" ;; + SITE_PGID) SITE_PGID="$val" ;; + BASE_DOMAIN) [ -z "$SITE_DOMAIN" ] && SITE_DOMAIN="$val" ;; + esac + done < "$cfg" + export SITE_TZ SITE_DOMAIN SITE_CADDY_NET SITE_PUID SITE_PGID +} + +save_site_config() { + local cfg="$DOCKER_DIR/.config" + mkdir -p "$(dirname "$cfg")" + { + echo "# ubuntu-post-install site defaults" + echo "# Re-run wizard: sudo ./setup.sh configure" + [ -n "$SITE_TZ" ] && echo "SITE_TZ=$SITE_TZ" + [ -n "$SITE_DOMAIN" ] && echo "SITE_DOMAIN=$SITE_DOMAIN" + [ -n "$SITE_CADDY_NET" ] && echo "SITE_CADDY_NET=$SITE_CADDY_NET" + [ -n "$SITE_PUID" ] && echo "SITE_PUID=$SITE_PUID" + [ -n "$SITE_PGID" ] && echo "SITE_PGID=$SITE_PGID" + # Backward-compat alias for services that still read BASE_DOMAIN directly + [ -n "$SITE_DOMAIN" ] && echo "BASE_DOMAIN=$SITE_DOMAIN" + } > "$cfg" + chmod 600 "$cfg" +} + +# Load immediately so all service modules inherit the values when sourced +load_site_config + # ── Pre-flight ─────────────────────────────────────────────────────────────── require_root() { if [ "${EUID:-$(id -u)}" -ne 0 ]; then diff --git a/services/actualbudget.sh b/services/actualbudget.sh index 8fa71f9..7da0e9c 100644 --- a/services/actualbudget.sh +++ b/services/actualbudget.sh @@ -24,7 +24,7 @@ install_actualbudget() { ensure_docker_dir_ownership "$AB_DIR" cd "$AB_DIR" || return 1 - local TZ_VAL; TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + local TZ_VAL; TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" cat > docker-compose.yml << 'AB_COMPOSE' name: actualbudget diff --git a/services/arm.sh b/services/arm.sh index 3d84efa..7f5dd19 100644 --- a/services/arm.sh +++ b/services/arm.sh @@ -45,7 +45,7 @@ install_arm() { cd "$ARM_DIR" || return 1 local TZ_VAL UID_VAL GID_VAL - TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" UID_VAL=$(id -u "$ACTUAL_USER"); GID_VAL=$(id -g "$ACTUAL_USER") cat > docker-compose.yml << ARM_COMPOSE diff --git a/services/audiobookshelf.sh b/services/audiobookshelf.sh index 6149f6f..c230602 100644 --- a/services/audiobookshelf.sh +++ b/services/audiobookshelf.sh @@ -30,7 +30,7 @@ install_audiobookshelf() { ensure_docker_dir_ownership "$ABS_DIR" cd "$ABS_DIR" || return 1 - local TZ_VAL; TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + local TZ_VAL; TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" cat > docker-compose.yml << ABS_COMPOSE name: audiobookshelf diff --git a/services/authelia.sh b/services/authelia.sh index 9a57e56..b11e138 100644 --- a/services/authelia.sh +++ b/services/authelia.sh @@ -36,9 +36,10 @@ install_authelia() { echo "" echo " Authelia needs a few details to configure." echo "" + local CADDY_NET="${SITE_CADDY_NET:-caddy_net}" local AUTHELIA_DOMAIN AUTHELIA_ADMIN_USER AUTHELIA_ADMIN_DISPLAY AUTHELIA_ADMIN_EMAIL local AUTHELIA_SMTP_HOST AUTHELIA_SMTP_PORT AUTHELIA_SMTP_USER AUTHELIA_SMTP_PASS AUTHELIA_TZ - prompt_text " Your domain (e.g., example.com):" "example.com" AUTHELIA_DOMAIN + prompt_text " Your domain (e.g., example.com):" "${SITE_DOMAIN:-example.com}" AUTHELIA_DOMAIN prompt_text " Admin username:" "admin" AUTHELIA_ADMIN_USER prompt_text " Admin display name:" "Administrator" AUTHELIA_ADMIN_DISPLAY prompt_text " Admin email:" "admin@${AUTHELIA_DOMAIN}" AUTHELIA_ADMIN_EMAIL @@ -46,7 +47,7 @@ install_authelia() { prompt_text " SMTP port:" "587" AUTHELIA_SMTP_PORT prompt_text " SMTP username (full email):" "authelia@${AUTHELIA_DOMAIN}" AUTHELIA_SMTP_USER prompt_text " SMTP password:" "" AUTHELIA_SMTP_PASS - prompt_text " Timezone (e.g., America/New_York):" "America/New_York" AUTHELIA_TZ + prompt_text " Timezone (e.g., America/New_York):" "${SITE_TZ:-America/New_York}" AUTHELIA_TZ # ── Secrets ────────────────────────────────────────────────────────────── echo "" @@ -81,7 +82,7 @@ install_authelia() { cat > .env << AUTHELIA_ENV MY_DOMAIN=${AUTHELIA_DOMAIN} SMTP_USER=${AUTHELIA_SMTP_USER} -DOCKER_MY_NETWORK=caddy_net +DOCKER_MY_NETWORK=${CADDY_NET} TZ=${AUTHELIA_TZ} AUTHELIA_ENV @@ -115,6 +116,7 @@ networks: caddy_net: external: true AUTHELIA_COMPOSE + [ "$CADDY_NET" != "caddy_net" ] && sed -i "s/caddy_net/${CADDY_NET}/g" docker-compose.yml # ── configuration.yml ──────────────────────────────────────────────────── cat > config/configuration.yml << AUTHELIA_CONFIG @@ -199,12 +201,12 @@ AUTHELIA_USERS chown -R 1000:1000 "$AUTHELIA_DIR/config" "$AUTHELIA_DIR/data" log_success "Authelia configured at $AUTHELIA_DIR" - # ── caddy_net network ──────────────────────────────────────────────────── - if ! docker network ls --format '{{.Name}}' | grep -q "^caddy_net$"; then - docker network create caddy_net >/dev/null 2>&1 && echo " ✓ Created docker network caddy_net" \ - || echo " ⚠ Failed to create caddy_net" + # ── Docker network ──────────────────────────────────────────────────────── + if ! docker network ls --format '{{.Name}}' | grep -q "^${CADDY_NET}$"; then + docker network create "$CADDY_NET" >/dev/null 2>&1 && echo " ✓ Created docker network ${CADDY_NET}" \ + || echo " ⚠ Failed to create ${CADDY_NET}" else - echo " ✓ Docker network caddy_net already exists" + echo " ✓ Docker network ${CADDY_NET} already exists" fi # ── Caddyfile forward-auth snippet + portal block ──────────────────────── diff --git a/services/ddclient.sh b/services/ddclient.sh index 49b9bd9..3d8aef5 100644 --- a/services/ddclient.sh +++ b/services/ddclient.sh @@ -25,7 +25,7 @@ install_ddclient() { ensure_docker_dir_ownership "$DDCLIENT_DIR" cd "$DDCLIENT_DIR" || return 1 - local TZ_VAL; TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + local TZ_VAL; TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" cat > docker-compose.yml << 'DDCLIENT_COMPOSE' name: ddclient diff --git a/services/emby.sh b/services/emby.sh index 280c31a..e8edbdc 100644 --- a/services/emby.sh +++ b/services/emby.sh @@ -34,7 +34,7 @@ install_emby() { cd "$EMBY_DIR" || return 1 local TZ_VAL UID_VAL GID_VAL - TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" UID_VAL=$(id -u "$ACTUAL_USER"); GID_VAL=$(id -g "$ACTUAL_USER") cat > docker-compose.yml << EMBY_COMPOSE diff --git a/services/filebrowser.sh b/services/filebrowser.sh index 0de8c4a..13ef4c3 100644 --- a/services/filebrowser.sh +++ b/services/filebrowser.sh @@ -33,7 +33,7 @@ services: environment: - PUID=$(id -u "$ACTUAL_USER") - PGID=$(id -g "$ACTUAL_USER") - - TZ=$(cat /etc/timezone 2>/dev/null || echo "UTC") + - TZ=${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)} volumes: - ${FB_PATH}:/srv - ./database/filebrowser.db:/database/filebrowser.db diff --git a/services/frigate-audio.sh b/services/frigate-audio.sh index 9406975..4cad85f 100644 --- a/services/frigate-audio.sh +++ b/services/frigate-audio.sh @@ -98,8 +98,7 @@ install_frigate-audio() { # ── Frigate public URL ───────────────────────────────────────────────────── echo "" - local BASE_DOMAIN="" - [ -f "$DOCKER_DIR/.config" ] && BASE_DOMAIN=$(grep '^BASE_DOMAIN=' "$DOCKER_DIR/.config" 2>/dev/null | cut -d= -f2-) + local BASE_DOMAIN="${SITE_DOMAIN:-}" local FRIGATE_PUBLIC_URL="" if [ -n "$BASE_DOMAIN" ]; then local _PFX="" diff --git a/services/frigate.sh b/services/frigate.sh index 836961b..98557d9 100644 --- a/services/frigate.sh +++ b/services/frigate.sh @@ -33,7 +33,7 @@ install_frigate() { ensure_docker_dir_ownership "$FRIGATE_DIR" cd "$FRIGATE_DIR" || return 1 - local TZ_VAL; TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + local TZ_VAL; TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" # Hardware detection: include /dev/dri only when a render node exists local DEVICE_BLOCK="" diff --git a/services/homeassistant.sh b/services/homeassistant.sh index 624ae25..060daf9 100644 --- a/services/homeassistant.sh +++ b/services/homeassistant.sh @@ -50,7 +50,7 @@ services: restart: unless-stopped privileged: true environment: - - TZ=$(cat /etc/timezone 2>/dev/null || echo "UTC") + - TZ=${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)} volumes: - ./config:/config - /run/dbus:/run/dbus:ro diff --git a/services/immich.sh b/services/immich.sh index 0bce477..80b60c3 100644 --- a/services/immich.sh +++ b/services/immich.sh @@ -112,7 +112,7 @@ install_immich() { # ── Generate DB password ──────────────────────────────────────────────── local DB_PASS TZ_VAL DB_PASS=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 32) - TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" # ── Write docker-compose.yml ──────────────────────────────────────────── if [ -n "$EXTERNAL_LIBRARY" ]; then diff --git a/services/jellyfin.sh b/services/jellyfin.sh index 8211150..a1d4d67 100644 --- a/services/jellyfin.sh +++ b/services/jellyfin.sh @@ -33,7 +33,7 @@ install_jellyfin() { ensure_docker_dir_ownership "$JELLYFIN_DIR" cd "$JELLYFIN_DIR" || return 1 - local TZ_VAL; TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + local TZ_VAL; TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" # Hardware acceleration: only wire /dev/dri through if a render node exists, # otherwise the container would fail to start on a GPU-less host. diff --git a/services/lyrion.sh b/services/lyrion.sh index ecdea8c..1844009 100644 --- a/services/lyrion.sh +++ b/services/lyrion.sh @@ -33,7 +33,7 @@ install_lyrion() { cd "$LYRION_DIR" || return 1 local TZ_VAL UID_VAL GID_VAL - TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" UID_VAL=$(id -u "$ACTUAL_USER"); GID_VAL=$(id -g "$ACTUAL_USER") cat > docker-compose.yml << LYRION_COMPOSE diff --git a/services/magicmirror.sh b/services/magicmirror.sh index e9d0851..e0bb4e5 100644 --- a/services/magicmirror.sh +++ b/services/magicmirror.sh @@ -33,7 +33,7 @@ install_magicmirror() { mkdir -p "$MM_BASE" chown "$ACTUAL_USER:$ACTUAL_USER" "$MM_BASE" - local TZ_VAL; TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + local TZ_VAL; TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" local i MM_PORT MM_DIR for i in $(seq 1 "$MM_COUNT"); do diff --git a/services/mealie.sh b/services/mealie.sh index 1db9eb3..7f07d28 100644 --- a/services/mealie.sh +++ b/services/mealie.sh @@ -26,7 +26,7 @@ install_mealie() { cd "$MEALIE_DIR" || return 1 local TZ_VAL UID_VAL GID_VAL - TZ_VAL=$(cat /etc/timezone 2>/dev/null || echo "UTC") + TZ_VAL="${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)}" UID_VAL=$(id -u "$ACTUAL_USER"); GID_VAL=$(id -g "$ACTUAL_USER") cat > docker-compose.yml << MEALIE_COMPOSE diff --git a/services/minecraft.sh b/services/minecraft.sh index ebd90b8..93a6daa 100644 --- a/services/minecraft.sh +++ b/services/minecraft.sh @@ -953,8 +953,7 @@ except: pass esac local MC_DOMAIN="" - local BASE_DOMAIN="" - [ -f "$DOCKER_DIR/.config" ] && BASE_DOMAIN=$(grep '^BASE_DOMAIN=' "$DOCKER_DIR/.config" 2>/dev/null | cut -d= -f2-) + local BASE_DOMAIN="${SITE_DOMAIN:-}" if [ "$USE_PLAYIT" = true ] || [ "$USE_PORTFORWARD" = true ]; then if [ -n "$BASE_DOMAIN" ]; then local _PREFIX="" diff --git a/services/ntfy.sh b/services/ntfy.sh index e5ecd46..fb004d4 100644 --- a/services/ntfy.sh +++ b/services/ntfy.sh @@ -38,7 +38,7 @@ services: NTFY_COMPOSE cat > .env << NTFY_ENV -TZ=$(cat /etc/timezone 2>/dev/null || echo "UTC") +TZ=${SITE_TZ:-$(cat /etc/timezone 2>/dev/null || echo UTC)} NTFY_ENV mkdir -p cache config diff --git a/services/sky-cam.sh b/services/sky-cam.sh index ef34630..985c0cd 100644 --- a/services/sky-cam.sh +++ b/services/sky-cam.sh @@ -72,7 +72,7 @@ install_sky-cam() { local LATITUDE="" LONGITUDE="" TIMEZONE="" BASE_DIR="" CAMERAS_LIST="" SUNRISE_CAM="" prompt_text " Latitude (decimal degrees) [0.0000]:" "0.0000" LATITUDE prompt_text " Longitude (decimal degrees) [0.0000]:" "0.0000" LONGITUDE - prompt_text " Timezone [America/New_York]:" "America/New_York" TIMEZONE + prompt_text " Timezone [${SITE_TZ:-America/New_York}]:" "${SITE_TZ:-America/New_York}" TIMEZONE echo "" echo " Camera names are short identifiers, e.g.: east north south west" diff --git a/setup.sh b/setup.sh index b57c78e..eaca02f 100755 --- a/setup.sh +++ b/setup.sh @@ -106,9 +106,42 @@ list_services() { echo "" } +# ── Site defaults wizard ────────────────────────────────────────────────────── +# Prompts for timezone, base domain, and Caddy network name; saves to .config. +# Run directly: sudo ./setup.sh configure +run_site_configure() { + local _sys_tz; _sys_tz=$(cat /etc/timezone 2>/dev/null || echo "UTC") + echo "" + echo "╔══════════════════════════════════════════════════════════════╗" + echo "║ Site defaults · pre-filled into every service prompt ║" + echo "╚══════════════════════════════════════════════════════════════╝" + echo "" + echo " These become the default answer each time a service asks for" + echo " timezone, domain, etc. Press Enter to keep the shown value." + echo "" + local _cur_tz="${SITE_TZ:-$_sys_tz}" + local _cur_dom="${SITE_DOMAIN:-}" + local _cur_net="${SITE_CADDY_NET:-caddy_net}" + prompt_text " Timezone [${_cur_tz}]:" "$_cur_tz" SITE_TZ + prompt_text " Base domain (e.g., example.com) [${_cur_dom:-}]:" "$_cur_dom" SITE_DOMAIN + prompt_text " Caddy Docker network [${_cur_net}]:" "$_cur_net" SITE_CADDY_NET + export SITE_TZ SITE_DOMAIN SITE_CADDY_NET + mkdir -p "$DOCKER_DIR" + save_site_config + log_success "Saved to $DOCKER_DIR/.config" + echo "" +} + # ── --list ─────────────────────────────────────────────────────────────────── if [ "$DO_LIST" = true ]; then list_services; exit 0; fi +# ── configure: show/update site-wide defaults ──────────────────────────────── +if [ "${REQUESTED[*]:-}" = "configure" ]; then + require_root + run_site_configure + exit 0 +fi + # ── Direct install: ./setup.sh caddy homeassistant ────────────────────────── if [ "${#REQUESTED[@]}" -gt 0 ]; then require_root @@ -146,7 +179,17 @@ if ! command -v docker >/dev/null 2>&1; then echo " Install with: curl -fsSL https://get.docker.com | sh" fi -# 3) Offer Caddy first (most services proxy through it). +# 3) Offer site defaults wizard if .config has no SITE_TZ yet (first run). +if ! grep -q '^SITE_TZ=' "$DOCKER_DIR/.config" 2>/dev/null; then + echo "" + echo " No site defaults found. Setting them now pre-fills timezone, domain," + echo " and Caddy network for every service — you type them once, not every time." + OFFER_CONFIG="" + prompt_yn "Configure site defaults now? (y/n):" "y" OFFER_CONFIG + [ "$OFFER_CONFIG" = "y" ] || [ "$OFFER_CONFIG" = "Y" ] && run_site_configure +fi + +# 4) Offer Caddy first (most services proxy through it). if [ -n "${SERVICE_GROUP[caddy]:-}" ] && ! is_installed caddy; then echo "" OFFER_CADDY="" @@ -154,7 +197,7 @@ if [ -n "${SERVICE_GROUP[caddy]:-}" ] && ! is_installed caddy; then [ "$OFFER_CADDY" = "y" ] || [ "$OFFER_CADDY" = "Y" ] && run_service caddy fi -# 4) Category menu loop: pick a category → checklist → install → back to menu. +# 5) Category menu loop: pick a category → checklist → install → back to menu. have_whiptail=false command -v whiptail >/dev/null 2>&1 && have_whiptail=true From 370c5136061de980498c93dd9d95ba8001f09f80 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 23:48:15 +0000 Subject: [PATCH 07/12] Skip required-packages step on re-run On second run, is_installed base (command -v ncdu) detects that base packages are already present and jumps straight to the service menu, skipping the required-setup banner, confirm prompt, and apt-get install. The first-run path is unchanged; `sudo ./setup.sh base` forces reinstall. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- setup.sh | 61 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/setup.sh b/setup.sh index eaca02f..a4e83c1 100755 --- a/setup.sh +++ b/setup.sh @@ -152,31 +152,44 @@ fi # ── Guided interactive flow ────────────────────────────────────────────────── require_root -# 1) Show the REQUIRED set and let the user cancel before anything happens. -echo "" -echo "╔══════════════════════════════════════════════════════════════╗" -echo "║ Ubuntu Post-Install · v$(cat "$HERE/VERSION" 2>/dev/null || echo '?')" -echo "╚══════════════════════════════════════════════════════════════╝" -echo "" -echo "REQUIRED (installed/verified first):" -echo " • Essential CLI packages: net-tools, git, curl, wget, htop, tree," -echo " ncdu, zip/unzip, jq, rsync, and glow (markdown reader)" -echo " • Docker presence check (needed by all containerized services)" -echo "" -echo "Then you'll get a category menu to pick optional services." -echo "" -PROCEED="" -prompt_yn "Proceed with the required setup? (y/n):" "y" PROCEED -if [ "$PROCEED" != "y" ] && [ "$PROCEED" != "Y" ]; then - echo "Cancelled. Nothing was changed." - exit 0 -fi +_VER="$(cat "$HERE/VERSION" 2>/dev/null || echo '?')" -# 2) Run required. -run_service base -if ! command -v docker >/dev/null 2>&1; then - log_warning "Docker is not installed. Containerized services need it." - echo " Install with: curl -fsSL https://get.docker.com | sh" +if is_installed base; then + # ── Re-run: base already present — skip required step ──────────────────── + echo "" + echo "╔══════════════════════════════════════════════════════════════╗" + echo "║ Ubuntu Post-Install · v${_VER}" + echo "╚══════════════════════════════════════════════════════════════╝" + echo "" + echo " Base packages already installed — skipping required setup." + echo " Use 'sudo ./setup.sh base' to force a reinstall." + echo "" +else + # ── First run: show required banner, confirm, install ──────────────────── + echo "" + echo "╔══════════════════════════════════════════════════════════════╗" + echo "║ Ubuntu Post-Install · v${_VER}" + echo "╚══════════════════════════════════════════════════════════════╝" + echo "" + echo "REQUIRED (installed/verified first):" + echo " • Essential CLI packages: net-tools, git, curl, wget, htop, tree," + echo " ncdu, zip/unzip, jq, rsync, and glow (markdown reader)" + echo " • Docker presence check (needed by all containerized services)" + echo "" + echo "Then you'll get a category menu to pick optional services." + echo "" + PROCEED="" + prompt_yn "Proceed with the required setup? (y/n):" "y" PROCEED + if [ "$PROCEED" != "y" ] && [ "$PROCEED" != "Y" ]; then + echo "Cancelled. Nothing was changed." + exit 0 + fi + + run_service base + if ! command -v docker >/dev/null 2>&1; then + log_warning "Docker is not installed. Containerized services need it." + echo " Install with: curl -fsSL https://get.docker.com | sh" + fi fi # 3) Offer site defaults wizard if .config has no SITE_TZ yet (first run). From ef08fef5402039fc0563fadd02abd72f78f3ed72 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 23:49:51 +0000 Subject: [PATCH 08/12] Add OS detection; surface version in header; centralise pip installs lib/common.sh: - detect_os(): reads /etc/os-release into OS_DISTRO, OS_VERSION, OS_CODENAME globals (exported, auto-called on source) - ubuntu_version_ge(): numeric version comparison helper - pip_user_install(): central wrapper for pip3 install --user so any future version-specific flags are in one place setup.sh: - Both header banners now show detected OS line (e.g., "Ubuntu 24.04 (noble)") - First-run path warns if not Ubuntu or < 24.04 services/sky-cam.sh, services/sync-cc.sh: - Replace inline pip3 invocations with pip_user_install helper https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- lib/common.sh | 35 +++++++++++++++++++++++++++++++++++ services/sky-cam.sh | 3 +-- services/sync-cc.sh | 4 +--- setup.sh | 12 ++++++++++-- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index a80ad63..ea68faa 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -96,6 +96,41 @@ save_site_config() { # Load immediately so all service modules inherit the values when sourced load_site_config +# ── OS detection ───────────────────────────────────────────────────────────── +OS_DISTRO="unknown" +OS_VERSION="unknown" +OS_CODENAME="unknown" + +detect_os() { + [ -f /etc/os-release ] || return 0 + local key val + while IFS='=' read -r key val; do + val="${val//\"/}" + case "$key" in + ID) OS_DISTRO="$val" ;; + VERSION_ID) OS_VERSION="$val" ;; + VERSION_CODENAME|UBUNTU_CODENAME) + [ "$OS_CODENAME" = "unknown" ] && OS_CODENAME="$val" ;; + esac + done < /etc/os-release + export OS_DISTRO OS_VERSION OS_CODENAME +} + +# Return 0 (true) if the detected Ubuntu version is >= the argument (e.g., "24.04"). +ubuntu_version_ge() { + [ "$OS_DISTRO" = "ubuntu" ] || return 1 + local a="${OS_VERSION//./}" b="${1//./}" + [ "${a:-0}" -ge "${b:-0}" ] 2>/dev/null +} + +# pip install --user as actual user. +# Centralised so any future version-specific pip flags land in one place. +pip_user_install() { + sudo -u "$ACTUAL_USER" pip3 install --user --quiet "$@" +} + +detect_os + # ── Pre-flight ─────────────────────────────────────────────────────────────── require_root() { if [ "${EUID:-$(id -u)}" -ne 0 ]; then diff --git a/services/sky-cam.sh b/services/sky-cam.sh index 985c0cd..fa27fba 100644 --- a/services/sky-cam.sh +++ b/services/sky-cam.sh @@ -43,8 +43,7 @@ install_sky-cam() { # ── Python packages ────────────────────────────────────────────────────── log_info "Installing Python packages..." - local PIP="pip3 install --user --quiet" - sudo -u "$ACTUAL_USER" $PIP suntime pytz requests skyfield Pillow numpy scipy \ + pip_user_install suntime pytz requests skyfield Pillow numpy scipy \ || log_warning "Some pip packages may have failed — check output above" log_success "Python packages installed" diff --git a/services/sync-cc.sh b/services/sync-cc.sh index 7117be7..4b3592c 100644 --- a/services/sync-cc.sh +++ b/services/sync-cc.sh @@ -48,10 +48,8 @@ install_sync-cc() { log_success "System packages installed" # ── pip packages ───────────────────────────────────────────────────────── - # Install as the actual (non-root) user so packages land in ~/.local log_info "Installing Python packages (openai-whisper, ffsubsync)..." - local PIP_CMD="pip3 install --user --quiet openai-whisper ffsubsync" - if sudo -u "$ACTUAL_USER" $PIP_CMD; then + if pip_user_install openai-whisper ffsubsync; then log_success "Python packages installed" else log_warning "pip install reported errors — the tool may still work if packages were partially installed" diff --git a/setup.sh b/setup.sh index a4e83c1..dc082e7 100755 --- a/setup.sh +++ b/setup.sh @@ -153,12 +153,13 @@ fi require_root _VER="$(cat "$HERE/VERSION" 2>/dev/null || echo '?')" +_OS_LINE="${OS_DISTRO^} ${OS_VERSION} (${OS_CODENAME})" if is_installed base; then # ── Re-run: base already present — skip required step ──────────────────── echo "" echo "╔══════════════════════════════════════════════════════════════╗" - echo "║ Ubuntu Post-Install · v${_VER}" + echo "║ Ubuntu Post-Install · v${_VER} · ${_OS_LINE}" echo "╚══════════════════════════════════════════════════════════════╝" echo "" echo " Base packages already installed — skipping required setup." @@ -168,9 +169,16 @@ else # ── First run: show required banner, confirm, install ──────────────────── echo "" echo "╔══════════════════════════════════════════════════════════════╗" - echo "║ Ubuntu Post-Install · v${_VER}" + echo "║ Ubuntu Post-Install · v${_VER} · ${_OS_LINE}" echo "╚══════════════════════════════════════════════════════════════╝" echo "" + if [ "$OS_DISTRO" != "ubuntu" ]; then + log_warning "Detected OS: ${_OS_LINE} — this script targets Ubuntu. Proceed with caution." + echo "" + elif ! ubuntu_version_ge "24.04"; then + log_warning "Ubuntu ${OS_VERSION} detected — tested on 24.04+. Some packages may differ." + echo "" + fi echo "REQUIRED (installed/verified first):" echo " • Essential CLI packages: net-tools, git, curl, wget, htop, tree," echo " ncdu, zip/unzip, jq, rsync, and glow (markdown reader)" From e8667f02cf5787515d511af83c183b49b8353ba0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Jun 2026 00:01:57 +0000 Subject: [PATCH 09/12] pip_user_install: add --break-system-packages on Ubuntu 24.04+ pip3 install --user alone does not reliably bypass PEP 668 in all 24.04 environments. --break-system-packages (pip 22.3+) is the correct override. Flag is only added when ubuntu_version_ge "24.04" so it does not run on Ubuntu 22.04 where pip 22.0 ships and the flag is not yet supported. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- lib/common.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index ea68faa..0b44ba5 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -124,9 +124,12 @@ ubuntu_version_ge() { } # pip install --user as actual user. -# Centralised so any future version-specific pip flags land in one place. +# --break-system-packages (pip 22.3+) is required on Ubuntu 24.04+ where PEP 668 +# marks the system Python as externally managed; --user alone is not always enough. pip_user_install() { - sudo -u "$ACTUAL_USER" pip3 install --user --quiet "$@" + local flags="--user --quiet" + ubuntu_version_ge "24.04" && flags="$flags --break-system-packages" + sudo -u "$ACTUAL_USER" pip3 install $flags "$@" } detect_os From 92e8866f13e168ce5043e40c3489561c7dce45d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Jun 2026 00:05:36 +0000 Subject: [PATCH 10/12] pip_user_install: capability probe instead of version check Probe for --break-system-packages support once (pip --help, cached in _PIP_HAS_BSP) rather than comparing Ubuntu version numbers. Works on any pip >= 22.3 regardless of distro; older pip (Ubuntu 22.04, pip 22.0) falls back to --user only, which is correct there since PEP 668 isn't enforced on 22.04 anyway. The flag name is scary but harmless with --user: installs go to ~/.local/ which apt never manages regardless. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- lib/common.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/common.sh b/lib/common.sh index 0b44ba5..7ae864c 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -124,11 +124,21 @@ ubuntu_version_ge() { } # pip install --user as actual user. -# --break-system-packages (pip 22.3+) is required on Ubuntu 24.04+ where PEP 668 -# marks the system Python as externally managed; --user alone is not always enough. +# --break-system-packages overrides PEP 668 ("externally managed environment"), +# required on Ubuntu 24.04+ — the flag name sounds alarming but with --user the +# install goes to ~/.local/ which apt never touches; nothing system-level is at risk. +# The flag was added in pip 22.3; probe once so older pip (Ubuntu 22.04) still works. +_PIP_HAS_BSP="" +_pip_probe() { + [ -n "$_PIP_HAS_BSP" ] && return + pip3 install --help 2>/dev/null | grep -q -- '--break-system-packages' \ + && _PIP_HAS_BSP=1 || _PIP_HAS_BSP=0 +} + pip_user_install() { + _pip_probe local flags="--user --quiet" - ubuntu_version_ge "24.04" && flags="$flags --break-system-packages" + [ "$_PIP_HAS_BSP" = "1" ] && flags="$flags --break-system-packages" sudo -u "$ACTUAL_USER" pip3 install $flags "$@" } From 2ab000833873006c0d2e1110f0843a7f7e4c2709 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Jun 2026 00:22:07 +0000 Subject: [PATCH 11/12] Bump NodeSource from Node 22 to Node 24 LTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both silent-send and immich already use NodeSource (not Ubuntu repos). Node 24 is the current active LTS; 22 moves to maintenance in 2025. Minimum version checks (>=18 and >=20) are unchanged — both services accept any sufficiently recent Node. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- services/immich.sh | 4 ++-- services/silent-send.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/immich.sh b/services/immich.sh index 80b60c3..e8d2f7d 100644 --- a/services/immich.sh +++ b/services/immich.sh @@ -492,9 +492,9 @@ fi if [ "$NODE_OK" = false ]; then echo " Immich CLI requires Node.js >= 20 (found: $(node -v 2>/dev/null || echo 'none'))." - read -r -p " Install Node.js 22 LTS now? (y/n): " INSTALL_NODE_YN + read -r -p " Install Node.js 24 LTS now? (y/n): " INSTALL_NODE_YN if [ "$INSTALL_NODE_YN" = "y" ] || [ "$INSTALL_NODE_YN" = "Y" ]; then - curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - 2>/dev/null + curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - 2>/dev/null sudo apt-get install -y -qq nodejs 2>/dev/null NODE_MAJOR=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) if [ "$NODE_MAJOR" -ge 20 ] 2>/dev/null; then diff --git a/services/silent-send.sh b/services/silent-send.sh index c6e6d11..76b6330 100644 --- a/services/silent-send.sh +++ b/services/silent-send.sh @@ -65,10 +65,10 @@ EOF if ! [ "$NODE_MAJOR" -ge 18 ] 2>/dev/null; then log_warning "Node.js >= 18 required for building/signing (found: $(node -v 2>/dev/null || echo none))." local INSTALL_NODE="" - prompt_yn " Install Node.js 22 LTS from NodeSource now? (y/n):" "y" INSTALL_NODE + prompt_yn " Install Node.js 24 LTS from NodeSource now? (y/n):" "y" INSTALL_NODE if [ "$INSTALL_NODE" = "y" ] || [ "$INSTALL_NODE" = "Y" ]; then - log_info "Installing Node.js 22 LTS..." - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - >/dev/null 2>&1 + log_info "Installing Node.js 24 LTS..." + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - >/dev/null 2>&1 apt-get install -y nodejs >/dev/null 2>&1 NODE_MAJOR=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) fi From 33f052ea0ba6fa67dd9a40f8928614952de43ee1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Jun 2026 00:26:09 +0000 Subject: [PATCH 12/12] Add bootstrap.sh and rewrite README for modular system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bootstrap.sh: one curl | sudo bash to get and run on a fresh box — installs git if missing, clones/updates the repo, execs setup.sh. README.md: complete rewrite. The old README described the monolithic script (--restore, --migrate flags, Samba, NetBird, etc.) which no longer exists. New README covers quick start, usage modes, service table, layout, and managing installed services. https://claude.ai/code/session_01Y4dMKtkqkpvmgDKoRdzhTG --- README.md | 1241 +++----------------------------------------------- bootstrap.sh | 47 ++ 2 files changed, 117 insertions(+), 1171 deletions(-) create mode 100755 bootstrap.sh diff --git a/README.md b/README.md index a96c517..7e068c7 100644 --- a/README.md +++ b/README.md @@ -1,1190 +1,89 @@ -# Ubuntu 24.04 Desktop Post-Installation Script +# ubuntu-post-install -Automated setup script for Ubuntu 24.04 Desktop that installs additional tools, configures SSH, and optionally sets up Docker, Samba file sharing, remote access tools (NetBird, RustDesk), and an automated backup system. +Modular post-install system for Ubuntu servers. One repo, one entry point, +install exactly what you need — interactively or by name. -**Key Features:** -- **Rerunnable** - Detects existing installations and offers to reinstall/reconfigure -- **Modular** - Every component is optional with y/n prompts -- **Dry-run mode** - Preview what would be installed without making changes -- **Unattended mode** - Run with defaults for automated/scripted installs -- **Disaster recovery** - One-click restore from Kopia backup after system failure -- **Logging** - All output logged to `/var/log/post-install.log` -- **Local backup** - rsync to 1-4 drives with customizable names -- **Cloud backup** - Encrypted backup to Google Drive, OneDrive, or 40+ providers - -## What This Script Does - -### Core Utilities Installed (Always) -- **net-tools** - Network utilities (ifconfig, netstat) -- **ncdu** - Disk usage analyzer with ncurses interface -- **git** - Version control system -- **curl & wget** - Download tools -- **htop** - Interactive process viewer -- **tree** - Directory structure visualizer -- **zip/unzip** - Archive utilities - -### SSH Configuration -- **OpenSSH Server** - Enables remote SSH access -- **SSH Key Generation** - Creates 4096-bit RSA key pair for this computer -- **Key Import** - Optionally imports public keys from GitHub and/or Launchpad -- **Security** - Automatically disables password authentication if keys are imported - -### Security Features (Optional) -- **fail2ban** (when password SSH enabled) - - Protects against SSH brute-force attacks - - Bans IPs after 5 failed attempts for 1 hour - - Only offered when SSH password authentication remains enabled - - **Note:** fail2ban provides no benefit with key-only SSH because SSH keys cannot be brute-forced (they're 4096-bit cryptographic keys, not passwords) -- **UFW Firewall** - - Simple firewall management - - Automatically allows SSH (port 22) - - Automatically allows Samba if installed - - Easy to add/remove port rules - -### Docker Installation (Optional) -- **Docker Engine** - Latest version from official Docker repository (not snap) -- **Docker Compose** - Installed as a plugin (modern method) -- **User Configuration** - Adds your user to docker group (run docker without sudo) -- Detects if already installed and offers to reinstall - -### Samba File Sharing (Optional) -- **Samba Server** - SMB/CIFS file server for network file sharing -- **Primary Drive Share** - Entire primary drive shared as "Primary" -- **User Configuration** - Creates Samba user matching your system username -- **Cross-Platform Access** - Works with Windows, Mac, and Linux -- Detects if already installed and offers to reconfigure - -### VPN Tools (Optional) -- **NetBird** - Mesh VPN for secure device connections - - Zero-config mesh VPN with built-in SSH - - Manages SSH keys automatically (no manual key setup) - - Detects if already installed -- **WireGuard** - Fast, modern VPN protocol - - Lightweight and high-performance - - Built into Linux kernel - - Manual configuration via config files -- **Tailscale** - Zero-config mesh VPN built on WireGuard - - Easy setup - just sign in - - Built-in SSH (Tailscale SSH) - no keys needed - - Automatic NAT traversal - -### Remote Desktop Tools (Optional) -- **RustDesk** - Open-source remote desktop software - - Self-hosted or use public servers - - Cross-platform -- **TeamViewer** - Commercial remote desktop (free tier available) - - Cross-platform (Windows, Mac, Linux, mobile) - - No port forwarding needed -- **MeshCentral Agent** - Open-source remote management - - Requires a MeshCentral server (self-hosted or public) - - Web-based remote desktop, terminal, and file transfer - -### Self-Hosted Docker Applications (Optional) -Install containerized applications to `~/docker/{appname}/`: - -- **Immich** - Self-hosted photo & video backup (like Google Photos) -- **Audiobookshelf** - Audiobook & podcast server with progress sync -- **Emby** - Media server for movies, TV, and music -- **A.R.M.** - Automatic Ripping Machine for DVDs/Blu-rays/CDs -- **Filebrowser** - Web-based file manager -- **Magic Mirror** - Smart mirror dashboard (up to 3 instances) -- **Lyrion Music Server** - Music streaming to Squeezebox/Chromecast -- **Mealie** - Recipe manager & meal planner -- **Minecraft Server** - Fabric server with configurable RAM limit -- **linux-to-sync** - Private repository setup -- **Jellyfin** - Free media server (alternative to Emby) -- **Frigate** - NVR with AI object detection -- **Caddy** - Reverse proxy with automatic HTTPS -- **ddclient** - Dynamic DNS updater -- **ntfy** - Self-hosted push notifications -- **Uptime Kuma** - Service uptime monitoring -- **wg-easy** - WireGuard VPN with web UI -- **Traccar** - GPS tracking server -- **Portainer** - Docker management UI -- **MeshCentral Server** - Self-hosted remote management server -- **FindMyDevice** - Self-hosted Android device tracking -- **Frigate-Notify** - Push notifications for Frigate AI events -- **Watchtower** - Container update monitoring (notify-only by default) - -### Container Backup & Restore (Kopia) -Backup all Docker container data (configs, databases, app data) to your backup drives for disaster recovery. Includes restore functionality to recover containers after OS drive failure. - -**What lives where:** -- `~/docker/*/` (OS drive) - App configs, databases, compose files → **Backed up by Kopia** -- `~/drives/primary/` (data drive) - Media files, photos, documents → **Backed up by rsync** -- `/var/lib/docker/` (OS drive) - Container images, runtime state → **Not backed up** (re-pulled on restore) - -### Backup System (Optional) - -**Local Backup (rsync):** -- Syncs your primary drive to 1-4 backup drives -- Delta transfers - only changed bytes are copied -- Customizable drive names (default: primary, backup1, backup2, etc.) -- Systemd timer for scheduled daily backups at 2 AM - -**Why rsync instead of RAID?** -- RAID mirrors corruption instantly - rsync gives you time to notice problems -- RAID requires identical drives - rsync works with any sizes -- RAID is complex to set up/recover - rsync is simple copy -- rsync can run on schedule - RAID is always-on (more wear) -- With rsync, backup drives can be disconnected for safety - -**Cloud Backup (rclone, optional):** -- Encrypted cloud backup to Google Drive, OneDrive, or 40+ providers -- Files are encrypted BEFORE upload - cloud provider cannot read them -- Guided setup for Google Drive and OneDrive with encryption -- rclone.conf automatically backed up to all local backup drives - -**Drive Mount Points (`~/drives/`):** -The script creates and manages mount points for your drives: -``` -~/drives/ -├── primary/ # Your main data drive -├── backup1/ # First backup drive -└── backup2/ # Second backup drive (split mode only) -``` - -**Interactive Drive Mounting:** -During backup setup, the script: -1. Shows available block devices (`lsblk`) -2. Asks for device paths (e.g., `/dev/sdb1`, `/dev/sdc1`) -3. Mounts drives to `~/drives/` directories -4. Optionally adds entries to `/etc/fstab` for auto-mount at boot - -**Features:** -- Systemd timer for scheduled daily backups at 2 AM -- Detects existing configuration and offers to reconfigure - -## Prerequisites - -- Fresh Ubuntu 24.04 Desktop installation -- Sudo/root access -- Internet connection -- (Optional) External drives for backup configuration - -## Quick Start - -### 1. Download the Script +## Quick start on a fresh box ```bash -# Clone the repository -git clone https://github.com/outis1one/post-ubuntu-install.git -cd post-ubuntu-install - -# Or download the script directly -wget https://raw.githubusercontent.com/outis1one/post-ubuntu-install/main/ubuntu-post-install.sh -O post-install.sh +curl -fsSL https://raw.githubusercontent.com/outis1one/ubuntu-post-install/main/bootstrap.sh | sudo bash ``` -### 2. Make Executable +That installs git (if missing), clones the repo to `~/ubuntu-post-install`, +and drops you into the interactive wizard. + +If you already have git: ```bash -chmod +x post-install.sh +git clone https://github.com/outis1one/ubuntu-post-install.git +cd ubuntu-post-install +sudo ./setup.sh ``` -### 3. Run the Script +## Usage ```bash -sudo ./post-install.sh +sudo ./setup.sh # interactive wizard +sudo ./setup.sh caddy immich # install specific services +sudo ./setup.sh configure # set site defaults (timezone, domain, Caddy network) +./setup.sh --list # list all services grouped by category +sudo ./setup.sh --dry-run immich # preview without making changes +sudo ./setup.sh --unattended base # non-interactive, use defaults ``` -### 4. Command-Line Options +## What the wizard does + +**First run:** +1. Installs essential CLI packages (`git`, `curl`, `htop`, `ncdu`, `jq`, `glow`, …) +2. Checks Docker is present (tells you how to install it if not) +3. Offers to set **site defaults** — timezone, base domain, Caddy Docker network — + so every service picks them up automatically instead of asking each time +4. Offers to install Caddy (the reverse proxy most services use) +5. Drops into a **category menu** — pick a group, tick services, install, repeat + +**Re-run:** skips steps 1–2 (already done), goes straight to the menu. + +**Site defaults** are saved to `~/docker/.config` and pre-fill every service prompt. +Update them any time with `sudo ./setup.sh configure`. + +## Services + +| Group | Services | +|-------|---------| +| `base` | `base`, `glow` | +| `homelab` | `caddy`, `crowdsec`, `authelia`, `homeassistant` | +| `utilities` | `actualbudget`, `ddclient`, `filebrowser`, `fmd`, `magicmirror`, `mealie`, `meshcentral`, `ntfy`, `portainer`, `traccar`, `uptimekuma`, `watchtower`, `wg-easy` | +| `media` | `arm`, `audiobookshelf`, `emby`, `immich`, `jellyfin`, `lyrion` | +| `cameras` | `frigate`, `frigate-audio`, `frigate-notify`, `sky-cam` | +| `gaming` | `js99er`, `minecraft`, `wolf`, `wolf-pair` | +| `extras` | `linux-to-sync`, `silent-send`, `sync-cc` | +| `backup` | `backup` | + +Run `./setup.sh --list` to see descriptions. + +## Layout + +``` +setup.sh dispatcher — wizard, direct install, --list, --dry-run +lib/common.sh shared helpers: logging, prompts, site config, OS detection +services/ one file per service (self-registering) +extras/ non-Docker assets bundled with the repo (e.g. sync_cc.py) +``` + +## Managing installed services + +Every Docker service installs to its own `~/docker//` folder: ```bash -# Interactive mode (default) -sudo ./post-install.sh - -# Preview what would be installed (no changes made) -sudo ./post-install.sh --dry-run - -# Automated install with defaults (no prompts) -sudo ./post-install.sh --unattended - -# Disaster recovery - restore from Kopia backup -sudo ./post-install.sh --restore - -# Show help -sudo ./post-install.sh --help +cd ~/docker/immich +docker compose up -d # start +docker compose logs -f # logs +docker compose pull && docker compose up -d # update +docker compose down # stop ``` -**Unattended mode defaults:** -- Skip SSH key generation -- No SSH key imports (password auth stays enabled) -- Install Docker -- Install fail2ban (since password auth is enabled) -- Enable UFW firewall +## Compatibility -## Disaster Recovery - -If your OS drive fails, you can restore everything from a Kopia backup. - -### One-Click Recovery - -```bash -# 1. Install fresh Ubuntu 24.04 -# 2. Connect your backup drive -# 3. Download and run the script - -wget https://raw.githubusercontent.com/outis1one/post-ubuntu-install/main/ubuntu-post-install.sh -chmod +x ubuntu-post-install.sh -sudo ./ubuntu-post-install.sh --restore -``` - -### What the Recovery Does - -1. **Installs core utilities** - openssh-server, git, curl, Kopia, etc. -2. **Finds your backup drive** - Shows available drives, auto-detects Kopia repo -3. **Gets Kopia password** - For repository access -4. **Installs Docker** - If not already installed -5. **Lists snapshots** - Shows all available backups, lets you choose -6. **Restores from backup** - Extracts files to temp location -7. **Selects services** - Whiptail checklist to pick which services to restore -8. **Starts containers** - Optionally starts all services immediately -9. **Reconnects Kopia** - Sets up ongoing backups to the same repository - -### What Gets Restored - -Everything in `~/docker/` that Kopia backed up: -- **App configs** - All settings, users, preferences -- **Databases** - Immich, Mealie, Traccar, etc. -- **Media metadata** - Emby/Jellyfin watch history, thumbnails -- **Minecraft worlds** - Saves, mods, permissions -- **Frigate** - Camera configs, detection settings -- **All other container data** - -### Recovery Requirements - -- Fresh Ubuntu 24.04 installation -- Backup drive with Kopia repository -- Kopia password (stored in `~/docker/kopia/.env` on backup, or remembered) - -### Interactive Mode - -When you run the script without `--restore`, you'll be asked: -``` -INSTALLATION MODE -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - - [N] Normal install - Fresh install or modify existing - [M] Migration - Import existing Docker containers - [R] Disaster recovery - Restore from Kopia backup - -Select mode (N/M/R) [N]: -``` - -## Migration Mode - -If you have existing Docker containers from another setup (different directory structure, another server, etc.), migration mode imports them without changing versions. - -### When to Use Migration - -- Moving from `/var/docker` or `/opt/docker` to `~/docker` -- Importing containers from another machine -- Adopting this script's structure for existing setups - -### What Migration Does - -1. **Scans source directory** - Auto-detects Docker directories on OS and mounted drives -2. **Shows containers found** - Lists each container with size -3. **Select what to migrate** - Whiptail checklist or text menu -4. **Stops containers** (optional) - Ensures clean copy of databases -5. **Migration method** - Copy, Symlink, or Use in-place (for mounted drives) -6. **Updates volume paths** - Detects old paths, suggests new ones, updates compose files -7. **Starts containers** - In the new location -8. **Offers additional services** - Continue with normal install for more apps - -### Volume Path Updates - -When migrating, the script detects volume mounts that don't exist on the new system: - -``` -Container: immich -Old path: /home/user1/media/driveb -Suggested: ~/drives/primary/driveb - -Options: - [Enter] Accept suggested path - [S] Skip - keep original path - [path] Enter custom path -``` - -- Scans all docker-compose.yml files for absolute volume paths -- Only shows paths that don't exist on the current system -- Suggests `~/drives/primary/{folder}` as default -- Updates the compose file and creates the directory - -### Migration vs Restore - -| Feature | Migration | Disaster Recovery | -|---------|-----------|-------------------| -| Source | Existing Docker directory | Kopia backup | -| Versions | Preserved exactly | Preserved exactly | -| After | Install more services | Reconnect backups | -| Use case | Restructuring setup | OS drive failure | - -### 5. Follow Interactive Prompts - -The script shows current system status and asks: -- **SSH Key Generation**: Generate new 4096-bit RSA key? (y/n) -- **Import SSH Keys**: GitHub username, Launchpad username (or leave blank) -- **fail2ban**: Install fail2ban? (only if password SSH remains enabled) -- **Docker**: Install Docker? (y/n) - or reinstall if detected -- **Samba File Sharing**: Install and configure Samba? (y/n) - - If yes: Set password for Samba user -- **NetBird**: Install NetBird mesh VPN? (y/n) -- **WireGuard**: Install WireGuard VPN? (y/n) -- **Tailscale**: Install Tailscale VPN? (y/n) -- **RustDesk**: Install RustDesk remote desktop? (y/n) -- **TeamViewer**: Install TeamViewer remote desktop? (y/n) -- **MeshCentral**: Install MeshCentral agent? (y/n) - - If yes: Provide MeshCentral server agent URL -- **Docker Apps**: Install self-hosted applications? (each individually) - - Immich, Audiobookshelf, Emby, A.R.M., Filebrowser - - Magic Mirror (1-3 instances), Lyrion, Mealie, Minecraft - - linux-to-sync (private repo) -- **Local Backup**: Set up local backup with rsync? (y/n) - - If yes: Configure drive names, mount drives, fstab configuration -- **Cloud Backup**: Set up encrypted cloud backup? (y/n) - - If yes: Choose provider (Google Drive, OneDrive, other), set up encryption -- **UFW Firewall**: Enable and configure UFW? (y/n) - -### 5. Post-Installation Steps - -**Required:** -```bash -# Log out and back in for docker group to take effect -logout -``` - -**If you enabled cloud backup:** -- Run `rclone config` if you need to reconfigure -- Keep your `~/.config/rclone/rclone.conf` backed up securely off-site - -## SSH Configuration - -### SSH Key Combinations Supported - -You can use **any combination** of: -- ✓ GitHub keys + Launchpad keys + NetBird SSH -- ✓ GitHub keys only -- ✓ Launchpad keys only -- ✓ NetBird SSH only -- ✓ Your generated key + any of the above -- ✓ Password authentication only (if no keys imported) - -### Traditional SSH vs NetBird SSH - -**Traditional SSH** (uses imported keys): -```bash -ssh user@hostname -ssh user@192.168.1.100 -``` - -**NetBird SSH** (manages keys automatically): -```bash -netbird ssh peer-name -``` - -These work independently - NetBird SSH works even if password auth is disabled. - -### Your Generated SSH Key - -After installation, find your public key: -```bash -cat ~/.ssh/id_rsa.pub -``` - -Use it to: -- Add to GitHub: Settings → SSH and GPG keys → New SSH key -- Add to other servers: Append to remote `~/.ssh/authorized_keys` -- Connect from this computer to other servers - -## Backup Configuration - -*This section applies if you chose to set up the backup system during installation.* - -### Local Backup (rsync) - -The local backup system uses rsync to sync your primary drive to one or more backup drives. - -**Run Backup:** -```bash -sudo /usr/local/bin/backup-scripts/local-backup.sh -``` - -**View Log:** -```bash -tail -f /var/log/rsync-backup.log -``` - -**Enable Automatic Daily Backups:** -```bash -sudo systemctl enable rsync-backup.timer -sudo systemctl start rsync-backup.timer - -# Check status -sudo systemctl list-timers | grep rsync -``` - -### Cloud Backup (rclone) - -If you set up cloud backup, your files are encrypted locally before being uploaded. - -**Run Cloud Backup:** -```bash -sudo /usr/local/bin/backup-scripts/cloud-backup.sh -``` - -**View Log:** -```bash -tail -f /var/log/cloud-backup.log -``` - -### Protecting Your rclone Configuration - -Your `~/.config/rclone/rclone.conf` file contains your encryption keys and cloud credentials. **Without this file, your encrypted cloud files cannot be decrypted.** - -**The script automatically:** -- Backs up rclone.conf to all local backup drives -- Reminds you to store a copy off-site - -**Recommended off-site backup methods for rclone.conf:** -- **Signal** - End-to-end encrypted; send to yourself or a trusted contact -- **Box.com** - Better privacy policy than some alternatives -- **Password manager** - 1Password, Bitwarden, etc. -- **Encrypted USB drive** - Store at another physical location - -### Restoring Files on Another Computer - -If you need to decrypt your cloud-backed files on a new machine: - -1. **Install rclone:** - ```bash - sudo apt install rclone - ``` - -2. **Copy your rclone.conf to the new machine:** - ```bash - mkdir -p ~/.config/rclone - # Copy your backed-up rclone.conf to ~/.config/rclone/rclone.conf - ``` - -3. **Download and decrypt files:** - ```bash - # List your encrypted remote - rclone ls cloud-crypt: - - # Download and decrypt to local folder - rclone copy cloud-crypt: /path/to/restore/ - ``` - -The "cloud-crypt" remote automatically decrypts files during download using the keys stored in rclone.conf. - -### Manual Drive Mounting - -If you skipped auto-mounting during installation: - -```bash -# Create mount points (already done by script) -mkdir -p ~/drives/primary ~/drives/backup1 ~/drives/backup2 - -# Find your drives -lsblk -f -sudo blkid - -# Mount drives -sudo mount /dev/sdb1 ~/drives/primary -sudo mount /dev/sdc1 ~/drives/backup1 -sudo mount /dev/sdd1 ~/drives/backup2 - -# Make permanent (add to /etc/fstab) -sudo nano /etc/fstab -``` - -Add lines like: -``` -UUID=xxxx-xxxx /home/username/drives/primary auto defaults 0 2 -UUID=yyyy-yyyy /home/username/drives/backup1 auto defaults 0 2 -UUID=zzzz-zzzz /home/username/drives/backup2 auto defaults 0 2 -``` - -## Drive Failure Recovery - -### If PRIMARY Drive Fails - -```bash -# 1. Get new drive (same size or larger) -# 2. Format and mount it -sudo mkfs.ext4 /dev/sdX1 -sudo mount /dev/sdX1 ~/drives/primary - -# 3. Restore from backup(s) -# Full mode: restore from backup1 -rsync -avh ~/drives/backup1/ ~/drives/primary/ # or rclone sync - -# Split mode: restore from BOTH backups -rsync -avh ~/drives/backup1/ ~/drives/primary/ -rsync -avh ~/drives/backup2/ ~/drives/primary/ - -# 4. Update /etc/fstab with new UUID -sudo blkid /dev/sdX1 -sudo nano /etc/fstab -``` - -### If BACKUP Drive Fails - -Your primary still has all data - it's safe. Just replace the backup drive and re-run the backup script: - -```bash -sudo mkfs.ext4 /dev/sdX1 -sudo mount /dev/sdX1 ~/drives/backup1 -sudo /usr/local/bin/backup-scripts/{tool}-backup.sh -``` - -**⚠️ Replace failed backup drives quickly!** While down, those folders have no redundancy. - -## Verification Commands - -### Check Backups Match Primary - -```bash -# Using rsync (dry-run shows differences) -rsync -avhn --delete ~/drives/primary/ ~/drives/backup1/ - -# Using rclone -rclone check ~/drives/primary ~/drives/backup1 -``` - -### Check Space Usage - -```bash -# See what's on each drive -du -sh ~/drives/primary/* -du -sh ~/drives/backup1/* - -# Check free space -df -h ~/drives/ -``` - -## VPN Setup - -### NetBird - -```bash -# 1. Connect to NetBird (opens browser for auth) -netbird up - -# 2. View connected peers -netbird status - -# 3. SSH via NetBird (if enabled in dashboard) -netbird ssh peer-name - -# 4. Configure ACLs and settings -# Visit: https://app.netbird.io -``` - -**NetBird SSH:** NetBird manages its own SSH keys automatically. Enable SSH in the NetBird dashboard, then use `netbird ssh ` to connect. No manual key configuration needed. - -### WireGuard - -```bash -# Generate keys -wg genkey | sudo tee /etc/wireguard/privatekey | wg pubkey | sudo tee /etc/wireguard/publickey - -# Create config -sudo nano /etc/wireguard/wg0.conf - -# Start VPN -sudo wg-quick up wg0 - -# Enable on boot -sudo systemctl enable wg-quick@wg0 - -# Check status -sudo wg show -``` - -### Tailscale - -```bash -# Connect (opens browser for auth) -sudo tailscale up - -# View connected devices -tailscale status - -# Get your Tailscale IP -tailscale ip - -# Tailscale SSH (enable in admin console first) -ssh user@device-name -``` - -**Tailscale SSH:** Enable in the Tailscale admin console. Uses Tailscale identity - no traditional SSH keys required. - -## Remote Desktop Setup - -### RustDesk - -After installation, launch RustDesk from the application menu. Note your ID and set a password for remote access. - -### TeamViewer - -```bash -# Launch TeamViewer -teamviewer - -# For unattended access: -# 1. Open TeamViewer -# 2. Go to Extras → Options → Security -# 3. Set personal password -# 4. Note your TeamViewer ID -``` - -### MeshCentral - -MeshCentral agent connects to your MeshCentral server automatically after installation. Check your server's web interface - the device should appear in "My Devices". - -To manually install/reinstall: -1. Log into your MeshCentral web interface -2. Go to "My Devices" → "Add Agent" -3. Download and run the Linux agent installer - -## Docker Applications - -Self-hosted applications are installed to `~/docker/{appname}/` with docker-compose. - -### Managing Docker Apps - -```bash -# Start an application -cd ~/docker/{appname} -docker compose up -d - -# View logs -docker compose logs -f - -# Stop an application -docker compose down - -# Update an application -docker compose pull -docker compose up -d -``` - -### Application Ports - -| Application | Port | URL | -|-------------|------|-----| -| Immich | 2283 | http://localhost:2283 | -| Audiobookshelf | 13378 | http://localhost:13378 | -| Emby | 8096 | http://localhost:8096 | -| Jellyfin | 8097 | http://localhost:8097 | -| A.R.M. | 8080 | http://localhost:8080 | -| Filebrowser | 8085 | http://localhost:8085 | -| Magic Mirror | 8081-8083 | http://localhost:808X | -| Lyrion (LMS) | 9000 | http://localhost:9000 | -| Mealie | 9925 | http://localhost:9925 | -| Minecraft | 25565 | localhost:25565 | -| Frigate | 5000 | http://localhost:5000 | -| Caddy | 80, 443 | http://localhost | -| ntfy | 8090 | http://localhost:8090 | -| Uptime Kuma | 3001 | http://localhost:3001 | -| wg-easy | 51821 | http://localhost:51821 | -| Traccar | 8082 | http://localhost:8082 | -| Portainer | 9443 | https://localhost:9443 | -| FindMyDevice | 8084 | http://localhost:8084 | -| MeshCentral Server | 4430 | https://localhost:4430 | - -### Container Backup & Restore (Kopia) - -Backup all Docker container data to your backup drives for disaster recovery. - -**Run Container Backup:** -```bash -~/docker/kopia/backup-containers.sh -``` - -**Restore Containers (after OS drive failure):** -```bash -~/docker/kopia/restore-containers.sh -``` - -**What Gets Backed Up:** -- All container configs and databases -- Immich facial recognition data and memories -- Emby/Jellyfin metadata and watch history -- Minecraft worlds, mods, and permissions -- Mealie recipes, Audiobookshelf progress -- All application state and settings - -**Kopia Repository Location:** Your backup drive(s) in `~/drives/backupX/container-backups/` - -### Frigate + ntfy Notifications - -If you installed Frigate, ntfy, and Frigate-Notify, they work together: - -1. **Frigate** detects objects (person, car, etc.) on your cameras -2. **Frigate-Notify** monitors Frigate for events -3. **ntfy** sends push notifications to your phone - -**Subscribe to alerts:** -```bash -# On your phone: Install ntfy app, add topic "frigate-alerts" -# Or visit: http://localhost:8090/frigate-alerts -``` - -**Customize alerts:** Edit `~/docker/frigate-notify/config.yml` -- Change which objects trigger alerts (person, car, dog, package) -- Set quiet hours for no notifications -- Add multiple notification services (Discord, Pushover, etc.) - -### Caddy Reverse Proxy Network - -To route traffic through Caddy, containers must be on the `caddy_net` network: - -```yaml -# Add to any container's docker-compose.yml: -networks: - default: - name: caddy_net - external: true -``` - -Then uncomment the service in `~/docker/caddy/Caddyfile`. - -### Private Repository (linux-to-sync) - -To clone a private GitHub repository, you need authentication: - -**Option 1: SSH Key (Recommended)** -```bash -# Your SSH key must be added to GitHub -cat ~/.ssh/id_rsa.pub -# Add at: https://github.com/settings/keys -``` - -**Option 2: Personal Access Token** -```bash -# Create token at: https://github.com/settings/tokens/new -# Select 'repo' scope -``` - -## Samba File Sharing - -If you chose to install Samba, it shares your **entire primary drive** via SMB/CIFS. - -### Share Details - -- **Share name**: Primary -- **Path**: `~/drives/primary` -- **Username**: Your system username -- **Password**: The Samba password you set during installation (suggested to match your system password) -- **Permissions**: Read/Write access for the configured user - -### Accessing the Share - -**From Windows:** -``` -1. Open File Explorer -2. In the address bar, type: - \\hostname\Primary - Or use IP: \\192.168.1.100\Primary - -3. Enter credentials when prompted: - Username: your_username - Password: your_samba_password -``` - -**From macOS:** -``` -1. Open Finder -2. Press Cmd+K (or Go → Connect to Server) -3. Enter: - smb://hostname/Primary - Or: smb://192.168.1.100/Primary - -4. Click Connect and enter credentials -``` - -**From Linux:** -```bash -# Browse in file manager -smb://hostname/Primary - -# Or mount manually -sudo mkdir /mnt/primary-share -sudo mount -t cifs //hostname/Primary /mnt/primary-share -o username=your_username -``` - -### Find Your Hostname/IP - -```bash -# Show hostname -hostname - -# Show IP address -hostname -I -ip addr show -``` - -### Managing Samba - -```bash -# Restart Samba -sudo systemctl restart smbd nmbd - -# Check status -sudo systemctl status smbd - -# View share configuration -sudo nano /etc/samba/smb.conf - -# Change Samba password -sudo smbpasswd your_username - -# Add additional users -sudo smbpasswd -a new_username -``` - -### Add Additional Shares - -Edit `/etc/samba/smb.conf`: - -```bash -sudo nano /etc/samba/smb.conf -``` - -Add new share: -```ini -[ShareName] - comment = Description of share - path = /path/to/share - browseable = yes - read only = no - writable = yes - valid users = username - create mask = 0775 - directory mask = 0775 -``` - -Restart Samba: -```bash -sudo systemctl restart smbd nmbd -``` - -### Troubleshooting Samba - -**Can't connect to share:** -```bash -# Check if Samba is running -sudo systemctl status smbd - -# Check firewall (if enabled) -sudo ufw allow samba - -# Test configuration -testparm - -# View active connections -sudo smbstatus -``` - -**Permission denied:** -```bash -# Check share permissions -ls -la ~/drives/primary - -# Ensure Samba user exists -sudo pdbedit -L - -# Reset Samba password -sudo smbpasswd your_username -``` - -## Troubleshooting - -### View Installation Log - -```bash -# Check what was installed and any errors -cat /var/log/post-install.log - -# View last 50 lines -tail -50 /var/log/post-install.log -``` - -### Docker Permission Denied - -```bash -# If you get "permission denied" after install -# Log out and back in for group membership to take effect -logout -``` - -### SSH Key Already Exists - -If you see "key already exists" warning: -- Choose 'n' to keep existing key -- Or choose 'y' to overwrite (cannot be undone!) - -### Drive Won't Mount - -```bash -# Check if drive is recognized -lsblk -f - -# Check filesystem -sudo fsck /dev/sdX1 - -# Try manual mount -sudo mount -t auto /dev/sdX1 ~/drives/primary -``` - -### Backup Script Fails - -```bash -# Check if drives are mounted -df -h | grep drives - -# Check log for errors -tail -50 /var/log/rclone-backup.log - -# Verify directories exist on primary -ls -la ~/drives/primary/ -``` - -### NetBird Won't Connect - -```bash -# Check service status -sudo systemctl status netbird - -# Restart service -sudo systemctl restart netbird - -# Check logs -sudo journalctl -u netbird -f -``` - -### Samba Share Not Accessible - -```bash -# Verify Samba is running -sudo systemctl status smbd - -# Check share configuration -testparm - -# View Samba users -sudo pdbedit -L - -# Check if firewall is blocking -sudo ufw status -sudo ufw allow samba - -# Restart Samba -sudo systemctl restart smbd nmbd -``` - -### fail2ban Issues - -```bash -# Check if fail2ban is running -sudo systemctl status fail2ban - -# View SSH jail status -sudo fail2ban-client status sshd - -# Unban an IP address -sudo fail2ban-client set sshd unbanip 192.168.1.100 - -# Check fail2ban logs -sudo tail -50 /var/log/fail2ban.log -``` - -### UFW Firewall Issues - -```bash -# Check UFW status -sudo ufw status verbose - -# If locked out, disable UFW temporarily -sudo ufw disable - -# Re-enable with SSH allowed first -sudo ufw allow ssh -sudo ufw enable - -# List all rules with numbers -sudo ufw status numbered - -# Delete a specific rule -sudo ufw delete 3 -``` - -## Backup Strategy Summary - -### Local Backup (rsync) -✓ Simple setup - just specify your drives -✓ Delta transfers - only changed bytes copied (fast incremental backups) -✓ Supports 1-4 backup drives with custom names -✓ Time to notice corruption before it propagates (unlike RAID) -✓ Backup drives can be disconnected for safety -✓ Easy restore - just rsync back - -### Cloud Backup (rclone) -✓ Files encrypted locally before upload (cloud provider can't read them) -✓ Guided setup for Google Drive and OneDrive -✓ 40+ cloud providers supported -✓ Config automatically backed up to local drives -⚠️ Requires rclone.conf for decryption - keep it safe! - -## Files Created by This Script - -``` -# Always created -/var/log/post-install.log # Installation log -/etc/ssh/sshd_config.backup # SSH config backup (if modified) -~/.ssh/id_rsa # Private SSH key (if generated) -~/.ssh/id_rsa.pub # Public SSH key (if generated) -~/.ssh/authorized_keys # Imported SSH keys (if any) - -# If fail2ban is installed -/etc/fail2ban/jail.local # fail2ban SSH jail configuration - -# If Samba is installed -/etc/samba/smb.conf.backup-TIMESTAMP # Samba config backup - -# If local backup is set up -/usr/local/bin/backup-scripts/local-backup.sh # Local rsync backup script -/etc/systemd/system/rsync-backup.service # Systemd service -/etc/systemd/system/rsync-backup.timer # Systemd timer (daily at 2 AM) -/var/log/rsync-backup.log # Backup log -/etc/fstab.backup-TIMESTAMP # fstab backup (if modified) -~/drives/{your-drive-names}/ # Mount points (customizable names) - -# If cloud backup is set up -/usr/local/bin/backup-scripts/cloud-backup.sh # Cloud rclone backup script -~/.config/rclone/rclone.conf # rclone config (KEEP SAFE - has encryption keys!) -~/drives/*/rclone-config-backup/rclone.conf # Config backed up to each local drive -``` - -## Security Notes - -- **Private SSH key** (`~/.ssh/id_rsa`): Keep secret! Never share! -- **Public SSH key** (`~/.ssh/id_rsa.pub`): Safe to share -- **Password authentication**: Disabled if keys imported (more secure) -- **Docker group**: Equivalent to root access - only add trusted users -- **Samba password** (if installed): Stored separately from system password; change with `sudo smbpasswd username` -- **Samba shares** (if installed): Only accessible to configured users; ensure strong passwords -- **Network security** (if Samba installed): Samba shares are accessible to anyone on your local network who has credentials -- **rclone.conf** (if cloud backup enabled): Contains encryption keys - without it, cloud files cannot be decrypted. Back up securely off-site! -- **Backup drives** (if backup enabled): Consider encrypting sensitive data - -## Support & Feedback - -This script continues even if individual packages fail. Check the output for warnings or errors. - -To report issues or improve the script: -- Review log files in `/var/log/` -- Check systemd service status -- Verify drive mounts with `df -h` - -## License - -This script is provided as-is for Ubuntu 24.04 Desktop installations. - -## Changelog - -- **v2.10**: Migration mode for existing Docker setups - - New **Migration mode** - Import existing Docker containers from any directory - - Auto-detects Docker directories and scans for compose files - - Preserves container versions (no unwanted upgrades) - - Whiptail checklist for selecting which containers to migrate - - Option to stop containers for clean database copy - - After migration, offers to install additional services - - Three modes now: Normal install, Migration, Disaster Recovery -- **v2.9**: Immich photo library, Watchtower, recovery improvements - - **Immich**: Now asks for photo storage location (default: `~/drives/primary/photos`) - - **Immich**: External library support for existing photos (read-only, no duplication) - - **Immich**: Storage template guidance for yyyy/mm folder organization - - Added Watchtower for container update monitoring (notify-only by default) - - Documented what Docker data lives where and what gets backed up - - **Recovery**: Now installs Kopia during recovery (Step 1) - - **Recovery**: Reconnects Kopia repository after restore (Step 9) for ongoing backups -- **v2.8**: MeshCentral Server and improved recovery - - Added MeshCentral Server (self-hosted remote management, web-based RDP/terminal) - - Recovery mode now installs core utilities first (openssh-server, git, curl, etc.) - - Added whiptail checklist for service selection during restore (Ubuntu-server style) - - Recovery supports restoring some/none/all services instead of all-or-nothing -- **v2.7**: Disaster recovery mode - - **One-click restore** from Kopia backup after system failure - - New `--restore` flag for disaster recovery mode - - Interactive mode selector at script start: Normal install or Disaster recovery - - Auto-detects Kopia repository on backup drives - - Auto-detects and restores all backed-up Docker services - - Installs Docker if needed, starts all containers after restore - - Recovery flow: Mount drive → Find repo → Enter password → Select snapshot → Restore → Start -- **v2.6**: FindMyDevice, Frigate-Notify, and resilient install - - Added FindMyDevice server (self-hosted Android tracking) - - Added Frigate-Notify (push alerts for Frigate AI detections) - - Caddy now asks for domain and creates comprehensive Caddyfile - - **Resilient install pattern**: Install first, configure with defaults, continue on errors - - All Docker apps now use "install → try config → use defaults if fail" approach - - Config templates include clear "EDIT THIS FILE" warnings - - Script won't stop if configuration prompts fail - uses sensible defaults -- **v2.5**: Additional Docker apps and container backup - - Added Jellyfin (free media server with hardware acceleration) - - Added Frigate NVR (AI-powered object detection) - - Added Caddy (reverse proxy with automatic HTTPS) - - Added ddclient (dynamic DNS updater) - - Added ntfy (self-hosted push notifications) - - Added Uptime Kuma (service monitoring) - - Added wg-easy (WireGuard with web UI) - - Added Traccar (GPS tracking server) - - Added Portainer (Docker management UI) - - Added Kopia backup for all Docker containers - - Added container import/restore for disaster recovery -- **v2.4**: Self-hosted Docker applications - - Added Immich (photo/video backup) - - Added Audiobookshelf (audiobook server) - - Added Emby (media server) - - Added A.R.M. (automatic ripping machine) - - Added Filebrowser (web file manager) - - Added Magic Mirror (up to 3 instances) - - Added Lyrion Music Server (LMS) - - Added Mealie (recipe manager) - - Added Minecraft Server (Fabric, RAM-limited) - - Added linux-to-sync private repo setup - - All apps use docker-compose in ~/docker/{appname}/ -- **v2.3**: Additional VPN and remote desktop options - - Added WireGuard VPN installation - - Added Tailscale VPN installation (with Tailscale SSH info) - - Added TeamViewer remote desktop installation - - Added MeshCentral agent installation - - Updated NetBird documentation to clarify SSH key management -- **v2.2**: Backup system overhaul - - Local backups now use rsync exclusively (simpler, better for local drives) - - Support for 1-4 backup drives with customizable names - - Cloud backup added as separate option using rclone with encryption - - Guided setup for Google Drive and OneDrive cloud backups - - rclone.conf automatically backed up to all local drives - - Added guidance for secure off-site config backup (Signal, Box.com, password managers) - - Documentation: why rsync instead of RAID, why fail2ban with key-only SSH is unnecessary -- **v2.1**: QoL improvements - - Added `--dry-run` flag to preview installations without changes - - Added `--unattended` flag for automated/scripted installs - - Added logging to `/var/log/post-install.log` - - Added fail2ban (offered when SSH password auth is enabled) - - Added UFW firewall configuration - - All prompts support unattended mode with sensible defaults -- **v2.0**: Major update - - Script is now rerunnable - detects existing installations - - All components optional with y/n prompts (Docker, Samba, NetBird, RustDesk) - - Backup system: choice of rsync or rclone - - Backup modes: full (one drive) or split (two drives) - - Shows current system status at start -- **v1.0**: Initial version - - SSH (with key generation and import), Docker, Samba file sharing - - NetBird, RustDesk, split-backup with rclone +Tested on **Ubuntu 24.04 LTS** and **26.04 LTS**. +Works on any Ubuntu LTS ≥ 22.04; non-LTS releases also work. +The wizard shows the detected OS in the header and warns on unknown versions. diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..0e79a28 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# bootstrap.sh — get and run ubuntu-post-install on a fresh system. +# +# One command to paste into a new Ubuntu box: +# curl -fsSL https://raw.githubusercontent.com/outis1one/ubuntu-post-install/main/bootstrap.sh | sudo bash +# +# What it does: +# 1. Installs git if missing (the only hard dependency) +# 2. Clones (or updates) the repo to ~/ubuntu-post-install +# 3. Launches the interactive setup wizard +set -euo pipefail + +REPO_URL="https://github.com/outis1one/ubuntu-post-install.git" +DEST="${HOME:-/root}/ubuntu-post-install" + +# Resolve actual user home when running under sudo +if [ -n "${SUDO_USER:-}" ]; then + ACTUAL_HOME="$(getent passwd "$SUDO_USER" | cut -d: -f6)" + DEST="$ACTUAL_HOME/ubuntu-post-install" +fi + +echo "" +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ ubuntu-post-install · bootstrap ║" +echo "╚══════════════════════════════════════════════════════════════╝" +echo "" + +# 1) Ensure git is available +if ! command -v git >/dev/null 2>&1; then + echo "Installing git..." + apt-get update -qq && apt-get install -y git +fi + +# 2) Clone or update +if [ -d "$DEST/.git" ]; then + echo "Repo already exists at $DEST — pulling latest..." + git -C "$DEST" pull --ff-only || echo " (pull failed — continuing with existing version)" +else + echo "Cloning to $DEST ..." + git clone "$REPO_URL" "$DEST" +fi + +echo "" +echo "Launching setup..." +echo "" + +exec bash "$DEST/setup.sh"