Add per-camera capture watchdog with stall and recovery notifications

capture-watchdog.sh: long-running script that checks the newest JPEG mtime
in each camera's output directory every CAPTURE_STALE_SECS/3 seconds (min 5s).
Sends a stall notification via notify.sh if no new frame has arrived within
CAPTURE_STALE_SECS. Sends a recovery notification once new frames resume.
Only alerts if the camera was previously working (avoids false positives on
first start or overnight before the first frame of the day).

sky-cam.conf: CAPTURE_STALE_SECS=30 added next to CAPTURE_INTERVAL with a
note that it should be at least 2x the interval.

install.sh: generates sky-cam-watchdog-<cam>.service for every camera that
has a RTSP URL configured. The watchdog service starts after and alongside
the capture service, restarts on failure, and is enabled/started with the
same loop as the capture service.

README.md: note about watchdog and journalctl command for its logs.

https://claude.ai/code/session_01C4jbd3waXG3eKZYbGUjLUQ

README.md

@@ -242,6 +242,11 @@ systemctl --user status sky-cam-capture-east.service
 journalctl --user -u sky-cam-capture-east.service -f
 ```
 
+The capture watchdog runs alongside each camera and sends a notification (via `notify.sh`) if no new frame arrives within `CAPTURE_STALE_SECS` seconds, and a second notification when capture resumes. Check watchdog logs with:
+```bash
+journalctl --user -u sky-cam-watchdog-east.service -f
+```
+
 **Check logs**:
 ```bash
 journalctl --user -u sky-cam-sunrise.service

capture-watchdog.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# capture-watchdog.sh — monitor image output for a camera and send a
+# notification if no new frame arrives within CAPTURE_STALE_SECS seconds.
+# Sends a recovery notification once capture resumes.
+#
+# Runs as a long-running systemd service alongside sky-cam-capture-<cam>.
+# One instance per camera, camera name passed as $1.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR/sky-cam.conf"
+export TZ="$TIMEZONE"
+
+CAM="${1:-$SUNRISE_CAM}"
+STALE_SECS="${CAPTURE_STALE_SECS:-30}"
+CHECK_INTERVAL=$(( STALE_SECS / 3 ))
+[ "$CHECK_INTERVAL" -lt 5 ] && CHECK_INTERVAL=5
+
+echo "Watchdog started for ${CAM} — alerting after ${STALE_SECS}s without a new frame"
+
+last_good=0       # epoch seconds of the newest frame we have seen
+stale_notified=false
+
+while true; do
+    sleep "$CHECK_INTERVAL"
+
+    today=$(date +%Y-%m-%d)
+    dir="$BASE_DIR/$CAM/$today"
+    now=$(date +%s)
+
+    # Find newest JPEG in today's directory
+    if [ -d "$dir" ]; then
+        latest_file=$(ls -t "$dir"/*.jpg 2>/dev/null | head -1 || true)
+        if [ -n "$latest_file" ]; then
+            latest_mtime=$(stat -c %Y "$latest_file")
+            if [ "$latest_mtime" -gt "$last_good" ]; then
+                last_good=$latest_mtime
+                # Recovery — was stale, now seeing new frames again
+                if $stale_notified; then
+                    stale_for=$(( now - last_good + STALE_SECS ))
+                    echo "Capture resumed for ${CAM} after ~${stale_for}s stall"
+                    "$SCRIPT_DIR/notify.sh" "Capture resumed: ${CAM}" \
+                        "New frames arriving again after ~${stale_for}s without capture" || true
+                    stale_notified=false
+                fi
+            fi
+        fi
+    fi
+
+    # Only alert if we have seen at least one frame (camera was working)
+    # and now there's a gap longer than the threshold
+    if [ "$last_good" -gt 0 ]; then
+        age=$(( now - last_good ))
+        if [ "$age" -gt "$STALE_SECS" ] && ! $stale_notified; then
+            echo "Capture stalled for ${CAM} — last frame ${age}s ago"
+            "$SCRIPT_DIR/notify.sh" "WARNING: capture stalled — ${CAM}" \
+                "No new frames for ${age}s — camera may be offline or stream lost" || true
+            stale_notified=true
+        fi
+    fi
+done

install.sh

@@ -137,6 +137,26 @@ WantedBy=default.target
 EOF
         echo "  wrote sky-cam-capture-${cam}.service"
         capture_services+=("sky-cam-capture-${cam}")
+
+        cat > "$UNIT_DIR/sky-cam-watchdog-${cam}.service" <<EOF
+[Unit]
+Description=Sky-Cam — ${cam}: capture watchdog (stall detection)
+After=sky-cam-capture-${cam}.service
+Wants=sky-cam-capture-${cam}.service
+
+[Service]
+Type=simple
+ExecStart=$SCRIPT_DIR/capture-watchdog.sh ${cam}
+Restart=on-failure
+RestartSec=10
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=default.target
+EOF
+        echo "  wrote sky-cam-watchdog-${cam}.service"
+        capture_services+=("sky-cam-watchdog-${cam}")
     else
         echo "  WARNING: CAM_RTSP_${cam} not set — skipping capture service for ${cam}"
         echo "           (set CAM_RTSP_${cam}=rtsp://... in sky-cam.conf to enable)"

sky-cam.conf

@@ -279,6 +279,11 @@ MONTAGE_ATTR_FADE=1    # attribution fade-in and fade-out length (seconds)
 # 10 = one frame every 10 s → 8640 frames/day → good timelapse density.
 CAPTURE_INTERVAL=10
 
+# Seconds without a new frame before the watchdog sends a stall notification.
+# Should be at least 2× CAPTURE_INTERVAL.  A recovery notification fires
+# automatically when frames start arriving again.
+CAPTURE_STALE_SECS=30
+
 # ── Sunrise audio capture ──────────────────────────────────────────────────────
 # Records the camera's RTSP audio stream during the sunrise window so that
 # daily_sunrise_video.sh can mix in natural ambient sound (birds, rain, wind)