From 3265ed80fdd7baffefb56c620c9cd0e467365f7c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Apr 2026 01:13:49 +0000 Subject: [PATCH] Add per-camera capture watchdog with stall and recovery notifications capture-watchdog.sh: long-running script that checks the newest JPEG mtime in each camera's output directory every CAPTURE_STALE_SECS/3 seconds (min 5s). Sends a stall notification via notify.sh if no new frame has arrived within CAPTURE_STALE_SECS. Sends a recovery notification once new frames resume. Only alerts if the camera was previously working (avoids false positives on first start or overnight before the first frame of the day). sky-cam.conf: CAPTURE_STALE_SECS=30 added next to CAPTURE_INTERVAL with a note that it should be at least 2x the interval. install.sh: generates sky-cam-watchdog-.service for every camera that has a RTSP URL configured. The watchdog service starts after and alongside the capture service, restarts on failure, and is enabled/started with the same loop as the capture service. README.md: note about watchdog and journalctl command for its logs. https://claude.ai/code/session_01C4jbd3waXG3eKZYbGUjLUQ --- README.md | 5 ++++ capture-watchdog.sh | 62 +++++++++++++++++++++++++++++++++++++++++++++ install.sh | 20 +++++++++++++++ sky-cam.conf | 5 ++++ 4 files changed, 92 insertions(+) create mode 100755 capture-watchdog.sh diff --git a/README.md b/README.md index 70ba407..165397b 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,11 @@ systemctl --user status sky-cam-capture-east.service journalctl --user -u sky-cam-capture-east.service -f ``` +The capture watchdog runs alongside each camera and sends a notification (via `notify.sh`) if no new frame arrives within `CAPTURE_STALE_SECS` seconds, and a second notification when capture resumes. Check watchdog logs with: +```bash +journalctl --user -u sky-cam-watchdog-east.service -f +``` + **Check logs**: ```bash journalctl --user -u sky-cam-sunrise.service diff --git a/capture-watchdog.sh b/capture-watchdog.sh new file mode 100755 index 0000000..19c459d --- /dev/null +++ b/capture-watchdog.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# capture-watchdog.sh — monitor image output for a camera and send a +# notification if no new frame arrives within CAPTURE_STALE_SECS seconds. +# Sends a recovery notification once capture resumes. +# +# Runs as a long-running systemd service alongside sky-cam-capture-. +# One instance per camera, camera name passed as $1. + +set -euo pipefail + +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR/sky-cam.conf" +export TZ="$TIMEZONE" + +CAM="${1:-$SUNRISE_CAM}" +STALE_SECS="${CAPTURE_STALE_SECS:-30}" +CHECK_INTERVAL=$(( STALE_SECS / 3 )) +[ "$CHECK_INTERVAL" -lt 5 ] && CHECK_INTERVAL=5 + +echo "Watchdog started for ${CAM} — alerting after ${STALE_SECS}s without a new frame" + +last_good=0 # epoch seconds of the newest frame we have seen +stale_notified=false + +while true; do + sleep "$CHECK_INTERVAL" + + today=$(date +%Y-%m-%d) + dir="$BASE_DIR/$CAM/$today" + now=$(date +%s) + + # Find newest JPEG in today's directory + if [ -d "$dir" ]; then + latest_file=$(ls -t "$dir"/*.jpg 2>/dev/null | head -1 || true) + if [ -n "$latest_file" ]; then + latest_mtime=$(stat -c %Y "$latest_file") + if [ "$latest_mtime" -gt "$last_good" ]; then + last_good=$latest_mtime + # Recovery — was stale, now seeing new frames again + if $stale_notified; then + stale_for=$(( now - last_good + STALE_SECS )) + echo "Capture resumed for ${CAM} after ~${stale_for}s stall" + "$SCRIPT_DIR/notify.sh" "Capture resumed: ${CAM}" \ + "New frames arriving again after ~${stale_for}s without capture" || true + stale_notified=false + fi + fi + fi + fi + + # Only alert if we have seen at least one frame (camera was working) + # and now there's a gap longer than the threshold + if [ "$last_good" -gt 0 ]; then + age=$(( now - last_good )) + if [ "$age" -gt "$STALE_SECS" ] && ! $stale_notified; then + echo "Capture stalled for ${CAM} — last frame ${age}s ago" + "$SCRIPT_DIR/notify.sh" "WARNING: capture stalled — ${CAM}" \ + "No new frames for ${age}s — camera may be offline or stream lost" || true + stale_notified=true + fi + fi +done diff --git a/install.sh b/install.sh index a43ec26..a6e5823 100755 --- a/install.sh +++ b/install.sh @@ -137,6 +137,26 @@ WantedBy=default.target EOF echo " wrote sky-cam-capture-${cam}.service" capture_services+=("sky-cam-capture-${cam}") + + cat > "$UNIT_DIR/sky-cam-watchdog-${cam}.service" <