#!/bin/bash # capture-watchdog.sh — monitor image output for a camera and send a # notification if no new frame arrives within CAPTURE_STALE_SECS seconds. # Sends a recovery notification once capture resumes. # # Runs as a long-running systemd service alongside sky-cam-capture-. # One instance per camera, camera name passed as $1. set -euo pipefail SCRIPT_DIR="$(dirname "$(realpath "$0")")" source "$SCRIPT_DIR/sky-cam.conf" export TZ="$TIMEZONE" CAM="${1:-}" if [ -z "$CAM" ]; then echo "Usage: $0 " exit 1 fi STALE_SECS="${CAPTURE_STALE_SECS:-30}" CHECK_INTERVAL=$(( STALE_SECS / 3 )) [ "$CHECK_INTERVAL" -lt 5 ] && CHECK_INTERVAL=5 echo "Watchdog started for ${CAM} — alerting after ${STALE_SECS}s without a new frame" last_good=0 # epoch seconds of the newest frame we have seen stale_notified=false while true; do sleep "$CHECK_INTERVAL" today=$(date +%Y-%m-%d) dir="$BASE_DIR/$CAM/$today" now=$(date +%s) # Find newest JPEG in today's directory if [ -d "$dir" ]; then latest_file=$(ls -t "$dir"/*.jpg 2>/dev/null | head -1 || true) if [ -n "$latest_file" ]; then latest_mtime=$(stat -c %Y "$latest_file") if [ "$latest_mtime" -gt "$last_good" ]; then last_good=$latest_mtime # Recovery — was stale, now seeing new frames again if $stale_notified; then stale_for=$(( now - last_good + STALE_SECS )) echo "Capture resumed for ${CAM} after ~${stale_for}s stall" "$SCRIPT_DIR/notify.sh" "Capture resumed: ${CAM}" \ "New frames arriving again after ~${stale_for}s without capture" || true stale_notified=false fi fi fi fi # Only alert if we have seen at least one frame (camera was working) # and now there's a gap longer than the threshold if [ "$last_good" -gt 0 ]; then age=$(( now - last_good )) if [ "$age" -gt "$STALE_SECS" ] && ! $stale_notified; then echo "Capture stalled for ${CAM} — last frame ${age}s ago" "$SCRIPT_DIR/notify.sh" "WARNING: capture stalled — ${CAM}" \ "No new frames for ${age}s — camera may be offline or stream lost" || true stale_notified=true fi fi done