Add per-step resilience and notifications across all video pipeline scripts
Each pipeline now saves intermediate files permanently before the step most likely to fail, so a partial failure leaves a recoverable artifact: - daily_sunrise_video.sh: speed-adjusted video (*-sped.mp4) survives if the drawtext overlay step fails; deleted automatically on success - montage-mvt.sh: speed-adjusted video (*-Sped.mp4) survives if the music-mux + attribution overlay step fails; deleted on success - 4-seasons.sh / montage-mvt.sh / year-end-join.sh: explicit ffmpeg error handling with targeted notify.sh calls naming the failed step and the surviving artifact path - montage-mvt.sh / 4-seasons.sh / year-end-join.sh: clip-duration-sum vs music-duration drift check warns when cumulative float error exceeds 2 s https://claude.ai/code/session_01C4jbd3waXG3eKZYbGUjLUQ
This commit is contained in:
Claude
+18
-2
@@ -90,16 +90,32 @@ speed_factor=$(echo "scale=6; $raw_duration / $target_per_clip" | bc)
|
||||
echo "Step 2/2: speed factor $speed_factor..."
|
||||
|
||||
final_file="$output_dir/${yesterday}_Mvt${MVT_NUM}-Day${DAY_OF_MVT}of${DAYS_IN_MVT}-final.mp4"
|
||||
ffmpeg -loglevel warning \
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-i "$temp_video" \
|
||||
-vf "setpts=PTS/$speed_factor" \
|
||||
-c:v libx264 -pix_fmt yuv420p -crf "$CRF_SEASONS_FINAL" \
|
||||
-t "$target_per_clip" -an \
|
||||
-y "$final_file"
|
||||
-y "$final_file"; then
|
||||
rm -f "$final_file"
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: $SEASON Mvt$MVT_NUM Day$DAY_OF_MVT speed-adjust ($yesterday)" \
|
||||
"ffmpeg speed-adjust failed — re-run 4-seasons.sh $CAM_NAME while JPGs exist" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
adjusted=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$final_file")
|
||||
echo "Daily clip: $final_file ($adjusted s)"
|
||||
|
||||
# Warn if clip duration deviates more than 0.5 s from target (float drift check)
|
||||
drift=$(echo "scale=3; $adjusted - $target_per_clip" | bc | sed 's/^-//')
|
||||
if awk "BEGIN{exit !($drift > 0.5)}"; then
|
||||
echo "WARNING: clip duration ${adjusted}s differs from target ${target_per_clip}s by ${drift}s"
|
||||
"$SCRIPT_DIR/notify.sh" "WARNING: $SEASON Mvt$MVT_NUM Day$DAY_OF_MVT duration drift ($yesterday)" \
|
||||
"Clip ${adjusted}s vs target ${target_per_clip}s (drift ${drift}s)" || true
|
||||
fi
|
||||
|
||||
"$SCRIPT_DIR/notify.sh" "$SEASON Mvt$MVT_NUM Day$DAY_OF_MVT/$DAYS_IN_MVT saved ($yesterday)" \
|
||||
"$(basename "$final_file") — ${adjusted}s" || true
|
||||
|
||||
# ── Auto-trigger montage on last day of movement ──────────────────────────────
|
||||
if [ "$IS_LAST_DAY" = "true" ]; then
|
||||
echo "Last day of $SEASON Mvt $MVT_NUM — triggering montage compilation..."
|
||||
|
||||
+54
-10
@@ -4,6 +4,12 @@
|
||||
# side, and write the final video. Upload is handled by a separate systemd
|
||||
# service (sky-cam-sunrise-upload) triggered via OnSuccess= so the two jobs
|
||||
# have independent log entries and failure states.
|
||||
#
|
||||
# Resilience: the speed-adjusted video is saved permanently before the overlay
|
||||
# step. If overlay fails, the sped video survives at:
|
||||
# <output_dir>/<date>-daily-sunrise-sped.mp4
|
||||
# Re-run this script once the issue is resolved; the sped file is deleted
|
||||
# automatically when the overlay step succeeds.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -22,6 +28,8 @@ mkdir -p "$output_dir"
|
||||
# ── Sunrise time ──────────────────────────────────────────────────────────────
|
||||
sunrise_time=$(python3 "$SCRIPT_DIR/sunrise.py")
|
||||
if [[ -z "$sunrise_time" ]]; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: sunrise $current_date" \
|
||||
"Could not retrieve sunrise time from sunrise.py" || true
|
||||
echo "Error: Failed to retrieve sunrise time."
|
||||
exit 1
|
||||
fi
|
||||
@@ -40,11 +48,14 @@ echo "Window: $((start_sec/3600)):$(printf '%02d' $(((start_sec%3600)/60))) →
|
||||
|
||||
# ── Collect images in window ──────────────────────────────────────────────────
|
||||
if [ ! -d "$image_dir" ]; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: sunrise $current_date" \
|
||||
"Image directory not found: $image_dir" || true
|
||||
echo "Error: image directory not found: $image_dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
temp_list=$(mktemp --suffix=.txt)
|
||||
raw_video=""
|
||||
trap 'rm -f "$temp_list" "$raw_video" 2>/dev/null || true' EXIT
|
||||
|
||||
find "$image_dir" -type f -name "*.jpg" | sort | while read -r img; do
|
||||
@@ -56,6 +67,8 @@ done > "$temp_list"
|
||||
|
||||
num_images=$(wc -l < "$temp_list")
|
||||
if [[ $num_images -lt 1 ]]; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: sunrise $current_date" \
|
||||
"No images found in sunrise window" || true
|
||||
echo "No images found in sunrise window."
|
||||
exit 1
|
||||
fi
|
||||
@@ -78,17 +91,39 @@ fi
|
||||
|
||||
# ── Step 1: Raw video from images ─────────────────────────────────────────────
|
||||
raw_video=$(mktemp --suffix=.mp4)
|
||||
echo "Step 1/2: encoding raw video..."
|
||||
ffmpeg -loglevel warning \
|
||||
echo "Step 1/3: encoding raw video..."
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-f concat -safe 0 -i "$temp_list" \
|
||||
-c:v libx264 -pix_fmt yuv420p -crf "$CRF_SUNRISE" -vsync 2 -an \
|
||||
-y "$raw_video"
|
||||
-y "$raw_video"; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: sunrise step 1/3 (encode) $current_date" \
|
||||
"ffmpeg raw encode from images failed — check: journalctl -u sky-cam-sunrise.service" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
raw_dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$raw_video")
|
||||
speed_factor=$(echo "scale=3; $raw_dur / $SUNRISE_TARGET_SECS" | bc)
|
||||
echo "Raw: ${raw_dur}s speed factor: ${speed_factor}x"
|
||||
|
||||
# ── Step 2: Speed-adjust + sunrise time overlay ───────────────────────────────
|
||||
# ── Step 2: Speed-adjust — saved permanently so overlay failure is recoverable ─
|
||||
# Deleted automatically if step 3 succeeds.
|
||||
sped_video="$output_dir/$current_date-daily-sunrise-sped.mp4"
|
||||
echo "Step 2/3: speed-adjust → $sped_video"
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-i "$raw_video" \
|
||||
-vf "setpts=PTS/${speed_factor}" \
|
||||
-c:v libx264 -pix_fmt yuv420p -crf "$CRF_SUNRISE" -an \
|
||||
-y "$sped_video"; then
|
||||
rm -f "$sped_video"
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: sunrise step 2/3 (speed) $current_date" \
|
||||
"speed-adjust failed — no video saved" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sped_dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$sped_video")
|
||||
echo "Speed-adjusted: ${sped_dur}s (saved — overlay still pending)"
|
||||
|
||||
# ── Build overlay filter ──────────────────────────────────────────────────────
|
||||
# Time displayed as stacked characters down the right side (e.g. 0/7/:/2/3),
|
||||
# semi-transparent with a soft drop shadow so it reads on any background.
|
||||
SR_TIME=$(echo "$sunrise_time_local" | cut -d'-' -f1,2 | tr '-' ':')
|
||||
@@ -104,14 +139,23 @@ DT="${DT}:line_spacing=4"
|
||||
DT="${DT}:x=w-tw-18:y=(h-th)/2"
|
||||
DT="${DT}:shadowcolor=black@0.55:shadowx=1:shadowy=1"
|
||||
|
||||
# ── Step 3: Overlay — sunrise time burned in ──────────────────────────────────
|
||||
# If this fails, $sped_video survives at its permanent path for manual recovery.
|
||||
final_video="$output_dir/$current_date-daily-sunrise.mp4"
|
||||
echo "Step 2/2: speed-adjust + overlay → $final_video"
|
||||
|
||||
ffmpeg -loglevel warning \
|
||||
-i "$raw_video" \
|
||||
-vf "setpts=PTS/${speed_factor},${DT}" \
|
||||
echo "Step 3/3: overlay → $final_video"
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-i "$sped_video" \
|
||||
-vf "${DT}" \
|
||||
-c:v libx264 -pix_fmt yuv420p -crf "$CRF_SUNRISE" -an \
|
||||
-y "$final_video"
|
||||
-y "$final_video"; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: sunrise overlay $current_date" \
|
||||
"Overlay failed — speed-only video saved: $(basename "$sped_video")" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
actual_dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$final_video")
|
||||
echo "Done: $final_video (${actual_dur}s, sunrise at ${SR_TIME})"
|
||||
|
||||
rm -f "$sped_video"
|
||||
"$SCRIPT_DIR/notify.sh" "Sunrise ready: $current_date" \
|
||||
"$(basename "$final_video") — ${actual_dur}s, sunrise at ${SR_TIME}" || true
|
||||
|
||||
+39
-8
@@ -51,6 +51,8 @@ mkdir -p "$output_dir"
|
||||
|
||||
music_file=$(find "$music_base_dir" -type f -iname "*${SEASON}*Mvt*${MVT_NUM}*" | sort | head -n 1)
|
||||
if [ -z "$music_file" ]; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: montage $SEASON Mvt$MVT_NUM ($ASTRO_YEAR)" \
|
||||
"Music file not found in $music_base_dir" || true
|
||||
echo "ERROR: Music file not found for $SEASON Mvt $MVT_NUM in $music_base_dir"
|
||||
exit 1
|
||||
fi
|
||||
@@ -60,11 +62,28 @@ echo "Music: $(basename "$music_file") (${music_duration}s)"
|
||||
# ── Collect sorted daily clips ────────────────────────────────────────────────
|
||||
mapfile -d '' daily_clips < <(find "$video_dir" -maxdepth 1 -name "*-final.mp4" -print0 | sort -z)
|
||||
if [ "${#daily_clips[@]}" -eq 0 ]; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: montage $SEASON Mvt$MVT_NUM ($ASTRO_YEAR)" \
|
||||
"No *-final.mp4 clips found in $video_dir" || true
|
||||
echo "ERROR: No *-final.mp4 clips in $video_dir"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found ${#daily_clips[@]} daily clips"
|
||||
|
||||
# ── Validate clip duration sum against music ──────────────────────────────────
|
||||
total_clip_dur=0
|
||||
for clip in "${daily_clips[@]}"; do
|
||||
d=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$clip")
|
||||
total_clip_dur=$(echo "scale=3; $total_clip_dur + $d" | bc)
|
||||
done
|
||||
clip_drift=$(echo "scale=3; $total_clip_dur - $music_duration" | bc | sed 's/^-//')
|
||||
echo "Sum of clips: ${total_clip_dur}s Music: ${music_duration}s Drift: ${clip_drift}s"
|
||||
if awk "BEGIN{exit !($clip_drift > 2.0)}"; then
|
||||
echo "WARNING: clip total differs from music by ${clip_drift}s — speed pass will compensate"
|
||||
"$SCRIPT_DIR/notify.sh" \
|
||||
"WARNING: $SEASON Mvt$MVT_NUM clip drift ${clip_drift}s ($ASTRO_YEAR)" \
|
||||
"Clips sum ${total_clip_dur}s vs music ${music_duration}s" || true
|
||||
fi
|
||||
|
||||
# ── Source resolution ─────────────────────────────────────────────────────────
|
||||
VIDEO_W=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of csv=p=0 "${daily_clips[0]}")
|
||||
VIDEO_H=$(ffprobe -v error -select_streams v:0 -show_entries stream=height -of csv=p=0 "${daily_clips[0]}")
|
||||
@@ -105,18 +124,25 @@ pad=${VIDEO_W}:${VIDEO_H}:(ow-iw)/2:(oh-ih)/2,setsar=1" \
|
||||
-c:v libx264 -pix_fmt yuv420p -crf "$CRF_MONTAGE" -an \
|
||||
-y "$temp_concat"
|
||||
|
||||
# ── Step 2: Speed-adjust to match music duration exactly ──────────────────────
|
||||
# ── Step 2: Speed-adjust — saved permanently so music/overlay failure is recoverable ─
|
||||
# Deleted automatically if step 3 succeeds.
|
||||
concat_dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$temp_concat")
|
||||
speed_factor=$(echo "scale=6; $concat_dur / $music_duration" | bc)
|
||||
echo "Step 2/3: speed $speed_factor (raw=${concat_dur}s → music=${music_duration}s)"
|
||||
|
||||
temp_sped=$(mktemp --suffix=.mp4); TMPFILES+=("$temp_sped")
|
||||
ffmpeg -loglevel warning \
|
||||
sped_file="$output_dir/${MVT_START}_${SEASON}_Mvt${MVT_NUM}-Sped.mp4"
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-i "$temp_concat" \
|
||||
-vf "setpts=PTS/$speed_factor" \
|
||||
-c:v libx264 -pix_fmt yuv420p -crf "$CRF_MONTAGE" -an \
|
||||
-t "$music_duration" -y "$temp_sped"
|
||||
-t "$music_duration" -y "$sped_file"; then
|
||||
rm -f "$sped_file"
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: montage step 2/3 (speed) $SEASON Mvt$MVT_NUM ($ASTRO_YEAR)" \
|
||||
"speed-adjust failed — concat temp lost; re-run montage-mvt.sh" || true
|
||||
exit 1
|
||||
fi
|
||||
rm "$temp_concat"
|
||||
echo "Speed-adjusted: $sped_file (saved — music+overlay still pending)"
|
||||
|
||||
# ── Step 3: Music + fades + attribution overlay ───────────────────────────────
|
||||
# Attribution overlay: semi-transparent bar across the top for the first
|
||||
@@ -152,16 +178,21 @@ DT="${DT}:alpha='${ALPHA}':enable='${ENABLE}'"
|
||||
|
||||
output_file="$output_dir/${MVT_START}_${SEASON}_Mvt${MVT_NUM}-Montage.mp4"
|
||||
|
||||
ffmpeg -loglevel warning \
|
||||
-i "$temp_sped" -i "$music_file" \
|
||||
# If this fails, $sped_file survives for manual recovery (re-run step 3 only).
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-i "$sped_file" -i "$music_file" \
|
||||
-filter_complex \
|
||||
"[0:v]fade=t=in:st=0:d=${FADE_DUR},fade=t=out:st=${fade_out_start}:d=${FADE_DUR},${DT}[vout];\
|
||||
[1:a]afade=t=in:st=0:d=${FADE_DUR},afade=t=out:st=${fade_out_start}:d=${FADE_DUR}[aout]" \
|
||||
-map "[vout]" -map "[aout]" \
|
||||
-c:v libx264 -pix_fmt yuv420p -c:a aac -b:a "$AUDIO_BITRATE" \
|
||||
-t "$music_duration" -y "$output_file"
|
||||
rm "$temp_sped"
|
||||
-t "$music_duration" -y "$output_file"; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: montage step 3/3 (music+overlay) $SEASON Mvt$MVT_NUM ($ASTRO_YEAR)" \
|
||||
"Music/overlay failed — sped video saved: $(basename "$sped_file")" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f "$sped_file"
|
||||
echo "Done: $output_file"
|
||||
|
||||
# ── Notify ────────────────────────────────────────────────────────────────────
|
||||
|
||||
+19
-3
@@ -32,6 +32,7 @@ concat_list=$(mktemp --suffix=.txt)
|
||||
trap 'rm -f "$concat_list"' EXIT
|
||||
|
||||
missing=0
|
||||
total_expected_dur=0
|
||||
for season in Winter Spring Summer Autumn; do
|
||||
for mvt in 1 2 3; do
|
||||
mvt_dir="$year_dir/$season/Mvt$mvt/montage"
|
||||
@@ -41,11 +42,14 @@ for season in Winter Spring Summer Autumn; do
|
||||
echo "WARNING: missing montage for $season Mvt$mvt ($mvt_dir)"
|
||||
missing=$((missing + 1))
|
||||
else
|
||||
echo " $season Mvt$mvt → $(basename "$f")"
|
||||
d=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$f")
|
||||
total_expected_dur=$(echo "scale=3; $total_expected_dur + $d" | bc)
|
||||
echo " $season Mvt$mvt → $(basename "$f") (${d}s)"
|
||||
printf "file '%s'\n" "$f" >> "$concat_list"
|
||||
fi
|
||||
done
|
||||
done
|
||||
echo "Expected total duration: ${total_expected_dur}s ($(echo "scale=1; $total_expected_dur/60" | bc) min)"
|
||||
|
||||
found=$(wc -l < "$concat_list")
|
||||
echo "$found of 12 movements found ($missing missing)"
|
||||
@@ -63,14 +67,26 @@ fi
|
||||
output_file="$year_dir/${CAM_NAME}-${ASTRO_YEAR}.mp4"
|
||||
echo "Output: $output_file"
|
||||
|
||||
ffmpeg -loglevel warning \
|
||||
if ! ffmpeg -loglevel warning \
|
||||
-f concat -safe 0 -i "$concat_list" \
|
||||
-c copy -y "$output_file"
|
||||
-c copy -y "$output_file"; then
|
||||
"$SCRIPT_DIR/notify.sh" "FAILED: year-end join $CAM_NAME $ASTRO_YEAR" \
|
||||
"ffmpeg concat failed — the 12 movement montages are still intact" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
total_dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$output_file")
|
||||
total_min=$(echo "scale=1; $total_dur / 60" | bc)
|
||||
echo "Done: $output_file (${total_min} min)"
|
||||
|
||||
# Sanity-check actual vs expected duration
|
||||
dur_drift=$(echo "scale=3; $total_dur - $total_expected_dur" | bc | sed 's/^-//')
|
||||
if awk "BEGIN{exit !($dur_drift > 2.0)}"; then
|
||||
echo "WARNING: output ${total_dur}s differs from expected ${total_expected_dur}s by ${dur_drift}s"
|
||||
"$SCRIPT_DIR/notify.sh" "WARNING: year-end duration drift $CAM_NAME $ASTRO_YEAR" \
|
||||
"Output ${total_dur}s vs expected ${total_expected_dur}s (drift ${dur_drift}s)" || true
|
||||
fi
|
||||
|
||||
# ── Notify ────────────────────────────────────────────────────────────────────
|
||||
"$SCRIPT_DIR/notify.sh" \
|
||||
"Four Seasons $ASTRO_YEAR complete" \
|
||||
|
||||
Reference in New Issue
Block a user