Merge pull request #39 from outis1one/clDemoe/seasonal-sunrise-montage-nn268

ClDemoe/seasonal sunrise montage nn268

4-seasons.sh

@@ -106,7 +106,7 @@ echo "Raw duration: $raw_duration s"
 speed_factor=$(echo "scale=6; $raw_duration / $target_per_clip" | bc)
 echo "Step 2/2: speed factor $speed_factor..."
 
-final_file="$output_dir/${yesterday}_${CAM_NAME}_Mvt${MVT_NUM}-Day${DAY_OF_MVT}of${DAYS_IN_MVT}-final.mp4"
+final_file="$output_dir/${CAM_NAME}-${yesterday}_Mvt${MVT_NUM}-Day${DAY_OF_MVT}of${DAYS_IN_MVT}-final.mp4"
 if ! ffmpeg -loglevel warning \
     -i "$temp_video" \
     -vf "setpts=PTS/$speed_factor,fps=25" \

ambient-record.sh

@@ -37,8 +37,9 @@ fi
 
 CHUNK="${AMBIENT_CHUNK_SECS:-1800}"
 BITRATE="${AMBIENT_BITRATE:-96k}"
-RETAIN="${AMBIENT_RETENTION_DAYS:-30}"
+retain_var="AMBIENT_RETENTION_DAYS_${CAM}"
-SAVE_DIR="${AMBIENT_DIR:-$BASE_DIR/ambient}"
+RETAIN="${!retain_var:-${AMBIENT_RETENTION_DAYS:-30}}"
+SAVE_DIR="${AUDIO_DIR:-$BASE_DIR/audio}"
 
 echo "Ambient recorder started: cam=$CAM  chunk=${CHUNK}s  bitrate=$BITRATE  retain=${RETAIN}d"
 echo "Saving to: $SAVE_DIR/$CAM/"
@@ -49,7 +50,7 @@ while true; do
     mkdir -p "$dir"
 
     timestamp=$(date +%H-%M-%S)
-    outfile="$dir/${timestamp}.m4a"
+    outfile="$dir/${CAM}-${timestamp}.m4a"
 
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] Recording → $outfile  (${CHUNK}s)"
 

daily_sunrise_video.sh

@@ -54,9 +54,8 @@ if ! $TEST_MODE; then
         echo "Error: Failed to retrieve sunrise time."
         exit 1
     fi
-    echo "Sunrise (UTC): $sunrise_time"
+    echo "Sunrise (local): $sunrise_time"
     sunrise_time_local=$(TZ="$TIMEZONE" date -d "$sunrise_time" +"%H-%M-%S")
-    echo "Sunrise (local): $sunrise_time_local"
     SR_TIME=$(echo "$sunrise_time_local" | cut -d'-' -f1,2 | tr '-' ':')
 fi
 
@@ -111,7 +110,18 @@ if [ "${#images[@]}" -lt 1 ]; then
     echo "No images found."
     exit 1
 fi
-echo "Images: ${#images[@]}"
+first_t=$(basename "${images[0]}"  .jpg | tr '-' ':')
+last_t=$(basename  "${images[-1]}" .jpg | tr '-' ':')
+echo "Images: ${#images[@]}  (${first_t} → ${last_t}  overlay: ${SR_TIME})"
+
+if ! $TEST_MODE; then
+    last_sec=$(time_to_seconds "$(basename "${images[-1]}" .jpg)")
+    gap=$(( end_sec - last_sec ))
+    if [ "$gap" -gt $(( ${CAPTURE_INTERVAL:-10} * 3 )) ]; then
+        echo "WARNING: last frame is $(( gap / 60 ))min ${gap}s before window end — capture gap near sunrise"
+        echo "         Check: systemctl --user status sky-cam-capture-${CAM_NAME}.service"
+    fi
+fi
 
 # Concat list with explicit frame duration so each frame represents real elapsed
 # time.  Without this, all frames collapse to timestamp 0 → one-frame video.
@@ -168,7 +178,7 @@ if $TEST_MODE; then
     sped_video=$(mktemp --suffix=-sped.mp4)
     echo "Step 2/3: speed-adjust (test)..."
 else
-    sped_video="$output_dir/$current_date-daily-sunrise-sped.mp4"
+    sped_video="$output_dir/$CAM_NAME-$current_date-daily-sunrise-sped.mp4"
     echo "Step 2/3: speed-adjust → $sped_video"
 fi
 if ! ffmpeg -loglevel warning \
@@ -232,7 +242,10 @@ if $TEST_MODE; then
         echo "Audio: CAM_RTSP_${CAM_NAME} not set — skipping"
     fi
 else
-    cam_audio="$image_dir/sunrise-audio.m4a"
+    # New path: AUDIO_DIR/sunrise/<cam>/<date>/sunrise-audio.m4a
+    # Fallback to old path (BASE_DIR/<cam>/<date>/sunrise-audio.m4a) for migration
+    cam_audio="${AUDIO_DIR:-$BASE_DIR/audio}/sunrise/$CAM_NAME/$current_date/sunrise-audio.m4a"
+    [ ! -f "$cam_audio" ] && cam_audio="$image_dir/sunrise-audio.m4a"
     if [ "${AUDIO_ENABLED:-false}" = "true" ]; then
         if [ -f "$cam_audio" ]; then
             audio_src="$cam_audio"
@@ -260,9 +273,11 @@ fi
 #   audio ✗  overlay ✓  → final has overlay, no audio
 #   audio ✗  overlay ✗  → sped video promoted (no audio, no overlay)
 if $TEST_MODE; then
-    final_video="$output_dir/$current_date-daily-sunrise-test.mp4"
+    demo_dir="${DEMO_DIR:-$MOVIES_DIR/demoio}/$CAM_NAME"
+    mkdir -p "$demo_dir"
+    final_video="$demo_dir/$CAM_NAME-$current_date-daily-sunrise-test.mp4"
 else
-    final_video="$output_dir/$current_date-daily-sunrise.mp4"
+    final_video="$output_dir/$CAM_NAME-$current_date-daily-sunrise.mp4"
 fi
 fade_out=$(echo "scale=1; $SUNRISE_TARGET_SECS - 0.5" | bc)
 
@@ -333,7 +348,17 @@ if $step3b_ok; then
     echo "Done: $final_video  (${actual_dur}s, time: ${SR_TIME})"
     rm -f "$sped_video" "$temp_audio" 2>/dev/null || true; temp_audio=""
 
-    if ! $TEST_MODE; then
+    if $TEST_MODE; then
+        # ── Demo retention ────────────────────────────────────────────────────
+        demo_retain="${DEMO_RETENTION_DAYS:-8}"
+        if [ "$demo_retain" -gt 0 ]; then
+            deleted_demo=$(find "${DEMO_DIR:-$MOVIES_DIR/demoio}" \
+                -name "*-daily-sunrise-test.mp4" \
+                -mtime +"$demo_retain" -print -delete 2>/dev/null | wc -l)
+            [ "$deleted_demo" -gt 0 ] && \
+                echo "Demo retention: deleted $deleted_demo test file(s) older than ${demo_retain} days"
+        fi
+    else
         [ -f "${cam_audio:-}" ] && rm -f "${cam_audio:-}"
         "$SCRIPT_DIR/notify.sh" "Sunrise ready: $current_date" \
             "$(basename "$final_video") — ${actual_dur}s, sunrise at ${SR_TIME} | $final_video" || true

install.sh

@@ -37,10 +37,13 @@ echo "Creating data directories..."
 for cam in "${CAMERAS[@]}"; do
     mkdir -p "$BASE_DIR/$cam"
     mkdir -p "$MOVIES_DIR/$cam"
+    mkdir -p "$AUDIO_DIR/$cam"
     echo "  $BASE_DIR/$cam"
     echo "  $MOVIES_DIR/$cam"
+    echo "  $AUDIO_DIR/$cam"
 done
 mkdir -p "$BASE_DIR/$SUNRISE_CAM"
+mkdir -p "$AUDIO_DIR/sunrise/$SUNRISE_CAM"
 echo ""
 
 # ── Helper: write a oneshot service unit ─────────────────────────────────────
@@ -262,6 +265,7 @@ env_example="$HERE/.env.example"
     echo "# different drive.  Leave commented to use the defaults next to the scripts."
     echo "#BASE_DIR=/path/to/images          # default: ~/sky-cam/data"
     echo "#MOVIES_DIR=/path/to/videos        # default: \$BASE_DIR/movies"
+    echo "#AUDIO_DIR=/path/to/audio          # default: \$BASE_DIR/audio"
     echo "#MUSIC_DIR=/path/to/music/4Seasons # default: ~/sky-cam/music"
     echo ""
     echo "# ── RTSP stream URLs ────────────────────────────────────────────────────────"

montage-mvt.sh

@@ -147,7 +147,7 @@ concat_dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$temp_c
 speed_factor=$(echo "scale=6; $concat_dur / $effective_duration" | bc)
 echo "Step 2/3: speed $speed_factor  (raw=${concat_dur}s → target=${effective_duration}s)"
 
-sped_file="$output_dir/${MVT_START}_${SEASON}_Mvt${MVT_NUM}-Sped.mp4"
+sped_file="$output_dir/${CAM_NAME}-${MVT_START}_${SEASON}_Mvt${MVT_NUM}-Sped.mp4"
 if ! ffmpeg -loglevel warning \
     -i "$temp_concat" \
     -vf "setpts=PTS/$speed_factor" \
@@ -193,7 +193,7 @@ DT="${DT}:fontcolor=0xCCCCCC:fontsize=h/36:x=(w-text_w)/2:y=h*0.113"
 DT="${DT}:box=1:boxcolor=black@0.55:boxborderw=5"
 DT="${DT}:alpha='${ALPHA}':enable='${ENABLE}'"
 
-output_file="$output_dir/${MVT_START}_${SEASON}_Mvt${MVT_NUM}-Montage.mp4"
+output_file="$output_dir/${CAM_NAME}-${MVT_START}_${SEASON}_Mvt${MVT_NUM}-Montage.mp4"
 
 # If music/overlay fails, the sped video is promoted to the output path so
 # year-end-join can still include this movement (as a silent, no-overlay clip).

sky-cam.conf

@@ -150,6 +150,9 @@ SCRIPT_DIR="${SCRIPT_DIR:-$(dirname "$(realpath "${BASH_SOURCE[0]}")")}"
 BASE_DIR="${BASE_DIR:-$SCRIPT_DIR/data}"
 MOVIES_DIR="${MOVIES_DIR:-$BASE_DIR/movies}"  # override if videos live on a different drive
 MUSIC_DIR="${MUSIC_DIR:-$SCRIPT_DIR/music}"
+AUDIO_DIR="${AUDIO_DIR:-$BASE_DIR/audio}"     # root for all audio recordings; override in .env
+#   Structure: AUDIO_DIR/<cam>/YYYY-MM-DD/HH-MM-SS.m4a         (ambient)
+#              AUDIO_DIR/sunrise/<cam>/YYYY-MM-DD/sunrise-audio.m4a  (sunrise clips)
 
 # ── Location & timezone ───────────────────────────────────────────────────────
 # Used by sunrise.py to calculate sunrise time each day.
@@ -225,6 +228,13 @@ SUNRISE_OVERLAY_OPACITY=0.45
 # Delete daily sunrise videos older than this many days (rolling window).
 # Set 0 to keep forever.
 SUNRISE_RETENTION_DAYS=10
+# Delete unpinned sunrise posts from Mattermost older than this many days.
+# Pinned posts are always kept regardless.  Set 0 to disable Mattermost cleanup.
+MM_SUNRISE_RETENTION_DAYS=8
+# Output directory for --test runs.  Kept separate from production sunrise videos.
+# Test files older than DEMO_RETENTION_DAYS are deleted automatically.
+DEMO_DIR="${DEMO_DIR:-$MOVIES_DIR/demoio}"
+DEMO_RETENTION_DAYS=8
 
 # ── Video encoding quality ────────────────────────────────────────────────────
 # FFmpeg CRF: lower = higher quality / larger files.  Typical range 18–30.
@@ -314,6 +324,9 @@ AUDIO_ENABLED=false           # set true if your camera has a working mic
 CAPTURE_AUDIO_BITRATE=96k     # bitrate for the sunrise audio recording
 # Audio is recorded for exactly SUNRISE_TARGET_SECS, centred on actual sunrise:
 # floor(TARGET/2) seconds before, ceil(TARGET/2) after (odd second → post-sunrise).
+# Clips are saved to AUDIO_DIR/sunrise/<cam>/YYYY-MM-DD/sunrise-audio.m4a.
+# After mixing into the video the clip is kept for SUNRISE_AUDIO_RETENTION_DAYS, then deleted.
+SUNRISE_AUDIO_RETENTION_DAYS=7
 
 # ── Ambient audio library ──────────────────────────────────────────────────────
 # Continuously records natural sounds (birds, rain, wind) from AMBIENT_CAMS to
@@ -323,10 +336,14 @@ CAPTURE_AUDIO_BITRATE=96k     # bitrate for the sunrise audio recording
 #
 AMBIENT_ENABLED=false
 AMBIENT_CAMS=(south)                    # cameras to record from
-AMBIENT_DIR="${BASE_DIR}/ambient"       # root storage directory
 AMBIENT_CHUNK_SECS=1800                 # seconds per file (default 30 min)
 AMBIENT_BITRATE=96k                     # AAC bitrate; 64–96k is plenty at 8 kHz
-AMBIENT_RETENTION_DAYS=30              # delete files older than this; 0 = keep forever
+# Retention — global default, then optional per-camera overrides.
+# Uncomment and set per-cam lines to keep different cameras for different durations.
+AMBIENT_RETENTION_DAYS=30              # global default; 0 = keep forever
+#AMBIENT_RETENTION_DAYS_east=14
+#AMBIENT_RETENTION_DAYS_north=30
+#AMBIENT_RETENTION_DAYS_south=60       # keep south longer for nature sound library
 
 # ── Mattermost — daily sunrise upload ─────────────────────────────────────────
 # mattermost_url, access_token, channel_id go in .env (see bottom of this file).

sunrise-audio-capture.sh

@@ -64,7 +64,7 @@ if [ "$wait_sec" -gt 0 ]; then
 fi
 
 today=$(date +%Y-%m-%d)
-dir="$BASE_DIR/$CAM/$today"
+dir="${AUDIO_DIR:-$BASE_DIR/audio}/sunrise/$CAM/$today"
 mkdir -p "$dir"
 output="$dir/sunrise-audio.m4a"
 
@@ -84,3 +84,11 @@ fi
 
 size=$(du -h "$output" | cut -f1)
 echo "Audio capture complete: $output  (${record_dur_sec}s, $size)"
+
+# Rolling retention for sunrise audio clips
+retain="${SUNRISE_AUDIO_RETENTION_DAYS:-7}"
+if [ "$retain" -gt 0 ]; then
+    audio_root="${AUDIO_DIR:-$BASE_DIR/audio}/sunrise/$CAM"
+    deleted=$(find "$audio_root" -name "sunrise-audio.m4a" -mtime +"$retain" -print -delete 2>/dev/null | wc -l)
+    [ "$deleted" -gt 0 ] && echo "Retention: deleted $deleted sunrise audio clip(s) older than ${retain}d"
+fi

sunrise2mm.py

@@ -1,127 +1,166 @@
 #!/usr/bin/env python3
-import requests
 import os
+import pathlib
+import re
 import subprocess
 from datetime import datetime
 
-import pathlib
+import requests
 
-# Locate sky-cam.conf next to this script (works regardless of cwd).
 _here = pathlib.Path(__file__).resolve().parent
-config_file_path = str(_here / 'sky-cam.conf')
 
-# Function to read configuration from the config file
+
-def read_config(config_file_path):
+def read_config(path):
-    config = {}
+    conf = {}
     try:
-        with open(config_file_path, 'r') as f:
+        with open(path) as f:
             for line in f:
-                # Ignore comments and empty lines
+                line = line.strip()
-                if line.strip() and not line.startswith('#'):
+                if not line or line.startswith('#') or '=' not in line:
-                    key, value = line.strip().split('=', 1)
+                    continue
-                    config[key.strip()] = value.strip()
+                k, v = line.split('=', 1)
+                k = k.strip()
+                v = v.strip().strip('"').strip("'")
+                m = re.match(r'^\$\{[^}]+:-([^}]*)\}$', v)
+                if m:
+                    v = m.group(1).strip('"').strip("'")
+                conf[k] = v
     except FileNotFoundError:
-        print(f"Error: The configuration file {config_file_path} was not found.")
+        pass
-        exit(1)
+    return conf
-    except Exception as e:
-        print(f"Error reading the configuration file: {e}")
-        exit(1)
-    return config
 
-# Read configuration — sky-cam.conf first, then .env overrides (credentials)
-config = read_config(config_file_path)
-_env_path = str(_here / '.env')
-if os.path.isfile(_env_path):
-    config.update(read_config(_env_path))
 
-# Get the values from the configuration file
+config = read_config(_here / 'sky-cam.conf')
-mattermost_url = config.get('mattermost_url')
+config.update(read_config(_here / '.env'))
-access_token = config.get('access_token')
-channel_id = config.get('channel_id')
 
-# Check if all required values are provided
+mattermost_url = config.get('mattermost_url', '').rstrip('/')
-if not mattermost_url or not access_token or not channel_id:
+access_token   = config.get('access_token', '')
-    print("Error: Missing one or more configuration values.")
+channel_id     = config.get('channel_id', '')
-    exit(1)
 
-# Get today's date
+if not all([mattermost_url, access_token, channel_id]):
-today = datetime.now()
+    print("Error: mattermost_url / access_token / channel_id not set in .env")
-today_date_str = today.strftime("%Y-%m-%d")  # e.g., "2024-11-05"
+    raise SystemExit(1)
-today_month_str = today.strftime("%Y-%m")     # e.g., "2024-11"
 
-# Set the full folder path for today's video
+headers = {'Authorization': f'Bearer {access_token}'}
-base_video_folder = os.path.join(config.get('MOVIES_DIR', ''), config.get('SUNRISE_CAM', 'sunrise'))
-output_dir = os.path.join(base_video_folder, today_month_str, "sunrise-only")
 
-# Check if the directory exists
+today_date_str  = datetime.now().strftime('%Y-%m-%d')
-if not os.path.isdir(output_dir):
+today_month_str = datetime.now().strftime('%Y-%m')
-    print(f"Error: The directory {output_dir} does not exist. The video may not have been created yet.")
-    exit(1)
 
-final_video_file = os.path.join(output_dir, f"{today_date_str}-daily-sunrise.mp4")
+movies_dir  = config.get('MOVIES_DIR', '')
+sunrise_cam = config.get('SUNRISE_CAM', 'east')
+output_dir  = os.path.join(movies_dir, sunrise_cam, today_month_str, 'sunrise-only')
+video_file  = os.path.join(output_dir, f'{sunrise_cam}-{today_date_str}-daily-sunrise.mp4')
 
-# Debugging: print the full path of the video file to make sure it's correct
+print(f"Looking for video file: {video_file}")
-print(f"Looking for video file: {final_video_file}")
 
-# Check if the video file exists
+if not os.path.isfile(video_file):
-if os.path.isfile(final_video_file):
+    print(f"No video file found at {video_file}.")
-    # Upload the video file
+    subprocess.run(
-    with open(final_video_file, 'rb') as f:
+        [str(_here / 'notify.sh'),
-        response = requests.post(
+         f'FAILED: sunrise upload {today_date_str}',
-            f"{mattermost_url}/api/v4/files",
+         f'Video file not found: {video_file}'],
-            headers={'Authorization': f'Bearer {access_token}'},
+        check=False
+    )
+    raise SystemExit(1)
+
+
+# ── Upload ────────────────────────────────────────────────────────────────────
+with open(video_file, 'rb') as f:
+    resp = requests.post(
+        f'{mattermost_url}/api/v4/files',
+        headers=headers,
         files={'files': f},
         data={'channel_id': channel_id}
     )
 
-    if response.status_code == 201:
+if resp.status_code != 201:
-        print(f"Successfully uploaded {final_video_file}")
+    print(f'Failed to upload: {resp.text}')
+    subprocess.run(
+        [str(_here / 'notify.sh'),
+         f'FAILED: sunrise upload {today_date_str}',
+         f'Mattermost file upload failed: {resp.status_code}'],
+        check=False
+    )
+    raise SystemExit(1)
 
-        # Get the file ID from the response
+print(f'Uploaded {video_file}')
-        file_id = response.json().get('file_infos', [{}])[0].get('id')
+file_id = resp.json()['file_infos'][0]['id']
 
-        # Send a message with the file reference
+post_resp = requests.post(
-        message_data = {
+    f'{mattermost_url}/api/v4/posts',
+    headers=headers,
+    json={
         'channel_id': channel_id,
-            'message': f"Here is a video of the sunrise for {today_date_str}",
+        'message': f'Sunrise {today_date_str}',
-            'file_ids': [file_id]  # Attach the uploaded file
+        'file_ids': [file_id],
     }
+)
 
-        message_response = requests.post(
+if post_resp.status_code == 201:
-            f"{mattermost_url}/api/v4/posts",
+    print('Message posted successfully.')
-            headers={'Authorization': f'Bearer {access_token}'},
-            json=message_data
-        )
-
-        if message_response.status_code == 201:
-            print("Message posted successfully.")
     subprocess.run(
         [str(_here / 'notify.sh'),
-                 f"Sunrise uploaded: {today_date_str}",
+         f'Sunrise uploaded: {today_date_str}',
-                 f"Video posted to Mattermost successfully"],
+         'Video posted to Mattermost successfully'],
-                check=False
-            )
-        else:
-            print(f"Failed to post message: {message_response.text}")
-            subprocess.run(
-                [str(_here / 'notify.sh'),
-                 f"FAILED: sunrise Mattermost post {today_date_str}",
-                 f"File uploaded but post failed: {message_response.status_code}"],
-                check=False
-            )
-    else:
-        print(f"Failed to upload {final_video_file}: {response.text}")
-        subprocess.run(
-            [str(_here / 'notify.sh'),
-             f"FAILED: sunrise upload {today_date_str}",
-             f"Mattermost file upload failed: {response.status_code}"],
         check=False
     )
 else:
-    print(f"No video file found at {final_video_file}.")
+    print(f'Failed to post message: {post_resp.text}')
     subprocess.run(
         [str(_here / 'notify.sh'),
-         f"FAILED: sunrise upload {today_date_str}",
+         f'FAILED: sunrise Mattermost post {today_date_str}',
-         f"Video file not found: {final_video_file}"],
+         f'File uploaded but post failed: {post_resp.status_code}'],
         check=False
     )
+    raise SystemExit(1)
+
+
+# ── Cleanup old unpinned posts ─────────────────────────────────────────────────
+retention_days = int(config.get('MM_SUNRISE_RETENTION_DAYS', 8))
+if retention_days <= 0:
+    raise SystemExit(0)
+
+cutoff_ms = (datetime.now().timestamp() - retention_days * 86400) * 1000
+deleted = 0
+page = 0
+
+while True:
+    r = requests.get(
+        f'{mattermost_url}/api/v4/channels/{channel_id}/posts',
+        headers=headers,
+        params={'page': page, 'per_page': 200}
+    )
+    if r.status_code != 200:
+        print(f'Cleanup: could not fetch posts ({r.status_code}) — skipping')
+        break
+
+    data  = r.json()
+    posts = data.get('posts', {})
+    order = data.get('order', [])
+    if not order:
+        break
+
+    for post_id in order:
+        post = posts[post_id]
+        if post.get('is_pinned'):
+            continue
+        if not post.get('file_ids'):
+            continue
+        # Only delete posts this script created — message starts with "Sunrise YYYY-MM-DD"
+        if not re.match(r'^Sunrise \d{4}-\d{2}-\d{2}', post.get('message', '')):
+            continue
+        if post['create_at'] >= cutoff_ms:
+            continue
+        del_r = requests.delete(
+            f'{mattermost_url}/api/v4/posts/{post_id}',
+            headers=headers
+        )
+        if del_r.status_code == 200:
+            deleted += 1
+
+    if len(order) < 200:
+        break
+    page += 1
+
+if deleted:
+    print(f'Cleanup: removed {deleted} unpinned post(s) older than {retention_days} days')