diff --git a/scripts/build-bay-map.sh b/scripts/build-bay-map.sh new file mode 100755 index 0000000..e07d98f --- /dev/null +++ b/scripts/build-bay-map.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Map physical drive bays to stable /dev/disk/by-id paths. +# Run on the Proxmox host after perc-nonraid.sh and a reboot. +# Output is a table you can paste into docs/hardware-layout.md. +# +# Requires: lsscsi, ledmon (apt install lsscsi ledmon) +# Optional: perccli for enclosure/slot info + +set -euo pipefail + +for cmd in lsscsi; do + command -v "$cmd" &>/dev/null || { echo "Missing: $cmd — run: apt install $cmd"; exit 1; } +done + +echo "=== Drive inventory ===" +echo "" +printf "%-12s %-10s %-30s %-20s %s\n" "DEVICE" "SIZE" "MODEL" "SERIAL" "BY-ID PATH" +printf "%-12s %-10s %-30s %-20s %s\n" "------" "----" "-----" "------" "---------" + +for dev in /dev/sd?; do + [[ -b "$dev" ]] || continue + + size=$(lsblk -dn -o SIZE "$dev" 2>/dev/null || echo "?") + model=$(cat "/sys/block/$(basename "$dev")/device/model" 2>/dev/null | tr -d ' ' || echo "?") + serial=$(cat "/sys/block/$(basename "$dev")/device/serial" 2>/dev/null | tr -d ' ' || echo "?") + + # Prefer WWN-based by-id, fall back to scsi- or ata- + byid=$(ls -1 /dev/disk/by-id/ 2>/dev/null \ + | grep -v "\-part" \ + | while read -r link; do + target=$(readlink -f "/dev/disk/by-id/$link") + [[ "$target" == "$dev" ]] && echo "$link" && break + done | head -1 || echo "not found") + + printf "%-12s %-10s %-30s %-20s %s\n" "$dev" "$size" "$model" "$serial" "/dev/disk/by-id/$byid" +done + +echo "" +echo "=== Bay identification via LED blink ===" +echo "" +echo "To confirm which physical bay a device is in, blink its LED:" +echo " apt install ledmon" +echo " ledctl locate=/dev/sdX # LED on" +echo " ledctl locate_off=/dev/sdX # LED off" +echo "" + +if command -v perccli &>/dev/null || command -v perccli64 &>/dev/null; then + PERCCLI=$(command -v perccli || command -v perccli64) + echo "=== PERC slot info ===" + $PERCCLI /c0 /eall /sall show | grep -E "^[0-9]|Drive's position|SN|WWN" || true +fi + +echo "" +echo "Copy the BY-ID paths into docs/hardware-layout.md." diff --git a/scripts/fan-control.service b/scripts/fan-control.service new file mode 100644 index 0000000..4e87a38 --- /dev/null +++ b/scripts/fan-control.service @@ -0,0 +1,19 @@ +[Unit] +Description=Dell R730xd fan speed control (third-party GPU) +# iDRAC sets fans to 100% when non-Dell PCIe cards are detected. +# This service overrides that and manages fan speed by inlet temperature. +After=network.target +# Restart if ipmitool fails transiently (e.g. iDRAC busy at boot) +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +Type=simple +ExecStart=/usr/local/sbin/fan-control.sh +# On stop, re-enable iDRAC auto control so fans are safe if service is removed +ExecStop=/usr/local/sbin/fan-control.sh --auto +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/fan-control.sh b/scripts/fan-control.sh new file mode 100755 index 0000000..dd901a2 --- /dev/null +++ b/scripts/fan-control.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Dell R730xd fan speed control for third-party PCIe cards (Quadro P2200, etc.) +# +# iDRAC detects non-Dell GPUs and slams fans to 100% indefinitely. +# This script disables iDRAC automatic fan control and manages speed based +# on inlet temperature, keeping the server quiet under normal load. +# +# Install: +# apt install ipmitool +# cp fan-control.sh /usr/local/sbin/fan-control.sh +# chmod +x /usr/local/sbin/fan-control.sh +# cp fan-control.service /etc/systemd/system/ +# systemctl daemon-reload && systemctl enable --now fan-control.service +# +# Manual speed test (without running as daemon): +# fan-control.sh --set 25 # set fans to 25% and exit +# fan-control.sh --auto # restore iDRAC automatic control and exit + +set -euo pipefail + +IPMI="ipmitool raw 0x30 0x30" + +# Fan speed thresholds by inlet temperature (°C → % speed) +# Tune these for your environment. Inlet temp sensor reads ambient air entering front. +declare -A SPEED_MAP=( + [0]=15 # < 30°C → 15% (near-silent) + [30]=20 # 30–39°C → 20% + [40]=30 # 40–44°C → 30% + [45]=40 # 45–49°C → 40% + [50]=55 # 50–54°C → 55% + [55]=75 # 55–59°C → 75% + [60]=100 # ≥ 60°C → 100% (safety) +) + +# Minimum speed floor — never go below this (protects drives and CPUs) +MIN_SPEED=15 + +get_inlet_temp() { + ipmitool sdr type Temperature 2>/dev/null \ + | grep -i "Inlet Temp\|Ambient\|Inlet" \ + | grep -oP '\d+(?= degrees)' \ + | head -1 || echo "35" # safe default if sensor read fails +} + +pct_to_hex() { + printf '0x%02x' "$(( $1 < 100 ? $1 : 100 ))" +} + +set_fan_speed() { + local pct=$1 + [[ $pct -lt $MIN_SPEED ]] && pct=$MIN_SPEED + local hex + hex=$(pct_to_hex "$pct") + $IPMI 0x02 0xff "$hex" +} + +disable_auto_fan() { + $IPMI 0x01 0x00 + echo "$(date): iDRAC automatic fan control DISABLED" +} + +enable_auto_fan() { + $IPMI 0x01 0x01 + echo "$(date): iDRAC automatic fan control RE-ENABLED" +} + +speed_for_temp() { + local temp=$1 + local speed=$MIN_SPEED + for threshold in $(echo "${!SPEED_MAP[@]}" | tr ' ' '\n' | sort -n); do + [[ $temp -ge $threshold ]] && speed=${SPEED_MAP[$threshold]} + done + echo "$speed" +} + +# ── Argument handling ───────────────────────────────────────────────────────── + +case "${1:-}" in + --auto) + enable_auto_fan + exit 0 + ;; + --set) + pct="${2:?Usage: fan-control.sh --set <0-100>}" + disable_auto_fan + set_fan_speed "$pct" + echo "$(date): Fans set to ${pct}%" + exit 0 + ;; + --temp) + echo "Inlet temp: $(get_inlet_temp)°C" + exit 0 + ;; + "") + # Daemon mode — fall through to loop + ;; + *) + echo "Usage: $0 [--auto | --set | --temp]" + exit 1 + ;; +esac + +# ── Daemon loop ─────────────────────────────────────────────────────────────── + +trap 'enable_auto_fan; exit 0' SIGTERM SIGINT + +echo "$(date): Fan control daemon starting" +disable_auto_fan + +last_speed=-1 + +while true; do + temp=$(get_inlet_temp) + target=$(speed_for_temp "$temp") + + if [[ $target -ne $last_speed ]]; then + set_fan_speed "$target" + echo "$(date): Inlet ${temp}°C → fans ${target}%" + last_speed=$target + fi + + sleep 30 +done diff --git a/scripts/gpu-passthrough-setup.sh b/scripts/gpu-passthrough-setup.sh new file mode 100755 index 0000000..8c20ca9 --- /dev/null +++ b/scripts/gpu-passthrough-setup.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Configure IOMMU + VFIO passthrough for two Quadro P2200 GPUs on Proxmox VE 9.x. +# Run on the Proxmox host. Requires a reboot to take effect. +# +# After running this script: +# 1. Reboot the host +# 2. Verify with: lspci -nnk | grep -A3 -i nvidia +# Driver should show 'vfio-pci', not 'nouveau' or 'nvidia' +# 3. Assign GPUs to VMs via qm set or the Proxmox UI + +set -euo pipefail + +CMDLINE_FILE="/etc/kernel/cmdline" +MODPROBE_VFIO="/etc/modprobe.d/vfio.conf" +MODPROBE_BLACKLIST="/etc/modprobe.d/blacklist-gpu.conf" +INITRAMFS_MODULES="/etc/initramfs-tools/modules" + +# ── Step 1: Check IOMMU groups ──────────────────────────────────────────────── + +echo "=== Current IOMMU groups (GPUs) ===" +for d in /sys/kernel/iommu_groups/*/devices/*; do + n=${d#*/iommu_groups/*}; n=${n%%/*} + dev=$(lspci -nns "${d##*/}" 2>/dev/null || true) + [[ "$dev" =~ VGA|3D|Display|Audio ]] && printf 'Group %3s %s\n' "$n" "$dev" +done +echo "" + +# ── Step 2: Collect GPU PCI IDs ─────────────────────────────────────────────── + +echo "=== Detected NVIDIA devices ===" +lspci -nn | grep -i nvidia +echo "" + +# Grab all NVIDIA PCI IDs (vendor:device) for vfio-pci binding +# This captures both the GPU (VGA) and its HDMI audio sibling +NVIDIA_IDS=$(lspci -nn | grep -i nvidia | grep -oP '\[\K[0-9a-f]{4}:[0-9a-f]{4}(?=\])' | sort -u | tr '\n' ',' | sed 's/,$//') + +if [[ -z "$NVIDIA_IDS" ]]; then + echo "ERROR: No NVIDIA devices found. Is the GPU installed and visible to lspci?" + exit 1 +fi + +echo "GPU PCI IDs to bind to vfio-pci: $NVIDIA_IDS" +echo "" +read -rp "Proceed with configuring VFIO passthrough? [y/N] " confirm +[[ "$confirm" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } + +# ── Step 3: Enable IOMMU in kernel cmdline ──────────────────────────────────── + +echo "" +echo "--- Configuring kernel cmdline for IOMMU ---" + +if [[ ! -f "$CMDLINE_FILE" ]]; then + echo "ERROR: $CMDLINE_FILE not found. Is this a Proxmox EFI system?" + echo "For legacy GRUB: edit /etc/default/grub GRUB_CMDLINE_LINUX_DEFAULT instead." + exit 1 +fi + +current_cmdline=$(cat "$CMDLINE_FILE") +new_cmdline="$current_cmdline" + +[[ "$new_cmdline" =~ intel_iommu=on ]] || new_cmdline="$new_cmdline intel_iommu=on" +[[ "$new_cmdline" =~ iommu=pt ]] || new_cmdline="$new_cmdline iommu=pt" + +# Deduplicate spaces +new_cmdline=$(echo "$new_cmdline" | tr -s ' ' | sed 's/^ //;s/ $//') + +echo "$new_cmdline" > "$CMDLINE_FILE" +echo "Written: $CMDLINE_FILE" +echo " $new_cmdline" + +proxmox-boot-tool refresh +echo "Boot tool refreshed." + +# ── Step 4: Blacklist host GPU drivers ──────────────────────────────────────── + +echo "" +echo "--- Blacklisting nouveau and nvidia on host ---" +cat > "$MODPROBE_BLACKLIST" < "$MODPROBE_VFIO" </dev/null || echo "$mod" >> "$INITRAMFS_MODULES" +done +echo "Updated: $INITRAMFS_MODULES" + +update-initramfs -u -k all +echo "Initramfs updated." + +# ── Step 7: Print PCI addresses for VM config ───────────────────────────────── + +echo "" +echo "=== GPU PCI addresses for VM assignment ===" +echo "Use these in qm set or the Proxmox UI (Hardware → Add → PCI Device):" +echo "" +lspci -nn | grep -i nvidia | while read -r line; do + addr=$(echo "$line" | awk '{print $1}') + desc=$(echo "$line" | cut -d' ' -f2-) + printf " hostpciN: 0000:%s,pcie=1 # %s\n" "$addr" "$desc" +done + +echo "" +echo "IMPORTANT: Pass each GPU + its HDMI audio sibling to the same VM." +echo "Example for GPU at 01:00.0 (audio at 01:00.1):" +echo " hostpci0: 0000:01:00,pcie=1,x-vga=1" +echo " (Proxmox will auto-include 01:00.1 when you use the .0 address)" +echo "" +echo "Done. Reboot the host to activate IOMMU and vfio-pci binding."