Add GPU passthrough, fan control, and bay mapping scripts
- scripts/build-bay-map.sh: maps physical drive bays to /dev/disk/by-id paths - scripts/gpu-passthrough-setup.sh: configures IOMMU + vfio-pci for two Quadro P2200s - scripts/fan-control.sh: ipmitool daemon to prevent iDRAC from running fans at 100% due to non-Dell PCIe cards; adjusts speed by inlet temperature - scripts/fan-control.service: systemd unit for fan-control.sh https://claude.ai/code/session_01TxTSnxDjgEVuNztdsWQjMD
This commit is contained in:
Claude
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Map physical drive bays to stable /dev/disk/by-id paths.
|
||||
# Run on the Proxmox host after perc-nonraid.sh and a reboot.
|
||||
# Output is a table you can paste into docs/hardware-layout.md.
|
||||
#
|
||||
# Requires: lsscsi, ledmon (apt install lsscsi ledmon)
|
||||
# Optional: perccli for enclosure/slot info
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
for cmd in lsscsi; do
|
||||
command -v "$cmd" &>/dev/null || { echo "Missing: $cmd — run: apt install $cmd"; exit 1; }
|
||||
done
|
||||
|
||||
echo "=== Drive inventory ==="
|
||||
echo ""
|
||||
printf "%-12s %-10s %-30s %-20s %s\n" "DEVICE" "SIZE" "MODEL" "SERIAL" "BY-ID PATH"
|
||||
printf "%-12s %-10s %-30s %-20s %s\n" "------" "----" "-----" "------" "---------"
|
||||
|
||||
for dev in /dev/sd?; do
|
||||
[[ -b "$dev" ]] || continue
|
||||
|
||||
size=$(lsblk -dn -o SIZE "$dev" 2>/dev/null || echo "?")
|
||||
model=$(cat "/sys/block/$(basename "$dev")/device/model" 2>/dev/null | tr -d ' ' || echo "?")
|
||||
serial=$(cat "/sys/block/$(basename "$dev")/device/serial" 2>/dev/null | tr -d ' ' || echo "?")
|
||||
|
||||
# Prefer WWN-based by-id, fall back to scsi- or ata-
|
||||
byid=$(ls -1 /dev/disk/by-id/ 2>/dev/null \
|
||||
| grep -v "\-part" \
|
||||
| while read -r link; do
|
||||
target=$(readlink -f "/dev/disk/by-id/$link")
|
||||
[[ "$target" == "$dev" ]] && echo "$link" && break
|
||||
done | head -1 || echo "not found")
|
||||
|
||||
printf "%-12s %-10s %-30s %-20s %s\n" "$dev" "$size" "$model" "$serial" "/dev/disk/by-id/$byid"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Bay identification via LED blink ==="
|
||||
echo ""
|
||||
echo "To confirm which physical bay a device is in, blink its LED:"
|
||||
echo " apt install ledmon"
|
||||
echo " ledctl locate=/dev/sdX # LED on"
|
||||
echo " ledctl locate_off=/dev/sdX # LED off"
|
||||
echo ""
|
||||
|
||||
if command -v perccli &>/dev/null || command -v perccli64 &>/dev/null; then
|
||||
PERCCLI=$(command -v perccli || command -v perccli64)
|
||||
echo "=== PERC slot info ==="
|
||||
$PERCCLI /c0 /eall /sall show | grep -E "^[0-9]|Drive's position|SN|WWN" || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Copy the BY-ID paths into docs/hardware-layout.md."
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Dell R730xd fan speed control (third-party GPU)
|
||||
# iDRAC sets fans to 100% when non-Dell PCIe cards are detected.
|
||||
# This service overrides that and manages fan speed by inlet temperature.
|
||||
After=network.target
|
||||
# Restart if ipmitool fails transiently (e.g. iDRAC busy at boot)
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/sbin/fan-control.sh
|
||||
# On stop, re-enable iDRAC auto control so fans are safe if service is removed
|
||||
ExecStop=/usr/local/sbin/fan-control.sh --auto
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
# Dell R730xd fan speed control for third-party PCIe cards (Quadro P2200, etc.)
|
||||
#
|
||||
# iDRAC detects non-Dell GPUs and slams fans to 100% indefinitely.
|
||||
# This script disables iDRAC automatic fan control and manages speed based
|
||||
# on inlet temperature, keeping the server quiet under normal load.
|
||||
#
|
||||
# Install:
|
||||
# apt install ipmitool
|
||||
# cp fan-control.sh /usr/local/sbin/fan-control.sh
|
||||
# chmod +x /usr/local/sbin/fan-control.sh
|
||||
# cp fan-control.service /etc/systemd/system/
|
||||
# systemctl daemon-reload && systemctl enable --now fan-control.service
|
||||
#
|
||||
# Manual speed test (without running as daemon):
|
||||
# fan-control.sh --set 25 # set fans to 25% and exit
|
||||
# fan-control.sh --auto # restore iDRAC automatic control and exit
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
IPMI="ipmitool raw 0x30 0x30"
|
||||
|
||||
# Fan speed thresholds by inlet temperature (°C → % speed)
|
||||
# Tune these for your environment. Inlet temp sensor reads ambient air entering front.
|
||||
declare -A SPEED_MAP=(
|
||||
[0]=15 # < 30°C → 15% (near-silent)
|
||||
[30]=20 # 30–39°C → 20%
|
||||
[40]=30 # 40–44°C → 30%
|
||||
[45]=40 # 45–49°C → 40%
|
||||
[50]=55 # 50–54°C → 55%
|
||||
[55]=75 # 55–59°C → 75%
|
||||
[60]=100 # ≥ 60°C → 100% (safety)
|
||||
)
|
||||
|
||||
# Minimum speed floor — never go below this (protects drives and CPUs)
|
||||
MIN_SPEED=15
|
||||
|
||||
get_inlet_temp() {
|
||||
ipmitool sdr type Temperature 2>/dev/null \
|
||||
| grep -i "Inlet Temp\|Ambient\|Inlet" \
|
||||
| grep -oP '\d+(?= degrees)' \
|
||||
| head -1 || echo "35" # safe default if sensor read fails
|
||||
}
|
||||
|
||||
pct_to_hex() {
|
||||
printf '0x%02x' "$(( $1 < 100 ? $1 : 100 ))"
|
||||
}
|
||||
|
||||
set_fan_speed() {
|
||||
local pct=$1
|
||||
[[ $pct -lt $MIN_SPEED ]] && pct=$MIN_SPEED
|
||||
local hex
|
||||
hex=$(pct_to_hex "$pct")
|
||||
$IPMI 0x02 0xff "$hex"
|
||||
}
|
||||
|
||||
disable_auto_fan() {
|
||||
$IPMI 0x01 0x00
|
||||
echo "$(date): iDRAC automatic fan control DISABLED"
|
||||
}
|
||||
|
||||
enable_auto_fan() {
|
||||
$IPMI 0x01 0x01
|
||||
echo "$(date): iDRAC automatic fan control RE-ENABLED"
|
||||
}
|
||||
|
||||
speed_for_temp() {
|
||||
local temp=$1
|
||||
local speed=$MIN_SPEED
|
||||
for threshold in $(echo "${!SPEED_MAP[@]}" | tr ' ' '\n' | sort -n); do
|
||||
[[ $temp -ge $threshold ]] && speed=${SPEED_MAP[$threshold]}
|
||||
done
|
||||
echo "$speed"
|
||||
}
|
||||
|
||||
# ── Argument handling ─────────────────────────────────────────────────────────
|
||||
|
||||
case "${1:-}" in
|
||||
--auto)
|
||||
enable_auto_fan
|
||||
exit 0
|
||||
;;
|
||||
--set)
|
||||
pct="${2:?Usage: fan-control.sh --set <0-100>}"
|
||||
disable_auto_fan
|
||||
set_fan_speed "$pct"
|
||||
echo "$(date): Fans set to ${pct}%"
|
||||
exit 0
|
||||
;;
|
||||
--temp)
|
||||
echo "Inlet temp: $(get_inlet_temp)°C"
|
||||
exit 0
|
||||
;;
|
||||
"")
|
||||
# Daemon mode — fall through to loop
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 [--auto | --set <pct> | --temp]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Daemon loop ───────────────────────────────────────────────────────────────
|
||||
|
||||
trap 'enable_auto_fan; exit 0' SIGTERM SIGINT
|
||||
|
||||
echo "$(date): Fan control daemon starting"
|
||||
disable_auto_fan
|
||||
|
||||
last_speed=-1
|
||||
|
||||
while true; do
|
||||
temp=$(get_inlet_temp)
|
||||
target=$(speed_for_temp "$temp")
|
||||
|
||||
if [[ $target -ne $last_speed ]]; then
|
||||
set_fan_speed "$target"
|
||||
echo "$(date): Inlet ${temp}°C → fans ${target}%"
|
||||
last_speed=$target
|
||||
fi
|
||||
|
||||
sleep 30
|
||||
done
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
# Configure IOMMU + VFIO passthrough for two Quadro P2200 GPUs on Proxmox VE 9.x.
|
||||
# Run on the Proxmox host. Requires a reboot to take effect.
|
||||
#
|
||||
# After running this script:
|
||||
# 1. Reboot the host
|
||||
# 2. Verify with: lspci -nnk | grep -A3 -i nvidia
|
||||
# Driver should show 'vfio-pci', not 'nouveau' or 'nvidia'
|
||||
# 3. Assign GPUs to VMs via qm set or the Proxmox UI
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CMDLINE_FILE="/etc/kernel/cmdline"
|
||||
MODPROBE_VFIO="/etc/modprobe.d/vfio.conf"
|
||||
MODPROBE_BLACKLIST="/etc/modprobe.d/blacklist-gpu.conf"
|
||||
INITRAMFS_MODULES="/etc/initramfs-tools/modules"
|
||||
|
||||
# ── Step 1: Check IOMMU groups ────────────────────────────────────────────────
|
||||
|
||||
echo "=== Current IOMMU groups (GPUs) ==="
|
||||
for d in /sys/kernel/iommu_groups/*/devices/*; do
|
||||
n=${d#*/iommu_groups/*}; n=${n%%/*}
|
||||
dev=$(lspci -nns "${d##*/}" 2>/dev/null || true)
|
||||
[[ "$dev" =~ VGA|3D|Display|Audio ]] && printf 'Group %3s %s\n' "$n" "$dev"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# ── Step 2: Collect GPU PCI IDs ───────────────────────────────────────────────
|
||||
|
||||
echo "=== Detected NVIDIA devices ==="
|
||||
lspci -nn | grep -i nvidia
|
||||
echo ""
|
||||
|
||||
# Grab all NVIDIA PCI IDs (vendor:device) for vfio-pci binding
|
||||
# This captures both the GPU (VGA) and its HDMI audio sibling
|
||||
NVIDIA_IDS=$(lspci -nn | grep -i nvidia | grep -oP '\[\K[0-9a-f]{4}:[0-9a-f]{4}(?=\])' | sort -u | tr '\n' ',' | sed 's/,$//')
|
||||
|
||||
if [[ -z "$NVIDIA_IDS" ]]; then
|
||||
echo "ERROR: No NVIDIA devices found. Is the GPU installed and visible to lspci?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "GPU PCI IDs to bind to vfio-pci: $NVIDIA_IDS"
|
||||
echo ""
|
||||
read -rp "Proceed with configuring VFIO passthrough? [y/N] " confirm
|
||||
[[ "$confirm" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
|
||||
|
||||
# ── Step 3: Enable IOMMU in kernel cmdline ────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Configuring kernel cmdline for IOMMU ---"
|
||||
|
||||
if [[ ! -f "$CMDLINE_FILE" ]]; then
|
||||
echo "ERROR: $CMDLINE_FILE not found. Is this a Proxmox EFI system?"
|
||||
echo "For legacy GRUB: edit /etc/default/grub GRUB_CMDLINE_LINUX_DEFAULT instead."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
current_cmdline=$(cat "$CMDLINE_FILE")
|
||||
new_cmdline="$current_cmdline"
|
||||
|
||||
[[ "$new_cmdline" =~ intel_iommu=on ]] || new_cmdline="$new_cmdline intel_iommu=on"
|
||||
[[ "$new_cmdline" =~ iommu=pt ]] || new_cmdline="$new_cmdline iommu=pt"
|
||||
|
||||
# Deduplicate spaces
|
||||
new_cmdline=$(echo "$new_cmdline" | tr -s ' ' | sed 's/^ //;s/ $//')
|
||||
|
||||
echo "$new_cmdline" > "$CMDLINE_FILE"
|
||||
echo "Written: $CMDLINE_FILE"
|
||||
echo " $new_cmdline"
|
||||
|
||||
proxmox-boot-tool refresh
|
||||
echo "Boot tool refreshed."
|
||||
|
||||
# ── Step 4: Blacklist host GPU drivers ────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Blacklisting nouveau and nvidia on host ---"
|
||||
cat > "$MODPROBE_BLACKLIST" <<EOF
|
||||
blacklist nouveau
|
||||
blacklist nvidia
|
||||
blacklist nvidiafb
|
||||
blacklist nvidia_drm
|
||||
blacklist nvidia_modeset
|
||||
options nouveau modeset=0
|
||||
EOF
|
||||
echo "Written: $MODPROBE_BLACKLIST"
|
||||
|
||||
# ── Step 5: Bind GPUs to vfio-pci ────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Binding GPU IDs to vfio-pci ---"
|
||||
cat > "$MODPROBE_VFIO" <<EOF
|
||||
options vfio-pci ids=$NVIDIA_IDS
|
||||
softdep nouveau pre: vfio-pci
|
||||
softdep nvidia pre: vfio-pci
|
||||
EOF
|
||||
echo "Written: $MODPROBE_VFIO"
|
||||
echo " IDs: $NVIDIA_IDS"
|
||||
|
||||
# ── Step 6: Load vfio modules early in initramfs ─────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "--- Adding vfio modules to initramfs ---"
|
||||
for mod in vfio vfio_iommu_type1 vfio_pci vfio_pci_core; do
|
||||
grep -qxF "$mod" "$INITRAMFS_MODULES" 2>/dev/null || echo "$mod" >> "$INITRAMFS_MODULES"
|
||||
done
|
||||
echo "Updated: $INITRAMFS_MODULES"
|
||||
|
||||
update-initramfs -u -k all
|
||||
echo "Initramfs updated."
|
||||
|
||||
# ── Step 7: Print PCI addresses for VM config ─────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "=== GPU PCI addresses for VM assignment ==="
|
||||
echo "Use these in qm set or the Proxmox UI (Hardware → Add → PCI Device):"
|
||||
echo ""
|
||||
lspci -nn | grep -i nvidia | while read -r line; do
|
||||
addr=$(echo "$line" | awk '{print $1}')
|
||||
desc=$(echo "$line" | cut -d' ' -f2-)
|
||||
printf " hostpciN: 0000:%s,pcie=1 # %s\n" "$addr" "$desc"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "IMPORTANT: Pass each GPU + its HDMI audio sibling to the same VM."
|
||||
echo "Example for GPU at 01:00.0 (audio at 01:00.1):"
|
||||
echo " hostpci0: 0000:01:00,pcie=1,x-vga=1"
|
||||
echo " (Proxmox will auto-include 01:00.1 when you use the .0 address)"
|
||||
echo ""
|
||||
echo "Done. Reboot the host to activate IOMMU and vfio-pci binding."
|
||||
Reference in New Issue
Block a user