diff --git a/README.md b/README.md index 652704a..bf44f6b 100644 --- a/README.md +++ b/README.md @@ -1 +1,31 @@ -# local_proxmox \ No newline at end of file +# local_proxmox + +Proxmox VE 9.1 configuration, scripts, and VM layouts for a Dell R730xd homelab. + +## Hardware + +- **Host:** Dell PowerEdge R730xd +- **RAID controller:** PERC H730 (configured in Non-RAID/per-disk passthrough mode) +- **GPUs:** 2x Quadro P3300 4GB (VFIO passthrough, one per VM) +- **TPU:** Google Coral USB (Frigate object detection) +- **Bays:** 12x (not all populated) + +## VM Layout + +| VM ID | Role | Drives | GPU | Coral | +|-------|------|--------|-----|-------| +| 100 | Frigate NVR | Bays 1–8 | Quadro #1 (VFIO) | Yes | +| 101 | TBD | Bays 9–10 | Quadro #2 (VFIO) | No | +| 102 | TBD | Bays 11–12 | None | No | + +## Setup Order + +1. [`scripts/perc-nonraid.sh`](scripts/perc-nonraid.sh) — convert H730 to per-disk non-RAID mode +2. [`scripts/build-bay-map.sh`](scripts/build-bay-map.sh) — map physical bays to stable `/dev/disk/by-id` paths +3. [`scripts/gpu-passthrough-setup.sh`](scripts/gpu-passthrough-setup.sh) — configure IOMMU + VFIO for both Quadros +4. Apply VM configs from [`vm-configs/`](vm-configs/) +5. Deploy Frigate from [`frigate/`](frigate/) + +## Docs + +- [`docs/hardware-layout.md`](docs/hardware-layout.md) — bay/device/VM mapping worksheet diff --git a/docs/hardware-layout.md b/docs/hardware-layout.md new file mode 100644 index 0000000..bbd1040 --- /dev/null +++ b/docs/hardware-layout.md @@ -0,0 +1,65 @@ +# Hardware Layout + +Dell R730xd — fill this in after running `scripts/build-bay-map.sh`. + +## Bay → Device → VM Map + +Run `scripts/build-bay-map.sh` on the Proxmox host to auto-populate the by-id column. +Confirm bay numbers by running `ledctl locate=` to blink the drive LED. + +| Bay | Assigned VM | Size | /dev/disk/by-id (fill in) | Notes | +|-----|-------------|------|---------------------------|-------| +| 1 | 100 | | | | +| 2 | 100 | | | | +| 3 | 100 | | | | +| 4 | 100 | | | | +| 5 | 100 | | | | +| 6 | 100 | | | | +| 7 | 100 | | | | +| 8 | 100 | | | | +| 9 | 101 | | | | +| 10 | 101 | | | | +| 11 | 102 | | | | +| 12 | 102 | | | | + +## GPU Map + +Run `lspci -nn | grep -i nvidia` on the Proxmox host and fill in the PCI addresses. + +| Slot | PCI Address (fill in) | Assigned VM | Notes | +|------|-----------------------|-------------|-------| +| GPU 1 | | 100 | Frigate decode + display | +| GPU 2 | | 101 | | + +The GPU's HDMI audio function (same address, function 1) must be passed through alongside the GPU. +Example: GPU at `01:00.0` → also pass `01:00.1`. + +## USB / TPU Map + +| Device | VID:PID | Assigned VM | Notes | +|--------|---------|-------------|-------| +| Coral (pre-init) | 1a6e:089a | 100 | Global Unichip — before first inference | +| Coral (post-init) | 18d1:9302 | 100 | Google — after first inference; pass both | + +## IOMMU Groups + +Run on Proxmox host to check groupings before passthrough: + +```bash +for d in /sys/kernel/iommu_groups/*/devices/*; do + n=${d#*/iommu_groups/*}; n=${n%%/*} + printf 'IOMMU Group %s ' "$n" + lspci -nns "${d##*/}" +done | sort -V +``` + +Each GPU should appear in its own group (with only its HDMI audio sibling). +If a GPU shares a group with other devices, those must be passed through together. + +## Network + +| VM | Interface | Bridge | Notes | +|-----|-----------|--------|-------| +| 100 | net0 | vmbr0 | | +| 101 | net0 | vmbr0 | | +| 102 | net0 | vmbr0 | | diff --git a/scripts/perc-nonraid.sh b/scripts/perc-nonraid.sh new file mode 100644 index 0000000..5dbd131 --- /dev/null +++ b/scripts/perc-nonraid.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Convert PERC H730 drives to Non-RAID (per-disk passthrough) mode. +# Run once on the Proxmox host before assigning drives to VMs. +# Requires: perccli (install from Dell's website or local .deb) +# +# Dell PERC H730 does not have a true HBA/IT mode. Non-RAID mode is the +# equivalent — each disk is presented directly to the OS with SMART intact. + +set -euo pipefail + +PERCCLI=$(command -v perccli || command -v perccli64 || true) +if [[ -z "$PERCCLI" ]]; then + echo "perccli not found. Install from:" + echo " https://www.dell.com/support (search 'PERCCLI')" + echo " or: dpkg -i perccli_*.deb" + exit 1 +fi + +echo "=== Controller overview ===" +$PERCCLI show + +echo "" +echo "=== Drives on controller 0 ===" +$PERCCLI /c0 /eall /sall show + +echo "" +read -rp "Proceed with converting all non-OS drives to Non-RAID mode? [y/N] " confirm +[[ "$confirm" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } + +# Identify enclosure IDs (typically 8 for the internal backplane on R730xd) +ENCLOSURES=$($PERCCLI /c0 /eall show | awk '/^[0-9]/{print $1}' | sort -u) + +for enc in $ENCLOSURES; do + echo "" + echo "=== Processing enclosure $enc ===" + SLOTS=$($PERCCLI /c0 /e"$enc" /sall show | awk '/UGood|Onln|JBOD|DHS/{print $2}' | sort -u) + + for slot in $SLOTS; do + echo -n " Slot $slot: setting to Good... " + $PERCCLI /c0 /e"$enc" /s"$slot" set good force 2>&1 | grep -i "success\|error\|already" || true + + echo -n " Slot $slot: setting to Non-RAID... " + $PERCCLI /c0 /e"$enc" /s"$slot" set nonraid 2>&1 | grep -i "success\|error\|already" || true + done +done + +echo "" +echo "=== Final drive state ===" +$PERCCLI /c0 /eall /sall show + +echo "" +echo "Done. Reboot the host for changes to take full effect." +echo "After reboot, run build-bay-map.sh to map bays to /dev/disk/by-id paths."