Handle old/low-VRAM GPUs and document nvidia-container-toolkit requirement
GPU tier table extended: ultra ≥16 GB → SDXL (unchanged) high 8-16 GB → SDXL (unchanged) medium 4-8 GB → SD 2.x (unchanged) legacy 2-4 GB → SD 1.5 (~1.7 GB fp16) ← new: GTX 970/1060/RX 580 etc. minimal <2 GB → SD 1.5 + sequential CPU offload ← new: very old/integrated GPUs gpu_detect.py: - Detects CUDA compute capability (CC); fp16 disabled for CC < 6.0 (pre-Pascal) - GpuInfo gains compute_capability and warnings fields - _make_warnings() emits human-readable warnings for low VRAM and old CC - model tier fallback updated from 'low' to 'legacy' local_diffusion.py: - minimal/legacy tiers use enable_sequential_cpu_offload() + enable_attention_slicing(1) - target resolution per tier: ultra/high=1024, medium=768, legacy/minimal=512 - .to(device) skipped when sequential CPU offload is active gpu_status.py: - Response now includes compute_capability and warnings docker-compose.gpu.yml: - Full nvidia-container-toolkit install instructions in header comment - nvidia-docker2 (legacy) fallback documented as comment block inline - AMD ROCm swap-in instructions added - GPU tier table documented in header scripts/gpu_setup.py: - Prints compute capability, fp16 status, tier, and model selection at startup - Prints per-tier warnings (old CC, low VRAM) https://claude.ai/code/session_01WVDg7amsy1TTtxvpku7bcM
This commit is contained in:
Claude
+5
-2
@@ -39,10 +39,13 @@ async def lifespan(app: FastAPI):
|
||||
):
|
||||
from app.services.gpu_detect import get_cached_gpu_info
|
||||
info = get_cached_gpu_info()
|
||||
cc_str = f" | CC={info.compute_capability}" if info.compute_capability else ""
|
||||
print(
|
||||
f"[gpu] {info.device_name} | {info.vram_gb:.1f} GB | tier={info.tier} | "
|
||||
f"backend={info.backend}"
|
||||
f"[gpu] {info.device_name} | {info.vram_gb:.1f} GB{cc_str} | "
|
||||
f"tier={info.tier} | fp16={info.fp16}"
|
||||
)
|
||||
for w in info.warnings:
|
||||
print(f"[gpu] ⚠ {w}")
|
||||
if settings.auto_download_models:
|
||||
# Download model weight files to disk cache in background so first
|
||||
# user request loads from local disk instead of the internet.
|
||||
|
||||
@@ -26,8 +26,10 @@ async def gpu_status():
|
||||
"backend": info.backend,
|
||||
"device_name": info.device_name,
|
||||
"vram_gb": info.vram_gb,
|
||||
"compute_capability": info.compute_capability,
|
||||
"tier": info.tier,
|
||||
"fp16": info.fp16,
|
||||
"warnings": info.warnings,
|
||||
"capabilities": info.capabilities,
|
||||
"models": {
|
||||
op: {"model_id": mid, "available": mid is not None}
|
||||
|
||||
@@ -11,30 +11,46 @@ from typing import Optional
|
||||
|
||||
|
||||
# Model IDs per VRAM tier — all publicly available on HuggingFace, no auth needed.
|
||||
# SDXL variants are used for high/ultra; SD 2.x for medium/low (smaller VRAM footprint).
|
||||
#
|
||||
# Tier selection by VRAM:
|
||||
# ultra ≥16 GB → SDXL (best quality)
|
||||
# high 8–16 GB → SDXL
|
||||
# medium 4–8 GB → SD 2.x
|
||||
# legacy 2–4 GB → SD 1.5 (older / budget GPUs like GTX 970/1060/RX 580)
|
||||
# minimal <2 GB → SD 1.5 with heavy memory offloading (very slow, but functional)
|
||||
#
|
||||
# SD 1.5 uses ~1.7 GB VRAM in fp16; SD 2.x uses ~3.5 GB; SDXL uses ~6.5 GB.
|
||||
_MODEL_TIERS: dict[str, dict[str, str]] = {
|
||||
"ultra": { # ≥16 GB VRAM
|
||||
"ultra": {
|
||||
"inpaint": "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
|
||||
"txt2img": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"img2img": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"upscale": "stabilityai/stable-diffusion-x4-upscaler",
|
||||
},
|
||||
"high": { # 8–16 GB VRAM
|
||||
"high": {
|
||||
"inpaint": "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
|
||||
"txt2img": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"img2img": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"upscale": "stabilityai/stable-diffusion-x4-upscaler",
|
||||
},
|
||||
"medium": { # 4–8 GB VRAM
|
||||
"medium": {
|
||||
"inpaint": "stabilityai/stable-diffusion-2-inpainting",
|
||||
"txt2img": "stabilityai/stable-diffusion-2-1",
|
||||
"img2img": "stabilityai/stable-diffusion-2-1",
|
||||
"upscale": None,
|
||||
},
|
||||
"low": { # <4 GB or CPU
|
||||
"inpaint": "stabilityai/stable-diffusion-2-inpainting",
|
||||
"txt2img": "stabilityai/stable-diffusion-2-1-base",
|
||||
"img2img": "stabilityai/stable-diffusion-2-1-base",
|
||||
# GTX 970 / GTX 1060 6 GB / RX 580 / etc. — 2–4 GB VRAM
|
||||
"legacy": {
|
||||
"inpaint": "runwayml/stable-diffusion-inpainting",
|
||||
"txt2img": "stable-diffusion-v1-5/stable-diffusion-v1-5",
|
||||
"img2img": "stable-diffusion-v1-5/stable-diffusion-v1-5",
|
||||
"upscale": None,
|
||||
},
|
||||
# Very old / integrated GPUs with <2 GB — runs but slowly; warns user.
|
||||
"minimal": {
|
||||
"inpaint": "runwayml/stable-diffusion-inpainting",
|
||||
"txt2img": "stable-diffusion-v1-5/stable-diffusion-v1-5",
|
||||
"img2img": "stable-diffusion-v1-5/stable-diffusion-v1-5",
|
||||
"upscale": None,
|
||||
},
|
||||
}
|
||||
@@ -45,26 +61,35 @@ class GpuInfo:
|
||||
backend: str # cuda | mps | cpu
|
||||
device_name: str = "CPU"
|
||||
vram_gb: float = 0.0
|
||||
tier: str = "low" # ultra | high | medium | low
|
||||
compute_capability: str = "" # e.g. "8.6" for RTX 3070
|
||||
tier: str = "legacy" # ultra | high | medium | legacy | minimal
|
||||
fp16: bool = False
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
capabilities: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def detect_gpu() -> GpuInfo:
|
||||
"""Detect available compute backend, VRAM, and assign a capability tier."""
|
||||
"""Detect available compute backend, VRAM, compute capability, and assign tier."""
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
props = torch.cuda.get_device_properties(0)
|
||||
vram_gb = props.total_memory / (1024 ** 3)
|
||||
cc = f"{props.major}.{props.minor}"
|
||||
# fp16 inference is reliable on Pascal (6.0) and newer.
|
||||
# Maxwell (5.x) technically works but is slower in fp16 than fp32 on some ops.
|
||||
use_fp16 = props.major >= 6
|
||||
tier = _vram_to_tier(vram_gb)
|
||||
warnings = _make_warnings(tier, vram_gb, cc, use_fp16)
|
||||
return GpuInfo(
|
||||
backend="cuda",
|
||||
device_name=props.name,
|
||||
vram_gb=round(vram_gb, 1),
|
||||
compute_capability=cc,
|
||||
tier=tier,
|
||||
fp16=True,
|
||||
fp16=use_fp16,
|
||||
warnings=warnings,
|
||||
capabilities=_caps_for_tier(tier),
|
||||
)
|
||||
|
||||
@@ -75,8 +100,9 @@ def detect_gpu() -> GpuInfo:
|
||||
backend="mps",
|
||||
device_name="Apple Silicon",
|
||||
vram_gb=round(vram_gb, 1),
|
||||
compute_capability="mps",
|
||||
tier=tier,
|
||||
fp16=False, # MPS is more stable with fp32
|
||||
fp16=False, # MPS diffusion is more stable with fp32
|
||||
capabilities=_caps_for_tier(tier),
|
||||
)
|
||||
|
||||
@@ -87,8 +113,9 @@ def detect_gpu() -> GpuInfo:
|
||||
backend="cpu",
|
||||
device_name="CPU (no GPU detected)",
|
||||
vram_gb=0.0,
|
||||
tier="low",
|
||||
tier="minimal",
|
||||
fp16=False,
|
||||
warnings=["No GPU found — running on CPU. Inference will be very slow (minutes per image)."],
|
||||
capabilities=["txt2img", "inpaint", "img2img", "outpaint"],
|
||||
)
|
||||
|
||||
@@ -100,7 +127,30 @@ def _vram_to_tier(vram_gb: float) -> str:
|
||||
return "high"
|
||||
if vram_gb >= 4:
|
||||
return "medium"
|
||||
return "low"
|
||||
if vram_gb >= 2:
|
||||
return "legacy"
|
||||
return "minimal"
|
||||
|
||||
|
||||
def _make_warnings(tier: str, vram_gb: float, cc: str, fp16: bool) -> list[str]:
|
||||
"""Generate human-readable warnings for suboptimal GPU configurations."""
|
||||
warns = []
|
||||
if tier == "minimal":
|
||||
warns.append(
|
||||
f"Very low VRAM ({vram_gb:.1f} GB) — inference will be slow and may OOM. "
|
||||
"Sequential CPU offloading will be enabled automatically."
|
||||
)
|
||||
elif tier == "legacy":
|
||||
warns.append(
|
||||
f"Limited VRAM ({vram_gb:.1f} GB) — using SD 1.5 models (smaller, lower quality "
|
||||
"than SD 2.x/SDXL). Still fully functional."
|
||||
)
|
||||
if not fp16:
|
||||
warns.append(
|
||||
f"GPU compute capability {cc} is below 6.0 — using fp32 (doubles VRAM use). "
|
||||
"Consider upgrading to a Pascal-era (GTX 1000) or newer GPU for fp16 support."
|
||||
)
|
||||
return warns
|
||||
|
||||
|
||||
def _apple_usable_gb() -> float:
|
||||
@@ -126,7 +176,7 @@ def _caps_for_tier(tier: str) -> list[str]:
|
||||
|
||||
def get_model_ids(tier: str) -> dict[str, Optional[str]]:
|
||||
"""Return the model-ID map for a given tier."""
|
||||
return dict(_MODEL_TIERS.get(tier, _MODEL_TIERS["low"]))
|
||||
return dict(_MODEL_TIERS.get(tier, _MODEL_TIERS["legacy"]))
|
||||
|
||||
|
||||
# Process-level singleton — detect once, reuse everywhere.
|
||||
|
||||
@@ -189,26 +189,37 @@ class LocalDiffusionProvider(RemoteAIProvider):
|
||||
|
||||
pipe = cls.from_pretrained(model_id, **kwargs)
|
||||
|
||||
# Move to device unless using CPU offload
|
||||
if tier != "low" or device != "cpu":
|
||||
pipe = pipe.to(device)
|
||||
|
||||
# Memory optimisations
|
||||
if tier in ("low", "medium"):
|
||||
try:
|
||||
pipe.enable_attention_slicing()
|
||||
except Exception:
|
||||
pass
|
||||
if tier == "low" and device == "cuda":
|
||||
try:
|
||||
pipe.enable_sequential_cpu_offload()
|
||||
except Exception:
|
||||
pass
|
||||
# Memory optimisations — applied based on VRAM tier:
|
||||
# minimal/legacy: full aggressive offloading (sequential CPU offload)
|
||||
# medium: attention slicing + VAE slicing
|
||||
# high/ultra: VAE slicing only (VRAM is plentiful)
|
||||
try:
|
||||
pipe.enable_vae_slicing()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if tier in ("minimal", "legacy", "medium"):
|
||||
try:
|
||||
pipe.enable_attention_slicing(1) # slice_size=1 = most aggressive
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if tier in ("minimal", "legacy"):
|
||||
# Sequential CPU offload keeps only the active layer on GPU — very low VRAM
|
||||
# but adds overhead per-step. Skip .to(device) when this is active.
|
||||
if device == "cuda":
|
||||
try:
|
||||
pipe.enable_sequential_cpu_offload()
|
||||
except Exception:
|
||||
# Fallback: model stays on CPU entirely
|
||||
pass
|
||||
elif device == "cpu":
|
||||
pass # already on CPU
|
||||
else:
|
||||
pipe = pipe.to(device)
|
||||
else:
|
||||
pipe = pipe.to(device)
|
||||
|
||||
_set_state(pipe_type, state="ready", progress=100.0, message="Ready")
|
||||
return pipe
|
||||
|
||||
@@ -246,6 +257,7 @@ class LocalDiffusionProvider(RemoteAIProvider):
|
||||
target = 1024 if info.tier in ("ultra", "high") else 512
|
||||
img_r, mask_r = _resize_pair(img, mask, target)
|
||||
|
||||
|
||||
steps = int(params.get("steps", 30))
|
||||
cfg = float(params.get("cfg_scale", 7.5))
|
||||
neg = params.get("negative_prompt", "") or None
|
||||
@@ -269,7 +281,7 @@ class LocalDiffusionProvider(RemoteAIProvider):
|
||||
pipe = await self._get_pipeline("txt2img")
|
||||
info = self._info
|
||||
|
||||
max_dim = 1024 if info.tier in ("ultra", "high") else 768
|
||||
max_dim = 1024 if info.tier in ("ultra", "high") else (768 if info.tier == "medium" else 512)
|
||||
w = min(width, max_dim) // 8 * 8
|
||||
h = min(height, max_dim) // 8 * 8
|
||||
|
||||
@@ -303,7 +315,7 @@ class LocalDiffusionProvider(RemoteAIProvider):
|
||||
|
||||
img = Image.open(BytesIO(image_bytes)).convert("RGB")
|
||||
orig_size = img.size
|
||||
target = 1024 if info.tier in ("ultra", "high") else 512
|
||||
target = 1024 if info.tier in ("ultra", "high") else (768 if info.tier == "medium" else 512)
|
||||
img_r = _resize_square(img, target)
|
||||
|
||||
steps = int(params.get("steps", 30))
|
||||
|
||||
Reference in New Issue
Block a user