root/PaintPlus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add per-operation AI provider routing

Browse Source 
Each operation (inpaint, txt2img, img2img, outpaint) can now use a different
provider. Resolution order: per-op override → global AI_PROVIDER default.

Example: txt2img→openai, inpaint→invokeai, everything else→invokeai default.

Backend:
- config.py: add AI_PROVIDER_INPAINT / TXT2IMG / IMG2IMG / OUTPAINT settings
- remote_provider.py: get_remote_provider(operation) resolves override then default;
  _build_provider() extracted as shared factory; _OP_FIELD maps op→setting name
- ai_tools.py: each endpoint passes its operation to _require_remote();
  GET /api/config runs per-op health checks concurrently, returns operations map
  and overrides; POST /api/config accepts and applies per-op override fields

Frontend:
- ai_provider_settings.js: four new selects (inpaint/txt2img/img2img/outpaint);
  persists to localStorage and sends per-op fields to POST /api/config
- provider-badge.js: shows override summary (e.g. "invokeai · txt2img→openai")
  and per-op health in tooltip
- .env.example: document per-op override env vars with examples

https://claude.ai/code/session_01B58MaJCU1R6KwBDJCp8AfN
This commit is contained in:
Claude
2026-06-09 18:14:12 +00:00
parent b2a12c356f
commit d01c11f948
6 changed files with 191 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+7
View File
@@ -26,6 +26,13 @@
AI_PROVIDER=replicate AI_PROVIDER=replicate
# Per-operation provider overrides (optional — blank means use AI_PROVIDER above)
# Example: use OpenAI for text-to-image (best quality) but InvokeAI for everything else
#AI_PROVIDER_TXT2IMG=openai
#AI_PROVIDER_INPAINT=invokeai
#AI_PROVIDER_IMG2IMG=invokeai
#AI_PROVIDER_OUTPAINT=invokeai
# ============================================================================= # =============================================================================
# STEP 2: Get Your API Key # STEP 2: Get Your API Key
backend/app/config.py
+10 -1
View File
@@ -13,9 +13,18 @@ class Settings(BaseSettings):
# AI Provider # AI Provider
# Local: blank or "mock" — always available, no config needed # Local: blank or "mock" — always available, no config needed
# Remote (set ONE): openai | invokeai | comfyui | replicate | stability # Remote default (used for any operation without a specific override):
# openai | invokeai | comfyui | replicate | stability
ai_provider: str = "mock" ai_provider: str = "mock"
# Per-operation provider overrides — blank means use ai_provider default.
# Operations: inpaint, txt2img, img2img, outpaint
# Example: AI_PROVIDER_TXT2IMG=openai (use OpenAI for text-to-image only)
ai_provider_inpaint: str = "" # remote inpaint / replace selection
ai_provider_txt2img: str = "" # text-to-image
ai_provider_img2img: str = "" # image-to-image
ai_provider_outpaint: str = "" # expand canvas
# Provider API Keys # Provider API Keys
openai_api_key: str = "" openai_api_key: str = ""
openai_model: str = "dall-e-3" openai_model: str = "dall-e-3"
backend/app/routers/ai_tools.py
+68 -42
View File
@@ -75,13 +75,15 @@ def _encode(data: bytes) -> str:
return base64.b64encode(data).decode() return base64.b64encode(data).decode()
def _require_remote(): def _require_remote(operation: str = None):
from app.services.remote_provider import get_remote_provider from app.services.remote_provider import get_remote_provider
provider = get_remote_provider() provider = get_remote_provider(operation)
if provider is None: if provider is None:
op_hint = f"AI_PROVIDER_{operation.upper()} or " if operation else ""
raise HTTPException( raise HTTPException(
status_code=503, status_code=503,
detail="No remote AI provider configured. Set AI_PROVIDER in .env (openai / invokeai / comfyui)." detail=f"No remote AI provider configured for '{operation or 'default'}'. "
f"Set {op_hint}AI_PROVIDER in .env (openai / invokeai / comfyui)."
) )
return provider return provider
@@ -173,7 +175,7 @@ async def background_remove(req: BgRemoveRequest):
@router.post("/inpaint/remote") @router.post("/inpaint/remote")
async def inpaint_remote(req: InpaintRemoteRequest): async def inpaint_remote(req: InpaintRemoteRequest):
"""Inpaint via configured remote provider (InvokeAI / ComfyUI / OpenAI).""" """Inpaint via configured remote provider (InvokeAI / ComfyUI / OpenAI)."""
provider = _require_remote() provider = _require_remote("inpaint")
try: try:
params = { params = {
"negative_prompt": req.negative_prompt or "", "negative_prompt": req.negative_prompt or "",
@@ -192,7 +194,7 @@ async def inpaint_remote(req: InpaintRemoteRequest):
@router.post("/generate/txt2img") @router.post("/generate/txt2img")
async def txt2img(req: Txt2ImgRequest): async def txt2img(req: Txt2ImgRequest):
"""Text-to-image via configured remote provider.""" """Text-to-image via configured remote provider."""
provider = _require_remote() provider = _require_remote("txt2img")
try: try:
params = { params = {
"negative_prompt": req.negative_prompt or "", "negative_prompt": req.negative_prompt or "",
@@ -212,7 +214,7 @@ async def txt2img(req: Txt2ImgRequest):
@router.post("/generate/img2img") @router.post("/generate/img2img")
async def img2img(req: Img2ImgRequest): async def img2img(req: Img2ImgRequest):
"""Image-to-image via configured remote provider.""" """Image-to-image via configured remote provider."""
provider = _require_remote() provider = _require_remote("img2img")
try: try:
params = { params = {
"negative_prompt": req.negative_prompt or "", "negative_prompt": req.negative_prompt or "",
@@ -231,7 +233,7 @@ async def img2img(req: Img2ImgRequest):
@router.post("/generate/outpaint") @router.post("/generate/outpaint")
async def outpaint(req: OutpaintRequest): async def outpaint(req: OutpaintRequest):
"""Expand canvas in given direction via remote provider.""" """Expand canvas in given direction via remote provider."""
provider = _require_remote() provider = _require_remote("outpaint")
if req.direction not in ("left", "right", "top", "bottom"): if req.direction not in ("left", "right", "top", "bottom"):
raise HTTPException(status_code=400, detail="direction must be left/right/top/bottom") raise HTTPException(status_code=400, detail="direction must be left/right/top/bottom")
try: try:
@@ -246,6 +248,12 @@ async def outpaint(req: OutpaintRequest):
class ConfigUpdateRequest(BaseModel): class ConfigUpdateRequest(BaseModel):
ai_provider: Optional[str] = None ai_provider: Optional[str] = None
# Per-operation overrides (blank = use default)
ai_provider_inpaint: Optional[str] = None
ai_provider_txt2img: Optional[str] = None
ai_provider_img2img: Optional[str] = None
ai_provider_outpaint: Optional[str] = None
# Credentials / URLs
openai_api_key: Optional[str] = None openai_api_key: Optional[str] = None
openai_model: Optional[str] = None openai_model: Optional[str] = None
invokeai_url: Optional[str] = None invokeai_url: Optional[str] = None
@@ -265,49 +273,59 @@ async def update_config(req: ConfigUpdateRequest):
""" """
from app.config import settings from app.config import settings
if req.ai_provider is not None: _str_fields = [
settings.ai_provider = req.ai_provider "ai_provider", "ai_provider_inpaint", "ai_provider_txt2img",
if req.openai_api_key: "ai_provider_img2img", "ai_provider_outpaint",
settings.openai_api_key = req.openai_api_key "openai_api_key", "openai_model",
if req.openai_model: "invokeai_url", "invokeai_default_model",
settings.openai_model = req.openai_model "comfyui_url", "comfyui_default_model",
if req.invokeai_url is not None: "replicate_api_key", "stability_api_key",
settings.invokeai_url = req.invokeai_url ]
if req.invokeai_default_model: for field in _str_fields:
settings.invokeai_default_model = req.invokeai_default_model val = getattr(req, field, None)
if req.comfyui_url is not None: if val is not None:
settings.comfyui_url = req.comfyui_url setattr(settings, field, val)
if req.comfyui_default_model:
settings.comfyui_default_model = req.comfyui_default_model
if req.replicate_api_key:
settings.replicate_api_key = req.replicate_api_key
if req.stability_api_key:
settings.stability_api_key = req.stability_api_key
return {"status": "ok", "ai_provider": settings.ai_provider} return {
"status": "ok",
"ai_provider": settings.ai_provider,
"overrides": {
"inpaint": settings.ai_provider_inpaint or None,
"txt2img": settings.ai_provider_txt2img or None,
"img2img": settings.ai_provider_img2img or None,
"outpaint": settings.ai_provider_outpaint or None,
}
}
async def _check_provider(operation: str) -> dict:
"""Health-check the provider for a specific operation."""
from app.services.remote_provider import get_remote_provider
try:
p = get_remote_provider(operation)
if p is None:
return {"provider": None, "healthy": False}
healthy = await asyncio.wait_for(p.health(), timeout=5.0)
return {"provider": p.__class__.__name__.replace("Provider", "").lower(), "healthy": healthy}
except Exception:
return {"provider": None, "healthy": False}
@router.get("/config") @router.get("/config")
async def get_config(): async def get_config():
""" """
Return capability flags so the frontend can show/hide tools. Return capability flags so the frontend can show/hide tools.
Frontend reads this on load. Includes per-operation provider assignments and health status.
""" """
from app.services.remote_provider import get_remote_provider
from app.config import settings from app.config import settings
remote_caps: list[str] = [] # Run health checks for each operation concurrently
remote_healthy = False ops = ["inpaint", "txt2img", "img2img", "outpaint"]
provider_name = (settings.ai_provider or "").lower() results = await asyncio.gather(*[_check_provider(op) for op in ops])
op_status = dict(zip(ops, results))
if provider_name in ("openai", "invokeai", "comfyui"): # Default provider for display (used when no per-op override)
try: default_name = (settings.ai_provider or "").lower() or None
provider = get_remote_provider()
if provider:
remote_caps = provider.capabilities()
remote_healthy = await asyncio.wait_for(provider.health(), timeout=5.0)
except Exception:
remote_healthy = False
return { return {
"local": { "local": {
@@ -317,8 +335,16 @@ async def get_config():
"gpu_detected": gpu_available(), "gpu_detected": gpu_available(),
}, },
"remote": { "remote": {
"provider": provider_name or None, "default_provider": default_name,
"capabilities": remote_caps, # Legacy field kept for backwards compat with badge/capabilities checks
"healthy": remote_healthy, "provider": default_name,
"healthy": any(v["healthy"] for v in op_status.values()),
"operations": op_status,
"overrides": {
"inpaint": settings.ai_provider_inpaint or None,
"txt2img": settings.ai_provider_txt2img or None,
"img2img": settings.ai_provider_img2img or None,
"outpaint": settings.ai_provider_outpaint or None,
},
} }
} }
backend/app/services/remote_provider.py
+41 -6
View File
@@ -407,25 +407,60 @@ class ComfyUIProvider(RemoteAIProvider):
return ["inpaint", "txt2img", "img2img", "outpaint"] return ["inpaint", "txt2img", "img2img", "outpaint"]
def get_remote_provider() -> Optional[RemoteAIProvider]: def _build_provider(name: str) -> Optional[RemoteAIProvider]:
"""Return the configured remote provider, or None if not configured.""" """Instantiate a named provider from current settings."""
from app.config import settings from app.config import settings
provider = (settings.ai_provider or "").lower() name = (name or "").lower().strip()
if provider == "openai": if name == "openai":
if not settings.openai_api_key: if not settings.openai_api_key:
return None return None
return OpenAIRemoteProvider(settings.openai_api_key, settings.openai_model) return OpenAIRemoteProvider(settings.openai_api_key, settings.openai_model)
if provider == "invokeai": if name == "invokeai":
if not settings.invokeai_url: if not settings.invokeai_url:
return None return None
return InvokeAIProvider(settings.invokeai_url, settings.invokeai_default_model) return InvokeAIProvider(settings.invokeai_url, settings.invokeai_default_model)
if provider == "comfyui": if name == "comfyui":
if not settings.comfyui_url: if not settings.comfyui_url:
return None return None
return ComfyUIProvider(settings.comfyui_url, settings.comfyui_default_model) return ComfyUIProvider(settings.comfyui_url, settings.comfyui_default_model)
return None return None
# Map operation names to the settings field that holds the override
_OP_FIELD = {
"inpaint": "ai_provider_inpaint",
"txt2img": "ai_provider_txt2img",
"img2img": "ai_provider_img2img",
"outpaint": "ai_provider_outpaint",
}
def get_remote_provider(operation: Optional[str] = None) -> Optional[RemoteAIProvider]:
"""
Return the provider for a given operation.
Resolution order:
1. Per-operation override (AI_PROVIDER_INPAINT, AI_PROVIDER_TXT2IMG, etc.)
2. Global default (AI_PROVIDER)
3. None (local-only mode)
Example .env for mixed setup:
AI_PROVIDER=invokeai # default for inpaint/img2img/outpaint
AI_PROVIDER_TXT2IMG=openai # use OpenAI only for text-to-image
"""
from app.config import settings
if operation and operation in _OP_FIELD:
override = getattr(settings, _OP_FIELD[operation], "")
if override:
provider = _build_provider(override)
if provider is not None:
return provider
# override configured but not usable (missing key/url) — fall through to default
return _build_provider(settings.ai_provider)
frontend/src/js/core/components/provider-badge.js
+13 -2
View File
@@ -34,8 +34,19 @@ export async function mountProviderBadge(container) {
dot.style.background = '#44cc44'; dot.style.background = '#44cc44';
badge.style.background = '#1a2a1a'; badge.style.background = '#1a2a1a';
badge.style.color = '#aaffaa'; badge.style.color = '#aaffaa';
label.textContent = remote.provider + (local.gpu_detected ? ' · GPU' : ' · CPU');
badge.title = 'Remote provider: ' + remote.provider + '\nCapabilities: ' + (remote.capabilities || []).join(', '); // Show override summary if any operations use different providers
var overrides = remote.overrides || {};
var overrideEntries = Object.entries(overrides).filter(([, v]) => v);
var overrideStr = overrideEntries.length
? ' · ' + overrideEntries.map(([k, v]) => `${k}→${v}`).join(', ')
: '';
label.textContent = remote.provider + overrideStr + (local.gpu_detected ? ' · GPU' : '');
var opLines = Object.entries(remote.operations || {})
.map(([op, s]) => `${op}: ${s.provider || remote.provider} ${s.healthy ? '✓' : '✗'}`)
.join('\n');
badge.title = opLines || ('Provider: ' + remote.provider);
} else if (remote.provider && !remote.healthy) { } else if (remote.provider && !remote.healthy) {
dot.style.background = '#ffaa00'; dot.style.background = '#ffaa00';
badge.style.background = '#2a2000'; badge.style.background = '#2a2000';
frontend/src/js/modules/tools/ai_provider_settings.js
+52 -11
View File
@@ -47,11 +47,44 @@ class Tools_ai_provider_settings_class {
}, },
{ {
name: 'provider', name: 'provider',
title: 'Remote provider:', title: 'Default provider (used unless overridden below):',
value: ls_get('provider', remote.provider || ''), value: ls_get('provider', remote.provider || ''),
values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'], values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'],
type: 'select', type: 'select',
}, },
// ── Per-operation overrides ───────────────────────────────
{
title: '',
html: '<div style="font-size:11px;color:#888;margin:2px 0 6px;">Per-operation overrides — blank = use default above</div>',
},
{
name: 'provider_inpaint',
title: 'Inpaint / Replace Selection:',
value: ls_get('provider_inpaint', remote.overrides?.inpaint || ''),
values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'],
type: 'select',
},
{
name: 'provider_txt2img',
title: 'Text → Image:',
value: ls_get('provider_txt2img', remote.overrides?.txt2img || ''),
values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'],
type: 'select',
},
{
name: 'provider_img2img',
title: 'Image → Image:',
value: ls_get('provider_img2img', remote.overrides?.img2img || ''),
values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'],
type: 'select',
},
{
name: 'provider_outpaint',
title: 'Expand Canvas (Outpaint):',
value: ls_get('provider_outpaint', remote.overrides?.outpaint || ''),
values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'],
type: 'select',
},
// ── OpenAI ──────────────────────────────────────────────── // ── OpenAI ────────────────────────────────────────────────
{ {
name: 'openai_key', name: 'openai_key',
@@ -108,7 +141,11 @@ class Tools_ai_provider_settings_class {
async _save(params) { async _save(params) {
// Persist to localStorage // Persist to localStorage
ls_set('provider', params.provider || ''); ls_set('provider', params.provider || '');
ls_set('provider_inpaint', params.provider_inpaint || '');
ls_set('provider_txt2img', params.provider_txt2img || '');
ls_set('provider_img2img', params.provider_img2img || '');
ls_set('provider_outpaint', params.provider_outpaint || '');
ls_set('openai_key', params.openai_key || ''); ls_set('openai_key', params.openai_key || '');
ls_set('openai_model', params.openai_model || 'dall-e-3'); ls_set('openai_model', params.openai_model || 'dall-e-3');
ls_set('invokeai_url', params.invokeai_url || ''); ls_set('invokeai_url', params.invokeai_url || '');
@@ -117,17 +154,21 @@ class Tools_ai_provider_settings_class {
ls_set('comfyui_model', params.comfyui_model || 'v1-5-pruned-emaonly.ckpt'); ls_set('comfyui_model', params.comfyui_model || 'v1-5-pruned-emaonly.ckpt');
ls_set('replicate_key', params.replicate_key || ''); ls_set('replicate_key', params.replicate_key || '');
// Push to backend (requires a running server that accepts runtime config) // Push to backend
try { try {
var payload = { var payload = {
ai_provider: params.provider || '', ai_provider: params.provider || '',
openai_api_key: params.openai_key || '', ai_provider_inpaint: params.provider_inpaint || '',
openai_model: params.openai_model || 'dall-e-3', ai_provider_txt2img: params.provider_txt2img || '',
invokeai_url: params.invokeai_url || '', ai_provider_img2img: params.provider_img2img || '',
invokeai_default_model: params.invokeai_model || 'flux-dev', ai_provider_outpaint: params.provider_outpaint || '',
comfyui_url: params.comfyui_url || '', openai_api_key: params.openai_key || '',
comfyui_default_model: params.comfyui_model || '', openai_model: params.openai_model || 'dall-e-3',
replicate_api_key: params.replicate_key || '', invokeai_url: params.invokeai_url || '',
invokeai_default_model: params.invokeai_model || 'flux-dev',
comfyui_url: params.comfyui_url || '',
comfyui_default_model: params.comfyui_model || '',
replicate_api_key: params.replicate_key || '',
}; };
var base = window.API_BASE_URL || ''; var base = window.API_BASE_URL || '';
var r = await fetch(`${base}/api/config`, { var r = await fetch(`${base}/api/config`, {