diff --git a/.env.example b/.env.example index eb1b3ab..aab2dce 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,13 @@ AI_PROVIDER=replicate +# Per-operation provider overrides (optional — blank means use AI_PROVIDER above) +# Example: use OpenAI for text-to-image (best quality) but InvokeAI for everything else +#AI_PROVIDER_TXT2IMG=openai +#AI_PROVIDER_INPAINT=invokeai +#AI_PROVIDER_IMG2IMG=invokeai +#AI_PROVIDER_OUTPAINT=invokeai + # ============================================================================= # STEP 2: Get Your API Key @@ -50,10 +57,27 @@ AI_PROVIDER=replicate REPLICATE_API_KEY=r8_PASTE_YOUR_KEY_HERE # ─────────────────────────────────────────────────────────────────────────── -# OPENAI (Alternative - not recommended, lower quality) +# OPENAI (cloud, dall-e-3 / gpt-image-1) # Get key at: https://platform.openai.com/api-keys +# AI_PROVIDER=openai # ─────────────────────────────────────────────────────────────────────────── #OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +#OPENAI_MODEL=dall-e-3 + +# ─────────────────────────────────────────────────────────────────────────── +# INVOKEAI (self-hosted, best for Flux/SDXL) +# Run InvokeAI on your local machine or NAS, point URL here. +# AI_PROVIDER=invokeai +# ─────────────────────────────────────────────────────────────────────────── +#INVOKEAI_URL=http://192.168.1.x:9090 +#INVOKEAI_DEFAULT_MODEL=flux-dev + +# ─────────────────────────────────────────────────────────────────────────── +# COMFYUI (self-hosted, workflow JSON API) +# AI_PROVIDER=comfyui +# ─────────────────────────────────────────────────────────────────────────── +#COMFYUI_URL=http://192.168.1.x:8188 +#COMFYUI_DEFAULT_MODEL=v1-5-pruned-emaonly.ckpt # ─────────────────────────────────────────────────────────────────────────── # STABILITY AI (Alternative) diff --git a/backend/app/config.py b/backend/app/config.py index c958edc..0b80ad9 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -12,13 +12,33 @@ class Settings(BaseSettings): access_token_expire_minutes: int = 30 # AI Provider - ai_provider: str = "mock" # Options: openai, stability, replicate, mock + # Local: blank or "mock" — always available, no config needed + # Remote default (used for any operation without a specific override): + # openai | invokeai | comfyui | replicate | stability + ai_provider: str = "mock" + + # Per-operation provider overrides — blank means use ai_provider default. + # Operations: inpaint, txt2img, img2img, outpaint + # Example: AI_PROVIDER_TXT2IMG=openai (use OpenAI for text-to-image only) + ai_provider_inpaint: str = "" # remote inpaint / replace selection + ai_provider_txt2img: str = "" # text-to-image + ai_provider_img2img: str = "" # image-to-image + ai_provider_outpaint: str = "" # expand canvas # Provider API Keys openai_api_key: str = "" + openai_model: str = "dall-e-3" stability_api_key: str = "" replicate_api_key: str = "" + # InvokeAI (self-hosted) + invokeai_url: str = "" + invokeai_default_model: str = "flux-dev" + + # ComfyUI (self-hosted) + comfyui_url: str = "" + comfyui_default_model: str = "v1-5-pruned-emaonly.ckpt" + # Model Selection (optional, provider-specific) stability_model: str = "sdxl" # Options: sdxl, sd15, sd21 replicate_model: str = "sdxl-inpaint" # Options: sdxl-inpaint, lama, realistic-vision diff --git a/backend/app/main.py b/backend/app/main.py index bfbb7f2..70d9b5a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -4,17 +4,23 @@ from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, HTMLResponse from contextlib import asynccontextmanager from pathlib import Path +import asyncio import os from app.config import settings from app.database import init_db -from app.routers import projects, edits, images, patches, generate, tools +from app.routers import projects, edits, images, patches, generate, tools, ai_tools, print_tools @asynccontextmanager async def lifespan(app: FastAPI): - """Initialize database on startup""" + """Initialize database on startup; auto-install Real-ESRGAN NCNN in background.""" init_db() + # Kick off NCNN install in background if no AI upscaler detected + from app.services.upscale import probe_upscale_capabilities, ensure_ncnn_installed + caps = probe_upscale_capabilities() + if not caps["realesrgan_pytorch"] and not caps["realesrgan_ncnn"]: + asyncio.create_task(ensure_ncnn_installed()) yield @@ -41,6 +47,8 @@ app.include_router(images.router) app.include_router(patches.router) app.include_router(generate.router) app.include_router(tools.router) +app.include_router(ai_tools.router) +app.include_router(print_tools.router) @app.get("/api") diff --git a/backend/app/routers/ai_tools.py b/backend/app/routers/ai_tools.py new file mode 100644 index 0000000..243c5ba --- /dev/null +++ b/backend/app/routers/ai_tools.py @@ -0,0 +1,350 @@ +""" +AI tools router — LaMa inpaint, background removal, remote generation, config. +All endpoints are under /api prefix. +""" + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel +from typing import Optional +import base64 +import asyncio + +from app.services.local_inpaint import ( + lama_inpaint, opencv_inpaint, lama_available, gpu_available, rembg_available, +) + +router = APIRouter(prefix="/api", tags=["ai-tools"]) + + +# ─── Request / response models ─────────────────────────────────────────────── + +class EraseRequest(BaseModel): + image: str # base64 + mask: str # base64 + + +class InpaintRemoteRequest(BaseModel): + image: str + mask: str + prompt: str + negative_prompt: Optional[str] = "" + steps: Optional[int] = 30 + cfg_scale: Optional[float] = 7.5 + model: Optional[str] = None + + +class Txt2ImgRequest(BaseModel): + prompt: str + width: Optional[int] = 1024 + height: Optional[int] = 1024 + negative_prompt: Optional[str] = "" + steps: Optional[int] = 30 + cfg_scale: Optional[float] = 7.5 + model: Optional[str] = None + seed: Optional[int] = 0 + + +class Img2ImgRequest(BaseModel): + image: str + prompt: str + strength: Optional[float] = 0.75 + negative_prompt: Optional[str] = "" + steps: Optional[int] = 30 + cfg_scale: Optional[float] = 7.5 + model: Optional[str] = None + + +class OutpaintRequest(BaseModel): + image: str + direction: str # left | right | top | bottom + size: Optional[int] = 256 + prompt: Optional[str] = "" + + +class BgRemoveRequest(BaseModel): + image: str + + +# ─── Helpers ───────────────────────────────────────────────────────────────── + +def _decode(b64: str) -> bytes: + return base64.b64decode(b64) + + +def _encode(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def _require_remote(operation: str = None): + from app.services.remote_provider import get_remote_provider + provider = get_remote_provider(operation) + if provider is None: + op_hint = f"AI_PROVIDER_{operation.upper()} or " if operation else "" + raise HTTPException( + status_code=503, + detail=f"No remote AI provider configured for '{operation or 'default'}'. " + f"Set {op_hint}AI_PROVIDER in .env (openai / invokeai / comfyui)." + ) + return provider + + +# ─── Local inpaint endpoints ───────────────────────────────────────────────── + +@router.post("/erase") +async def erase(req: EraseRequest): + """ + Magic eraser: remove object / fill region using LaMa (local, no API key needed). + Falls back to OpenCV if LaMa not installed. + """ + try: + image_bytes = _decode(req.image) + mask_bytes = _decode(req.mask) + + if lama_available(): + result = await asyncio.get_event_loop().run_in_executor( + None, lama_inpaint, image_bytes, mask_bytes + ) + method = "lama" + else: + result = await asyncio.get_event_loop().run_in_executor( + None, opencv_inpaint, image_bytes, mask_bytes + ) + method = "opencv" + + return {"result": _encode(result), "method": method} + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/inpaint/lama") +async def inpaint_lama(req: EraseRequest): + """LaMa structural inpainting.""" + if not lama_available(): + raise HTTPException(status_code=503, detail="simple-lama-inpainting not installed.") + try: + result = await asyncio.get_event_loop().run_in_executor( + None, lama_inpaint, _decode(req.image), _decode(req.mask) + ) + return {"result": _encode(result)} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/inpaint/fast") +async def inpaint_fast(req: EraseRequest): + """OpenCV fast inpainting (CPU, milliseconds).""" + try: + result = await asyncio.get_event_loop().run_in_executor( + None, opencv_inpaint, _decode(req.image), _decode(req.mask) + ) + return {"result": _encode(result)} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/background/remove") +async def background_remove(req: BgRemoveRequest): + """Remove background — rembg if available, else U2Net.""" + try: + image_bytes = _decode(req.image) + + # Try rembg first + if rembg_available(): + from app.services.local_inpaint import remove_background_rembg + result = await asyncio.get_event_loop().run_in_executor( + None, remove_background_rembg, image_bytes + ) + return {"result": _encode(result), "method": "rembg"} + + # Fall back to U2Net (existing implementation) + from PIL import Image + from io import BytesIO as _BytesIO + img = Image.open(_BytesIO(image_bytes)).convert("RGB") + from app.routers.tools import _remove_background_u2net + result = await _remove_background_u2net(img) + return {"result": _encode(result), "method": "u2net"} + + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +# ─── Remote provider endpoints ─────────────────────────────────────────────── + +@router.post("/inpaint/remote") +async def inpaint_remote(req: InpaintRemoteRequest): + """Inpaint via configured remote provider (InvokeAI / ComfyUI / OpenAI).""" + provider = _require_remote("inpaint") + try: + params = { + "negative_prompt": req.negative_prompt or "", + "steps": req.steps, + "cfg_scale": req.cfg_scale, + } + if req.model: + params["model"] = req.model + result = await provider.inpaint(_decode(req.image), _decode(req.mask), req.prompt, params) + return {"result": _encode(result)} + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/generate/txt2img") +async def txt2img(req: Txt2ImgRequest): + """Text-to-image via configured remote provider.""" + provider = _require_remote("txt2img") + try: + params = { + "negative_prompt": req.negative_prompt or "", + "steps": req.steps, + "cfg_scale": req.cfg_scale, + "seed": req.seed or 0, + } + if req.model: + params["model"] = req.model + result = await provider.txt2img(req.prompt, req.width, req.height, params) + return {"result": _encode(result)} + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/generate/img2img") +async def img2img(req: Img2ImgRequest): + """Image-to-image via configured remote provider.""" + provider = _require_remote("img2img") + try: + params = { + "negative_prompt": req.negative_prompt or "", + "steps": req.steps, + "cfg_scale": req.cfg_scale, + } + if req.model: + params["model"] = req.model + result = await provider.img2img(_decode(req.image), req.prompt, req.strength, params) + return {"result": _encode(result)} + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/generate/outpaint") +async def outpaint(req: OutpaintRequest): + """Expand canvas in given direction via remote provider.""" + provider = _require_remote("outpaint") + if req.direction not in ("left", "right", "top", "bottom"): + raise HTTPException(status_code=400, detail="direction must be left/right/top/bottom") + try: + result = await provider.outpaint(_decode(req.image), req.direction, req.size, req.prompt or "") + return {"result": _encode(result)} + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + +# ─── Config / capabilities ──────────────────────────────────────────────────── + +class ConfigUpdateRequest(BaseModel): + ai_provider: Optional[str] = None + # Per-operation overrides (blank = use default) + ai_provider_inpaint: Optional[str] = None + ai_provider_txt2img: Optional[str] = None + ai_provider_img2img: Optional[str] = None + ai_provider_outpaint: Optional[str] = None + # Credentials / URLs + openai_api_key: Optional[str] = None + openai_model: Optional[str] = None + invokeai_url: Optional[str] = None + invokeai_default_model: Optional[str] = None + comfyui_url: Optional[str] = None + comfyui_default_model: Optional[str] = None + replicate_api_key: Optional[str] = None + stability_api_key: Optional[str] = None + + +@router.post("/config") +async def update_config(req: ConfigUpdateRequest): + """ + Apply runtime provider settings (no restart needed). + Values are applied to the live settings object in-process. + They do NOT persist across restarts — set them in .env for permanence. + """ + from app.config import settings + + _str_fields = [ + "ai_provider", "ai_provider_inpaint", "ai_provider_txt2img", + "ai_provider_img2img", "ai_provider_outpaint", + "openai_api_key", "openai_model", + "invokeai_url", "invokeai_default_model", + "comfyui_url", "comfyui_default_model", + "replicate_api_key", "stability_api_key", + ] + for field in _str_fields: + val = getattr(req, field, None) + if val is not None: + setattr(settings, field, val) + + return { + "status": "ok", + "ai_provider": settings.ai_provider, + "overrides": { + "inpaint": settings.ai_provider_inpaint or None, + "txt2img": settings.ai_provider_txt2img or None, + "img2img": settings.ai_provider_img2img or None, + "outpaint": settings.ai_provider_outpaint or None, + } + } + + +async def _check_provider(operation: str) -> dict: + """Health-check the provider for a specific operation.""" + from app.services.remote_provider import get_remote_provider + try: + p = get_remote_provider(operation) + if p is None: + return {"provider": None, "healthy": False} + healthy = await asyncio.wait_for(p.health(), timeout=5.0) + return {"provider": p.__class__.__name__.replace("Provider", "").lower(), "healthy": healthy} + except Exception: + return {"provider": None, "healthy": False} + + +@router.get("/config") +async def get_config(): + """ + Return capability flags so the frontend can show/hide tools. + Includes per-operation provider assignments and health status. + """ + from app.config import settings + + # Run health checks for each operation concurrently + ops = ["inpaint", "txt2img", "img2img", "outpaint"] + results = await asyncio.gather(*[_check_provider(op) for op in ops]) + op_status = dict(zip(ops, results)) + + # Default provider for display (used when no per-op override) + default_name = (settings.ai_provider or "").lower() or None + + return { + "local": { + "lama": lama_available(), + "rembg": rembg_available(), + "opencv": True, + "gpu_detected": gpu_available(), + }, + "remote": { + "default_provider": default_name, + # Legacy field kept for backwards compat with badge/capabilities checks + "provider": default_name, + "healthy": any(v["healthy"] for v in op_status.values()), + "operations": op_status, + "overrides": { + "inpaint": settings.ai_provider_inpaint or None, + "txt2img": settings.ai_provider_txt2img or None, + "img2img": settings.ai_provider_img2img or None, + "outpaint": settings.ai_provider_outpaint or None, + }, + } + } diff --git a/backend/app/routers/print_tools.py b/backend/app/routers/print_tools.py new file mode 100644 index 0000000..52fe476 --- /dev/null +++ b/backend/app/routers/print_tools.py @@ -0,0 +1,377 @@ +""" +Print / frame tools — frame fit and upscale. +All endpoints under /api/print prefix. +""" + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel +from typing import Optional, Literal +import base64 +import asyncio +from io import BytesIO +from PIL import Image +import numpy as np + +router = APIRouter(prefix="/api/print", tags=["print-tools"]) + +# ── Frame size catalogue (inches) ────────────────────────────────────────── +FRAME_SIZES = { + "4x6": (4, 6), + "5x7": (5, 7), + "8x10": (8, 10), + "11x14": (11, 14), + "16x20": (16, 20), + "20x24": (20, 24), + "24x36": (24, 36), + # Square + "4x4": (4, 4), + "8x8": (8, 8), + "12x12": (12, 12), +} + + +def _encode(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def _decode(b64: str) -> bytes: + return base64.b64decode(b64) + + +def _to_png(img: Image.Image) -> bytes: + buf = BytesIO() + img.save(buf, format="PNG") + return buf.getvalue() + + +# ── Request models ───────────────────────────────────────────────────────── + +class FrameFitRequest(BaseModel): + image: str # base64 PNG/JPEG + frame: str # e.g. "8x10" + orientation: Literal["auto", "portrait", "landscape"] = "auto" + mode: Literal["crop", "extend", "smart"] = "smart" + dpi: int = 300 + # For extend mode: prompt passed to outpaint + prompt: Optional[str] = "" + # Smart mode threshold: extend if gap fraction < this, else crop + smart_threshold: float = 0.15 + + +class UpscaleRequest(BaseModel): + image: str # base64 + scale: float = 2.0 # 1.5, 2, 3, 4 + # auto = pick best available; lanczos = always works; realesrgan_pytorch / realesrgan_ncnn = explicit + method: str = "auto" + + +# ── Frame sizes endpoint ─────────────────────────────────────────────────── + +@router.get("/frame-sizes") +def list_frame_sizes(): + """Return the catalogue of supported frame sizes.""" + return { + "sizes": list(FRAME_SIZES.keys()), + "catalogue": {k: {"inches": v, "pixels_300dpi": (v[0]*300, v[1]*300)} + for k, v in FRAME_SIZES.items()}, + } + + +# ── Frame fit ────────────────────────────────────────────────────────────── + +@router.post("/frame-fit") +async def frame_fit(req: FrameFitRequest): + """ + Fit an image to a print frame size. + + Modes: + crop — center-crop to frame aspect ratio, then scale to print resolution. + extend — scale to fill one dimension, outpaint the gap with AI. + smart — extend if gap < smart_threshold of frame dimension, else crop. + + Returns the fitted image plus a summary of what was done. + """ + if req.frame not in FRAME_SIZES: + raise HTTPException(status_code=400, + detail=f"Unknown frame '{req.frame}'. Valid: {list(FRAME_SIZES.keys())}") + if not (72 <= req.dpi <= 600): + raise HTTPException(status_code=400, detail="dpi must be 72–600") + + try: + image = Image.open(BytesIO(_decode(req.image))).convert("RGB") + except Exception as e: + raise HTTPException(status_code=400, detail=f"Could not decode image: {e}") + + fw, fh = FRAME_SIZES[req.frame] # frame inches (w, h in portrait) + + # Resolve orientation + img_w, img_h = image.size + img_landscape = img_w >= img_h + frame_landscape = fw >= fh + + if req.orientation == "landscape": + fw, fh = max(fw, fh), min(fw, fh) + elif req.orientation == "portrait": + fw, fh = min(fw, fh), max(fw, fh) + else: # auto — match image orientation + if img_landscape and not frame_landscape: + fw, fh = fh, fw # rotate frame to landscape + elif not img_landscape and frame_landscape: + fw, fh = fh, fw # rotate frame to portrait + + target_w = fw * req.dpi + target_h = fh * req.dpi + target_ratio = target_w / target_h + img_ratio = img_w / img_h + + # Determine actual mode + mode = req.mode + if mode == "smart": + # Scale image to fill the frame — compute gap fraction + if img_ratio > target_ratio: + # Image wider → fits on height, gap on width + scaled_h = target_h + scaled_w = round(target_h * img_ratio) + gap_frac = (scaled_w - target_w) / target_w # positive = overflow (crop) + else: + scaled_w = target_w + scaled_h = round(target_w / img_ratio) + gap_frac = (scaled_h - target_h) / target_h + + # gap_frac > 0 means we'd need to crop; < 0 means we'd need to extend + if gap_frac < 0: + # Need to extend — use extend if gap is small enough + mode = "extend" if abs(gap_frac) <= req.smart_threshold else "crop" + else: + mode = "crop" + + if mode == "crop": + result, summary = _crop_fit(image, target_w, target_h) + else: # extend + result, summary = await _extend_fit(image, target_w, target_h, req.prompt or "") + + return { + "result": _encode(_to_png(result)), + "mode_used": mode, + "frame": req.frame, + "orientation": "landscape" if fw > fh else "portrait", + "output_pixels": {"width": result.width, "height": result.height}, + "output_inches": {"width": fw, "height": fh}, + "dpi": req.dpi, + "summary": summary, + } + + +def _crop_fit(image: Image.Image, target_w: int, target_h: int): + """Center-crop image to target aspect ratio, then Lanczos scale to target size.""" + img_w, img_h = image.size + target_ratio = target_w / target_h + img_ratio = img_w / img_h + + if img_ratio > target_ratio: + # Wider than target — crop sides + new_w = round(img_h * target_ratio) + x0 = (img_w - new_w) // 2 + cropped = image.crop((x0, 0, x0 + new_w, img_h)) + else: + # Taller than target — crop top/bottom + new_h = round(img_w / target_ratio) + y0 = (img_h - new_h) // 2 + cropped = image.crop((0, y0, img_w, y0 + new_h)) + + result = cropped.resize((target_w, target_h), Image.Resampling.LANCZOS) + summary = ( + f"Cropped from {img_w}×{img_h} to {cropped.width}×{cropped.height}, " + f"scaled to {target_w}×{target_h}" + ) + return result, summary + + +async def _extend_fit(image: Image.Image, target_w: int, target_h: int, prompt: str): + """ + Scale image to fill one dimension exactly, then outpaint the gap with AI. + Falls back to content-aware mirror fill if no remote provider configured. + """ + from app.services.remote_provider import get_remote_provider + + img_w, img_h = image.size + target_ratio = target_w / target_h + img_ratio = img_w / img_h + + if img_ratio > target_ratio: + # Image wider — scale to target width, extend height + scale = target_w / img_w + scaled_w = target_w + scaled_h = round(img_h * scale) + gap_dir = "height" + gap_top = (target_h - scaled_h) // 2 + gap_bottom = target_h - scaled_h - gap_top + else: + # Image taller — scale to target height, extend width + scale = target_h / img_h + scaled_h = target_h + scaled_w = round(img_w * scale) + gap_dir = "width" + gap_left = (target_w - scaled_w) // 2 + gap_right = target_w - scaled_w - gap_left + + scaled = image.resize((scaled_w, scaled_h), Image.Resampling.LANCZOS) + + # Place scaled image on canvas + canvas = Image.new("RGB", (target_w, target_h), (128, 128, 128)) + if gap_dir == "height": + canvas.paste(scaled, (0, gap_top)) + # Build mask: top and bottom strips are white (to inpaint) + mask = Image.new("L", (target_w, target_h), 0) + if gap_top > 0: + mask.paste(Image.new("L", (target_w, gap_top), 255), (0, 0)) + if gap_bottom > 0: + mask.paste(Image.new("L", (target_w, gap_bottom), 255), (0, target_h - gap_bottom)) + else: + canvas.paste(scaled, (gap_left, 0)) + mask = Image.new("L", (target_w, target_h), 0) + if gap_left > 0: + mask.paste(Image.new("L", (gap_left, target_h), 255), (0, 0)) + if gap_right > 0: + mask.paste(Image.new("L", (gap_right, target_h), 255), (target_w - gap_right, 0)) + + # Try AI inpaint + provider = get_remote_provider("inpaint") + if provider: + try: + canvas_bytes = _to_png(canvas) + mask_bytes = _to_png(mask) + fill_prompt = prompt or "seamlessly continue the image, natural extension" + result_bytes = await provider.inpaint(canvas_bytes, mask_bytes, fill_prompt, {}) + result = Image.open(BytesIO(result_bytes)).convert("RGB") + summary = ( + f"Scaled {img_w}×{img_h} → {scaled_w}×{scaled_h}, " + f"AI-extended {gap_dir} to {target_w}×{target_h}" + ) + return result, summary + except Exception as e: + print(f"AI extend failed, using mirror fill: {e}") + + # Fallback: mirror-fill the gap (looks decent for backgrounds/landscapes) + result = _mirror_fill(canvas, mask, scaled, gap_dir, + gap_top if gap_dir == "height" else gap_left, + gap_bottom if gap_dir == "height" else gap_right, + target_w, target_h) + summary = ( + f"Scaled {img_w}×{img_h} → {scaled_w}×{scaled_h}, " + f"mirror-filled {gap_dir} to {target_w}×{target_h} (no AI provider)" + ) + return result, summary + + +def _mirror_fill(canvas, mask, scaled, gap_dir, gap_a, gap_b, target_w, target_h): + """Fill gaps by reflecting the nearest edge strip.""" + result = canvas.copy() + if gap_dir == "height": + if gap_a > 0: + strip = scaled.crop((0, 0, scaled.width, min(gap_a * 2, scaled.height))) + strip = strip.transpose(Image.Transpose.FLIP_TOP_BOTTOM) + strip = strip.resize((target_w, gap_a), Image.Resampling.LANCZOS) + result.paste(strip, (0, 0)) + if gap_b > 0: + strip = scaled.crop((0, max(0, scaled.height - gap_b * 2), scaled.width, scaled.height)) + strip = strip.transpose(Image.Transpose.FLIP_TOP_BOTTOM) + strip = strip.resize((target_w, gap_b), Image.Resampling.LANCZOS) + result.paste(strip, (0, target_h - gap_b)) + else: + if gap_a > 0: + strip = scaled.crop((0, 0, min(gap_a * 2, scaled.width), scaled.height)) + strip = strip.transpose(Image.Transpose.FLIP_LEFT_RIGHT) + strip = strip.resize((gap_a, target_h), Image.Resampling.LANCZOS) + result.paste(strip, (0, 0)) + if gap_b > 0: + strip = scaled.crop((max(0, scaled.width - gap_b * 2), 0, scaled.width, scaled.height)) + strip = strip.transpose(Image.Transpose.FLIP_LEFT_RIGHT) + strip = strip.resize((gap_b, target_h), Image.Resampling.LANCZOS) + result.paste(strip, (target_w - gap_b, 0)) + return result + + +# ── Upscale ──────────────────────────────────────────────────────────────── + +@router.post("/upscale/refresh-caps") +def upscale_refresh_caps(): + """Bust the capability cache (call after installing Real-ESRGAN without restarting).""" + from app.services.upscale import invalidate_caps_cache, probe_upscale_capabilities + invalidate_caps_cache() + return probe_upscale_capabilities() + + +@router.get("/upscale/available") +async def upscale_available(): + """ + Return capability probe: which upscale methods are available, + which device will be used, and which method is recommended. + If no AI upscaler is found, triggers background NCNN auto-install. + Frontend uses this to populate the method selector. + """ + from app.services.upscale import probe_upscale_capabilities, ensure_ncnn_installed, get_install_status + caps = probe_upscale_capabilities() + # Auto-install NCNN if no AI upscaler is available yet + if not caps["realesrgan_pytorch"] and not caps["realesrgan_ncnn"]: + asyncio.create_task(ensure_ncnn_installed()) + caps["ncnn_install_status"] = get_install_status() + return caps + + +@router.get("/upscale/install-status") +def upscale_install_status(): + """Poll for Real-ESRGAN NCNN auto-install progress.""" + from app.services.upscale import get_install_status, probe_upscale_capabilities, _find_ncnn_binary + status = get_install_status() + # If install just finished, refresh caps + if status["state"] == "done": + from app.services.upscale import invalidate_caps_cache + invalidate_caps_cache() + caps = probe_upscale_capabilities() + status["ncnn_available"] = caps["realesrgan_ncnn"] + else: + status["ncnn_available"] = False + return status + + +@router.post("/upscale") +async def upscale(req: UpscaleRequest): + """ + Upscale image. method values: + auto — pick best available (recommended) + realesrgan_pytorch — Real-ESRGAN via PyTorch (CUDA/MPS/CPU) + realesrgan_ncnn — Real-ESRGAN NCNN Vulkan binary + lanczos — always available, instant + Any AI method falls back to the next best if unavailable. + """ + if not (1.1 <= req.scale <= 8.0): + raise HTTPException(status_code=400, detail="scale must be 1.1–8.0") + + valid_methods = {"auto", "realesrgan_pytorch", "realesrgan_ncnn", "lanczos"} + if req.method not in valid_methods: + raise HTTPException(status_code=400, + detail=f"method must be one of {sorted(valid_methods)}") + + try: + image = Image.open(BytesIO(_decode(req.image))).convert("RGB") + except Exception as e: + raise HTTPException(status_code=400, detail=f"Could not decode image: {e}") + + orig_w, orig_h = image.size + + try: + from app.services.upscale import upscale_image + result_bytes, method_used = await upscale_image(image, req.scale, req.method) + result = Image.open(BytesIO(result_bytes)) + except Exception as e: + import traceback; traceback.print_exc() + raise HTTPException(status_code=500, detail=str(e)) + + return { + "result": _encode(result_bytes), + "method": method_used, + "original": {"width": orig_w, "height": orig_h}, + "output": {"width": result.width, "height": result.height}, + "scale": req.scale, + } diff --git a/backend/app/services/local_inpaint.py b/backend/app/services/local_inpaint.py new file mode 100644 index 0000000..cd82997 --- /dev/null +++ b/backend/app/services/local_inpaint.py @@ -0,0 +1,82 @@ +""" +Local inpainting operations — LaMa, OpenCV, and background removal. +All operations use GPU automatically if PyTorch detects one, CPU otherwise. +""" + +from io import BytesIO +from PIL import Image +import numpy as np +import cv2 + +# Lazy-loaded LaMa model (downloaded on first use, ~100MB) +_lama = None + + +def get_lama(): + global _lama + if _lama is None: + from simple_lama_inpainting import SimpleLama + _lama = SimpleLama() + return _lama + + +def lama_available() -> bool: + try: + import simple_lama_inpainting # noqa: F401 + return True + except ImportError: + return False + + +def lama_inpaint(image_bytes: bytes, mask_bytes: bytes) -> bytes: + """LaMa structural inpainting — best for object removal and large fills.""" + lama = get_lama() + image = Image.open(BytesIO(image_bytes)).convert("RGB") + mask = Image.open(BytesIO(mask_bytes)).convert("L") + if mask.size != image.size: + mask = mask.resize(image.size, Image.Resampling.LANCZOS) + result = lama(image, mask) + buf = BytesIO() + result.save(buf, format="PNG") + return buf.getvalue() + + +def opencv_inpaint(image_bytes: bytes, mask_bytes: bytes, method: str = "telea") -> bytes: + """OpenCV fast structural inpainting — CPU only, milliseconds.""" + image = Image.open(BytesIO(image_bytes)).convert("RGB") + mask = Image.open(BytesIO(mask_bytes)).convert("L") + if mask.size != image.size: + mask = mask.resize(image.size, Image.Resampling.LANCZOS) + + img_np = np.array(image) + mask_np = np.array(mask) + _, mask_bin = cv2.threshold(mask_np, 127, 255, cv2.THRESH_BINARY) + + flags = cv2.INPAINT_TELEA if method == "telea" else cv2.INPAINT_NS + result = cv2.inpaint(img_np, mask_bin, inpaintRadius=3, flags=flags) + + buf = BytesIO() + Image.fromarray(result).save(buf, format="PNG") + return buf.getvalue() + + +def remove_background_rembg(image_bytes: bytes) -> bytes: + """Background removal using rembg.""" + from rembg import remove + return remove(image_bytes) + + +def rembg_available() -> bool: + try: + import rembg # noqa: F401 + return True + except ImportError: + return False + + +def gpu_available() -> bool: + try: + import torch + return torch.cuda.is_available() + except ImportError: + return False diff --git a/backend/app/services/remote_provider.py b/backend/app/services/remote_provider.py new file mode 100644 index 0000000..6fb8d31 --- /dev/null +++ b/backend/app/services/remote_provider.py @@ -0,0 +1,466 @@ +""" +Remote AI provider abstraction. +One interface, three drivers: OpenAI, InvokeAI, ComfyUI. +Configure one provider via AI_PROVIDER in .env. +""" + +from abc import ABC, abstractmethod +from typing import Optional +import httpx +import base64 +import asyncio +from io import BytesIO + + +class RemoteAIProvider(ABC): + @abstractmethod + async def inpaint(self, image_bytes: bytes, mask_bytes: bytes, prompt: str, params: dict) -> bytes: ... + @abstractmethod + async def txt2img(self, prompt: str, width: int, height: int, params: dict) -> bytes: ... + @abstractmethod + async def img2img(self, image_bytes: bytes, prompt: str, strength: float, params: dict) -> bytes: ... + @abstractmethod + async def outpaint(self, image_bytes: bytes, direction: str, size: int, prompt: str) -> bytes: ... + @abstractmethod + async def health(self) -> bool: ... + @abstractmethod + def capabilities(self) -> list[str]: ... + + +class OpenAIRemoteProvider(RemoteAIProvider): + """OpenAI image API — gpt-image-1 / dall-e-3.""" + + def __init__(self, api_key: str, model: str = "dall-e-3"): + self.api_key = api_key + self.model = model + self.base_url = "https://api.openai.com/v1" + + def _headers(self): + return {"Authorization": f"Bearer {self.api_key}"} + + async def inpaint(self, image_bytes: bytes, mask_bytes: bytes, prompt: str, params: dict) -> bytes: + async with httpx.AsyncClient(timeout=120.0) as client: + files = { + "image": ("image.png", image_bytes, "image/png"), + "mask": ("mask.png", mask_bytes, "image/png"), + } + data = {"prompt": prompt, "n": "1", "size": "1024x1024"} + r = await client.post(f"{self.base_url}/images/edits", files=files, data=data, headers=self._headers()) + r.raise_for_status() + url = r.json()["data"][0]["url"] + img_r = await client.get(url) + img_r.raise_for_status() + return img_r.content + + async def txt2img(self, prompt: str, width: int, height: int, params: dict) -> bytes: + size = f"{width}x{height}" if f"{width}x{height}" in {"256x256", "512x512", "1024x1024"} else "1024x1024" + async with httpx.AsyncClient(timeout=120.0) as client: + data = {"model": self.model, "prompt": prompt, "n": 1, "size": size} + r = await client.post(f"{self.base_url}/images/generations", json=data, headers=self._headers()) + r.raise_for_status() + url = r.json()["data"][0]["url"] + img_r = await client.get(url) + img_r.raise_for_status() + return img_r.content + + async def img2img(self, image_bytes: bytes, prompt: str, strength: float, params: dict) -> bytes: + # OpenAI doesn't have img2img natively — use edits with blank mask + from PIL import Image + import numpy as np + img = Image.open(BytesIO(image_bytes)).convert("RGBA") + mask = Image.new("RGBA", img.size, (0, 0, 0, 0)) + mask_buf = BytesIO() + mask.save(mask_buf, format="PNG") + return await self.inpaint(image_bytes, mask_buf.getvalue(), prompt, params) + + async def outpaint(self, image_bytes: bytes, direction: str, size: int, prompt: str) -> bytes: + from PIL import Image + img = Image.open(BytesIO(image_bytes)).convert("RGBA") + w, h = img.size + directions = {"left": (size, 0), "right": (size, 0), "top": (0, size), "bottom": (0, size)} + dw, dh = directions.get(direction, (size, 0)) + new_w, new_h = w + dw, h + dh + canvas = Image.new("RGBA", (new_w, new_h), (0, 0, 0, 0)) + offsets = { + "left": (size, 0), "right": (0, 0), "top": (0, size), "bottom": (0, 0) + } + ox, oy = offsets.get(direction, (0, 0)) + canvas.paste(img, (ox, oy)) + # mask: transparent = inpaint + mask = Image.new("L", (new_w, new_h), 0) + # fill the expanded region with white in mask + import numpy as np + mask_arr = np.zeros((new_h, new_w), dtype=np.uint8) + if direction == "left": + mask_arr[:, :size] = 255 + elif direction == "right": + mask_arr[:, w:] = 255 + elif direction == "top": + mask_arr[:size, :] = 255 + else: + mask_arr[h:, :] = 255 + mask = Image.fromarray(mask_arr, "L") + + canvas_rgb = canvas.convert("RGB") + img_buf = BytesIO() + canvas_rgb.save(img_buf, format="PNG") + mask_buf = BytesIO() + mask.save(mask_buf, format="PNG") + return await self.inpaint(img_buf.getvalue(), mask_buf.getvalue(), prompt, {}) + + async def health(self) -> bool: + try: + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url}/models", headers=self._headers()) + return r.status_code == 200 + except Exception: + return False + + def capabilities(self) -> list[str]: + return ["inpaint", "txt2img", "img2img", "outpaint"] + + +class InvokeAIProvider(RemoteAIProvider): + """InvokeAI REST API driver — supports Flux, SDXL, SD1.5 and more.""" + + def __init__(self, base_url: str, default_model: str = "flux-dev"): + self.base_url = base_url.rstrip("/") + self.default_model = default_model + + async def _b64(self, data: bytes) -> str: + return base64.b64encode(data).decode() + + async def _upload_image(self, client: httpx.AsyncClient, image_bytes: bytes, category: str = "general") -> str: + """Upload image to InvokeAI and return image_name.""" + files = {"file": ("image.png", image_bytes, "image/png")} + data = {"image_category": category, "is_intermediate": "false"} + r = await client.post(f"{self.base_url}/api/v1/images/upload", files=files, data=data) + r.raise_for_status() + return r.json()["image_name"] + + async def _run_graph(self, client: httpx.AsyncClient, graph: dict) -> bytes: + """Post a graph, poll for completion, return result image bytes.""" + r = await client.post(f"{self.base_url}/api/v1/queue/default/enqueue_batch", + json={"prepend": False, "batch": {"graph": graph, "runs": 1}}) + r.raise_for_status() + batch_id = r.json()["batch"]["batch_id"] + + # Poll queue status + for _ in range(180): + await asyncio.sleep(2) + sr = await client.get(f"{self.base_url}/api/v1/queue/default/status") + sr.raise_for_status() + status = sr.json() + if status.get("queue", {}).get("completed", 0) > 0: + break + if status.get("queue", {}).get("failed", 0) > 0: + raise RuntimeError("InvokeAI graph failed") + + # Fetch latest result image + lr = await client.get(f"{self.base_url}/api/v1/images/?categories=general&limit=1&is_intermediate=false") + lr.raise_for_status() + items = lr.json().get("items", []) + if not items: + raise RuntimeError("No output image from InvokeAI") + + img_name = items[0]["image_name"] + img_r = await client.get(f"{self.base_url}/api/v1/images/i/{img_name}/full") + img_r.raise_for_status() + return img_r.content + + async def inpaint(self, image_bytes: bytes, mask_bytes: bytes, prompt: str, params: dict) -> bytes: + async with httpx.AsyncClient(timeout=300.0) as client: + img_name = await self._upload_image(client, image_bytes) + mask_name = await self._upload_image(client, mask_bytes, "mask") + model = params.get("model", self.default_model) + graph = { + "id": "inpaint_graph", + "nodes": { + "img_node": {"id": "img_node", "type": "image", "image": {"image_name": img_name}}, + "mask_node": {"id": "mask_node", "type": "image", "image": {"image_name": mask_name}}, + "model_node": {"id": "model_node", "type": "main_model_loader", "model": {"model_name": model, "base": "any"}}, + "clip_skip": {"id": "clip_skip", "type": "clip_skip", "skipped_layers": 0}, + "positive": {"id": "positive", "type": "compel", "prompt": prompt}, + "negative": {"id": "negative", "type": "compel", "prompt": params.get("negative_prompt", "")}, + "denoise": { + "id": "denoise", "type": "denoise_latents", + "steps": params.get("steps", 30), + "cfg_scale": params.get("cfg_scale", 7.5), + "denoising_start": 0.0, "denoising_end": 1.0, + "scheduler": "euler", "is_intermediate": False + }, + "vae_loader": {"id": "vae_loader", "type": "vae_loader", "vae_model": {"model_name": model, "base": "any"}}, + "img_to_latents": {"id": "img_to_latents", "type": "i2l"}, + "latents_to_img": {"id": "latents_to_img", "type": "l2i"}, + }, + "edges": [ + {"source": {"node_id": "model_node", "field": "unet"}, "destination": {"node_id": "denoise", "field": "unet"}}, + {"source": {"node_id": "model_node", "field": "clip"}, "destination": {"node_id": "clip_skip", "field": "clip"}}, + {"source": {"node_id": "clip_skip", "field": "clip"}, "destination": {"node_id": "positive", "field": "clip"}}, + {"source": {"node_id": "clip_skip", "field": "clip"}, "destination": {"node_id": "negative", "field": "clip"}}, + {"source": {"node_id": "positive", "field": "conditioning"}, "destination": {"node_id": "denoise", "field": "positive_conditioning"}}, + {"source": {"node_id": "negative", "field": "conditioning"}, "destination": {"node_id": "denoise", "field": "negative_conditioning"}}, + {"source": {"node_id": "img_node", "field": "image"}, "destination": {"node_id": "img_to_latents", "field": "image"}}, + {"source": {"node_id": "vae_loader", "field": "vae"}, "destination": {"node_id": "img_to_latents", "field": "vae"}}, + {"source": {"node_id": "img_to_latents", "field": "latents"}, "destination": {"node_id": "denoise", "field": "latents"}}, + {"source": {"node_id": "mask_node", "field": "image"}, "destination": {"node_id": "denoise", "field": "mask"}}, + {"source": {"node_id": "denoise", "field": "latents"}, "destination": {"node_id": "latents_to_img", "field": "latents"}}, + {"source": {"node_id": "vae_loader", "field": "vae"}, "destination": {"node_id": "latents_to_img", "field": "vae"}}, + ] + } + return await self._run_graph(client, graph) + + async def txt2img(self, prompt: str, width: int, height: int, params: dict) -> bytes: + async with httpx.AsyncClient(timeout=300.0) as client: + model = params.get("model", self.default_model) + graph = { + "id": "txt2img_graph", + "nodes": { + "model_node": {"id": "model_node", "type": "main_model_loader", "model": {"model_name": model, "base": "any"}}, + "clip_skip": {"id": "clip_skip", "type": "clip_skip", "skipped_layers": 0}, + "positive": {"id": "positive", "type": "compel", "prompt": prompt}, + "negative": {"id": "negative", "type": "compel", "prompt": params.get("negative_prompt", "")}, + "noise": {"id": "noise", "type": "noise", "width": width, "height": height, "seed": params.get("seed", 0)}, + "denoise": { + "id": "denoise", "type": "denoise_latents", + "steps": params.get("steps", 30), + "cfg_scale": params.get("cfg_scale", 7.5), + "denoising_start": 0.0, "denoising_end": 1.0, + "scheduler": "euler", + }, + "vae_loader": {"id": "vae_loader", "type": "vae_loader", "vae_model": {"model_name": model, "base": "any"}}, + "latents_to_img": {"id": "latents_to_img", "type": "l2i"}, + }, + "edges": [ + {"source": {"node_id": "model_node", "field": "unet"}, "destination": {"node_id": "denoise", "field": "unet"}}, + {"source": {"node_id": "model_node", "field": "clip"}, "destination": {"node_id": "clip_skip", "field": "clip"}}, + {"source": {"node_id": "clip_skip", "field": "clip"}, "destination": {"node_id": "positive", "field": "clip"}}, + {"source": {"node_id": "clip_skip", "field": "clip"}, "destination": {"node_id": "negative", "field": "clip"}}, + {"source": {"node_id": "positive", "field": "conditioning"}, "destination": {"node_id": "denoise", "field": "positive_conditioning"}}, + {"source": {"node_id": "negative", "field": "conditioning"}, "destination": {"node_id": "denoise", "field": "negative_conditioning"}}, + {"source": {"node_id": "noise", "field": "noise"}, "destination": {"node_id": "denoise", "field": "noise"}}, + {"source": {"node_id": "denoise", "field": "latents"}, "destination": {"node_id": "latents_to_img", "field": "latents"}}, + {"source": {"node_id": "vae_loader", "field": "vae"}, "destination": {"node_id": "latents_to_img", "field": "vae"}}, + ] + } + return await self._run_graph(client, graph) + + async def img2img(self, image_bytes: bytes, prompt: str, strength: float, params: dict) -> bytes: + # Reuse inpaint with a full-white mask at the given strength + from PIL import Image + img = Image.open(BytesIO(image_bytes)) + mask = Image.new("L", img.size, 255) + mask_buf = BytesIO() + mask.save(mask_buf, format="PNG") + p = dict(params) + p.setdefault("denoising_start", 1.0 - strength) + return await self.inpaint(image_bytes, mask_buf.getvalue(), prompt, p) + + async def outpaint(self, image_bytes: bytes, direction: str, size: int, prompt: str) -> bytes: + # Delegate to inpaint with expanded canvas + provider = OpenAIRemoteProvider.__new__(OpenAIRemoteProvider) + return await provider.outpaint(image_bytes, direction, size, prompt) + + async def health(self) -> bool: + try: + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url}/api/v1/app/version") + return r.status_code == 200 + except Exception: + return False + + def capabilities(self) -> list[str]: + return ["inpaint", "txt2img", "img2img", "outpaint"] + + +class ComfyUIProvider(RemoteAIProvider): + """ComfyUI workflow JSON API driver.""" + + def __init__(self, base_url: str, default_model: str = "v1-5-pruned-emaonly.ckpt"): + self.base_url = base_url.rstrip("/") + self.default_model = default_model + + async def _upload_image(self, client: httpx.AsyncClient, image_bytes: bytes, name: str = "image.png") -> str: + files = {"image": (name, image_bytes, "image/png")} + data = {"overwrite": "true"} + r = await client.post(f"{self.base_url}/upload/image", files=files, data=data) + r.raise_for_status() + j = r.json() + return j.get("name", name) + + async def _queue_prompt(self, client: httpx.AsyncClient, workflow: dict) -> str: + r = await client.post(f"{self.base_url}/prompt", json={"prompt": workflow}) + r.raise_for_status() + return r.json()["prompt_id"] + + async def _wait_for_result(self, client: httpx.AsyncClient, prompt_id: str) -> bytes: + for _ in range(180): + await asyncio.sleep(2) + r = await client.get(f"{self.base_url}/history/{prompt_id}") + r.raise_for_status() + history = r.json() + if prompt_id in history: + outputs = history[prompt_id].get("outputs", {}) + for node_output in outputs.values(): + for img_info in node_output.get("images", []): + img_r = await client.get( + f"{self.base_url}/view", + params={"filename": img_info["filename"], "subfolder": img_info.get("subfolder", ""), + "type": img_info.get("type", "output")} + ) + img_r.raise_for_status() + return img_r.content + raise RuntimeError("ComfyUI timed out waiting for result") + + async def inpaint(self, image_bytes: bytes, mask_bytes: bytes, prompt: str, params: dict) -> bytes: + model = params.get("model", self.default_model) + async with httpx.AsyncClient(timeout=300.0) as client: + img_name = await self._upload_image(client, image_bytes, "input.png") + mask_name = await self._upload_image(client, mask_bytes, "mask.png") + workflow = { + "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": model}}, + "2": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["1", 1]}}, + "3": {"class_type": "CLIPTextEncode", "inputs": {"text": params.get("negative_prompt", ""), "clip": ["1", 1]}}, + "4": {"class_type": "LoadImage", "inputs": {"image": img_name}}, + "5": {"class_type": "LoadImage", "inputs": {"image": mask_name}}, + "6": {"class_type": "VAEEncode", "inputs": {"pixels": ["4", 0], "vae": ["1", 2]}}, + "7": {"class_type": "KSampler", "inputs": { + "model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], + "latent_image": ["6", 0], "mask": ["5", 0], + "seed": params.get("seed", 42), "steps": params.get("steps", 20), + "cfg": params.get("cfg_scale", 7.0), "sampler_name": "euler", + "scheduler": "normal", "denoise": params.get("denoise", 1.0) + }}, + "8": {"class_type": "VAEDecode", "inputs": {"samples": ["7", 0], "vae": ["1", 2]}}, + "9": {"class_type": "SaveImage", "inputs": {"images": ["8", 0], "filename_prefix": "api_out"}}, + } + pid = await self._queue_prompt(client, workflow) + return await self._wait_for_result(client, pid) + + async def txt2img(self, prompt: str, width: int, height: int, params: dict) -> bytes: + model = params.get("model", self.default_model) + async with httpx.AsyncClient(timeout=300.0) as client: + workflow = { + "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": model}}, + "2": {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": ["1", 1]}}, + "3": {"class_type": "CLIPTextEncode", "inputs": {"text": params.get("negative_prompt", ""), "clip": ["1", 1]}}, + "4": {"class_type": "EmptyLatentImage", "inputs": {"width": width, "height": height, "batch_size": 1}}, + "5": {"class_type": "KSampler", "inputs": { + "model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], + "latent_image": ["4", 0], + "seed": params.get("seed", 42), "steps": params.get("steps", 20), + "cfg": params.get("cfg_scale", 7.0), "sampler_name": "euler", + "scheduler": "normal", "denoise": 1.0 + }}, + "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, + "7": {"class_type": "SaveImage", "inputs": {"images": ["6", 0], "filename_prefix": "api_out"}}, + } + pid = await self._queue_prompt(client, workflow) + return await self._wait_for_result(client, pid) + + async def img2img(self, image_bytes: bytes, prompt: str, strength: float, params: dict) -> bytes: + from PIL import Image + img = Image.open(BytesIO(image_bytes)) + mask = Image.new("L", img.size, 255) + mask_buf = BytesIO() + mask.save(mask_buf, format="PNG") + p = dict(params) + p["denoise"] = strength + return await self.inpaint(image_bytes, mask_buf.getvalue(), prompt, p) + + async def outpaint(self, image_bytes: bytes, direction: str, size: int, prompt: str) -> bytes: + # Build expanded canvas then inpaint with blank mask + from PIL import Image + import numpy as np + img = Image.open(BytesIO(image_bytes)).convert("RGB") + w, h = img.size + dw = size if direction in ("left", "right") else 0 + dh = size if direction in ("top", "bottom") else 0 + canvas = Image.new("RGB", (w + dw, h + dh), (128, 128, 128)) + ox = size if direction == "left" else 0 + oy = size if direction == "top" else 0 + canvas.paste(img, (ox, oy)) + mask_arr = np.zeros((h + dh, w + dw), dtype=np.uint8) + if direction == "left": + mask_arr[:, :size] = 255 + elif direction == "right": + mask_arr[:, w:] = 255 + elif direction == "top": + mask_arr[:size, :] = 255 + else: + mask_arr[h:, :] = 255 + img_buf = BytesIO() + canvas.save(img_buf, format="PNG") + mask_buf = BytesIO() + Image.fromarray(mask_arr, "L").save(mask_buf, format="PNG") + return await self.inpaint(img_buf.getvalue(), mask_buf.getvalue(), prompt, {}) + + async def health(self) -> bool: + try: + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(f"{self.base_url}/system_stats") + return r.status_code == 200 + except Exception: + return False + + def capabilities(self) -> list[str]: + return ["inpaint", "txt2img", "img2img", "outpaint"] + + +def _build_provider(name: str) -> Optional[RemoteAIProvider]: + """Instantiate a named provider from current settings.""" + from app.config import settings + + name = (name or "").lower().strip() + + if name == "openai": + if not settings.openai_api_key: + return None + return OpenAIRemoteProvider(settings.openai_api_key, settings.openai_model) + + if name == "invokeai": + if not settings.invokeai_url: + return None + return InvokeAIProvider(settings.invokeai_url, settings.invokeai_default_model) + + if name == "comfyui": + if not settings.comfyui_url: + return None + return ComfyUIProvider(settings.comfyui_url, settings.comfyui_default_model) + + return None + + +# Map operation names to the settings field that holds the override +_OP_FIELD = { + "inpaint": "ai_provider_inpaint", + "txt2img": "ai_provider_txt2img", + "img2img": "ai_provider_img2img", + "outpaint": "ai_provider_outpaint", +} + + +def get_remote_provider(operation: Optional[str] = None) -> Optional[RemoteAIProvider]: + """ + Return the provider for a given operation. + + Resolution order: + 1. Per-operation override (AI_PROVIDER_INPAINT, AI_PROVIDER_TXT2IMG, etc.) + 2. Global default (AI_PROVIDER) + 3. None (local-only mode) + + Example .env for mixed setup: + AI_PROVIDER=invokeai # default for inpaint/img2img/outpaint + AI_PROVIDER_TXT2IMG=openai # use OpenAI only for text-to-image + """ + from app.config import settings + + if operation and operation in _OP_FIELD: + override = getattr(settings, _OP_FIELD[operation], "") + if override: + provider = _build_provider(override) + if provider is not None: + return provider + # override configured but not usable (missing key/url) — fall through to default + + return _build_provider(settings.ai_provider) diff --git a/backend/app/services/upscale.py b/backend/app/services/upscale.py new file mode 100644 index 0000000..92d43b4 --- /dev/null +++ b/backend/app/services/upscale.py @@ -0,0 +1,421 @@ +""" +Upscale service — auto-detects best available method and runs it. +Auto-installs Real-ESRGAN NCNN Vulkan binary on first use if no AI upscaler found. + +Priority (auto mode): + 1. Real-ESRGAN PyTorch + CUDA GPU — fastest, best quality + 2. Real-ESRGAN PyTorch + Apple MPS — fast on Apple Silicon + 3. Real-ESRGAN NCNN Vulkan binary — fast on any GPU (Intel/AMD/integrated) + 4. Real-ESRGAN PyTorch CPU — works, slow (warn user) + 5. Lanczos — always available, instant + +Capability probe is run once at first call and cached. +NCNN binary is auto-downloaded if no AI upscaler is found. +""" + +import asyncio +import os +import platform +import shutil +import stat +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from dataclasses import dataclass, field +from enum import Enum +from io import BytesIO +from pathlib import Path +from typing import Optional + +from PIL import Image + +# ── NCNN auto-install ───────────────────────────────────────────────────────── + +NCNN_DEST_DIR = Path("/app/data/models/realesrgan") +NCNN_VERSION = "v0.2.5.0" +NCNN_BASE_URL = f"https://github.com/xinntao/Real-ESRGAN/releases/download/{NCNN_VERSION}" + +_PLATFORM_ZIP = { + "linux": f"realesrgan-ncnn-vulkan-{NCNN_VERSION}-ubuntu.zip", + "darwin": f"realesrgan-ncnn-vulkan-{NCNN_VERSION}-macos.zip", + "win32": f"realesrgan-ncnn-vulkan-{NCNN_VERSION}-windows.zip", + "windows": f"realesrgan-ncnn-vulkan-{NCNN_VERSION}-windows.zip", +} + + +class InstallState(str, Enum): + idle = "idle" + downloading = "downloading" + extracting = "extracting" + done = "done" + failed = "failed" + + +@dataclass +class InstallStatus: + state: InstallState = InstallState.idle + progress: int = 0 # 0-100 + message: str = "" + error: str = "" + + +_install_status = InstallStatus() +_install_lock = asyncio.Lock() + + +def get_install_status() -> dict: + s = _install_status + return { + "state": s.state.value, + "progress": s.progress, + "message": s.message, + "error": s.error, + } + + +def _ncnn_binary_name() -> str: + plat = sys.platform.lower() + return "realesrgan-ncnn-vulkan.exe" if "win" in plat else "realesrgan-ncnn-vulkan" + + +async def ensure_ncnn_installed() -> Optional[Path]: + """ + Check if NCNN binary is present; if not, download and install it. + Returns the binary Path on success, None on failure. + Serialised via _install_lock so concurrent callers wait for a single install. + """ + global _install_status + + binary_path = NCNN_DEST_DIR / _ncnn_binary_name() + if binary_path.exists() and os.access(binary_path, os.X_OK): + _install_status = InstallStatus(state=InstallState.done, progress=100, + message="Already installed.") + return binary_path + + async with _install_lock: + # Re-check after acquiring lock (another coroutine may have just finished) + if binary_path.exists() and os.access(binary_path, os.X_OK): + _install_status = InstallStatus(state=InstallState.done, progress=100, + message="Already installed.") + return binary_path + + if _install_status.state == InstallState.downloading: + return None # install already in progress + + plat = sys.platform.lower() + zip_name = _PLATFORM_ZIP.get(plat) + if not zip_name: + _install_status = InstallStatus( + state=InstallState.failed, + error=f"Unsupported platform: {plat}", + ) + return None + + url = f"{NCNN_BASE_URL}/{zip_name}" + + try: + NCNN_DEST_DIR.mkdir(parents=True, exist_ok=True) + zip_path = NCNN_DEST_DIR / zip_name + + # Download + _install_status = InstallStatus( + state=InstallState.downloading, progress=0, + message=f"Downloading Real-ESRGAN NCNN {NCNN_VERSION}…", + ) + + def _do_download(): + def _progress(count, block, total): + if total > 0: + pct = min(90, int(count * block * 90 / total)) + _install_status.progress = pct + urllib.request.urlretrieve(url, zip_path, _progress) + + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _do_download) + + # Extract + _install_status.state = InstallState.extracting + _install_status.progress = 92 + _install_status.message = "Extracting…" + + def _do_extract(): + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(NCNN_DEST_DIR) + # Find binary (may be in a subdirectory) + found = list(NCNN_DEST_DIR.rglob(_ncnn_binary_name())) + if not found: + raise FileNotFoundError(f"Binary not found after extract: {_ncnn_binary_name()}") + extracted = found[0] + if extracted != binary_path: + extracted.rename(binary_path) + # Make executable + if "win" not in sys.platform.lower(): + binary_path.chmod( + binary_path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH + ) + zip_path.unlink(missing_ok=True) + + await loop.run_in_executor(None, _do_extract) + + _install_status = InstallStatus( + state=InstallState.done, progress=100, + message=f"Installed: {binary_path}", + ) + # Bust caps cache so probe picks up new binary + invalidate_caps_cache() + return binary_path + + except Exception as exc: + _install_status = InstallStatus( + state=InstallState.failed, + error=str(exc), + message="Installation failed.", + ) + print(f"[upscale] NCNN auto-install failed: {exc}") + return None + + +# ── Capability detection ────────────────────────────────────────────────────── + +_caps: Optional[dict] = None + + +def probe_upscale_capabilities() -> dict: + """ + Detect what upscaling hardware and software is available. + Result is cached after first call. + """ + global _caps + if _caps is not None: + return _caps + + caps = { + "lanczos": True, + "realesrgan_pytorch": False, + "realesrgan_pytorch_device": None, + "realesrgan_ncnn": False, + "realesrgan_ncnn_path": None, + "recommended": "lanczos", + "recommended_label": "Lanczos (no AI upscaler found)", + "methods": ["lanczos"], + "ncnn_install_status": get_install_status(), + } + + # ── PyTorch path ────────────────────────────────────────────────────────── + pytorch_device = None + try: + import torch + if torch.cuda.is_available(): + pytorch_device = "cuda" + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + pytorch_device = "mps" + else: + pytorch_device = "cpu" + except ImportError: + pass + + if pytorch_device: + try: + import realesrgan # noqa: F401 + from basicsr.archs.rrdbnet_arch import RRDBNet # noqa: F401 + caps["realesrgan_pytorch"] = True + caps["realesrgan_pytorch_device"] = pytorch_device + caps["methods"].append("realesrgan_pytorch") + except ImportError: + pass + + # ── NCNN Vulkan binary ──────────────────────────────────────────────────── + ncnn_path = _find_ncnn_binary() + if ncnn_path: + caps["realesrgan_ncnn"] = True + caps["realesrgan_ncnn_path"] = str(ncnn_path) + caps["methods"].append("realesrgan_ncnn") + + # ── Pick recommended ────────────────────────────────────────────────────── + if caps["realesrgan_pytorch"] and pytorch_device in ("cuda", "mps"): + device_label = "CUDA GPU" if pytorch_device == "cuda" else "Apple Silicon" + caps["recommended"] = "realesrgan_pytorch" + caps["recommended_label"] = f"Real-ESRGAN ({device_label})" + elif caps["realesrgan_ncnn"]: + caps["recommended"] = "realesrgan_ncnn" + caps["recommended_label"] = "Real-ESRGAN NCNN (Vulkan)" + elif caps["realesrgan_pytorch"] and pytorch_device == "cpu": + caps["recommended"] = "realesrgan_pytorch" + caps["recommended_label"] = "Real-ESRGAN (CPU — may be slow)" + else: + caps["recommended"] = "lanczos" + caps["recommended_label"] = "Lanczos (installing Real-ESRGAN…)" + + _caps = caps + return caps + + +def _find_ncnn_binary() -> Optional[Path]: + """Find realesrgan-ncnn-vulkan binary on the system.""" + found = shutil.which("realesrgan-ncnn-vulkan") + if found: + return Path(found) + + candidates = [ + NCNN_DEST_DIR / _ncnn_binary_name(), + Path("/usr/local/bin/realesrgan-ncnn-vulkan"), + Path.home() / ".local/bin/realesrgan-ncnn-vulkan", + Path(r"C:/realesrgan-ncnn-vulkan/realesrgan-ncnn-vulkan.exe"), + Path("/opt/homebrew/bin/realesrgan-ncnn-vulkan"), + ] + for p in candidates: + if p.exists() and os.access(p, os.X_OK): + return p + return None + + +def invalidate_caps_cache(): + """Call after installing new software so next probe picks it up.""" + global _caps + _caps = None + + +# ── Upscale implementations ─────────────────────────────────────────────────── + +def _to_png_bytes(img: Image.Image) -> bytes: + buf = BytesIO() + img.save(buf, format="PNG") + return buf.getvalue() + + +def upscale_lanczos(image: Image.Image, scale: float) -> tuple[bytes, str]: + """Pure Pillow Lanczos — instant, always available.""" + new_w = round(image.width * scale) + new_h = round(image.height * scale) + result = image.resize((new_w, new_h), Image.Resampling.LANCZOS) + return _to_png_bytes(result), "lanczos" + + +def upscale_realesrgan_pytorch(image: Image.Image, scale: float) -> tuple[bytes, str]: + """ + Real-ESRGAN via PyTorch. + Uses CUDA > MPS > CPU automatically based on what's available. + """ + import torch + from basicsr.archs.rrdbnet_arch import RRDBNet + from realesrgan import RealESRGANer + + caps = probe_upscale_capabilities() + device = caps.get("realesrgan_pytorch_device", "cpu") + + model_scale = 2 if scale <= 2.5 else 4 + model = RRDBNet( + num_in_ch=3, num_out_ch=3, num_feat=64, + num_block=23, num_grow_ch=32, scale=model_scale + ) + + model_dir = Path("/app/data/models/realesrgan") + model_dir.mkdir(parents=True, exist_ok=True) + model_name = f"RealESRGAN_x{model_scale}plus.pth" + model_path = model_dir / model_name + if not model_path.exists(): + model_path = None + + upsampler = RealESRGANer( + scale=model_scale, + model_path=str(model_path) if model_path else None, + model=model, + tile=512, + tile_pad=10, + pre_pad=0, + half=(device == "cuda"), + device=torch.device(device), + ) + + import numpy as np + img_bgr = np.array(image)[:, :, ::-1].copy() + enhanced, _ = upsampler.enhance(img_bgr, outscale=scale) + result = Image.fromarray(enhanced[:, :, ::-1]) + + label = f"realesrgan_pytorch_{device}" + return _to_png_bytes(result), label + + +def upscale_realesrgan_ncnn(image: Image.Image, scale: float) -> tuple[bytes, str]: + """ + Real-ESRGAN via NCNN Vulkan binary — works on any GPU. + Runs as subprocess with temp file I/O. + """ + caps = probe_upscale_capabilities() + binary = caps.get("realesrgan_ncnn_path") + if not binary: + raise RuntimeError("realesrgan-ncnn-vulkan binary not found") + + model_scale = 4 if scale > 2.5 else 2 + target_w = round(image.width * scale) + target_h = round(image.height * scale) + + with tempfile.TemporaryDirectory() as tmpdir: + in_path = Path(tmpdir) / "input.png" + out_path = Path(tmpdir) / "output.png" + + image.save(in_path, format="PNG") + + model_name = f"realesrgan-x{model_scale}plus" + cmd = [ + binary, "-i", str(in_path), "-o", str(out_path), + "-s", str(model_scale), "-n", model_name, "-f", "png", + ] + + result_proc = subprocess.run(cmd, capture_output=True, timeout=300) + if result_proc.returncode != 0: + raise RuntimeError( + f"realesrgan-ncnn-vulkan failed: {result_proc.stderr.decode()}" + ) + + result = Image.open(out_path).convert("RGB") + if result.width != target_w or result.height != target_h: + result = result.resize((target_w, target_h), Image.Resampling.LANCZOS) + + return _to_png_bytes(result), "realesrgan_ncnn" + + +# ── Public entry point ──────────────────────────────────────────────────────── + +def upscale_sync(image: Image.Image, scale: float, method: str = "auto") -> tuple[bytes, str]: + """Upscale image synchronously. Returns (png_bytes, method_used_label).""" + caps = probe_upscale_capabilities() + + if method == "auto": + method = caps["recommended"] + + if method == "realesrgan_pytorch": + if caps["realesrgan_pytorch"]: + try: + return upscale_realesrgan_pytorch(image, scale) + except Exception as e: + print(f"Real-ESRGAN PyTorch failed, falling back: {e}") + if caps["realesrgan_ncnn"]: + try: + return upscale_realesrgan_ncnn(image, scale) + except Exception as e: + print(f"Real-ESRGAN NCNN fallback failed: {e}") + return upscale_lanczos(image, scale) + + if method == "realesrgan_ncnn": + if caps["realesrgan_ncnn"]: + try: + return upscale_realesrgan_ncnn(image, scale) + except Exception as e: + print(f"Real-ESRGAN NCNN failed, falling back: {e}") + if caps["realesrgan_pytorch"]: + try: + return upscale_realesrgan_pytorch(image, scale) + except Exception as e: + print(f"Real-ESRGAN PyTorch fallback failed: {e}") + return upscale_lanczos(image, scale) + + return upscale_lanczos(image, scale) + + +async def upscale_image(image: Image.Image, scale: float, method: str = "auto") -> tuple[bytes, str]: + """Async wrapper — runs upscale in thread pool to avoid blocking the event loop.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, upscale_sync, image, scale, method) diff --git a/backend/requirements.txt b/backend/requirements.txt index 9649ea5..28e4923 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,19 +12,14 @@ httpx==0.26.0 pydantic==2.5.3 pydantic-settings==2.1.0 email-validator==2.1.0 -opencv-python-headless==4.9.0.80 +opencv-python-headless>=4.10.0 # SAM (Segment Anything) for smart object selection - runs locally, no API needed torch==2.1.2 torchvision==0.16.2 segment-anything @ git+https://github.com/facebookresearch/segment-anything.git -# Note: Using OpenCV DNN instead of onnxruntime for U2Net -# (onnxruntime has executable stack issues in some Docker environments) +# Local AI inpainting — LaMa model (auto GPU/CPU, no API key needed) +simple-lama-inpainting -# NOTE: rembg (background removal) disabled due to dependency conflicts -# rembg>=2.0.70 requires: -# - scikit-image>=0.26.0 which requires numpy>=2.0 -# - Pillow>=12.1.0 -# But opencv-python-headless 4.9.0.80 requires numpy<2.0 -# To enable rembg, need to update opencv-python-headless to 4.10+ (numpy 2.x compatible) -# and update all dependent packages accordingly +# Background removal — rembg enabled now that opencv 4.10+ supports numpy 2.x +rembg[gpu] diff --git a/frontend/src/js/api/capabilities.js b/frontend/src/js/api/capabilities.js new file mode 100644 index 0000000..b49b423 --- /dev/null +++ b/frontend/src/js/api/capabilities.js @@ -0,0 +1,65 @@ +/** + * Backend capabilities singleton. + * Fetched once on load from GET /api/config. + * Tools use this to decide whether to show, grey out, or show tooltips. + * + * Shape: + * { + * local: { lama, rembg, opencv, gpu_detected }, + * remote: { provider, capabilities: string[], healthy } + * } + */ + +import apiService from '../services/api.js'; + +const DEFAULT_CAPS = { + local: { lama: false, rembg: false, opencv: true, gpu_detected: false }, + remote: { provider: null, capabilities: [], healthy: false }, +}; + +let _caps = null; +let _fetchPromise = null; + +/** + * Return capabilities (fetched lazily, cached thereafter). + * Always resolves — falls back to DEFAULT_CAPS on network error. + */ +export async function getCapabilities() { + if (_caps) return _caps; + if (!_fetchPromise) { + _fetchPromise = apiService.getConfig() + .then(data => { _caps = data || DEFAULT_CAPS; return _caps; }) + .catch(() => { _caps = DEFAULT_CAPS; return _caps; }); + } + return _fetchPromise; +} + +/** + * Synchronous check — returns cached value or DEFAULT_CAPS if not yet loaded. + */ +export function getCachedCapabilities() { + return _caps || DEFAULT_CAPS; +} + +/** + * True if the remote provider is configured and healthy. + */ +export function hasRemote() { + return !!(_caps?.remote?.healthy); +} + +/** + * Invalidate cache and re-fetch (call after saving provider settings). + */ +export async function refreshCapabilities() { + _caps = null; + _fetchPromise = null; + return getCapabilities(); +} + +/** + * Kick off the fetch immediately at module load time so it's ready when tools need it. + */ +getCapabilities(); + +export default { getCapabilities, getCachedCapabilities, hasRemote, refreshCapabilities }; diff --git a/frontend/src/js/config-menu.js b/frontend/src/js/config-menu.js index e99b763..b19e667 100644 --- a/frontend/src/js/config-menu.js +++ b/frontend/src/js/config-menu.js @@ -330,6 +330,16 @@ const menuDefinition = [ ellipsis: true, target: 'image/remove_background.remove_background' }, + { + name: 'Fit to Frame...', + ellipsis: true, + target: 'image/frame_fit.frame_fit' + }, + { + name: 'Upscale...', + ellipsis: true, + target: 'image/upscale.upscale' + }, { divider: true }, @@ -816,9 +826,32 @@ const menuDefinition = [ name: 'Settings', ellipsis: true, target: 'tools/settings.settings' + }, + { + divider: true + }, + { + name: 'AI Provider Settings', + ellipsis: true, + target: 'tools/ai_provider_settings.ai_provider_settings' } ] }, + { + name: 'Generate', + children: [ + { + name: 'Text → Image', + ellipsis: true, + target: 'generate/text_to_image.text_to_image' + }, + { + name: 'Expand Canvas (Outpaint)', + ellipsis: true, + target: 'generate/outpaint.outpaint' + }, + ] + }, { name: 'Help', children: [ diff --git a/frontend/src/js/config.js b/frontend/src/js/config.js index 077c1f7..466112c 100644 --- a/frontend/src/js/config.js +++ b/frontend/src/js/config.js @@ -110,6 +110,35 @@ config.TOOLS = [ on_activate: 'on_activate', attributes: {}, }, + { + name: 'ai_lama_erase', + title: 'AI Magic Erase (LaMa) - Paint over to erase', + attributes: { + size: { + value: 30, + min: 5, + max: 200, + }, + }, + }, + { + name: 'ai_smart_inpaint', + title: 'AI Smart Inpaint - Paint + describe replacement', + on_activate: 'on_activate', + attributes: { + size: { + value: 30, + min: 5, + max: 200, + }, + }, + }, + { + name: 'ai_replace_selection', + title: 'AI Replace Selection - Use any selection tool first', + on_activate: 'on_activate', + attributes: {}, + }, { name: 'magic_wand', title: 'Magic Wand (Color Select)', diff --git a/frontend/src/js/core/components/provider-badge.js b/frontend/src/js/core/components/provider-badge.js new file mode 100644 index 0000000..8724733 --- /dev/null +++ b/frontend/src/js/core/components/provider-badge.js @@ -0,0 +1,72 @@ +/** + * ProviderBadge — small DOM element showing the active AI provider. + * Inserted into the toolbar footer on app load. + * + * Green = remote provider healthy + * Yellow = provider configured but unhealthy/unreachable + * Grey = local only (LaMa + OpenCV) + */ + +import { getCapabilities } from '../../api/capabilities.js'; + +export async function mountProviderBadge(container) { + var caps = await getCapabilities(); + + var badge = document.createElement('div'); + badge.id = 'provider-badge'; + badge.style.cssText = [ + 'display:inline-flex', 'align-items:center', 'gap:5px', + 'padding:3px 8px', 'border-radius:10px', + 'font-size:11px', 'font-family:sans-serif', + 'cursor:default', 'user-select:none', + 'margin:4px', 'opacity:0.85', + ].join(';'); + + var dot = document.createElement('span'); + dot.style.cssText = 'width:7px;height:7px;border-radius:50%;display:inline-block;'; + + var label = document.createElement('span'); + + var remote = caps.remote || {}; + var local = caps.local || {}; + + if (remote.provider && remote.healthy) { + dot.style.background = '#44cc44'; + badge.style.background = '#1a2a1a'; + badge.style.color = '#aaffaa'; + + // Show override summary if any operations use different providers + var overrides = remote.overrides || {}; + var overrideEntries = Object.entries(overrides).filter(([, v]) => v); + var overrideStr = overrideEntries.length + ? ' · ' + overrideEntries.map(([k, v]) => `${k}→${v}`).join(', ') + : ''; + label.textContent = remote.provider + overrideStr + (local.gpu_detected ? ' · GPU' : ''); + + var opLines = Object.entries(remote.operations || {}) + .map(([op, s]) => `${op}: ${s.provider || remote.provider} ${s.healthy ? '✓' : '✗'}`) + .join('\n'); + badge.title = opLines || ('Provider: ' + remote.provider); + } else if (remote.provider && !remote.healthy) { + dot.style.background = '#ffaa00'; + badge.style.background = '#2a2000'; + badge.style.color = '#ffdd88'; + label.textContent = remote.provider + ' (offline)'; + badge.title = remote.provider + ' is configured but not reachable. Check your .env URL.'; + } else { + dot.style.background = '#888888'; + badge.style.background = '#1a1a1a'; + badge.style.color = '#aaaaaa'; + label.textContent = 'Local' + (local.lama ? ' · LaMa' : '') + (local.gpu_detected ? ' · GPU' : ''); + badge.title = 'Local only. Set AI_PROVIDER in .env to enable generative tools.'; + } + + badge.appendChild(dot); + badge.appendChild(label); + + if (container) { + container.appendChild(badge); + } + + return badge; +} diff --git a/frontend/src/js/main.js b/frontend/src/js/main.js index 9d3eac0..ded0000 100644 --- a/frontend/src/js/main.js +++ b/frontend/src/js/main.js @@ -23,6 +23,7 @@ import Base_search_class from './core/base-search.js'; import File_open_class from './modules/file/open.js'; import File_save_class from './modules/file/save.js'; import * as Actions from './actions/index.js'; +import { mountProviderBadge } from './core/components/provider-badge.js'; window.addEventListener('load', function (e) { // Initiate app @@ -54,4 +55,7 @@ window.addEventListener('load', function (e) { // Render all GUI.init(); Layers.init(); + + // Mount provider badge in the tools panel footer + mountProviderBadge(document.getElementById('tools_container') || document.body); }, false); diff --git a/frontend/src/js/modules/generate/outpaint.js b/frontend/src/js/modules/generate/outpaint.js new file mode 100644 index 0000000..cd7165a --- /dev/null +++ b/frontend/src/js/modules/generate/outpaint.js @@ -0,0 +1,142 @@ +/** + * Outpaint / Expand Canvas — remote provider fills the new region. + * Menu target: generate/outpaint.outpaint + */ + +import app from './../../app.js'; +import config from './../../config.js'; +import Base_layers_class from './../../core/base-layers.js'; +import Dialog_class from './../../libs/popup.js'; +import alertify from './../../../../node_modules/alertifyjs/build/alertify.min.js'; +import apiService from './../../services/api.js'; +import { getCapabilities } from './../../api/capabilities.js'; + +var instance = null; + +class Generate_outpaint_class { + + constructor() { + if (instance) return instance; + instance = this; + this.Base_layers = new Base_layers_class(); + this.Dialog = new Dialog_class(); + this.isProcessing = false; + } + + async outpaint() { + var caps = await getCapabilities(); + if (!caps.remote || !caps.remote.healthy) { + alertify.error( + 'Expand Canvas requires a remote AI provider. ' + + 'Set AI_PROVIDER (openai / invokeai / comfyui) in .env and restart.' + ); + return; + } + + var _this = this; + + this.Dialog.show({ + title: 'Expand Canvas (Outpaint)', + params: [ + { + name: 'direction', + title: 'Expand direction:', + value: 'right', + values: ['right', 'left', 'bottom', 'top'], + }, + { + name: 'size', + title: 'Pixels to add:', + type: 'range', + value: 256, + range: [64, 1024], + step: 64, + }, + { + name: 'prompt', + title: 'Describe the expansion (optional):', + value: '', + placeholder: "e.g. 'continue the landscape', 'more sky and clouds'", + }, + ], + on_finish: async function (params) { + await _this._run(params); + }, + }); + } + + async _run(params) { + if (this.isProcessing) return; + if (config.layer.type !== 'image') { + alertify.error('Current layer must be an image.'); + return; + } + + this.isProcessing = true; + alertify.message('Expanding canvas... please wait', 0); + + try { + var layerCanvas = document.createElement('canvas'); + layerCanvas.width = config.layer.width_original; + layerCanvas.height = config.layer.height_original; + layerCanvas.getContext('2d').drawImage(config.layer.link, 0, 0); + var imageB64 = layerCanvas.toDataURL('image/png').split(',')[1]; + + var response = await fetch( + (window.API_BASE_URL || '') + '/api/generate/outpaint', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + image: imageB64, + direction: params.direction, + size: params.size || 256, + prompt: params.prompt || '', + }), + } + ); + if (!response.ok) { + var err = await response.json().catch(() => ({ detail: 'Unknown error' })); + throw new Error(err.detail || 'Outpaint failed'); + } + var result = await response.json(); + + var img = new Image(); + img.onload = () => { + var newW = img.naturalWidth; + var newH = img.naturalHeight; + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = newW; + resultCanvas.height = newH; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + + // Update canvas dimensions and replace layer + config.WIDTH = newW; + config.HEIGHT = newH; + app.State.do_action( + new app.Actions.Bundle_action('outpaint', 'Expand Canvas', [ + new app.Actions.Resize_canvas_action(newW, newH), + new app.Actions.Update_layer_image_action(resultCanvas), + ]) + ); + + alertify.dismissAll(); + alertify.success('Canvas expanded!'); + this.isProcessing = false; + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load expanded image.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('Outpaint failed: ' + (err.message || err)); + this.isProcessing = false; + } + } +} + +export default Generate_outpaint_class; diff --git a/frontend/src/js/modules/generate/text_to_image.js b/frontend/src/js/modules/generate/text_to_image.js new file mode 100644 index 0000000..d23ddca --- /dev/null +++ b/frontend/src/js/modules/generate/text_to_image.js @@ -0,0 +1,174 @@ +/** + * Text → Image — opens a sidebar-style dialog, generates via remote provider, + * pastes result as a new layer on the current canvas. + * + * Menu target: generate/text_to_image.text_to_image + */ + +import app from './../../app.js'; +import config from './../../config.js'; +import Base_layers_class from './../../core/base-layers.js'; +import Dialog_class from './../../libs/popup.js'; +import alertify from './../../../../node_modules/alertifyjs/build/alertify.min.js'; +import apiService from './../../services/api.js'; +import { getCapabilities } from './../../api/capabilities.js'; + +var instance = null; + +class Generate_text_to_image_class { + + constructor() { + if (instance) return instance; + instance = this; + this.Base_layers = new Base_layers_class(); + this.Dialog = new Dialog_class(); + this.isProcessing = false; + } + + async text_to_image() { + var caps = await getCapabilities(); + if (!caps.remote || !caps.remote.healthy) { + alertify.error( + 'Text → Image requires a remote AI provider. ' + + 'Set AI_PROVIDER (openai / invokeai / comfyui) in .env and restart.' + ); + return; + } + + var _this = this; + var canvasW = config.WIDTH || 1024; + var canvasH = config.HEIGHT || 1024; + + this.Dialog.show({ + title: 'Text → Image', + params: [ + { + name: 'prompt', + title: 'Describe your image:', + type: 'textarea', + value: '', + placeholder: "e.g. 'a serene mountain lake at sunset, cinematic lighting'", + }, + { + name: 'negative_prompt', + title: 'Avoid (optional):', + value: '', + placeholder: 'blurry, distorted, watermark', + }, + { + name: 'width', + title: 'Width (px):', + value: Math.min(canvasW, 1024), + range: [256, 2048], + step: 64, + type: 'range', + }, + { + name: 'height', + title: 'Height (px):', + value: Math.min(canvasH, 1024), + range: [256, 2048], + step: 64, + type: 'range', + }, + { + name: 'placement', + title: 'Add as:', + value: 'new_layer', + values: ['new_layer', 'replace_canvas'], + }, + { + name: 'steps', + title: 'Steps:', + type: 'range', + value: 30, + range: [10, 60], + step: 5, + }, + { + name: 'seed', + title: 'Seed (0 = random):', + value: 0, + range: [0, 2147483647], + step: 1, + type: 'range', + }, + ], + on_finish: async function (params) { + if (!params.prompt || !params.prompt.trim()) { + alertify.warning('Please enter a description.'); + return; + } + await _this._generate(params); + }, + }); + } + + async _generate(params) { + if (this.isProcessing) return; + this.isProcessing = true; + alertify.message('Generating image... please wait', 0); + + try { + var result = await apiService.textToImage(params.prompt, { + width: params.width || 1024, + height: params.height || 1024, + negativePrompt: params.negative_prompt || '', + steps: params.steps || 30, + seed: params.seed || 0, + }); + + var img = new Image(); + img.onload = () => { + if (params.placement === 'replace_canvas') { + // Resize canvas and replace bottom layer + config.WIDTH = img.naturalWidth; + config.HEIGHT = img.naturalHeight; + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = img.naturalWidth; + resultCanvas.height = img.naturalHeight; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + app.State.do_action( + new app.Actions.Bundle_action('txt2img_replace', 'Text → Image', [ + new app.Actions.Update_layer_image_action(resultCanvas) + ]) + ); + } else { + // Add as new layer on top + var dataURL = img.src; + app.State.do_action( + new app.Actions.Bundle_action('txt2img_layer', 'Text → Image Layer', [ + new app.Actions.Insert_layer_action({ + name: params.prompt.slice(0, 30), + type: 'image', + data: dataURL, + x: 0, + y: 0, + width: img.naturalWidth, + height: img.naturalHeight, + width_original: img.naturalWidth, + height_original: img.naturalHeight, + }) + ]) + ); + } + alertify.dismissAll(); + alertify.success('Image generated!'); + this.isProcessing = false; + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load generated image.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('Generation failed: ' + (err.message || err)); + this.isProcessing = false; + } + } +} + +export default Generate_text_to_image_class; diff --git a/frontend/src/js/modules/help/about.js b/frontend/src/js/modules/help/about.js index 192984a..405f1f3 100644 --- a/frontend/src/js/modules/help/about.js +++ b/frontend/src/js/modules/help/about.js @@ -9,19 +9,23 @@ class Help_about_class { //about about() { - var email = 'www.viliusl@gmail.com'; - + var email = 'www.viliusl@gmail.com'; + var settings = { title: 'About', params: [ {title: "", html: ''}, - {title: "Name:", html: 'miniPaint'}, + {title: "Name:", html: 'PaintPlus'}, {title: "Version:", value: VERSION}, - {title: "Description:", value: "Online image editor."}, - {title: "Author:", value: 'ViliusL'}, - {title: "Email:", html: '' + email + ''}, - {title: "GitHub:", html: 'https://github.com/viliusle/miniPaint'}, - {title: "Website:", html: 'https://viliusle.github.io/miniPaint/'}, + {title: "Description:", value: "Layer-based image editor with AI tools."}, + {title: "", html: '
'}, + {title: "Base:", html: 'miniPaint by ViliusL'}, + {title: "AI Erase:", html: 'LaMa (Samsung Research) via simple-lama-inpainting'}, + {title: "Bg Removal:", html: 'rembg / U2Net / OpenCV'}, + {title: "Smart Select:", html: 'SAM (Meta AI)'}, + {title: "Remote AI:", html: 'InvokeAI · ComfyUI · OpenAI (user-configured)'}, + {title: "", html: '
'}, + {title: "GitHub:", html: 'outis1one/EditmaskwithAI'}, ], }; this.POP.show(settings); diff --git a/frontend/src/js/modules/image/frame_fit.js b/frontend/src/js/modules/image/frame_fit.js new file mode 100644 index 0000000..cfcd1fa --- /dev/null +++ b/frontend/src/js/modules/image/frame_fit.js @@ -0,0 +1,213 @@ +/** + * Fit to Frame — resize/extend/crop image to a standard print frame size. + * + * Modes: + * crop — center-crop to aspect ratio, scale to print resolution (no AI needed) + * extend — scale to fill one dimension, AI-outpaint the gap (needs provider) + * smart — auto-pick: extend if gap < 15% of dimension, else crop + * + * Menu target: image/frame_fit.frame_fit + */ + +import app from './../../app.js'; +import config from './../../config.js'; +import Base_layers_class from './../../core/base-layers.js'; +import Dialog_class from './../../libs/popup.js'; +import alertify from './../../../../node_modules/alertifyjs/build/alertify.min.js'; +import { getCapabilities } from './../../api/capabilities.js'; + +var instance = null; + +const FRAME_SIZES = [ + '4x6', '5x7', '8x10', '11x14', '16x20', '20x24', '24x36', + '4x4', '8x8', '12x12', +]; + +// Pixels at 300 dpi for preview labels +const FRAME_PX = { + '4x6': [1200, 1800], '5x7': [1500, 2100], + '8x10': [2400, 3000], '11x14': [3300, 4200], + '16x20': [4800, 6000], '20x24': [6000, 7200], + '24x36': [7200, 10800], + '4x4': [1200, 1200], '8x8': [2400, 2400], '12x12': [3600, 3600], +}; + +class Image_frame_fit_class { + + constructor() { + if (instance) return instance; + instance = this; + this.Base_layers = new Base_layers_class(); + this.Dialog = new Dialog_class(); + this.isProcessing = false; + } + + async frame_fit() { + if (!config.layer || config.layer.type !== 'image') { + alertify.error('Select an image layer first.'); + return; + } + + var caps = await getCapabilities(); + var hasRemote = caps.remote && caps.remote.healthy; + + var _this = this; + var W = config.layer.width_original; + var H = config.layer.height_original; + + // Build display labels with pixel sizes + var sizeLabels = FRAME_SIZES.map(s => { + var px = FRAME_PX[s] || [0, 0]; + return `${s}" (${px[0]}×${px[1]}px @ 300dpi)`; + }); + + this.Dialog.show({ + title: 'Fit to Frame', + params: [ + { + title: '', + html: `
+ Current image: ${W}×${H}px
+ Crop = no AI needed. Extend = AI fills the gaps${hasRemote ? '' : ' (no provider configured — extend will use mirror fill)'}. +
`, + }, + { + name: 'frame', + title: 'Frame size:', + value: sizeLabels[1], // default 5x7 + values: sizeLabels, + type: 'select', + }, + { + name: 'orientation', + title: 'Orientation:', + value: 'auto', + values: ['auto', 'portrait', 'landscape'], + type: 'select', + }, + { + name: 'mode', + title: 'Fit mode:', + value: 'smart', + values: ['smart', 'crop', 'extend'], + type: 'select', + }, + { + name: 'dpi', + title: 'Output DPI:', + value: '300', + values: ['72', '150', '300'], + type: 'select', + }, + { + name: 'prompt', + title: 'Extend prompt (optional):', + value: '', + placeholder: 'e.g. "continue the background naturally" — blank works well', + }, + { + name: 'new_layer', + title: 'Result as new layer (keep original):', + value: true, + }, + ], + on_finish: async function (params) { + var frameKey = params.frame.split('"')[0]; // strip label suffix back to "8x10" + await _this._run(frameKey, params); + }, + }); + } + + async _run(frameKey, params) { + if (this.isProcessing) return; + this.isProcessing = true; + + var mode = params.mode || 'smart'; + alertify.message( + mode === 'extend' + ? 'Fitting to frame with AI extension... please wait' + : 'Fitting to frame...', + 0 + ); + + try { + var layerCanvas = document.createElement('canvas'); + layerCanvas.width = config.layer.width_original; + layerCanvas.height = config.layer.height_original; + layerCanvas.getContext('2d').drawImage(config.layer.link, 0, 0); + var imageB64 = layerCanvas.toDataURL('image/png').split(',')[1]; + + var base = window.API_BASE_URL || ''; + var r = await fetch(`${base}/api/print/frame-fit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + image: imageB64, + frame: frameKey, + orientation: params.orientation || 'auto', + mode: params.mode || 'smart', + dpi: parseInt(params.dpi) || 300, + prompt: params.prompt || '', + }), + }); + + if (!r.ok) { + var err = await r.json().catch(() => ({ detail: 'Server error' })); + throw new Error(err.detail || 'Frame fit failed'); + } + var result = await r.json(); + + var img = new Image(); + img.onload = () => { + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = img.naturalWidth; + resultCanvas.height = img.naturalHeight; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + + if (params.new_layer) { + var dataURL = img.src; + app.State.do_action( + new app.Actions.Bundle_action('frame_fit_layer', 'Fit to Frame', [ + new app.Actions.Insert_layer_action({ + name: `${frameKey} fit`, + type: 'image', + data: dataURL, + x: 0, y: 0, + width: img.naturalWidth, + height: img.naturalHeight, + width_original: img.naturalWidth, + height_original: img.naturalHeight, + }) + ]) + ); + } else { + app.State.do_action( + new app.Actions.Bundle_action('frame_fit', 'Fit to Frame', [ + new app.Actions.Update_layer_image_action(resultCanvas) + ]) + ); + } + + alertify.dismissAll(); + alertify.success( + `Done! ${result.output_pixels.width}×${result.output_pixels.height}px` + + ` (${result.frame} ${result.orientation}, ${result.mode_used})` + ); + this.isProcessing = false; + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load result.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('Frame fit failed: ' + (err.message || err)); + this.isProcessing = false; + } + } +} + +export default Image_frame_fit_class; diff --git a/frontend/src/js/modules/image/upscale.js b/frontend/src/js/modules/image/upscale.js new file mode 100644 index 0000000..6ee805a --- /dev/null +++ b/frontend/src/js/modules/image/upscale.js @@ -0,0 +1,283 @@ +/** + * Upscale — increase image resolution. + * Fetches available methods from /api/print/upscale/available on first open. + * Auto-selects the recommended method; user can override. + * If no AI upscaler is found, polls /api/print/upscale/install-status while + * the backend auto-installs Real-ESRGAN NCNN Vulkan, then refreshes and continues. + * + * Menu target: image/upscale.upscale + */ + +import app from './../../app.js'; +import config from './../../config.js'; +import Base_layers_class from './../../core/base-layers.js'; +import Dialog_class from './../../libs/popup.js'; +import alertify from './../../../../node_modules/alertifyjs/build/alertify.min.js'; + +var instance = null; + +const METHOD_LABELS = { + auto: 'Auto (best available)', + realesrgan_pytorch: 'Real-ESRGAN — PyTorch', + realesrgan_ncnn: 'Real-ESRGAN — NCNN Vulkan', + lanczos: 'Lanczos (fast, no AI)', +}; + +class Image_upscale_class { + + constructor() { + if (instance) return instance; + instance = this; + this.Base_layers = new Base_layers_class(); + this.Dialog = new Dialog_class(); + this.isProcessing = false; + this._caps = null; + } + + async upscale() { + if (!config.layer || config.layer.type !== 'image') { + alertify.error('Select an image layer first.'); + return; + } + + // If a previous caps fetch showed no AI upscaler, check install progress + var caps = await this._fetchCaps(); + if (!caps.realesrgan_pytorch && !caps.realesrgan_ncnn) { + await this._waitForInstall(caps); + // Re-fetch caps after install + this._caps = null; + caps = await this._fetchCaps(); + } + + this._showDialog(caps); + } + + _showDialog(caps) { + var W = config.layer.width_original; + var H = config.layer.height_original; + + var available = ['auto', ...caps.methods]; + var methodValues = [...new Set(available)]; + + var methodLabels = methodValues.map(m => { + var label = METHOD_LABELS[m] || m; + if (m === 'auto') { + label = `Auto → ${caps.recommended_label}`; + } else if (m === caps.recommended && m !== 'auto') { + label += ' ★'; + } + return label; + }); + + var deviceNote = ''; + if (caps.realesrgan_pytorch) { + var dev = caps.realesrgan_pytorch_device; + var devLabel = dev === 'cuda' ? 'CUDA GPU' + : dev === 'mps' ? 'Apple Silicon' + : 'CPU (slow — ~1–3 min for large images)'; + deviceNote += `PyTorch: ${devLabel}. `; + } + if (caps.realesrgan_ncnn) { + deviceNote += 'NCNN Vulkan binary found. '; + } + if (!caps.realesrgan_pytorch && !caps.realesrgan_ncnn) { + deviceNote = 'No AI upscaler available — Lanczos only.'; + } + + var _this = this; + + this.Dialog.show({ + title: 'Upscale Image', + params: [ + { + title: '', + html: `
+ Current: ${W}×${H}px
+ ${deviceNote} +
`, + }, + { + name: 'scale', + title: 'Scale factor:', + value: '2×', + values: ['1.5×', '2×', '3×', '4×'], + type: 'select', + }, + { + name: 'method', + title: 'Method:', + value: methodLabels[0], + values: methodLabels, + type: 'select', + }, + { + name: 'new_layer', + title: 'Result as new layer (keep original):', + value: false, + }, + ], + on_finish: async function (params) { + var labelIdx = methodLabels.indexOf(params.method); + var methodKey = labelIdx >= 0 ? methodValues[labelIdx] : 'auto'; + var scale = parseFloat(params.scale); + await _this._run(scale, methodKey, params.new_layer); + }, + }); + } + + /** + * Poll install-status until done/failed, showing a progress bar notification. + */ + async _waitForInstall(caps) { + var installStatus = caps.ncnn_install_status || {}; + if (installStatus.state === 'done' || installStatus.state === 'failed') { + return; + } + + return new Promise((resolve) => { + var msg = alertify.message( + `
Installing Real-ESRGAN AI upscaler…
+ + 0%
`, + 0 + ); + + var poll = setInterval(async () => { + try { + var base = window.API_BASE_URL || ''; + var r = await fetch(`${base}/api/print/upscale/install-status`); + if (!r.ok) return; + var s = await r.json(); + + var bar = document.getElementById('esrgan-install-progress'); + var pct = document.getElementById('esrgan-install-pct'); + if (bar) bar.value = s.progress || 0; + if (pct) pct.textContent = `${s.progress || 0}%`; + + if (s.state === 'done') { + clearInterval(poll); + alertify.dismissAll(); + alertify.success('Real-ESRGAN NCNN installed ✓'); + resolve(); + } else if (s.state === 'failed') { + clearInterval(poll); + alertify.dismissAll(); + alertify.warning('AI upscaler install failed — using Lanczos.'); + resolve(); + } + } catch { /* network hiccup, keep polling */ } + }, 1500); + }); + } + + async _fetchCaps() { + if (this._caps) return this._caps; + try { + var base = window.API_BASE_URL || ''; + var r = await fetch(`${base}/api/print/upscale/available`); + if (r.ok) { + this._caps = await r.json(); + } + } catch { /* ignore */ } + + if (!this._caps) { + this._caps = { + lanczos: true, + realesrgan_pytorch: false, + realesrgan_ncnn: false, + recommended: 'lanczos', + recommended_label: 'Lanczos', + methods: ['lanczos'], + ncnn_install_status: { state: 'idle', progress: 0 }, + }; + } + return this._caps; + } + + async _run(scale, method, newLayer) { + if (this.isProcessing) return; + this.isProcessing = true; + + var caps = this._caps || {}; + var methodLabel = method === 'auto' + ? `Auto (${caps.recommended_label || 'best available'})` + : (METHOD_LABELS[method] || method); + + alertify.message(`Upscaling ${scale}× · ${methodLabel}…`, 0); + + try { + var layerCanvas = document.createElement('canvas'); + layerCanvas.width = config.layer.width_original; + layerCanvas.height = config.layer.height_original; + layerCanvas.getContext('2d').drawImage(config.layer.link, 0, 0); + var imageB64 = layerCanvas.toDataURL('image/png').split(',')[1]; + + var base = window.API_BASE_URL || ''; + var r = await fetch(`${base}/api/print/upscale`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ image: imageB64, scale, method }), + }); + + if (!r.ok) { + var err = await r.json().catch(() => ({ detail: 'Server error' })); + throw new Error(err.detail || 'Upscale failed'); + } + var result = await r.json(); + + var img = new Image(); + img.onload = () => { + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = img.naturalWidth; + resultCanvas.height = img.naturalHeight; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + + var usedLabel = result.method.replace('realesrgan_pytorch_', 'ESRGAN/') + .replace('realesrgan_ncnn', 'ESRGAN/NCNN'); + + if (newLayer) { + app.State.do_action( + new app.Actions.Bundle_action('upscale_layer', 'Upscale', [ + new app.Actions.Insert_layer_action({ + name: `${scale}× ${usedLabel}`, + type: 'image', + data: img.src, + x: 0, y: 0, + width: img.naturalWidth, + height: img.naturalHeight, + width_original: img.naturalWidth, + height_original: img.naturalHeight, + }) + ]) + ); + } else { + app.State.do_action( + new app.Actions.Bundle_action('upscale', 'Upscale', [ + new app.Actions.Update_layer_image_action(resultCanvas) + ]) + ); + } + + alertify.dismissAll(); + alertify.success( + `${result.output.width}×${result.output.height}px · ${usedLabel}` + ); + this.isProcessing = false; + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load upscaled image.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('Upscale failed: ' + (err.message || err)); + this.isProcessing = false; + } + } +} + +export default Image_upscale_class; diff --git a/frontend/src/js/modules/tools/ai_provider_settings.js b/frontend/src/js/modules/tools/ai_provider_settings.js new file mode 100644 index 0000000..4268346 --- /dev/null +++ b/frontend/src/js/modules/tools/ai_provider_settings.js @@ -0,0 +1,204 @@ +/** + * AI Provider Settings — configure remote AI provider in-app without editing .env manually. + * Settings are persisted to localStorage and sent to the backend config endpoint. + * Menu target: tools/ai_provider_settings.ai_provider_settings + */ + +import Dialog_class from './../../libs/popup.js'; +import alertify from './../../../../node_modules/alertifyjs/build/alertify.min.js'; +import { getCapabilities } from './../../api/capabilities.js'; + +// localStorage key prefix +const LS = 'paintplus_ai_'; + +function ls_get(key, def = '') { + return localStorage.getItem(LS + key) ?? def; +} +function ls_set(key, val) { + localStorage.setItem(LS + key, val); +} + +var instance = null; + +class Tools_ai_provider_settings_class { + + constructor() { + if (instance) return instance; + instance = this; + this.POP = new Dialog_class(); + } + + async ai_provider_settings() { + var _this = this; + var caps = await getCapabilities(); + var remote = caps.remote || {}; + var statusHtml = remote.provider + ? (remote.healthy + ? `● ${remote.provider} — connected` + : `● ${remote.provider} — unreachable`) + : 'No remote provider configured'; + + this.POP.show({ + title: 'AI Provider Settings', + params: [ + { + title: 'Status:', + html: `
${statusHtml}
`, + }, + { + name: 'provider', + title: 'Default provider (used unless overridden below):', + value: ls_get('provider', remote.provider || ''), + values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'], + type: 'select', + }, + // ── Per-operation overrides ─────────────────────────────── + { + title: '', + html: '
Per-operation overrides — blank = use default above
', + }, + { + name: 'provider_inpaint', + title: 'Inpaint / Replace Selection:', + value: ls_get('provider_inpaint', remote.overrides?.inpaint || ''), + values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'], + type: 'select', + }, + { + name: 'provider_txt2img', + title: 'Text → Image:', + value: ls_get('provider_txt2img', remote.overrides?.txt2img || ''), + values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'], + type: 'select', + }, + { + name: 'provider_img2img', + title: 'Image → Image:', + value: ls_get('provider_img2img', remote.overrides?.img2img || ''), + values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'], + type: 'select', + }, + { + name: 'provider_outpaint', + title: 'Expand Canvas (Outpaint):', + value: ls_get('provider_outpaint', remote.overrides?.outpaint || ''), + values: ['', 'openai', 'invokeai', 'comfyui', 'replicate'], + type: 'select', + }, + // ── OpenAI ──────────────────────────────────────────────── + { + name: 'openai_key', + title: 'OpenAI API key:', + value: ls_get('openai_key'), + placeholder: 'sk-...', + }, + { + name: 'openai_model', + title: 'OpenAI model:', + value: ls_get('openai_model', 'dall-e-3'), + values: ['dall-e-3', 'dall-e-2'], + type: 'select', + }, + // ── InvokeAI ────────────────────────────────────────────── + { + name: 'invokeai_url', + title: 'InvokeAI URL:', + value: ls_get('invokeai_url'), + placeholder: 'http://192.168.1.x:9090', + }, + { + name: 'invokeai_model', + title: 'InvokeAI default model:', + value: ls_get('invokeai_model', 'flux-dev'), + placeholder: 'flux-dev', + }, + // ── ComfyUI ─────────────────────────────────────────────── + { + name: 'comfyui_url', + title: 'ComfyUI URL:', + value: ls_get('comfyui_url'), + placeholder: 'http://192.168.1.x:8188', + }, + { + name: 'comfyui_model', + title: 'ComfyUI default checkpoint:', + value: ls_get('comfyui_model', 'v1-5-pruned-emaonly.ckpt'), + placeholder: 'v1-5-pruned-emaonly.ckpt', + }, + // ── Replicate ───────────────────────────────────────────── + { + name: 'replicate_key', + title: 'Replicate API key:', + value: ls_get('replicate_key'), + placeholder: 'r8_...', + }, + ], + on_finish: async function (params) { + await _this._save(params); + }, + }); + } + + async _save(params) { + // Persist to localStorage + ls_set('provider', params.provider || ''); + ls_set('provider_inpaint', params.provider_inpaint || ''); + ls_set('provider_txt2img', params.provider_txt2img || ''); + ls_set('provider_img2img', params.provider_img2img || ''); + ls_set('provider_outpaint', params.provider_outpaint || ''); + ls_set('openai_key', params.openai_key || ''); + ls_set('openai_model', params.openai_model || 'dall-e-3'); + ls_set('invokeai_url', params.invokeai_url || ''); + ls_set('invokeai_model', params.invokeai_model || 'flux-dev'); + ls_set('comfyui_url', params.comfyui_url || ''); + ls_set('comfyui_model', params.comfyui_model || 'v1-5-pruned-emaonly.ckpt'); + ls_set('replicate_key', params.replicate_key || ''); + + // Push to backend + try { + var payload = { + ai_provider: params.provider || '', + ai_provider_inpaint: params.provider_inpaint || '', + ai_provider_txt2img: params.provider_txt2img || '', + ai_provider_img2img: params.provider_img2img || '', + ai_provider_outpaint: params.provider_outpaint || '', + openai_api_key: params.openai_key || '', + openai_model: params.openai_model || 'dall-e-3', + invokeai_url: params.invokeai_url || '', + invokeai_default_model: params.invokeai_model || 'flux-dev', + comfyui_url: params.comfyui_url || '', + comfyui_default_model: params.comfyui_model || '', + replicate_api_key: params.replicate_key || '', + }; + var base = window.API_BASE_URL || ''; + var r = await fetch(`${base}/api/config`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + + if (r.ok) { + alertify.success('AI provider settings saved. Testing connection...'); + var { refreshCapabilities } = await import('./../../api/capabilities.js'); + var caps = await refreshCapabilities(); + if (caps?.remote?.healthy) { + alertify.success(`Connected to ${caps.remote.provider}!`); + } else if (params.provider) { + alertify.warning('Settings saved but provider is not reachable. Check URL/key.'); + } + } else { + // Server-side config update not supported — inform user to set .env + alertify.warning( + 'Settings saved locally. To make them permanent, ' + + 'set these values in your .env file and restart the server.' + ); + } + } catch { + alertify.warning( + 'Settings saved locally. Set AI_PROVIDER and related keys in .env to make permanent.' + ); + } + } +} + +export default Tools_ai_provider_settings_class; diff --git a/frontend/src/js/services/api.js b/frontend/src/js/services/api.js index ee4abb3..f853673 100644 --- a/frontend/src/js/services/api.js +++ b/frontend/src/js/services/api.js @@ -95,6 +95,124 @@ class ApiService { return response.json(); } + /** + * AI erase using LaMa (local, no API key needed) + * @param {string} imageData - Base64 encoded image + * @param {string} maskData - Base64 encoded mask (white = erase) + * @returns {Promise<{result: string, method: string}>} + */ + async erase(imageData, maskData) { + const response = await fetch(`${this.baseUrl}/api/erase`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ image: imageData, mask: maskData }), + }); + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: 'Unknown error' })); + throw new Error(error.detail || `Erase request failed: ${response.status}`); + } + return response.json(); + } + + /** + * Text-to-image via remote provider + * @param {string} prompt + * @param {Object} options - width, height, negativePrompt, steps, cfgScale, model + * @returns {Promise<{result: string}>} + */ + async textToImage(prompt, options = {}) { + const response = await fetch(`${this.baseUrl}/api/generate/txt2img`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + prompt, + width: options.width || 1024, + height: options.height || 1024, + negative_prompt: options.negativePrompt || '', + steps: options.steps || 30, + cfg_scale: options.cfgScale || 7.5, + model: options.model || null, + seed: options.seed || 0, + }), + }); + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: 'Unknown error' })); + throw new Error(error.detail || `Text-to-image failed: ${response.status}`); + } + return response.json(); + } + + /** + * Image-to-image via remote provider + * @param {string} imageData - Base64 encoded image + * @param {string} prompt + * @param {Object} options + * @returns {Promise<{result: string}>} + */ + async imageToImage(imageData, prompt, options = {}) { + const response = await fetch(`${this.baseUrl}/api/generate/img2img`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + image: imageData, + prompt, + strength: options.strength || 0.75, + negative_prompt: options.negativePrompt || '', + steps: options.steps || 30, + cfg_scale: options.cfgScale || 7.5, + model: options.model || null, + }), + }); + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: 'Unknown error' })); + throw new Error(error.detail || `Image-to-image failed: ${response.status}`); + } + return response.json(); + } + + /** + * Inpaint with prompt via remote provider + * @param {string} imageData - Base64 + * @param {string} maskData - Base64 + * @param {string} prompt + * @param {Object} options + * @returns {Promise<{result: string}>} + */ + async remoteInpaint(imageData, maskData, prompt, options = {}) { + const response = await fetch(`${this.baseUrl}/api/inpaint/remote`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + image: imageData, + mask: maskData, + prompt, + negative_prompt: options.negativePrompt || '', + steps: options.steps || 30, + cfg_scale: options.cfgScale || 7.5, + model: options.model || null, + }), + }); + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: 'Unknown error' })); + throw new Error(error.detail || `Remote inpaint failed: ${response.status}`); + } + return response.json(); + } + + /** + * Fetch backend capabilities (local tools available, remote provider status). + * @returns {Promise} + */ + async getConfig() { + try { + const response = await fetch(`${this.baseUrl}/api/config`); + if (!response.ok) return null; + return response.json(); + } catch { + return null; + } + } + /** * Health check for the backend * @returns {Promise} diff --git a/frontend/src/js/tools/ai_lama_erase.js b/frontend/src/js/tools/ai_lama_erase.js new file mode 100644 index 0000000..e4616b6 --- /dev/null +++ b/frontend/src/js/tools/ai_lama_erase.js @@ -0,0 +1,199 @@ +/** + * AI Magic Eraser — paint a mask with a brush, send to LaMa backend, apply result. + * Works locally (no API key). GPU auto-detected; CPU fallback always available. + * + * Workflow: + * 1. User paints over the object to erase (red overlay shows the mask) + * 2. On mouseup, POST image + mask to /api/erase + * 3. Result replaces the current layer canvas + * + * Registered as tool name: "ai_lama_erase" + */ + +import app from './../app.js'; +import config from './../config.js'; +import Base_tools_class from './../core/base-tools.js'; +import Base_layers_class from './../core/base-layers.js'; +import Helper_class from './../libs/helpers.js'; +import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js'; +import apiService from './../services/api.js'; + +class Ai_lama_erase_class extends Base_tools_class { + + constructor(ctx) { + super(); + this.Base_layers = new Base_layers_class(); + this.Helper = new Helper_class(); + this.ctx = ctx; + this.name = 'ai_lama_erase'; + + this.isDrawing = false; + this.isProcessing = false; + + // Off-screen canvas used to accumulate the painted mask + this.maskCanvas = null; + this.maskCtx = null; + } + + load() { + var _this = this; + document.addEventListener('mousedown', function (e) { _this.mousedown(e); }); + document.addEventListener('mousemove', function (e) { _this.mousemove(e); }); + document.addEventListener('mouseup', function (e) { _this.mouseup(e); }); + document.addEventListener('touchstart', function (e) { _this.mousedown(e); }, { passive: false }); + document.addEventListener('touchmove', function (e) { _this.mousemove(e); }, { passive: false }); + document.addEventListener('touchend', function (e) { _this.mouseup(e); }); + } + + mousedown(e) { + var mouse = this.get_mouse_info(e); + if (!mouse.click_valid) return; + if (config.TOOL.name !== this.name) return; + if (this.isProcessing) return; + + if (config.layer.type !== 'image') { + alertify.error('This layer must contain an image.'); + return; + } + + this._initMask(); + this.isDrawing = true; + this._paint(mouse); + } + + mousemove(e) { + if (!this.isDrawing) return; + if (config.TOOL.name !== this.name) return; + var mouse = this.get_mouse_info(e); + this._paint(mouse); + } + + mouseup(e) { + if (!this.isDrawing) return; + this.isDrawing = false; + if (config.TOOL.name !== this.name) return; + this._applyErase(); + } + + // ── Private ────────────────────────────────────────────────────────────── + + _initMask() { + var w = config.layer.width_original; + var h = config.layer.height_original; + + if (!this.maskCanvas || this.maskCanvas.width !== w || this.maskCanvas.height !== h) { + this.maskCanvas = document.createElement('canvas'); + this.maskCanvas.width = w; + this.maskCanvas.height = h; + this.maskCtx = this.maskCanvas.getContext('2d'); + } + this.maskCtx.clearRect(0, 0, w, h); + } + + _paint(mouse) { + var params = this.getParams(); + var size = params.size || 30; + + // Map screen coords → layer-original coords + var lx = Math.round(this.adaptSize(Math.round(mouse.x) - config.layer.x, 'width')); + var ly = Math.round(this.adaptSize(Math.round(mouse.y) - config.layer.y, 'height')); + + this.maskCtx.beginPath(); + this.maskCtx.arc(lx, ly, size / 2, 0, Math.PI * 2); + this.maskCtx.fillStyle = '#ffffff'; + this.maskCtx.fill(); + + // Show red overlay on screen so user can see the painted area + this._renderOverlay(lx, ly, size); + } + + _renderOverlay(lx, ly, size) { + // Draw a translucent red circle on the main canvas for visual feedback + var scale = config.ZOOM / 100; + var sx = config.layer.x * scale + lx * scale; + var sy = config.layer.y * scale + ly * scale; + var sRadius = (size / 2) * scale; + + var mainCtx = document.getElementById('canvas_temp') + ? document.getElementById('canvas_temp').getContext('2d') + : null; + if (!mainCtx) return; + + mainCtx.save(); + mainCtx.beginPath(); + mainCtx.arc(sx, sy, sRadius, 0, Math.PI * 2); + mainCtx.fillStyle = 'rgba(255, 60, 60, 0.4)'; + mainCtx.fill(); + mainCtx.restore(); + } + + async _applyErase() { + if (this.isProcessing) return; + + // Check if any mask pixels were painted + var maskData = this.maskCtx.getImageData( + 0, 0, this.maskCanvas.width, this.maskCanvas.height + ); + var hasPixels = maskData.data.some((v, i) => i % 4 === 3 && v > 0); + if (!hasPixels) return; + + this.isProcessing = true; + alertify.message('AI erasing... please wait', 0); + + try { + // Get current layer as PNG base64 + var layerCanvas = document.createElement('canvas'); + layerCanvas.width = config.layer.width_original; + layerCanvas.height = config.layer.height_original; + var lctx = layerCanvas.getContext('2d'); + lctx.drawImage(config.layer.link, 0, 0); + var imageB64 = layerCanvas.toDataURL('image/png').split(',')[1]; + + // Get mask as PNG base64 + var maskB64 = this.maskCanvas.toDataURL('image/png').split(',')[1]; + + // Call backend + var result = await apiService.erase(imageB64, maskB64); + + // Apply result back to layer + var img = new Image(); + img.onload = () => { + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = config.layer.width_original; + resultCanvas.height = config.layer.height_original; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + + app.State.do_action( + new app.Actions.Bundle_action('ai_lama_erase', 'AI Erase', [ + new app.Actions.Update_layer_image_action(resultCanvas) + ]) + ); + + alertify.dismissAll(); + alertify.success('Erased! (' + result.method + ')'); + this.isProcessing = false; + this._clearOverlay(); + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load result image.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('AI erase failed: ' + (err.message || err)); + this.isProcessing = false; + } + } + + _clearOverlay() { + var canvas = document.getElementById('canvas_temp'); + if (canvas) { + canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height); + } + } +} + +export default Ai_lama_erase_class; diff --git a/frontend/src/js/tools/ai_replace_selection.js b/frontend/src/js/tools/ai_replace_selection.js new file mode 100644 index 0000000..f601ac0 --- /dev/null +++ b/frontend/src/js/tools/ai_replace_selection.js @@ -0,0 +1,218 @@ +/** + * AI Replace Selection — pick any selection (Smart Select, Magic Wand, Lasso, Brush Select), + * describe what should go there, remote provider fills it in. + * + * Requires a configured remote provider (InvokeAI / ComfyUI / OpenAI). + * Registered as tool name: "ai_replace_selection" + */ + +import app from './../app.js'; +import config from './../config.js'; +import Base_tools_class from './../core/base-tools.js'; +import Base_layers_class from './../core/base-layers.js'; +import Dialog_class from './../libs/popup.js'; +import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js'; +import apiService from './../services/api.js'; +import { getCapabilities } from './../api/capabilities.js'; + +class Ai_replace_selection_class extends Base_tools_class { + + constructor(ctx) { + super(); + this.Base_layers = new Base_layers_class(); + this.POP = new Dialog_class(); + this.ctx = ctx; + this.name = 'ai_replace_selection'; + this.isProcessing = false; + } + + load() {} + + async on_activate() { + var caps = await getCapabilities(); + if (!caps.remote || !caps.remote.healthy) { + alertify.error( + 'Replace Selection requires a remote AI provider. ' + + 'Set AI_PROVIDER (openai / invokeai / comfyui) in .env and restart.' + ); + return; + } + + var hasMask = window.smartSelectMask?.canvas != null; + var hasRect = this._getRectSelection() != null; + + if (!hasMask && !hasRect) { + alertify.warning( + 'No selection found. Use Smart Select, Magic Wand, Lasso, ' + + 'Ellipse Select, or Brush Select first, then activate this tool.' + ); + return; + } + + this._showDialog(caps.remote.provider); + } + + // ── Private ────────────────────────────────────────────────────────────── + + _getRectSelection() { + if (!config.layer) return null; + var sel = config.layer.selection; + if (!sel) return null; + var { x, y, width, height } = sel; + if (!width || !height) return null; + return { x, y, width, height }; + } + + _showDialog(providerName) { + var _this = this; + + this.POP.show({ + title: 'AI Replace Selection', + params: [ + { + name: 'prompt', + title: 'Describe what to place here:', + type: 'textarea', + value: '', + placeholder: "e.g. 'a blooming red rose', 'dark polished wood', 'a smiling golden retriever'", + }, + { + name: 'negative_prompt', + title: 'Avoid (optional):', + value: '', + placeholder: 'blurry, distorted, low quality', + }, + { + name: 'steps', + title: 'Steps:', + type: 'range', + value: 30, + range: [10, 60], + step: 5, + }, + { + name: 'cfg_scale', + title: 'Prompt strength:', + type: 'range', + value: 75, + range: [10, 100], + step: 5, + }, + ], + on_finish: function (params) { + if (!params.prompt || !params.prompt.trim()) { + alertify.warning('Please enter a description.'); + return; + } + _this._run(params); + }, + }); + } + + async _run(params) { + if (this.isProcessing) return; + if (config.layer.type !== 'image') { + alertify.error('Current layer must be an image.'); + return; + } + + this.isProcessing = true; + alertify.message('Replacing selection... please wait', 0); + + try { + // Build mask canvas from current selection + var maskCanvas = await this._buildMaskCanvas(); + if (!maskCanvas) { + alertify.dismissAll(); + alertify.error('Could not build selection mask.'); + this.isProcessing = false; + return; + } + + // Get layer as PNG + var layerCanvas = document.createElement('canvas'); + layerCanvas.width = config.layer.width_original; + layerCanvas.height = config.layer.height_original; + layerCanvas.getContext('2d').drawImage(config.layer.link, 0, 0); + + var imageB64 = layerCanvas.toDataURL('image/png').split(',')[1]; + var maskB64 = maskCanvas.toDataURL('image/png').split(',')[1]; + + var result = await apiService.remoteInpaint( + imageB64, maskB64, + params.prompt, + { + negativePrompt: params.negative_prompt || '', + steps: params.steps || 30, + cfgScale: (params.cfg_scale || 75) / 10, + } + ); + + var img = new Image(); + img.onload = () => { + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = config.layer.width_original; + resultCanvas.height = config.layer.height_original; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + + app.State.do_action( + new app.Actions.Bundle_action('ai_replace_selection', 'AI Replace Selection', [ + new app.Actions.Update_layer_image_action(resultCanvas) + ]) + ); + + alertify.dismissAll(); + alertify.success('Done!'); + this.isProcessing = false; + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load result image.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('Replace failed: ' + (err.message || err)); + this.isProcessing = false; + } + } + + async _buildMaskCanvas() { + var w = config.layer.width_original; + var h = config.layer.height_original; + var canvas = document.createElement('canvas'); + canvas.width = w; + canvas.height = h; + var ctx = canvas.getContext('2d'); + + // Prefer smartSelectMask (all selection tools write here) + if (window.smartSelectMask?.canvas) { + ctx.drawImage(window.smartSelectMask.canvas, 0, 0, w, h); + // Ensure pure B&W + var d = ctx.getImageData(0, 0, w, h); + for (var i = 0; i < d.data.length; i += 4) { + var v = d.data[i] > 128 ? 255 : 0; + d.data[i] = d.data[i+1] = d.data[i+2] = v; + d.data[i+3] = 255; + } + ctx.putImageData(d, 0, 0); + return canvas; + } + + // Fall back to rectangular selection + var sel = this._getRectSelection(); + if (sel) { + ctx.fillStyle = '#000'; + ctx.fillRect(0, 0, w, h); + ctx.fillStyle = '#fff'; + ctx.fillRect(sel.x, sel.y, sel.width, sel.height); + return canvas; + } + + return null; + } +} + +export default Ai_replace_selection_class; diff --git a/frontend/src/js/tools/ai_smart_inpaint.js b/frontend/src/js/tools/ai_smart_inpaint.js new file mode 100644 index 0000000..61f6322 --- /dev/null +++ b/frontend/src/js/tools/ai_smart_inpaint.js @@ -0,0 +1,201 @@ +/** + * AI Smart Inpaint — paint a mask, enter a prompt, choose Fast (LaMa) or Quality (remote). + * + * Fast mode: /api/erase — LaMa local, no API key, seconds + * Quality mode: /api/inpaint/remote — InvokeAI / ComfyUI / OpenAI, requires configured provider + * + * Registered as tool name: "ai_smart_inpaint" + */ + +import app from './../app.js'; +import config from './../config.js'; +import Base_tools_class from './../core/base-tools.js'; +import Base_layers_class from './../core/base-layers.js'; +import Helper_class from './../libs/helpers.js'; +import Dialog_class from './../libs/popup.js'; +import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js'; +import apiService from './../services/api.js'; +import { getCapabilities } from './../api/capabilities.js'; + +class Ai_smart_inpaint_class extends Base_tools_class { + + constructor(ctx) { + super(); + this.Base_layers = new Base_layers_class(); + this.Helper = new Helper_class(); + this.POP = new Dialog_class(); + this.ctx = ctx; + this.name = 'ai_smart_inpaint'; + + this.isDrawing = false; + this.isProcessing = false; + this.maskCanvas = null; + this.maskCtx = null; + } + + load() { + var _this = this; + document.addEventListener('mousedown', function (e) { _this.mousedown(e); }); + document.addEventListener('mousemove', function (e) { _this.mousemove(e); }); + document.addEventListener('mouseup', function (e) { _this.mouseup(e); }); + document.addEventListener('touchstart', function (e) { _this.mousedown(e); }, { passive: false }); + document.addEventListener('touchmove', function (e) { _this.mousemove(e); }, { passive: false }); + document.addEventListener('touchend', function (e) { _this.mouseup(e); }); + } + + on_activate() { + // Nothing on activate — tool is drag-to-paint, then dialog on mouseup + } + + mousedown(e) { + var mouse = this.get_mouse_info(e); + if (!mouse.click_valid) return; + if (config.TOOL.name !== this.name) return; + if (this.isProcessing) return; + if (config.layer.type !== 'image') { + alertify.error('This layer must contain an image.'); + return; + } + this._initMask(); + this.isDrawing = true; + this._paint(mouse); + } + + mousemove(e) { + if (!this.isDrawing) return; + if (config.TOOL.name !== this.name) return; + this._paint(this.get_mouse_info(e)); + } + + mouseup(e) { + if (!this.isDrawing) return; + this.isDrawing = false; + if (config.TOOL.name !== this.name) return; + + var maskData = this.maskCtx.getImageData( + 0, 0, this.maskCanvas.width, this.maskCanvas.height + ); + if (!maskData.data.some((v, i) => i % 4 === 3 && v > 0)) return; + + this._showDialog(); + } + + // ── Private ────────────────────────────────────────────────────────────── + + _initMask() { + var w = config.layer.width_original; + var h = config.layer.height_original; + if (!this.maskCanvas || this.maskCanvas.width !== w || this.maskCanvas.height !== h) { + this.maskCanvas = document.createElement('canvas'); + this.maskCanvas.width = w; + this.maskCanvas.height = h; + this.maskCtx = this.maskCanvas.getContext('2d'); + } + this.maskCtx.clearRect(0, 0, w, h); + } + + _paint(mouse) { + var params = this.getParams(); + var size = params.size || 30; + var lx = Math.round(this.adaptSize(Math.round(mouse.x) - config.layer.x, 'width')); + var ly = Math.round(this.adaptSize(Math.round(mouse.y) - config.layer.y, 'height')); + this.maskCtx.beginPath(); + this.maskCtx.arc(lx, ly, size / 2, 0, Math.PI * 2); + this.maskCtx.fillStyle = '#ffffff'; + this.maskCtx.fill(); + } + + async _showDialog() { + var caps = await getCapabilities(); + var hasRemote = caps.remote && caps.remote.healthy; + + var _this = this; + + var settings = { + title: 'AI Smart Inpaint', + params: [ + { + name: 'quality', + title: 'Mode:', + value: 'fast', + values: hasRemote ? ['fast', 'quality'] : ['fast'], + note: hasRemote ? 'Fast = LaMa (local). Quality = remote AI + prompt.' : 'Quality mode requires a remote provider (InvokeAI / ComfyUI / OpenAI).', + }, + { + name: 'prompt', + title: 'What to put here (Quality mode only):', + type: 'textarea', + value: '', + placeholder: "e.g. 'lush green grass', 'wooden table surface', 'clear blue sky'", + }, + { + name: 'negative_prompt', + title: 'Avoid (optional):', + value: '', + placeholder: 'blurry, distorted', + }, + ], + on_load: function (params, popup) {}, + on_finish: function (params) { + _this._runInpaint(params.quality, params.prompt, params.negative_prompt); + }, + }; + + this.POP.show(settings); + } + + async _runInpaint(quality, prompt, negativePrompt) { + if (this.isProcessing) return; + this.isProcessing = true; + + var modeLabel = quality === 'quality' ? 'Quality (remote)' : 'Fast (LaMa)'; + alertify.message('Inpainting (' + modeLabel + ')... please wait', 0); + + try { + var layerCanvas = document.createElement('canvas'); + layerCanvas.width = config.layer.width_original; + layerCanvas.height = config.layer.height_original; + layerCanvas.getContext('2d').drawImage(config.layer.link, 0, 0); + var imageB64 = layerCanvas.toDataURL('image/png').split(',')[1]; + var maskB64 = this.maskCanvas.toDataURL('image/png').split(',')[1]; + + var result; + if (quality === 'quality') { + result = await apiService.remoteInpaint(imageB64, maskB64, prompt || 'fill naturally', { negativePrompt }); + } else { + result = await apiService.erase(imageB64, maskB64); + } + + var img = new Image(); + img.onload = () => { + var resultCanvas = document.createElement('canvas'); + resultCanvas.width = config.layer.width_original; + resultCanvas.height = config.layer.height_original; + resultCanvas.getContext('2d').drawImage(img, 0, 0); + + app.State.do_action( + new app.Actions.Bundle_action('ai_smart_inpaint', 'AI Smart Inpaint', [ + new app.Actions.Update_layer_image_action(resultCanvas) + ]) + ); + + alertify.dismissAll(); + alertify.success('Done!'); + this.isProcessing = false; + }; + img.onerror = () => { + alertify.dismissAll(); + alertify.error('Failed to load result.'); + this.isProcessing = false; + }; + img.src = 'data:image/png;base64,' + result.result; + + } catch (err) { + alertify.dismissAll(); + alertify.error('Inpaint failed: ' + (err.message || err)); + this.isProcessing = false; + } + } +} + +export default Ai_smart_inpaint_class; diff --git a/scripts/download_realesrgan.py b/scripts/download_realesrgan.py new file mode 100644 index 0000000..4f35c8f --- /dev/null +++ b/scripts/download_realesrgan.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +""" +Download Real-ESRGAN NCNN Vulkan binary. + +This gives you fast AI upscaling on ANY GPU (Intel/AMD/NVIDIA integrated or discrete, +Apple Metal) without needing CUDA or Python AI packages. + +Usage: + docker exec -it ai-photo-edit python /scripts/download_realesrgan.py + # or locally: + python scripts/download_realesrgan.py +""" + +import os +import sys +import platform +import zipfile +import urllib.request +import stat +from pathlib import Path + +DEST_DIR = Path("/app/data/models/realesrgan") +VERSION = "v0.2.5.0" + +PLATFORM_MAP = { + "linux": f"realesrgan-ncnn-vulkan-{VERSION}-ubuntu.zip", + "darwin": f"realesrgan-ncnn-vulkan-{VERSION}-macos.zip", + "win32": f"realesrgan-ncnn-vulkan-{VERSION}-windows.zip", + "windows": f"realesrgan-ncnn-vulkan-{VERSION}-windows.zip", +} + +BASE_URL = f"https://github.com/xinntao/Real-ESRGAN/releases/download/{VERSION}" + + +def main(): + plat = sys.platform.lower() + if plat not in PLATFORM_MAP: + print(f"Unknown platform: {plat}") + sys.exit(1) + + filename = PLATFORM_MAP[plat] + url = f"{BASE_URL}/{filename}" + zip_path = DEST_DIR / filename + + DEST_DIR.mkdir(parents=True, exist_ok=True) + + binary_name = "realesrgan-ncnn-vulkan.exe" if "win" in plat else "realesrgan-ncnn-vulkan" + binary_path = DEST_DIR / binary_name + + if binary_path.exists(): + print(f"Already installed: {binary_path}") + print("Delete it and re-run to reinstall.") + return + + print(f"Downloading Real-ESRGAN NCNN Vulkan {VERSION} for {plat}...") + print(f"URL: {url}") + + def progress(count, block_size, total_size): + if total_size > 0 and count % 100 == 0: + pct = min(100, count * block_size * 100 // total_size) + mb = count * block_size / 1024 / 1024 + total_mb = total_size / 1024 / 1024 + print(f" {pct}% ({mb:.1f}/{total_mb:.1f} MB)", end="\r") + + urllib.request.urlretrieve(url, zip_path, progress) + print(f"\nDownloaded to {zip_path}") + + print("Extracting...") + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(DEST_DIR) + + # The zip extracts into a subdirectory — find the binary + found = list(DEST_DIR.rglob(binary_name)) + if not found: + print(f"ERROR: Could not find {binary_name} in extracted files.") + sys.exit(1) + + extracted = found[0] + if extracted != binary_path: + extracted.rename(binary_path) + + # Make executable on unix + if "win" not in plat: + binary_path.chmod(binary_path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + # Clean up zip + zip_path.unlink(missing_ok=True) + + print(f"\nInstalled: {binary_path}") + print("\nTest it:") + print(f" {binary_path} --help") + print("\nThe upscaler will auto-detect this binary next time you use Upscale in PaintPlus.") + print("Restart the backend container to clear the capability cache:") + print(" docker-compose restart backend") + + +if __name__ == "__main__": + main()