feat: real-world selection actions — scale %, AI edit, clipboard paste
Backend (3 new endpoints under /api/image/): - POST /api/image/scale-selection — scale selected object by any % in-place; LaMa/OpenCV fills the exposed gap so the scene looks natural - POST /api/image/ai-edit-region — AI redraws the masked region via the configured inpaint provider (local_gpu / InvokeAI / ComfyUI / OpenAI) - POST /api/image/paste-into-selection — scales clipboard image to fit the selection bounding box, masks it to the selection shape, composites result Frontend (selection_actions.js + tool integration): - New SelectionActions panel: fixed bottom-center HUD that appears automatically after every SAM selection (click or paint) - Panel actions: Scale by % (default 3%), Make less symmetrical (AI), custom AI Edit prompt, Replace with clipboard, Copy/Cut to layer, Erase - Both smart_select.js and brush_select.js updated to show the panel, add updateLayerWithResult(), and hide panel on clearSelection/on_leave - brush_select: offerFloatSelection() replaced with richer action panel Real-world workflows now supported in one click after painting over object: "Make this 3% bigger" → scale-selection (LaMa fills gap) "Make this less symmetrical" → ai-edit-region with asymmetry prompt "Replace this with what I copied" → paste-into-selection https://claude.ai/code/session_01WVDg7amsy1TTtxvpku7bcM
This commit is contained in:
Claude
@@ -367,6 +367,164 @@ async def get_config():
|
||||
}
|
||||
|
||||
|
||||
# ─── Selection image operations ─────────────────────────────────────────────
|
||||
|
||||
class ScaleSelectionRequest(BaseModel):
|
||||
image: str # base64 full canvas
|
||||
mask: str # base64 selection mask (white = object)
|
||||
scale_pct: float = 103.0 # 103 = 3% bigger, 95 = 5% smaller
|
||||
|
||||
|
||||
class AiEditRegionRequest(BaseModel):
|
||||
image: str
|
||||
mask: str
|
||||
instruction: str
|
||||
negative_prompt: str = ""
|
||||
steps: int = 30
|
||||
cfg_scale: float = 7.5
|
||||
|
||||
|
||||
class PasteIntoSelectionRequest(BaseModel):
|
||||
image: str # base64 target canvas
|
||||
mask: str # base64 selection mask
|
||||
paste_image: str # base64 image to paste
|
||||
|
||||
|
||||
@router.post("/image/scale-selection")
|
||||
async def scale_selection(req: ScaleSelectionRequest):
|
||||
"""
|
||||
Scale the object selected by mask by scale_pct%, AI-fill the exposed gap.
|
||||
Works purely with local tools (LaMa/OpenCV) — no remote provider needed.
|
||||
"""
|
||||
try:
|
||||
import numpy as np
|
||||
from PIL import Image, ImageFilter
|
||||
except ImportError:
|
||||
raise HTTPException(status_code=500, detail="PIL/numpy not available")
|
||||
|
||||
img = Image.open(BytesIO(_decode(req.image))).convert("RGB")
|
||||
mask = Image.open(BytesIO(_decode(req.mask))).convert("L")
|
||||
if img.size != mask.size:
|
||||
mask = mask.resize(img.size, Image.LANCZOS)
|
||||
|
||||
mask_arr = np.array(mask)
|
||||
ys, xs = np.where(mask_arr > 128)
|
||||
if len(xs) == 0:
|
||||
raise HTTPException(status_code=400, detail="Empty mask — nothing to scale")
|
||||
|
||||
minx, maxx = int(xs.min()), int(xs.max())
|
||||
miny, maxy = int(ys.min()), int(ys.max())
|
||||
cx, cy = (minx + maxx) / 2.0, (miny + maxy) / 2.0
|
||||
obj_w, obj_h = maxx - minx + 1, maxy - miny + 1
|
||||
|
||||
scale = req.scale_pct / 100.0
|
||||
new_w = max(1, round(obj_w * scale))
|
||||
new_h = max(1, round(obj_h * scale))
|
||||
|
||||
# Extract masked object crop (RGBA with mask as alpha)
|
||||
img_rgba = img.convert("RGBA")
|
||||
obj_crop = img_rgba.crop((minx, miny, maxx + 1, maxy + 1))
|
||||
mask_crop = mask.crop((minx, miny, maxx + 1, maxy + 1))
|
||||
r, g, b, _ = obj_crop.split()
|
||||
obj_masked = Image.merge("RGBA", (r, g, b, mask_crop))
|
||||
scaled_obj = obj_masked.resize((new_w, new_h), Image.LANCZOS)
|
||||
|
||||
# AI-fill the original mask area (gap) with LaMa/OpenCV
|
||||
gap_mask = mask.filter(ImageFilter.MaxFilter(9)) # expand ~4px for clean seam
|
||||
gap_bytes = BytesIO()
|
||||
img.save(gap_bytes, format="PNG")
|
||||
gap_mask_bytes = BytesIO()
|
||||
gap_mask.save(gap_mask_bytes, format="PNG")
|
||||
|
||||
try:
|
||||
if lama_available():
|
||||
filled_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, lama_inpaint, gap_bytes.getvalue(), gap_mask_bytes.getvalue()
|
||||
)
|
||||
else:
|
||||
filled_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, opencv_inpaint, gap_bytes.getvalue(), gap_mask_bytes.getvalue()
|
||||
)
|
||||
filled = Image.open(BytesIO(filled_bytes)).convert("RGBA")
|
||||
except Exception as exc:
|
||||
print(f"[scale-selection] fill fallback: {exc}")
|
||||
filled = img.convert("RGBA")
|
||||
|
||||
# Paste scaled object centered on original centroid
|
||||
px = round(cx - new_w / 2)
|
||||
py = round(cy - new_h / 2)
|
||||
result = filled.copy()
|
||||
result.paste(scaled_obj, (px, py), scaled_obj.split()[3])
|
||||
|
||||
out = BytesIO()
|
||||
result.convert("RGB").save(out, format="PNG")
|
||||
return {"result": _encode(out.getvalue())}
|
||||
|
||||
|
||||
@router.post("/image/ai-edit-region")
|
||||
async def ai_edit_region(req: AiEditRegionRequest):
|
||||
"""
|
||||
AI-edit the selected region using the configured inpaint provider.
|
||||
Works with local_gpu, InvokeAI, ComfyUI, or OpenAI.
|
||||
"""
|
||||
provider = _require_remote("inpaint")
|
||||
result_bytes = await provider.inpaint(
|
||||
_decode(req.image),
|
||||
_decode(req.mask),
|
||||
req.instruction,
|
||||
{"negative_prompt": req.negative_prompt, "steps": req.steps, "cfg_scale": req.cfg_scale},
|
||||
)
|
||||
return {"result": _encode(result_bytes)}
|
||||
|
||||
|
||||
@router.post("/image/paste-into-selection")
|
||||
async def paste_into_selection(req: PasteIntoSelectionRequest):
|
||||
"""
|
||||
Scale a clipboard image to the selection bounding box, mask it to the
|
||||
selection shape, and composite it over the original canvas.
|
||||
"""
|
||||
try:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
raise HTTPException(status_code=500, detail="PIL/numpy not available")
|
||||
|
||||
img = Image.open(BytesIO(_decode(req.image))).convert("RGBA")
|
||||
mask = Image.open(BytesIO(_decode(req.mask))).convert("L")
|
||||
paste_img = Image.open(BytesIO(_decode(req.paste_image))).convert("RGBA")
|
||||
|
||||
if img.size != mask.size:
|
||||
mask = mask.resize(img.size, Image.LANCZOS)
|
||||
|
||||
mask_arr = np.array(mask)
|
||||
ys, xs = np.where(mask_arr > 128)
|
||||
if len(xs) == 0:
|
||||
raise HTTPException(status_code=400, detail="Empty mask")
|
||||
|
||||
minx, maxx = int(xs.min()), int(xs.max())
|
||||
miny, maxy = int(ys.min()), int(ys.max())
|
||||
target_w = maxx - minx + 1
|
||||
target_h = maxy - miny + 1
|
||||
|
||||
# Scale clipboard image to fit the selection bounding box
|
||||
paste_scaled = paste_img.resize((target_w, target_h), Image.LANCZOS)
|
||||
|
||||
# Clip paste to selection shape using mask
|
||||
mask_crop = mask.crop((minx, miny, maxx + 1, maxy + 1))
|
||||
r, g, b, a = paste_scaled.split()
|
||||
mask_np = np.array(mask_crop)
|
||||
alpha_np = np.array(a)
|
||||
combined = (alpha_np.astype(np.uint16) * mask_np.astype(np.uint16) // 255).astype(np.uint8)
|
||||
paste_final = Image.merge("RGBA", (r, g, b, Image.fromarray(combined)))
|
||||
|
||||
result = img.copy()
|
||||
result.paste(paste_final, (minx, miny), paste_final.split()[3])
|
||||
|
||||
out = BytesIO()
|
||||
result.convert("RGB").save(out, format="PNG")
|
||||
return {"result": _encode(out.getvalue())}
|
||||
|
||||
|
||||
# ─── SAM (Segment Anything) ──────────────────────────────────────────────────
|
||||
|
||||
class SegmentPointRequest(BaseModel):
|
||||
|
||||
Reference in New Issue
Block a user