Add SAM Smart Select and AI Inpaint tools to miniPaint
New features: - Smart Select tool: Click to select objects using SAM (Segment Anything) - AI Inpaint tool: Edit selected regions with text prompts Changes: - frontend/src/js/tools/smart_select.js: SAM-powered selection tool - frontend/src/js/tools/ai_inpaint.js: AI inpainting with prompt dialog - frontend/src/js/services/api.js: API service for backend communication - frontend/src/js/config.js: Register new tools - frontend/src/css/layout.css: Tool icon styles - frontend/images/icons/: SVG icons for new tools - backend/app/routers/tools.py: New base64 API endpoints - frontend/Dockerfile: Updated for miniPaint build - frontend/nginx.conf: Added /api prefix proxy
This commit is contained in:
Claude
@@ -8,6 +8,7 @@ import numpy as np
|
||||
import json
|
||||
import base64
|
||||
import cv2
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.project import Project
|
||||
@@ -16,6 +17,115 @@ from app.schemas import StatusResponse
|
||||
router = APIRouter(prefix="/tools", tags=["tools"])
|
||||
|
||||
|
||||
# Pydantic models for JSON API
|
||||
class SmartSelectRequest(BaseModel):
|
||||
image: str # Base64 encoded image
|
||||
point_x: int
|
||||
point_y: int
|
||||
|
||||
|
||||
class InpaintRequest(BaseModel):
|
||||
image: str # Base64 encoded image
|
||||
mask: str # Base64 encoded mask
|
||||
prompt: str
|
||||
negative_prompt: Optional[str] = ""
|
||||
strength: Optional[float] = 0.8
|
||||
guidance_scale: Optional[float] = 7.5
|
||||
|
||||
|
||||
@router.post("/smart-select-base64")
|
||||
async def smart_select_base64(request: SmartSelectRequest):
|
||||
"""
|
||||
Smart select using base64 encoded image (no project required).
|
||||
Used by miniPaint frontend.
|
||||
"""
|
||||
try:
|
||||
# Decode base64 image
|
||||
image_bytes = base64.b64decode(request.image)
|
||||
img = Image.open(BytesIO(image_bytes)).convert('RGB')
|
||||
img_array = np.array(img)
|
||||
|
||||
# Run SAM selection
|
||||
try:
|
||||
mask = await _sam_select(img_array, request.point_x, request.point_y)
|
||||
except Exception as e:
|
||||
print(f"SAM not available, using flood fill: {e}")
|
||||
mask = _flood_fill_select(img_array, request.point_x, request.point_y)
|
||||
|
||||
# Convert mask to base64 PNG
|
||||
mask_img = Image.fromarray((mask * 255).astype(np.uint8), mode='L')
|
||||
buffer = BytesIO()
|
||||
mask_img.save(buffer, format='PNG')
|
||||
mask_b64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
|
||||
# Get polygon and bbox
|
||||
polygon, bbox = _mask_to_polygon(mask)
|
||||
|
||||
return {
|
||||
"mask": mask_b64,
|
||||
"polygon": polygon,
|
||||
"bbox": bbox
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/inpaint")
|
||||
async def inpaint_base64(request: InpaintRequest):
|
||||
"""
|
||||
AI inpainting using base64 encoded image and mask.
|
||||
Used by miniPaint frontend.
|
||||
"""
|
||||
try:
|
||||
# Decode base64 image and mask
|
||||
image_bytes = base64.b64decode(request.image)
|
||||
mask_bytes = base64.b64decode(request.mask)
|
||||
|
||||
img = Image.open(BytesIO(image_bytes)).convert('RGB')
|
||||
mask_img = Image.open(BytesIO(mask_bytes)).convert('L')
|
||||
|
||||
# Resize mask to match image if needed
|
||||
if mask_img.size != img.size:
|
||||
mask_img = mask_img.resize(img.size, Image.Resampling.LANCZOS)
|
||||
|
||||
# Get the AI provider and run inpainting
|
||||
from app.services.ai_provider import get_ai_provider
|
||||
from app.config import settings
|
||||
|
||||
provider = get_ai_provider()
|
||||
|
||||
# Convert images to format expected by provider
|
||||
img_buffer = BytesIO()
|
||||
img.save(img_buffer, format='PNG')
|
||||
img_buffer.seek(0)
|
||||
|
||||
mask_buffer = BytesIO()
|
||||
mask_img.save(mask_buffer, format='PNG')
|
||||
mask_buffer.seek(0)
|
||||
|
||||
# Run inpainting
|
||||
result_bytes = await provider.inpaint(
|
||||
image=img_buffer,
|
||||
mask=mask_buffer,
|
||||
prompt=request.prompt,
|
||||
negative_prompt=request.negative_prompt,
|
||||
strength=request.strength
|
||||
)
|
||||
|
||||
# Convert result to base64
|
||||
result_b64 = base64.b64encode(result_bytes).decode('utf-8')
|
||||
|
||||
return {
|
||||
"result": result_b64
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/remove-background")
|
||||
async def remove_background(
|
||||
project_id: Optional[int] = Form(None),
|
||||
|
||||
+8
-5
@@ -3,22 +3,25 @@ FROM node:20-alpine as build
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files
|
||||
COPY package.json ./
|
||||
COPY package.json package-lock.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm install
|
||||
RUN npm ci
|
||||
|
||||
# Copy source
|
||||
COPY . .
|
||||
|
||||
# Build app
|
||||
# Build webpack bundle
|
||||
RUN npm run build
|
||||
|
||||
# Production stage
|
||||
FROM nginx:alpine
|
||||
|
||||
# Copy built files
|
||||
COPY --from=build /app/dist /usr/share/nginx/html
|
||||
# Copy all static files needed by miniPaint
|
||||
COPY --from=build /app/index.html /usr/share/nginx/html/
|
||||
COPY --from=build /app/dist /usr/share/nginx/html/dist
|
||||
COPY --from=build /app/images /usr/share/nginx/html/images
|
||||
COPY --from=build /app/src/css /usr/share/nginx/html/src/css
|
||||
|
||||
# Copy nginx config
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<!-- Paint brush -->
|
||||
<path d="M18.37 2.63L14 7l-1.59-1.59a2 2 0 00-2.82 0L8 7l9 9 1.59-1.59a2 2 0 000-2.82L17 10l4.37-4.37a2.12 2.12 0 10-3-3z"/>
|
||||
<path d="M9 8c-2 3-4 3.5-7 4l8 10c2-1 6-5 6-7"/>
|
||||
<path d="M14.5 17.5L4.5 15"/>
|
||||
<!-- AI sparkle -->
|
||||
<circle cx="19" cy="19" r="1" fill="currentColor"/>
|
||||
<path d="M19 16v1"/>
|
||||
<path d="M19 21v1"/>
|
||||
<path d="M16 19h1"/>
|
||||
<path d="M21 19h1"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 567 B |
@@ -0,0 +1,15 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<!-- Magic wand with AI sparkle -->
|
||||
<path d="M15 4V2"/>
|
||||
<path d="M15 16v-2"/>
|
||||
<path d="M8 9h2"/>
|
||||
<path d="M20 9h2"/>
|
||||
<path d="M17.8 11.8L19 13"/>
|
||||
<path d="M15 9h0"/>
|
||||
<path d="M17.8 6.2L19 5"/>
|
||||
<path d="M12.2 6.2L11 5"/>
|
||||
<path d="M3 21l9-9"/>
|
||||
<path d="M12.2 11.8L11 13"/>
|
||||
<!-- Selection indicator -->
|
||||
<rect x="2" y="2" width="6" height="6" rx="1" stroke-dasharray="2 1"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 563 B |
@@ -70,6 +70,18 @@ server {
|
||||
proxy_pass http://backend:8000;
|
||||
}
|
||||
|
||||
# API prefix for frontend (maps /api/tools to /tools, etc.)
|
||||
location /api/ {
|
||||
rewrite ^/api/(.*)$ /$1 break;
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
location /docs {
|
||||
proxy_pass http://backend:8000;
|
||||
}
|
||||
|
||||
@@ -321,6 +321,8 @@ IMPORTANT: any new icon should also must be added on /service-worker.js + its ve
|
||||
.sidebar_left .desaturate:after{ background-image: url('images/icons/desaturate.svg'); }
|
||||
.sidebar_left .bulge_pinch:after{ background-image: url('images/icons/bulge_pinch.svg'); }
|
||||
.sidebar_left .animation:after{ background-image: url('images/icons/animation.svg'); }
|
||||
.sidebar_left .smart_select:after{ background-image: url('images/icons/smart_select.svg'); }
|
||||
.sidebar_left .ai_inpaint:after{ background-image: url('images/icons/ai_inpaint.svg'); }
|
||||
|
||||
@media screen and (max-width:550px){
|
||||
#sidebar_left{
|
||||
|
||||
@@ -92,6 +92,17 @@ config.TOOLS = [
|
||||
attributes: {},
|
||||
on_leave: 'on_leave',
|
||||
},
|
||||
{
|
||||
name: 'smart_select',
|
||||
title: 'Smart Select (AI)',
|
||||
attributes: {},
|
||||
},
|
||||
{
|
||||
name: 'ai_inpaint',
|
||||
title: 'AI Inpaint',
|
||||
on_activate: 'on_activate',
|
||||
attributes: {},
|
||||
},
|
||||
{
|
||||
name: 'brush',
|
||||
attributes: {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* API Service for communicating with the FastAPI backend
|
||||
* Handles SAM selection and AI inpainting requests
|
||||
*/
|
||||
|
||||
class ApiService {
|
||||
constructor() {
|
||||
// Backend API base URL - adjust for your deployment
|
||||
this.baseUrl = window.API_BASE_URL || '/api';
|
||||
}
|
||||
|
||||
/**
|
||||
* Call SAM (Segment Anything Model) for smart selection
|
||||
* @param {string} imageData - Base64 encoded image data
|
||||
* @param {number} pointX - X coordinate of click point
|
||||
* @param {number} pointY - Y coordinate of click point
|
||||
* @returns {Promise<{mask: ImageData, polygon: Array}>}
|
||||
*/
|
||||
async smartSelect(imageData, pointX, pointY) {
|
||||
const response = await fetch(`${this.baseUrl}/tools/smart-select-base64`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
image: imageData,
|
||||
point_x: pointX,
|
||||
point_y: pointY,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: 'Unknown error' }));
|
||||
throw new Error(error.detail || `SAM request failed: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Call AI inpainting to edit a selected region
|
||||
* @param {string} imageData - Base64 encoded image data
|
||||
* @param {string} maskData - Base64 encoded mask data (white = area to edit)
|
||||
* @param {string} prompt - Text prompt describing desired edit
|
||||
* @param {Object} options - Additional options
|
||||
* @returns {Promise<{result: string}>} - Base64 encoded result image
|
||||
*/
|
||||
async inpaint(imageData, maskData, prompt, options = {}) {
|
||||
const response = await fetch(`${this.baseUrl}/tools/inpaint`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
image: imageData,
|
||||
mask: maskData,
|
||||
prompt: prompt,
|
||||
negative_prompt: options.negativePrompt || '',
|
||||
strength: options.strength || 0.8,
|
||||
guidance_scale: options.guidanceScale || 7.5,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: 'Unknown error' }));
|
||||
throw new Error(error.detail || `Inpaint request failed: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Health check for the backend
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async healthCheck() {
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/health`);
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
const apiService = new ApiService();
|
||||
export default apiService;
|
||||
@@ -0,0 +1,315 @@
|
||||
/**
|
||||
* AI Inpaint Tool - Edit selected regions using AI with text prompts
|
||||
* Works with Smart Select tool's mask or manual selection
|
||||
*/
|
||||
|
||||
import app from './../app.js';
|
||||
import config from './../config.js';
|
||||
import Base_tools_class from './../core/base-tools.js';
|
||||
import Base_layers_class from './../core/base-layers.js';
|
||||
import Helper_class from './../libs/helpers.js';
|
||||
import Dialog_class from './../libs/popup.js';
|
||||
import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js';
|
||||
import apiService from './../services/api.js';
|
||||
|
||||
class Ai_inpaint_class extends Base_tools_class {
|
||||
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.Base_layers = new Base_layers_class();
|
||||
this.Helper = new Helper_class();
|
||||
this.POP = new Dialog_class();
|
||||
this.ctx = ctx;
|
||||
this.name = 'ai_inpaint';
|
||||
this.isProcessing = false;
|
||||
}
|
||||
|
||||
load() {
|
||||
// No mouse events needed - this tool uses a dialog
|
||||
}
|
||||
|
||||
on_activate() {
|
||||
this.showInpaintDialog();
|
||||
}
|
||||
|
||||
/**
|
||||
* Show the inpainting dialog
|
||||
*/
|
||||
showInpaintDialog() {
|
||||
var _this = this;
|
||||
|
||||
// Check if we have a selection
|
||||
var hasMask = window.smartSelectMask != null;
|
||||
var hasRectSelection = this.getRectSelection() != null;
|
||||
|
||||
if (!hasMask && !hasRectSelection) {
|
||||
alertify.warning('No selection found. Use Smart Select or Selection tool first.');
|
||||
return;
|
||||
}
|
||||
|
||||
var settings = {
|
||||
title: 'AI Inpaint',
|
||||
params: [
|
||||
{
|
||||
name: "prompt",
|
||||
title: "Describe what you want:",
|
||||
type: "textarea",
|
||||
value: "",
|
||||
placeholder: "e.g., 'a red rose', 'remove the object', 'blue sky with clouds'"
|
||||
},
|
||||
{
|
||||
name: "negative_prompt",
|
||||
title: "What to avoid (optional):",
|
||||
value: "",
|
||||
placeholder: "e.g., 'blurry, distorted, low quality'"
|
||||
},
|
||||
{
|
||||
name: "strength",
|
||||
title: "Edit Strength:",
|
||||
type: "range",
|
||||
value: 80,
|
||||
range: [1, 100],
|
||||
step: 1
|
||||
}
|
||||
],
|
||||
on_finish: async function (params) {
|
||||
await _this.executeInpaint(params);
|
||||
},
|
||||
};
|
||||
|
||||
this.POP.show(settings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the inpainting operation
|
||||
*/
|
||||
async executeInpaint(params) {
|
||||
if (this.isProcessing) {
|
||||
alertify.warning('Already processing... please wait');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!params.prompt || params.prompt.trim() === '') {
|
||||
alertify.error('Please enter a prompt describing what you want');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we have an image layer
|
||||
if (config.layer.type != 'image') {
|
||||
alertify.error('Please select an image layer');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isProcessing = true;
|
||||
alertify.message('AI is generating... this may take a moment');
|
||||
|
||||
try {
|
||||
// Get image data
|
||||
var imageData = this.getLayerImageData();
|
||||
|
||||
// Get mask data (from Smart Select or rectangular selection)
|
||||
var maskData = this.getMaskData();
|
||||
|
||||
if (!maskData) {
|
||||
throw new Error('No valid selection/mask found');
|
||||
}
|
||||
|
||||
// Call inpaint API
|
||||
var result = await apiService.inpaint(
|
||||
imageData,
|
||||
maskData,
|
||||
params.prompt,
|
||||
{
|
||||
negativePrompt: params.negative_prompt || '',
|
||||
strength: params.strength / 100
|
||||
}
|
||||
);
|
||||
|
||||
// Apply result to layer
|
||||
await this.applyResult(result.result);
|
||||
|
||||
alertify.success('Inpainting complete!');
|
||||
|
||||
} catch (error) {
|
||||
console.error('Inpaint error:', error);
|
||||
alertify.error('Inpainting failed: ' + error.message);
|
||||
} finally {
|
||||
this.isProcessing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current layer's image data as base64
|
||||
*/
|
||||
getLayerImageData() {
|
||||
var canvas = document.createElement('canvas');
|
||||
var ctx = canvas.getContext('2d');
|
||||
|
||||
canvas.width = config.layer.width_original;
|
||||
canvas.height = config.layer.height_original;
|
||||
ctx.drawImage(config.layer.link, 0, 0);
|
||||
|
||||
return canvas.toDataURL('image/png').split(',')[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get mask data - either from Smart Select or rectangular selection
|
||||
*/
|
||||
getMaskData() {
|
||||
// First try Smart Select mask
|
||||
if (window.smartSelectMask && window.smartSelectMask.canvas) {
|
||||
var maskCanvas = window.smartSelectMask.canvas;
|
||||
return maskCanvas.toDataURL('image/png').split(',')[1];
|
||||
}
|
||||
|
||||
// Fall back to rectangular selection
|
||||
var selection = this.getRectSelection();
|
||||
if (selection) {
|
||||
return this.createRectMask(selection);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rectangular selection from miniPaint's selection tool
|
||||
*/
|
||||
getRectSelection() {
|
||||
// Try to get selection from selection tool
|
||||
var Selection = null;
|
||||
try {
|
||||
var GUI_tools = app.GUI?.GUI_tools || this.Base_layers?.Base_gui?.GUI_tools;
|
||||
if (GUI_tools && GUI_tools.tools_modules && GUI_tools.tools_modules.selection) {
|
||||
Selection = GUI_tools.tools_modules.selection.object;
|
||||
}
|
||||
} catch (e) {
|
||||
// Selection tool not available
|
||||
}
|
||||
|
||||
if (Selection && Selection.selection &&
|
||||
Selection.selection.width > 0 && Selection.selection.height > 0) {
|
||||
return Selection.selection;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a white rectangle mask from selection coordinates
|
||||
*/
|
||||
createRectMask(selection) {
|
||||
var canvas = document.createElement('canvas');
|
||||
var ctx = canvas.getContext('2d');
|
||||
|
||||
canvas.width = config.layer.width_original;
|
||||
canvas.height = config.layer.height_original;
|
||||
|
||||
// Fill with black (unselected)
|
||||
ctx.fillStyle = '#000000';
|
||||
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
// Calculate selection position relative to layer
|
||||
var x = selection.x - config.layer.x;
|
||||
var y = selection.y - config.layer.y;
|
||||
var width = selection.width;
|
||||
var height = selection.height;
|
||||
|
||||
// Scale to original image size
|
||||
var scaleX = config.layer.width_original / config.layer.width;
|
||||
var scaleY = config.layer.height_original / config.layer.height;
|
||||
|
||||
x = x * scaleX;
|
||||
y = y * scaleY;
|
||||
width = width * scaleX;
|
||||
height = height * scaleY;
|
||||
|
||||
// Draw white rectangle (selected area)
|
||||
ctx.fillStyle = '#FFFFFF';
|
||||
ctx.fillRect(x, y, width, height);
|
||||
|
||||
return canvas.toDataURL('image/png').split(',')[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the inpainted result to the current layer
|
||||
*/
|
||||
async applyResult(resultBase64) {
|
||||
var _this = this;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
var img = new Image();
|
||||
img.onload = function() {
|
||||
// Create canvas with result
|
||||
var canvas = document.createElement('canvas');
|
||||
canvas.width = config.layer.width_original;
|
||||
canvas.height = config.layer.height_original;
|
||||
var ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
|
||||
|
||||
// Update layer through action system for undo support
|
||||
app.State.do_action(
|
||||
new app.Actions.Bundle_action('ai_inpaint', 'AI Inpaint', [
|
||||
new app.Actions.Update_layer_image_action(canvas)
|
||||
])
|
||||
);
|
||||
|
||||
// Clear the smart select mask
|
||||
window.smartSelectMask = null;
|
||||
|
||||
config.need_render = true;
|
||||
resolve();
|
||||
};
|
||||
img.onerror = function() {
|
||||
reject(new Error('Failed to load result image'));
|
||||
};
|
||||
img.src = 'data:image/png;base64,' + resultBase64;
|
||||
});
|
||||
}
|
||||
|
||||
render_overlay(ctx) {
|
||||
// Show visual indicator if there's a selection ready for inpainting
|
||||
if (window.smartSelectMask && window.smartSelectMask.canvas) {
|
||||
// Draw a subtle border around the tool indicating mask is ready
|
||||
ctx.save();
|
||||
ctx.strokeStyle = '#00ff00';
|
||||
ctx.lineWidth = 2;
|
||||
ctx.setLineDash([5, 5]);
|
||||
|
||||
var scaleX = config.layer.width / config.layer.width_original;
|
||||
var scaleY = config.layer.height / config.layer.height_original;
|
||||
|
||||
// Get mask bounds
|
||||
var maskCanvas = window.smartSelectMask.canvas;
|
||||
var maskCtx = maskCanvas.getContext('2d');
|
||||
var imageData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
|
||||
|
||||
var minX = maskCanvas.width, minY = maskCanvas.height;
|
||||
var maxX = 0, maxY = 0;
|
||||
|
||||
for (var y = 0; y < maskCanvas.height; y += 4) { // Sample every 4th pixel for speed
|
||||
for (var x = 0; x < maskCanvas.width; x += 4) {
|
||||
var i = (y * maskCanvas.width + x) * 4;
|
||||
if (imageData.data[i] > 128) {
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x);
|
||||
maxY = Math.max(maxY, y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (maxX > minX && maxY > minY) {
|
||||
ctx.strokeRect(
|
||||
config.layer.x + minX * scaleX,
|
||||
config.layer.y + minY * scaleY,
|
||||
(maxX - minX) * scaleX,
|
||||
(maxY - minY) * scaleY
|
||||
);
|
||||
}
|
||||
|
||||
ctx.restore();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default Ai_inpaint_class;
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Smart Select Tool - Uses SAM (Segment Anything Model) for AI-powered selection
|
||||
* Click on any object to automatically select it
|
||||
*/
|
||||
|
||||
import app from './../app.js';
|
||||
import config from './../config.js';
|
||||
import Base_tools_class from './../core/base-tools.js';
|
||||
import Base_layers_class from './../core/base-layers.js';
|
||||
import Helper_class from './../libs/helpers.js';
|
||||
import Dialog_class from './../libs/popup.js';
|
||||
import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js';
|
||||
import apiService from './../services/api.js';
|
||||
|
||||
class Smart_select_class extends Base_tools_class {
|
||||
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.Base_layers = new Base_layers_class();
|
||||
this.Helper = new Helper_class();
|
||||
this.POP = new Dialog_class();
|
||||
this.ctx = ctx;
|
||||
this.name = 'smart_select';
|
||||
|
||||
// Store the current mask data
|
||||
this.currentMask = null;
|
||||
this.maskCanvas = null;
|
||||
this.isProcessing = false;
|
||||
}
|
||||
|
||||
load() {
|
||||
var _this = this;
|
||||
|
||||
// Mouse click event for selection
|
||||
document.addEventListener('mousedown', function (e) {
|
||||
_this.mousedown(e);
|
||||
});
|
||||
}
|
||||
|
||||
async mousedown(e) {
|
||||
var mouse = this.get_mouse_info(e);
|
||||
|
||||
if (config.TOOL.name != this.name) return;
|
||||
if (mouse.click_valid == false) return;
|
||||
if (this.isProcessing) {
|
||||
alertify.warning('Processing... please wait');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we have an image layer
|
||||
if (config.layer.type != 'image') {
|
||||
alertify.error('Please select an image layer first');
|
||||
return;
|
||||
}
|
||||
|
||||
// Get click coordinates relative to the image
|
||||
var x = mouse.x - config.layer.x;
|
||||
var y = mouse.y - config.layer.y;
|
||||
|
||||
// Adjust for layer scaling
|
||||
if (config.layer.width != config.layer.width_original) {
|
||||
x = x * (config.layer.width_original / config.layer.width);
|
||||
}
|
||||
if (config.layer.height != config.layer.height_original) {
|
||||
y = y * (config.layer.height_original / config.layer.height);
|
||||
}
|
||||
|
||||
// Make sure click is within image bounds
|
||||
if (x < 0 || y < 0 || x > config.layer.width_original || y > config.layer.height_original) {
|
||||
alertify.error('Click inside the image');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isProcessing = true;
|
||||
alertify.message('AI is analyzing the image...');
|
||||
|
||||
try {
|
||||
// Get image data as base64
|
||||
var imageData = this.getLayerImageData();
|
||||
|
||||
// Call SAM API
|
||||
var result = await apiService.smartSelect(imageData, Math.round(x), Math.round(y));
|
||||
|
||||
// Apply the mask as selection
|
||||
this.applyMask(result.mask, result.bbox);
|
||||
|
||||
alertify.success('Selection complete! Use AI Inpaint to edit.');
|
||||
|
||||
} catch (error) {
|
||||
console.error('Smart select error:', error);
|
||||
alertify.error('Selection failed: ' + error.message);
|
||||
} finally {
|
||||
this.isProcessing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current layer's image data as base64
|
||||
*/
|
||||
getLayerImageData() {
|
||||
var canvas = document.createElement('canvas');
|
||||
var ctx = canvas.getContext('2d');
|
||||
|
||||
canvas.width = config.layer.width_original;
|
||||
canvas.height = config.layer.height_original;
|
||||
|
||||
// Draw the layer's image
|
||||
ctx.drawImage(config.layer.link, 0, 0);
|
||||
|
||||
// Return as base64 (remove data:image/png;base64, prefix)
|
||||
return canvas.toDataURL('image/png').split(',')[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the SAM mask as a selection
|
||||
* @param {string} maskBase64 - Base64 encoded mask image
|
||||
* @param {Object} bbox - Bounding box {x, y, width, height}
|
||||
*/
|
||||
applyMask(maskBase64, bbox) {
|
||||
var _this = this;
|
||||
|
||||
// Create mask image
|
||||
var maskImage = new Image();
|
||||
maskImage.onload = function() {
|
||||
// Store mask for later use by inpaint tool
|
||||
_this.maskCanvas = document.createElement('canvas');
|
||||
_this.maskCanvas.width = config.layer.width_original;
|
||||
_this.maskCanvas.height = config.layer.height_original;
|
||||
var maskCtx = _this.maskCanvas.getContext('2d');
|
||||
maskCtx.drawImage(maskImage, 0, 0);
|
||||
|
||||
_this.currentMask = {
|
||||
canvas: _this.maskCanvas,
|
||||
bbox: bbox
|
||||
};
|
||||
|
||||
// Store globally for AI inpaint tool to access
|
||||
window.smartSelectMask = _this.currentMask;
|
||||
|
||||
// Visual feedback - render mask overlay
|
||||
_this.renderMaskOverlay();
|
||||
|
||||
config.need_render = true;
|
||||
};
|
||||
maskImage.src = 'data:image/png;base64,' + maskBase64;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a visual overlay showing the selected region
|
||||
*/
|
||||
renderMaskOverlay() {
|
||||
if (!this.maskCanvas) return;
|
||||
|
||||
// Create overlay layer or update existing
|
||||
// For now, we'll use the selection system
|
||||
var maskCtx = this.maskCanvas.getContext('2d');
|
||||
var imageData = maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
|
||||
|
||||
// Find bounding box of selection
|
||||
var minX = this.maskCanvas.width, minY = this.maskCanvas.height;
|
||||
var maxX = 0, maxY = 0;
|
||||
|
||||
for (var y = 0; y < this.maskCanvas.height; y++) {
|
||||
for (var x = 0; x < this.maskCanvas.width; x++) {
|
||||
var i = (y * this.maskCanvas.width + x) * 4;
|
||||
if (imageData.data[i] > 128) { // White pixel in mask
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x);
|
||||
maxY = Math.max(maxY, y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (maxX > minX && maxY > minY) {
|
||||
// Set selection using miniPaint's selection system
|
||||
var Selection = this.Base_layers.Base_gui?.GUI_tools?.tools_modules?.selection?.object;
|
||||
if (Selection) {
|
||||
Selection.selection = {
|
||||
x: config.layer.x + minX * (config.layer.width / config.layer.width_original),
|
||||
y: config.layer.y + minY * (config.layer.height / config.layer.height_original),
|
||||
width: (maxX - minX) * (config.layer.width / config.layer.width_original),
|
||||
height: (maxY - minY) * (config.layer.height / config.layer.height_original)
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
render_overlay(ctx) {
|
||||
// Render marching ants or highlight around selected region
|
||||
if (!this.currentMask || !this.maskCanvas) return;
|
||||
|
||||
var mouse = this.get_mouse_info(event);
|
||||
|
||||
// Draw semi-transparent overlay on non-selected areas
|
||||
ctx.save();
|
||||
ctx.globalAlpha = 0.3;
|
||||
ctx.fillStyle = '#000000';
|
||||
|
||||
// Scale to match layer
|
||||
var scaleX = config.layer.width / config.layer.width_original;
|
||||
var scaleY = config.layer.height / config.layer.height_original;
|
||||
|
||||
ctx.translate(config.layer.x, config.layer.y);
|
||||
ctx.scale(scaleX, scaleY);
|
||||
|
||||
// Draw inverse mask (darken unselected areas)
|
||||
var maskCtx = this.maskCanvas.getContext('2d');
|
||||
var imageData = maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
|
||||
|
||||
// Create inverse mask canvas
|
||||
var inverseCanvas = document.createElement('canvas');
|
||||
inverseCanvas.width = this.maskCanvas.width;
|
||||
inverseCanvas.height = this.maskCanvas.height;
|
||||
var inverseCtx = inverseCanvas.getContext('2d');
|
||||
|
||||
inverseCtx.fillStyle = '#000000';
|
||||
inverseCtx.fillRect(0, 0, inverseCanvas.width, inverseCanvas.height);
|
||||
inverseCtx.globalCompositeOperation = 'destination-out';
|
||||
inverseCtx.drawImage(this.maskCanvas, 0, 0);
|
||||
|
||||
ctx.drawImage(inverseCanvas, 0, 0);
|
||||
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the current selection
|
||||
*/
|
||||
clearSelection() {
|
||||
this.currentMask = null;
|
||||
this.maskCanvas = null;
|
||||
window.smartSelectMask = null;
|
||||
config.need_render = true;
|
||||
}
|
||||
|
||||
on_leave() {
|
||||
// Don't clear mask when switching tools - AI inpaint needs it
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export default Smart_select_class;
|
||||
Reference in New Issue
Block a user