Merge pull request #23 from outis1one/claude/migrate-to-minipaint-eYKWf
Add SAM Smart Select and AI Inpaint tools to miniPaint
This commit is contained in:
@@ -8,6 +8,7 @@ import numpy as np
|
|||||||
import json
|
import json
|
||||||
import base64
|
import base64
|
||||||
import cv2
|
import cv2
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.models.project import Project
|
from app.models.project import Project
|
||||||
@@ -16,6 +17,115 @@ from app.schemas import StatusResponse
|
|||||||
router = APIRouter(prefix="/tools", tags=["tools"])
|
router = APIRouter(prefix="/tools", tags=["tools"])
|
||||||
|
|
||||||
|
|
||||||
|
# Pydantic models for JSON API
|
||||||
|
class SmartSelectRequest(BaseModel):
|
||||||
|
image: str # Base64 encoded image
|
||||||
|
point_x: int
|
||||||
|
point_y: int
|
||||||
|
|
||||||
|
|
||||||
|
class InpaintRequest(BaseModel):
|
||||||
|
image: str # Base64 encoded image
|
||||||
|
mask: str # Base64 encoded mask
|
||||||
|
prompt: str
|
||||||
|
negative_prompt: Optional[str] = ""
|
||||||
|
strength: Optional[float] = 0.8
|
||||||
|
guidance_scale: Optional[float] = 7.5
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/smart-select-base64")
|
||||||
|
async def smart_select_base64(request: SmartSelectRequest):
|
||||||
|
"""
|
||||||
|
Smart select using base64 encoded image (no project required).
|
||||||
|
Used by miniPaint frontend.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Decode base64 image
|
||||||
|
image_bytes = base64.b64decode(request.image)
|
||||||
|
img = Image.open(BytesIO(image_bytes)).convert('RGB')
|
||||||
|
img_array = np.array(img)
|
||||||
|
|
||||||
|
# Run SAM selection
|
||||||
|
try:
|
||||||
|
mask = await _sam_select(img_array, request.point_x, request.point_y)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"SAM not available, using flood fill: {e}")
|
||||||
|
mask = _flood_fill_select(img_array, request.point_x, request.point_y)
|
||||||
|
|
||||||
|
# Convert mask to base64 PNG
|
||||||
|
mask_img = Image.fromarray((mask * 255).astype(np.uint8), mode='L')
|
||||||
|
buffer = BytesIO()
|
||||||
|
mask_img.save(buffer, format='PNG')
|
||||||
|
mask_b64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||||
|
|
||||||
|
# Get polygon and bbox
|
||||||
|
polygon, bbox = _mask_to_polygon(mask)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"mask": mask_b64,
|
||||||
|
"polygon": polygon,
|
||||||
|
"bbox": bbox
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/inpaint")
|
||||||
|
async def inpaint_base64(request: InpaintRequest):
|
||||||
|
"""
|
||||||
|
AI inpainting using base64 encoded image and mask.
|
||||||
|
Used by miniPaint frontend.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Decode base64 image and mask
|
||||||
|
image_bytes = base64.b64decode(request.image)
|
||||||
|
mask_bytes = base64.b64decode(request.mask)
|
||||||
|
|
||||||
|
img = Image.open(BytesIO(image_bytes)).convert('RGB')
|
||||||
|
mask_img = Image.open(BytesIO(mask_bytes)).convert('L')
|
||||||
|
|
||||||
|
# Resize mask to match image if needed
|
||||||
|
if mask_img.size != img.size:
|
||||||
|
mask_img = mask_img.resize(img.size, Image.Resampling.LANCZOS)
|
||||||
|
|
||||||
|
# Get the AI provider and run inpainting
|
||||||
|
from app.services.ai_provider import get_ai_provider
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
provider = get_ai_provider()
|
||||||
|
|
||||||
|
# Convert images to format expected by provider
|
||||||
|
img_buffer = BytesIO()
|
||||||
|
img.save(img_buffer, format='PNG')
|
||||||
|
img_buffer.seek(0)
|
||||||
|
|
||||||
|
mask_buffer = BytesIO()
|
||||||
|
mask_img.save(mask_buffer, format='PNG')
|
||||||
|
mask_buffer.seek(0)
|
||||||
|
|
||||||
|
# Run inpainting
|
||||||
|
result_bytes = await provider.inpaint(
|
||||||
|
image=img_buffer,
|
||||||
|
mask=mask_buffer,
|
||||||
|
prompt=request.prompt,
|
||||||
|
negative_prompt=request.negative_prompt,
|
||||||
|
strength=request.strength
|
||||||
|
)
|
||||||
|
|
||||||
|
# Convert result to base64
|
||||||
|
result_b64 = base64.b64encode(result_bytes).decode('utf-8')
|
||||||
|
|
||||||
|
return {
|
||||||
|
"result": result_b64
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
@router.post("/remove-background")
|
@router.post("/remove-background")
|
||||||
async def remove_background(
|
async def remove_background(
|
||||||
project_id: Optional[int] = Form(None),
|
project_id: Optional[int] = Form(None),
|
||||||
|
|||||||
+8
-5
@@ -3,22 +3,25 @@ FROM node:20-alpine as build
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Copy package files
|
# Copy package files
|
||||||
COPY package.json ./
|
COPY package.json package-lock.json ./
|
||||||
|
|
||||||
# Install dependencies
|
# Install dependencies
|
||||||
RUN npm install
|
RUN npm ci
|
||||||
|
|
||||||
# Copy source
|
# Copy source
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Build app
|
# Build webpack bundle
|
||||||
RUN npm run build
|
RUN npm run build
|
||||||
|
|
||||||
# Production stage
|
# Production stage
|
||||||
FROM nginx:alpine
|
FROM nginx:alpine
|
||||||
|
|
||||||
# Copy built files
|
# Copy all static files needed by miniPaint
|
||||||
COPY --from=build /app/dist /usr/share/nginx/html
|
COPY --from=build /app/index.html /usr/share/nginx/html/
|
||||||
|
COPY --from=build /app/dist /usr/share/nginx/html/dist
|
||||||
|
COPY --from=build /app/images /usr/share/nginx/html/images
|
||||||
|
COPY --from=build /app/src/css /usr/share/nginx/html/src/css
|
||||||
|
|
||||||
# Copy nginx config
|
# Copy nginx config
|
||||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||||
|
<!-- Paint brush -->
|
||||||
|
<path d="M18.37 2.63L14 7l-1.59-1.59a2 2 0 00-2.82 0L8 7l9 9 1.59-1.59a2 2 0 000-2.82L17 10l4.37-4.37a2.12 2.12 0 10-3-3z"/>
|
||||||
|
<path d="M9 8c-2 3-4 3.5-7 4l8 10c2-1 6-5 6-7"/>
|
||||||
|
<path d="M14.5 17.5L4.5 15"/>
|
||||||
|
<!-- AI sparkle -->
|
||||||
|
<circle cx="19" cy="19" r="1" fill="currentColor"/>
|
||||||
|
<path d="M19 16v1"/>
|
||||||
|
<path d="M19 21v1"/>
|
||||||
|
<path d="M16 19h1"/>
|
||||||
|
<path d="M21 19h1"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 567 B |
@@ -0,0 +1,15 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||||
|
<!-- Magic wand with AI sparkle -->
|
||||||
|
<path d="M15 4V2"/>
|
||||||
|
<path d="M15 16v-2"/>
|
||||||
|
<path d="M8 9h2"/>
|
||||||
|
<path d="M20 9h2"/>
|
||||||
|
<path d="M17.8 11.8L19 13"/>
|
||||||
|
<path d="M15 9h0"/>
|
||||||
|
<path d="M17.8 6.2L19 5"/>
|
||||||
|
<path d="M12.2 6.2L11 5"/>
|
||||||
|
<path d="M3 21l9-9"/>
|
||||||
|
<path d="M12.2 11.8L11 13"/>
|
||||||
|
<!-- Selection indicator -->
|
||||||
|
<rect x="2" y="2" width="6" height="6" rx="1" stroke-dasharray="2 1"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 563 B |
@@ -70,6 +70,18 @@ server {
|
|||||||
proxy_pass http://backend:8000;
|
proxy_pass http://backend:8000;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# API prefix for frontend (maps /api/tools to /tools, etc.)
|
||||||
|
location /api/ {
|
||||||
|
rewrite ^/api/(.*)$ /$1 break;
|
||||||
|
proxy_pass http://backend:8000;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection 'upgrade';
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_cache_bypass $http_upgrade;
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
}
|
||||||
|
|
||||||
location /docs {
|
location /docs {
|
||||||
proxy_pass http://backend:8000;
|
proxy_pass http://backend:8000;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -321,6 +321,8 @@ IMPORTANT: any new icon should also must be added on /service-worker.js + its ve
|
|||||||
.sidebar_left .desaturate:after{ background-image: url('images/icons/desaturate.svg'); }
|
.sidebar_left .desaturate:after{ background-image: url('images/icons/desaturate.svg'); }
|
||||||
.sidebar_left .bulge_pinch:after{ background-image: url('images/icons/bulge_pinch.svg'); }
|
.sidebar_left .bulge_pinch:after{ background-image: url('images/icons/bulge_pinch.svg'); }
|
||||||
.sidebar_left .animation:after{ background-image: url('images/icons/animation.svg'); }
|
.sidebar_left .animation:after{ background-image: url('images/icons/animation.svg'); }
|
||||||
|
.sidebar_left .smart_select:after{ background-image: url('images/icons/smart_select.svg'); }
|
||||||
|
.sidebar_left .ai_inpaint:after{ background-image: url('images/icons/ai_inpaint.svg'); }
|
||||||
|
|
||||||
@media screen and (max-width:550px){
|
@media screen and (max-width:550px){
|
||||||
#sidebar_left{
|
#sidebar_left{
|
||||||
|
|||||||
@@ -92,6 +92,17 @@ config.TOOLS = [
|
|||||||
attributes: {},
|
attributes: {},
|
||||||
on_leave: 'on_leave',
|
on_leave: 'on_leave',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'smart_select',
|
||||||
|
title: 'Smart Select (AI)',
|
||||||
|
attributes: {},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'ai_inpaint',
|
||||||
|
title: 'AI Inpaint',
|
||||||
|
on_activate: 'on_activate',
|
||||||
|
attributes: {},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'brush',
|
name: 'brush',
|
||||||
attributes: {
|
attributes: {
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
/**
|
||||||
|
* API Service for communicating with the FastAPI backend
|
||||||
|
* Handles SAM selection and AI inpainting requests
|
||||||
|
*/
|
||||||
|
|
||||||
|
class ApiService {
|
||||||
|
constructor() {
|
||||||
|
// Backend API base URL - adjust for your deployment
|
||||||
|
this.baseUrl = window.API_BASE_URL || '/api';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call SAM (Segment Anything Model) for smart selection
|
||||||
|
* @param {string} imageData - Base64 encoded image data
|
||||||
|
* @param {number} pointX - X coordinate of click point
|
||||||
|
* @param {number} pointY - Y coordinate of click point
|
||||||
|
* @returns {Promise<{mask: ImageData, polygon: Array}>}
|
||||||
|
*/
|
||||||
|
async smartSelect(imageData, pointX, pointY) {
|
||||||
|
const response = await fetch(`${this.baseUrl}/tools/smart-select-base64`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
image: imageData,
|
||||||
|
point_x: pointX,
|
||||||
|
point_y: pointY,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.json().catch(() => ({ detail: 'Unknown error' }));
|
||||||
|
throw new Error(error.detail || `SAM request failed: ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call AI inpainting to edit a selected region
|
||||||
|
* @param {string} imageData - Base64 encoded image data
|
||||||
|
* @param {string} maskData - Base64 encoded mask data (white = area to edit)
|
||||||
|
* @param {string} prompt - Text prompt describing desired edit
|
||||||
|
* @param {Object} options - Additional options
|
||||||
|
* @returns {Promise<{result: string}>} - Base64 encoded result image
|
||||||
|
*/
|
||||||
|
async inpaint(imageData, maskData, prompt, options = {}) {
|
||||||
|
const response = await fetch(`${this.baseUrl}/tools/inpaint`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
image: imageData,
|
||||||
|
mask: maskData,
|
||||||
|
prompt: prompt,
|
||||||
|
negative_prompt: options.negativePrompt || '',
|
||||||
|
strength: options.strength || 0.8,
|
||||||
|
guidance_scale: options.guidanceScale || 7.5,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.json().catch(() => ({ detail: 'Unknown error' }));
|
||||||
|
throw new Error(error.detail || `Inpaint request failed: ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Health check for the backend
|
||||||
|
* @returns {Promise<boolean>}
|
||||||
|
*/
|
||||||
|
async healthCheck() {
|
||||||
|
try {
|
||||||
|
const response = await fetch(`${this.baseUrl}/health`);
|
||||||
|
return response.ok;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Singleton instance
|
||||||
|
const apiService = new ApiService();
|
||||||
|
export default apiService;
|
||||||
@@ -0,0 +1,315 @@
|
|||||||
|
/**
|
||||||
|
* AI Inpaint Tool - Edit selected regions using AI with text prompts
|
||||||
|
* Works with Smart Select tool's mask or manual selection
|
||||||
|
*/
|
||||||
|
|
||||||
|
import app from './../app.js';
|
||||||
|
import config from './../config.js';
|
||||||
|
import Base_tools_class from './../core/base-tools.js';
|
||||||
|
import Base_layers_class from './../core/base-layers.js';
|
||||||
|
import Helper_class from './../libs/helpers.js';
|
||||||
|
import Dialog_class from './../libs/popup.js';
|
||||||
|
import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js';
|
||||||
|
import apiService from './../services/api.js';
|
||||||
|
|
||||||
|
class Ai_inpaint_class extends Base_tools_class {
|
||||||
|
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
this.Base_layers = new Base_layers_class();
|
||||||
|
this.Helper = new Helper_class();
|
||||||
|
this.POP = new Dialog_class();
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.name = 'ai_inpaint';
|
||||||
|
this.isProcessing = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
load() {
|
||||||
|
// No mouse events needed - this tool uses a dialog
|
||||||
|
}
|
||||||
|
|
||||||
|
on_activate() {
|
||||||
|
this.showInpaintDialog();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Show the inpainting dialog
|
||||||
|
*/
|
||||||
|
showInpaintDialog() {
|
||||||
|
var _this = this;
|
||||||
|
|
||||||
|
// Check if we have a selection
|
||||||
|
var hasMask = window.smartSelectMask != null;
|
||||||
|
var hasRectSelection = this.getRectSelection() != null;
|
||||||
|
|
||||||
|
if (!hasMask && !hasRectSelection) {
|
||||||
|
alertify.warning('No selection found. Use Smart Select or Selection tool first.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var settings = {
|
||||||
|
title: 'AI Inpaint',
|
||||||
|
params: [
|
||||||
|
{
|
||||||
|
name: "prompt",
|
||||||
|
title: "Describe what you want:",
|
||||||
|
type: "textarea",
|
||||||
|
value: "",
|
||||||
|
placeholder: "e.g., 'a red rose', 'remove the object', 'blue sky with clouds'"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative_prompt",
|
||||||
|
title: "What to avoid (optional):",
|
||||||
|
value: "",
|
||||||
|
placeholder: "e.g., 'blurry, distorted, low quality'"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "strength",
|
||||||
|
title: "Edit Strength:",
|
||||||
|
type: "range",
|
||||||
|
value: 80,
|
||||||
|
range: [1, 100],
|
||||||
|
step: 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
on_finish: async function (params) {
|
||||||
|
await _this.executeInpaint(params);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
this.POP.show(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute the inpainting operation
|
||||||
|
*/
|
||||||
|
async executeInpaint(params) {
|
||||||
|
if (this.isProcessing) {
|
||||||
|
alertify.warning('Already processing... please wait');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!params.prompt || params.prompt.trim() === '') {
|
||||||
|
alertify.error('Please enter a prompt describing what you want');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if we have an image layer
|
||||||
|
if (config.layer.type != 'image') {
|
||||||
|
alertify.error('Please select an image layer');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isProcessing = true;
|
||||||
|
alertify.message('AI is generating... this may take a moment');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get image data
|
||||||
|
var imageData = this.getLayerImageData();
|
||||||
|
|
||||||
|
// Get mask data (from Smart Select or rectangular selection)
|
||||||
|
var maskData = this.getMaskData();
|
||||||
|
|
||||||
|
if (!maskData) {
|
||||||
|
throw new Error('No valid selection/mask found');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call inpaint API
|
||||||
|
var result = await apiService.inpaint(
|
||||||
|
imageData,
|
||||||
|
maskData,
|
||||||
|
params.prompt,
|
||||||
|
{
|
||||||
|
negativePrompt: params.negative_prompt || '',
|
||||||
|
strength: params.strength / 100
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// Apply result to layer
|
||||||
|
await this.applyResult(result.result);
|
||||||
|
|
||||||
|
alertify.success('Inpainting complete!');
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Inpaint error:', error);
|
||||||
|
alertify.error('Inpainting failed: ' + error.message);
|
||||||
|
} finally {
|
||||||
|
this.isProcessing = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current layer's image data as base64
|
||||||
|
*/
|
||||||
|
getLayerImageData() {
|
||||||
|
var canvas = document.createElement('canvas');
|
||||||
|
var ctx = canvas.getContext('2d');
|
||||||
|
|
||||||
|
canvas.width = config.layer.width_original;
|
||||||
|
canvas.height = config.layer.height_original;
|
||||||
|
ctx.drawImage(config.layer.link, 0, 0);
|
||||||
|
|
||||||
|
return canvas.toDataURL('image/png').split(',')[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get mask data - either from Smart Select or rectangular selection
|
||||||
|
*/
|
||||||
|
getMaskData() {
|
||||||
|
// First try Smart Select mask
|
||||||
|
if (window.smartSelectMask && window.smartSelectMask.canvas) {
|
||||||
|
var maskCanvas = window.smartSelectMask.canvas;
|
||||||
|
return maskCanvas.toDataURL('image/png').split(',')[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to rectangular selection
|
||||||
|
var selection = this.getRectSelection();
|
||||||
|
if (selection) {
|
||||||
|
return this.createRectMask(selection);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get rectangular selection from miniPaint's selection tool
|
||||||
|
*/
|
||||||
|
getRectSelection() {
|
||||||
|
// Try to get selection from selection tool
|
||||||
|
var Selection = null;
|
||||||
|
try {
|
||||||
|
var GUI_tools = app.GUI?.GUI_tools || this.Base_layers?.Base_gui?.GUI_tools;
|
||||||
|
if (GUI_tools && GUI_tools.tools_modules && GUI_tools.tools_modules.selection) {
|
||||||
|
Selection = GUI_tools.tools_modules.selection.object;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Selection tool not available
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Selection && Selection.selection &&
|
||||||
|
Selection.selection.width > 0 && Selection.selection.height > 0) {
|
||||||
|
return Selection.selection;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a white rectangle mask from selection coordinates
|
||||||
|
*/
|
||||||
|
createRectMask(selection) {
|
||||||
|
var canvas = document.createElement('canvas');
|
||||||
|
var ctx = canvas.getContext('2d');
|
||||||
|
|
||||||
|
canvas.width = config.layer.width_original;
|
||||||
|
canvas.height = config.layer.height_original;
|
||||||
|
|
||||||
|
// Fill with black (unselected)
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||||||
|
|
||||||
|
// Calculate selection position relative to layer
|
||||||
|
var x = selection.x - config.layer.x;
|
||||||
|
var y = selection.y - config.layer.y;
|
||||||
|
var width = selection.width;
|
||||||
|
var height = selection.height;
|
||||||
|
|
||||||
|
// Scale to original image size
|
||||||
|
var scaleX = config.layer.width_original / config.layer.width;
|
||||||
|
var scaleY = config.layer.height_original / config.layer.height;
|
||||||
|
|
||||||
|
x = x * scaleX;
|
||||||
|
y = y * scaleY;
|
||||||
|
width = width * scaleX;
|
||||||
|
height = height * scaleY;
|
||||||
|
|
||||||
|
// Draw white rectangle (selected area)
|
||||||
|
ctx.fillStyle = '#FFFFFF';
|
||||||
|
ctx.fillRect(x, y, width, height);
|
||||||
|
|
||||||
|
return canvas.toDataURL('image/png').split(',')[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply the inpainted result to the current layer
|
||||||
|
*/
|
||||||
|
async applyResult(resultBase64) {
|
||||||
|
var _this = this;
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
var img = new Image();
|
||||||
|
img.onload = function() {
|
||||||
|
// Create canvas with result
|
||||||
|
var canvas = document.createElement('canvas');
|
||||||
|
canvas.width = config.layer.width_original;
|
||||||
|
canvas.height = config.layer.height_original;
|
||||||
|
var ctx = canvas.getContext('2d');
|
||||||
|
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
|
||||||
|
|
||||||
|
// Update layer through action system for undo support
|
||||||
|
app.State.do_action(
|
||||||
|
new app.Actions.Bundle_action('ai_inpaint', 'AI Inpaint', [
|
||||||
|
new app.Actions.Update_layer_image_action(canvas)
|
||||||
|
])
|
||||||
|
);
|
||||||
|
|
||||||
|
// Clear the smart select mask
|
||||||
|
window.smartSelectMask = null;
|
||||||
|
|
||||||
|
config.need_render = true;
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
img.onerror = function() {
|
||||||
|
reject(new Error('Failed to load result image'));
|
||||||
|
};
|
||||||
|
img.src = 'data:image/png;base64,' + resultBase64;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
render_overlay(ctx) {
|
||||||
|
// Show visual indicator if there's a selection ready for inpainting
|
||||||
|
if (window.smartSelectMask && window.smartSelectMask.canvas) {
|
||||||
|
// Draw a subtle border around the tool indicating mask is ready
|
||||||
|
ctx.save();
|
||||||
|
ctx.strokeStyle = '#00ff00';
|
||||||
|
ctx.lineWidth = 2;
|
||||||
|
ctx.setLineDash([5, 5]);
|
||||||
|
|
||||||
|
var scaleX = config.layer.width / config.layer.width_original;
|
||||||
|
var scaleY = config.layer.height / config.layer.height_original;
|
||||||
|
|
||||||
|
// Get mask bounds
|
||||||
|
var maskCanvas = window.smartSelectMask.canvas;
|
||||||
|
var maskCtx = maskCanvas.getContext('2d');
|
||||||
|
var imageData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
|
||||||
|
|
||||||
|
var minX = maskCanvas.width, minY = maskCanvas.height;
|
||||||
|
var maxX = 0, maxY = 0;
|
||||||
|
|
||||||
|
for (var y = 0; y < maskCanvas.height; y += 4) { // Sample every 4th pixel for speed
|
||||||
|
for (var x = 0; x < maskCanvas.width; x += 4) {
|
||||||
|
var i = (y * maskCanvas.width + x) * 4;
|
||||||
|
if (imageData.data[i] > 128) {
|
||||||
|
minX = Math.min(minX, x);
|
||||||
|
minY = Math.min(minY, y);
|
||||||
|
maxX = Math.max(maxX, x);
|
||||||
|
maxY = Math.max(maxY, y);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (maxX > minX && maxY > minY) {
|
||||||
|
ctx.strokeRect(
|
||||||
|
config.layer.x + minX * scaleX,
|
||||||
|
config.layer.y + minY * scaleY,
|
||||||
|
(maxX - minX) * scaleX,
|
||||||
|
(maxY - minY) * scaleY
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.restore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default Ai_inpaint_class;
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
/**
|
||||||
|
* Smart Select Tool - Uses SAM (Segment Anything Model) for AI-powered selection
|
||||||
|
* Click on any object to automatically select it
|
||||||
|
*/
|
||||||
|
|
||||||
|
import app from './../app.js';
|
||||||
|
import config from './../config.js';
|
||||||
|
import Base_tools_class from './../core/base-tools.js';
|
||||||
|
import Base_layers_class from './../core/base-layers.js';
|
||||||
|
import Helper_class from './../libs/helpers.js';
|
||||||
|
import Dialog_class from './../libs/popup.js';
|
||||||
|
import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js';
|
||||||
|
import apiService from './../services/api.js';
|
||||||
|
|
||||||
|
class Smart_select_class extends Base_tools_class {
|
||||||
|
|
||||||
|
constructor(ctx) {
|
||||||
|
super();
|
||||||
|
this.Base_layers = new Base_layers_class();
|
||||||
|
this.Helper = new Helper_class();
|
||||||
|
this.POP = new Dialog_class();
|
||||||
|
this.ctx = ctx;
|
||||||
|
this.name = 'smart_select';
|
||||||
|
|
||||||
|
// Store the current mask data
|
||||||
|
this.currentMask = null;
|
||||||
|
this.maskCanvas = null;
|
||||||
|
this.isProcessing = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
load() {
|
||||||
|
var _this = this;
|
||||||
|
|
||||||
|
// Mouse click event for selection
|
||||||
|
document.addEventListener('mousedown', function (e) {
|
||||||
|
_this.mousedown(e);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async mousedown(e) {
|
||||||
|
var mouse = this.get_mouse_info(e);
|
||||||
|
|
||||||
|
if (config.TOOL.name != this.name) return;
|
||||||
|
if (mouse.click_valid == false) return;
|
||||||
|
if (this.isProcessing) {
|
||||||
|
alertify.warning('Processing... please wait');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if we have an image layer
|
||||||
|
if (config.layer.type != 'image') {
|
||||||
|
alertify.error('Please select an image layer first');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get click coordinates relative to the image
|
||||||
|
var x = mouse.x - config.layer.x;
|
||||||
|
var y = mouse.y - config.layer.y;
|
||||||
|
|
||||||
|
// Adjust for layer scaling
|
||||||
|
if (config.layer.width != config.layer.width_original) {
|
||||||
|
x = x * (config.layer.width_original / config.layer.width);
|
||||||
|
}
|
||||||
|
if (config.layer.height != config.layer.height_original) {
|
||||||
|
y = y * (config.layer.height_original / config.layer.height);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make sure click is within image bounds
|
||||||
|
if (x < 0 || y < 0 || x > config.layer.width_original || y > config.layer.height_original) {
|
||||||
|
alertify.error('Click inside the image');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isProcessing = true;
|
||||||
|
alertify.message('AI is analyzing the image...');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get image data as base64
|
||||||
|
var imageData = this.getLayerImageData();
|
||||||
|
|
||||||
|
// Call SAM API
|
||||||
|
var result = await apiService.smartSelect(imageData, Math.round(x), Math.round(y));
|
||||||
|
|
||||||
|
// Apply the mask as selection
|
||||||
|
this.applyMask(result.mask, result.bbox);
|
||||||
|
|
||||||
|
alertify.success('Selection complete! Use AI Inpaint to edit.');
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Smart select error:', error);
|
||||||
|
alertify.error('Selection failed: ' + error.message);
|
||||||
|
} finally {
|
||||||
|
this.isProcessing = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current layer's image data as base64
|
||||||
|
*/
|
||||||
|
getLayerImageData() {
|
||||||
|
var canvas = document.createElement('canvas');
|
||||||
|
var ctx = canvas.getContext('2d');
|
||||||
|
|
||||||
|
canvas.width = config.layer.width_original;
|
||||||
|
canvas.height = config.layer.height_original;
|
||||||
|
|
||||||
|
// Draw the layer's image
|
||||||
|
ctx.drawImage(config.layer.link, 0, 0);
|
||||||
|
|
||||||
|
// Return as base64 (remove data:image/png;base64, prefix)
|
||||||
|
return canvas.toDataURL('image/png').split(',')[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply the SAM mask as a selection
|
||||||
|
* @param {string} maskBase64 - Base64 encoded mask image
|
||||||
|
* @param {Object} bbox - Bounding box {x, y, width, height}
|
||||||
|
*/
|
||||||
|
applyMask(maskBase64, bbox) {
|
||||||
|
var _this = this;
|
||||||
|
|
||||||
|
// Create mask image
|
||||||
|
var maskImage = new Image();
|
||||||
|
maskImage.onload = function() {
|
||||||
|
// Store mask for later use by inpaint tool
|
||||||
|
_this.maskCanvas = document.createElement('canvas');
|
||||||
|
_this.maskCanvas.width = config.layer.width_original;
|
||||||
|
_this.maskCanvas.height = config.layer.height_original;
|
||||||
|
var maskCtx = _this.maskCanvas.getContext('2d');
|
||||||
|
maskCtx.drawImage(maskImage, 0, 0);
|
||||||
|
|
||||||
|
_this.currentMask = {
|
||||||
|
canvas: _this.maskCanvas,
|
||||||
|
bbox: bbox
|
||||||
|
};
|
||||||
|
|
||||||
|
// Store globally for AI inpaint tool to access
|
||||||
|
window.smartSelectMask = _this.currentMask;
|
||||||
|
|
||||||
|
// Visual feedback - render mask overlay
|
||||||
|
_this.renderMaskOverlay();
|
||||||
|
|
||||||
|
config.need_render = true;
|
||||||
|
};
|
||||||
|
maskImage.src = 'data:image/png;base64,' + maskBase64;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render a visual overlay showing the selected region
|
||||||
|
*/
|
||||||
|
renderMaskOverlay() {
|
||||||
|
if (!this.maskCanvas) return;
|
||||||
|
|
||||||
|
// Create overlay layer or update existing
|
||||||
|
// For now, we'll use the selection system
|
||||||
|
var maskCtx = this.maskCanvas.getContext('2d');
|
||||||
|
var imageData = maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
|
||||||
|
|
||||||
|
// Find bounding box of selection
|
||||||
|
var minX = this.maskCanvas.width, minY = this.maskCanvas.height;
|
||||||
|
var maxX = 0, maxY = 0;
|
||||||
|
|
||||||
|
for (var y = 0; y < this.maskCanvas.height; y++) {
|
||||||
|
for (var x = 0; x < this.maskCanvas.width; x++) {
|
||||||
|
var i = (y * this.maskCanvas.width + x) * 4;
|
||||||
|
if (imageData.data[i] > 128) { // White pixel in mask
|
||||||
|
minX = Math.min(minX, x);
|
||||||
|
minY = Math.min(minY, y);
|
||||||
|
maxX = Math.max(maxX, x);
|
||||||
|
maxY = Math.max(maxY, y);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (maxX > minX && maxY > minY) {
|
||||||
|
// Set selection using miniPaint's selection system
|
||||||
|
var Selection = this.Base_layers.Base_gui?.GUI_tools?.tools_modules?.selection?.object;
|
||||||
|
if (Selection) {
|
||||||
|
Selection.selection = {
|
||||||
|
x: config.layer.x + minX * (config.layer.width / config.layer.width_original),
|
||||||
|
y: config.layer.y + minY * (config.layer.height / config.layer.height_original),
|
||||||
|
width: (maxX - minX) * (config.layer.width / config.layer.width_original),
|
||||||
|
height: (maxY - minY) * (config.layer.height / config.layer.height_original)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
render_overlay(ctx) {
|
||||||
|
// Render marching ants or highlight around selected region
|
||||||
|
if (!this.currentMask || !this.maskCanvas) return;
|
||||||
|
|
||||||
|
var mouse = this.get_mouse_info(event);
|
||||||
|
|
||||||
|
// Draw semi-transparent overlay on non-selected areas
|
||||||
|
ctx.save();
|
||||||
|
ctx.globalAlpha = 0.3;
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
|
||||||
|
// Scale to match layer
|
||||||
|
var scaleX = config.layer.width / config.layer.width_original;
|
||||||
|
var scaleY = config.layer.height / config.layer.height_original;
|
||||||
|
|
||||||
|
ctx.translate(config.layer.x, config.layer.y);
|
||||||
|
ctx.scale(scaleX, scaleY);
|
||||||
|
|
||||||
|
// Draw inverse mask (darken unselected areas)
|
||||||
|
var maskCtx = this.maskCanvas.getContext('2d');
|
||||||
|
var imageData = maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
|
||||||
|
|
||||||
|
// Create inverse mask canvas
|
||||||
|
var inverseCanvas = document.createElement('canvas');
|
||||||
|
inverseCanvas.width = this.maskCanvas.width;
|
||||||
|
inverseCanvas.height = this.maskCanvas.height;
|
||||||
|
var inverseCtx = inverseCanvas.getContext('2d');
|
||||||
|
|
||||||
|
inverseCtx.fillStyle = '#000000';
|
||||||
|
inverseCtx.fillRect(0, 0, inverseCanvas.width, inverseCanvas.height);
|
||||||
|
inverseCtx.globalCompositeOperation = 'destination-out';
|
||||||
|
inverseCtx.drawImage(this.maskCanvas, 0, 0);
|
||||||
|
|
||||||
|
ctx.drawImage(inverseCanvas, 0, 0);
|
||||||
|
|
||||||
|
ctx.restore();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear the current selection
|
||||||
|
*/
|
||||||
|
clearSelection() {
|
||||||
|
this.currentMask = null;
|
||||||
|
this.maskCanvas = null;
|
||||||
|
window.smartSelectMask = null;
|
||||||
|
config.need_render = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
on_leave() {
|
||||||
|
// Don't clear mask when switching tools - AI inpaint needs it
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default Smart_select_class;
|
||||||
Reference in New Issue
Block a user