Add SAM Smart Select and AI Inpaint tools to miniPaint

New features:
- Smart Select tool: Click to select objects using SAM (Segment Anything)
- AI Inpaint tool: Edit selected regions with text prompts

Changes:
- frontend/src/js/tools/smart_select.js: SAM-powered selection tool
- frontend/src/js/tools/ai_inpaint.js: AI inpainting with prompt dialog
- frontend/src/js/services/api.js: API service for backend communication
- frontend/src/js/config.js: Register new tools
- frontend/src/css/layout.css: Tool icon styles
- frontend/images/icons/: SVG icons for new tools
- backend/app/routers/tools.py: New base64 API endpoints
- frontend/Dockerfile: Updated for miniPaint build
- frontend/nginx.conf: Added /api prefix proxy
This commit is contained in:
Claude
2026-01-26 17:34:20 +00:00
parent d009a2c0f0
commit bf83ecc8ad
10 changed files with 816 additions and 5 deletions
+11
View File
@@ -92,6 +92,17 @@ config.TOOLS = [
attributes: {},
on_leave: 'on_leave',
},
{
name: 'smart_select',
title: 'Smart Select (AI)',
attributes: {},
},
{
name: 'ai_inpaint',
title: 'AI Inpaint',
on_activate: 'on_activate',
attributes: {},
},
{
name: 'brush',
attributes: {
+88
View File
@@ -0,0 +1,88 @@
/**
* API Service for communicating with the FastAPI backend
* Handles SAM selection and AI inpainting requests
*/
class ApiService {
constructor() {
// Backend API base URL - adjust for your deployment
this.baseUrl = window.API_BASE_URL || '/api';
}
/**
* Call SAM (Segment Anything Model) for smart selection
* @param {string} imageData - Base64 encoded image data
* @param {number} pointX - X coordinate of click point
* @param {number} pointY - Y coordinate of click point
* @returns {Promise<{mask: ImageData, polygon: Array}>}
*/
async smartSelect(imageData, pointX, pointY) {
const response = await fetch(`${this.baseUrl}/tools/smart-select-base64`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
image: imageData,
point_x: pointX,
point_y: pointY,
}),
});
if (!response.ok) {
const error = await response.json().catch(() => ({ detail: 'Unknown error' }));
throw new Error(error.detail || `SAM request failed: ${response.status}`);
}
return response.json();
}
/**
* Call AI inpainting to edit a selected region
* @param {string} imageData - Base64 encoded image data
* @param {string} maskData - Base64 encoded mask data (white = area to edit)
* @param {string} prompt - Text prompt describing desired edit
* @param {Object} options - Additional options
* @returns {Promise<{result: string}>} - Base64 encoded result image
*/
async inpaint(imageData, maskData, prompt, options = {}) {
const response = await fetch(`${this.baseUrl}/tools/inpaint`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
image: imageData,
mask: maskData,
prompt: prompt,
negative_prompt: options.negativePrompt || '',
strength: options.strength || 0.8,
guidance_scale: options.guidanceScale || 7.5,
}),
});
if (!response.ok) {
const error = await response.json().catch(() => ({ detail: 'Unknown error' }));
throw new Error(error.detail || `Inpaint request failed: ${response.status}`);
}
return response.json();
}
/**
* Health check for the backend
* @returns {Promise<boolean>}
*/
async healthCheck() {
try {
const response = await fetch(`${this.baseUrl}/health`);
return response.ok;
} catch {
return false;
}
}
}
// Singleton instance
const apiService = new ApiService();
export default apiService;
+315
View File
@@ -0,0 +1,315 @@
/**
* AI Inpaint Tool - Edit selected regions using AI with text prompts
* Works with Smart Select tool's mask or manual selection
*/
import app from './../app.js';
import config from './../config.js';
import Base_tools_class from './../core/base-tools.js';
import Base_layers_class from './../core/base-layers.js';
import Helper_class from './../libs/helpers.js';
import Dialog_class from './../libs/popup.js';
import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js';
import apiService from './../services/api.js';
class Ai_inpaint_class extends Base_tools_class {
constructor(ctx) {
super();
this.Base_layers = new Base_layers_class();
this.Helper = new Helper_class();
this.POP = new Dialog_class();
this.ctx = ctx;
this.name = 'ai_inpaint';
this.isProcessing = false;
}
load() {
// No mouse events needed - this tool uses a dialog
}
on_activate() {
this.showInpaintDialog();
}
/**
* Show the inpainting dialog
*/
showInpaintDialog() {
var _this = this;
// Check if we have a selection
var hasMask = window.smartSelectMask != null;
var hasRectSelection = this.getRectSelection() != null;
if (!hasMask && !hasRectSelection) {
alertify.warning('No selection found. Use Smart Select or Selection tool first.');
return;
}
var settings = {
title: 'AI Inpaint',
params: [
{
name: "prompt",
title: "Describe what you want:",
type: "textarea",
value: "",
placeholder: "e.g., 'a red rose', 'remove the object', 'blue sky with clouds'"
},
{
name: "negative_prompt",
title: "What to avoid (optional):",
value: "",
placeholder: "e.g., 'blurry, distorted, low quality'"
},
{
name: "strength",
title: "Edit Strength:",
type: "range",
value: 80,
range: [1, 100],
step: 1
}
],
on_finish: async function (params) {
await _this.executeInpaint(params);
},
};
this.POP.show(settings);
}
/**
* Execute the inpainting operation
*/
async executeInpaint(params) {
if (this.isProcessing) {
alertify.warning('Already processing... please wait');
return;
}
if (!params.prompt || params.prompt.trim() === '') {
alertify.error('Please enter a prompt describing what you want');
return;
}
// Check if we have an image layer
if (config.layer.type != 'image') {
alertify.error('Please select an image layer');
return;
}
this.isProcessing = true;
alertify.message('AI is generating... this may take a moment');
try {
// Get image data
var imageData = this.getLayerImageData();
// Get mask data (from Smart Select or rectangular selection)
var maskData = this.getMaskData();
if (!maskData) {
throw new Error('No valid selection/mask found');
}
// Call inpaint API
var result = await apiService.inpaint(
imageData,
maskData,
params.prompt,
{
negativePrompt: params.negative_prompt || '',
strength: params.strength / 100
}
);
// Apply result to layer
await this.applyResult(result.result);
alertify.success('Inpainting complete!');
} catch (error) {
console.error('Inpaint error:', error);
alertify.error('Inpainting failed: ' + error.message);
} finally {
this.isProcessing = false;
}
}
/**
* Get the current layer's image data as base64
*/
getLayerImageData() {
var canvas = document.createElement('canvas');
var ctx = canvas.getContext('2d');
canvas.width = config.layer.width_original;
canvas.height = config.layer.height_original;
ctx.drawImage(config.layer.link, 0, 0);
return canvas.toDataURL('image/png').split(',')[1];
}
/**
* Get mask data - either from Smart Select or rectangular selection
*/
getMaskData() {
// First try Smart Select mask
if (window.smartSelectMask && window.smartSelectMask.canvas) {
var maskCanvas = window.smartSelectMask.canvas;
return maskCanvas.toDataURL('image/png').split(',')[1];
}
// Fall back to rectangular selection
var selection = this.getRectSelection();
if (selection) {
return this.createRectMask(selection);
}
return null;
}
/**
* Get rectangular selection from miniPaint's selection tool
*/
getRectSelection() {
// Try to get selection from selection tool
var Selection = null;
try {
var GUI_tools = app.GUI?.GUI_tools || this.Base_layers?.Base_gui?.GUI_tools;
if (GUI_tools && GUI_tools.tools_modules && GUI_tools.tools_modules.selection) {
Selection = GUI_tools.tools_modules.selection.object;
}
} catch (e) {
// Selection tool not available
}
if (Selection && Selection.selection &&
Selection.selection.width > 0 && Selection.selection.height > 0) {
return Selection.selection;
}
return null;
}
/**
* Create a white rectangle mask from selection coordinates
*/
createRectMask(selection) {
var canvas = document.createElement('canvas');
var ctx = canvas.getContext('2d');
canvas.width = config.layer.width_original;
canvas.height = config.layer.height_original;
// Fill with black (unselected)
ctx.fillStyle = '#000000';
ctx.fillRect(0, 0, canvas.width, canvas.height);
// Calculate selection position relative to layer
var x = selection.x - config.layer.x;
var y = selection.y - config.layer.y;
var width = selection.width;
var height = selection.height;
// Scale to original image size
var scaleX = config.layer.width_original / config.layer.width;
var scaleY = config.layer.height_original / config.layer.height;
x = x * scaleX;
y = y * scaleY;
width = width * scaleX;
height = height * scaleY;
// Draw white rectangle (selected area)
ctx.fillStyle = '#FFFFFF';
ctx.fillRect(x, y, width, height);
return canvas.toDataURL('image/png').split(',')[1];
}
/**
* Apply the inpainted result to the current layer
*/
async applyResult(resultBase64) {
var _this = this;
return new Promise((resolve, reject) => {
var img = new Image();
img.onload = function() {
// Create canvas with result
var canvas = document.createElement('canvas');
canvas.width = config.layer.width_original;
canvas.height = config.layer.height_original;
var ctx = canvas.getContext('2d');
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
// Update layer through action system for undo support
app.State.do_action(
new app.Actions.Bundle_action('ai_inpaint', 'AI Inpaint', [
new app.Actions.Update_layer_image_action(canvas)
])
);
// Clear the smart select mask
window.smartSelectMask = null;
config.need_render = true;
resolve();
};
img.onerror = function() {
reject(new Error('Failed to load result image'));
};
img.src = 'data:image/png;base64,' + resultBase64;
});
}
render_overlay(ctx) {
// Show visual indicator if there's a selection ready for inpainting
if (window.smartSelectMask && window.smartSelectMask.canvas) {
// Draw a subtle border around the tool indicating mask is ready
ctx.save();
ctx.strokeStyle = '#00ff00';
ctx.lineWidth = 2;
ctx.setLineDash([5, 5]);
var scaleX = config.layer.width / config.layer.width_original;
var scaleY = config.layer.height / config.layer.height_original;
// Get mask bounds
var maskCanvas = window.smartSelectMask.canvas;
var maskCtx = maskCanvas.getContext('2d');
var imageData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
var minX = maskCanvas.width, minY = maskCanvas.height;
var maxX = 0, maxY = 0;
for (var y = 0; y < maskCanvas.height; y += 4) { // Sample every 4th pixel for speed
for (var x = 0; x < maskCanvas.width; x += 4) {
var i = (y * maskCanvas.width + x) * 4;
if (imageData.data[i] > 128) {
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
}
}
if (maxX > minX && maxY > minY) {
ctx.strokeRect(
config.layer.x + minX * scaleX,
config.layer.y + minY * scaleY,
(maxX - minX) * scaleX,
(maxY - minY) * scaleY
);
}
ctx.restore();
}
}
}
export default Ai_inpaint_class;
+243
View File
@@ -0,0 +1,243 @@
/**
* Smart Select Tool - Uses SAM (Segment Anything Model) for AI-powered selection
* Click on any object to automatically select it
*/
import app from './../app.js';
import config from './../config.js';
import Base_tools_class from './../core/base-tools.js';
import Base_layers_class from './../core/base-layers.js';
import Helper_class from './../libs/helpers.js';
import Dialog_class from './../libs/popup.js';
import alertify from './../../../node_modules/alertifyjs/build/alertify.min.js';
import apiService from './../services/api.js';
class Smart_select_class extends Base_tools_class {
constructor(ctx) {
super();
this.Base_layers = new Base_layers_class();
this.Helper = new Helper_class();
this.POP = new Dialog_class();
this.ctx = ctx;
this.name = 'smart_select';
// Store the current mask data
this.currentMask = null;
this.maskCanvas = null;
this.isProcessing = false;
}
load() {
var _this = this;
// Mouse click event for selection
document.addEventListener('mousedown', function (e) {
_this.mousedown(e);
});
}
async mousedown(e) {
var mouse = this.get_mouse_info(e);
if (config.TOOL.name != this.name) return;
if (mouse.click_valid == false) return;
if (this.isProcessing) {
alertify.warning('Processing... please wait');
return;
}
// Check if we have an image layer
if (config.layer.type != 'image') {
alertify.error('Please select an image layer first');
return;
}
// Get click coordinates relative to the image
var x = mouse.x - config.layer.x;
var y = mouse.y - config.layer.y;
// Adjust for layer scaling
if (config.layer.width != config.layer.width_original) {
x = x * (config.layer.width_original / config.layer.width);
}
if (config.layer.height != config.layer.height_original) {
y = y * (config.layer.height_original / config.layer.height);
}
// Make sure click is within image bounds
if (x < 0 || y < 0 || x > config.layer.width_original || y > config.layer.height_original) {
alertify.error('Click inside the image');
return;
}
this.isProcessing = true;
alertify.message('AI is analyzing the image...');
try {
// Get image data as base64
var imageData = this.getLayerImageData();
// Call SAM API
var result = await apiService.smartSelect(imageData, Math.round(x), Math.round(y));
// Apply the mask as selection
this.applyMask(result.mask, result.bbox);
alertify.success('Selection complete! Use AI Inpaint to edit.');
} catch (error) {
console.error('Smart select error:', error);
alertify.error('Selection failed: ' + error.message);
} finally {
this.isProcessing = false;
}
}
/**
* Get the current layer's image data as base64
*/
getLayerImageData() {
var canvas = document.createElement('canvas');
var ctx = canvas.getContext('2d');
canvas.width = config.layer.width_original;
canvas.height = config.layer.height_original;
// Draw the layer's image
ctx.drawImage(config.layer.link, 0, 0);
// Return as base64 (remove data:image/png;base64, prefix)
return canvas.toDataURL('image/png').split(',')[1];
}
/**
* Apply the SAM mask as a selection
* @param {string} maskBase64 - Base64 encoded mask image
* @param {Object} bbox - Bounding box {x, y, width, height}
*/
applyMask(maskBase64, bbox) {
var _this = this;
// Create mask image
var maskImage = new Image();
maskImage.onload = function() {
// Store mask for later use by inpaint tool
_this.maskCanvas = document.createElement('canvas');
_this.maskCanvas.width = config.layer.width_original;
_this.maskCanvas.height = config.layer.height_original;
var maskCtx = _this.maskCanvas.getContext('2d');
maskCtx.drawImage(maskImage, 0, 0);
_this.currentMask = {
canvas: _this.maskCanvas,
bbox: bbox
};
// Store globally for AI inpaint tool to access
window.smartSelectMask = _this.currentMask;
// Visual feedback - render mask overlay
_this.renderMaskOverlay();
config.need_render = true;
};
maskImage.src = 'data:image/png;base64,' + maskBase64;
}
/**
* Render a visual overlay showing the selected region
*/
renderMaskOverlay() {
if (!this.maskCanvas) return;
// Create overlay layer or update existing
// For now, we'll use the selection system
var maskCtx = this.maskCanvas.getContext('2d');
var imageData = maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
// Find bounding box of selection
var minX = this.maskCanvas.width, minY = this.maskCanvas.height;
var maxX = 0, maxY = 0;
for (var y = 0; y < this.maskCanvas.height; y++) {
for (var x = 0; x < this.maskCanvas.width; x++) {
var i = (y * this.maskCanvas.width + x) * 4;
if (imageData.data[i] > 128) { // White pixel in mask
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
}
}
if (maxX > minX && maxY > minY) {
// Set selection using miniPaint's selection system
var Selection = this.Base_layers.Base_gui?.GUI_tools?.tools_modules?.selection?.object;
if (Selection) {
Selection.selection = {
x: config.layer.x + minX * (config.layer.width / config.layer.width_original),
y: config.layer.y + minY * (config.layer.height / config.layer.height_original),
width: (maxX - minX) * (config.layer.width / config.layer.width_original),
height: (maxY - minY) * (config.layer.height / config.layer.height_original)
};
}
}
}
render_overlay(ctx) {
// Render marching ants or highlight around selected region
if (!this.currentMask || !this.maskCanvas) return;
var mouse = this.get_mouse_info(event);
// Draw semi-transparent overlay on non-selected areas
ctx.save();
ctx.globalAlpha = 0.3;
ctx.fillStyle = '#000000';
// Scale to match layer
var scaleX = config.layer.width / config.layer.width_original;
var scaleY = config.layer.height / config.layer.height_original;
ctx.translate(config.layer.x, config.layer.y);
ctx.scale(scaleX, scaleY);
// Draw inverse mask (darken unselected areas)
var maskCtx = this.maskCanvas.getContext('2d');
var imageData = maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
// Create inverse mask canvas
var inverseCanvas = document.createElement('canvas');
inverseCanvas.width = this.maskCanvas.width;
inverseCanvas.height = this.maskCanvas.height;
var inverseCtx = inverseCanvas.getContext('2d');
inverseCtx.fillStyle = '#000000';
inverseCtx.fillRect(0, 0, inverseCanvas.width, inverseCanvas.height);
inverseCtx.globalCompositeOperation = 'destination-out';
inverseCtx.drawImage(this.maskCanvas, 0, 0);
ctx.drawImage(inverseCanvas, 0, 0);
ctx.restore();
}
/**
* Clear the current selection
*/
clearSelection() {
this.currentMask = null;
this.maskCanvas = null;
window.smartSelectMask = null;
config.need_render = true;
}
on_leave() {
// Don't clear mask when switching tools - AI inpaint needs it
return [];
}
}
export default Smart_select_class;