diff --git a/backend/app/routers/tools.py b/backend/app/routers/tools.py index 062a8ae..86a81d5 100644 --- a/backend/app/routers/tools.py +++ b/backend/app/routers/tools.py @@ -6,6 +6,8 @@ from PIL import Image from io import BytesIO import numpy as np import json +import base64 +import cv2 from app.database import get_db from app.models.project import Project @@ -133,11 +135,12 @@ async def smart_select( project_id: int = Form(...), point_x: int = Form(...), point_y: int = Form(...), + return_format: str = Form("json"), # "json" (default) or "image" db: Session = Depends(get_db) ): """ Use SAM (Segment Anything) to select object at given point. - Returns mask for the selected object. + Returns mask and polygon data for the selected object. Note: Requires SAM model to be downloaded. Falls back to simple flood-fill selection if SAM unavailable. @@ -163,13 +166,62 @@ async def smart_select( # Convert mask to PNG mask_img = Image.fromarray((mask * 255).astype(np.uint8), mode='L') + if return_format == "image": + buffer = BytesIO() + mask_img.save(buffer, format='PNG') + return Response( + content=buffer.getvalue(), + media_type="image/png" + ) + + # Return JSON with polygon and bbox + polygon, bbox = _mask_to_polygon(mask) + + # Also return mask as base64 for potential use buffer = BytesIO() mask_img.save(buffer, format='PNG') + mask_b64 = base64.b64encode(buffer.getvalue()).decode('utf-8') - return Response( - content=buffer.getvalue(), - media_type="image/png" - ) + return { + "polygon": polygon, + "bbox": bbox, + "mask_base64": mask_b64, + } + + +def _mask_to_polygon(mask: np.ndarray) -> tuple: + """ + Convert a binary mask to a simplified polygon and bounding box. + + Returns: + (polygon, bbox) where: + - polygon: list of [x, y] points (simplified contour) + - bbox: dict with x, y, width, height + """ + # Ensure mask is binary uint8 + mask_uint8 = (mask * 255).astype(np.uint8) if mask.max() <= 1 else mask.astype(np.uint8) + + # Find contours + contours, _ = cv2.findContours(mask_uint8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not contours: + return [], {"x": 0, "y": 0, "width": 0, "height": 0} + + # Get largest contour + largest = max(contours, key=cv2.contourArea) + + # Get bounding box + x, y, w, h = cv2.boundingRect(largest) + bbox = {"x": int(x), "y": int(y), "width": int(w), "height": int(h)} + + # Simplify contour to reduce points (epsilon = 1% of arc length) + epsilon = 0.01 * cv2.arcLength(largest, True) + simplified = cv2.approxPolyDP(largest, epsilon, True) + + # Convert to list of [x, y] points + polygon = [[int(pt[0][0]), int(pt[0][1])] for pt in simplified] + + return polygon, bbox # Global SAM model cache (loaded once, reused) @@ -387,11 +439,12 @@ async def color_select( color_g: int = Form(...), color_b: int = Form(...), tolerance: int = Form(30), + return_format: str = Form("json"), # "json" (default) or "image" db: Session = Depends(get_db) ): """ Select all pixels similar to the given color. - Returns a mask of selected areas. + Returns a mask and polygon data for selected areas. """ project = db.query(Project).filter(Project.id == project_id).first() if not project: @@ -412,18 +465,33 @@ async def color_select( distance = np.sum(diff, axis=2) # Create mask where distance is within tolerance - mask = (distance <= tolerance * 3).astype(np.uint8) * 255 + mask = (distance <= tolerance * 3).astype(np.uint8) # Convert to PNG - mask_img = Image.fromarray(mask, mode='L') + mask_img = Image.fromarray(mask * 255, mode='L') + + if return_format == "image": + buffer = BytesIO() + mask_img.save(buffer, format='PNG') + return Response( + content=buffer.getvalue(), + media_type="image/png" + ) + + # Return JSON with polygon and bbox + polygon, bbox = _mask_to_polygon(mask) buffer = BytesIO() mask_img.save(buffer, format='PNG') + mask_b64 = base64.b64encode(buffer.getvalue()).decode('utf-8') - return Response( - content=buffer.getvalue(), - media_type="image/png" - ) + return { + "polygon": polygon, + "bbox": bbox, + "mask_base64": mask_b64, + "color": {"r": color_r, "g": color_g, "b": color_b}, + "tolerance": tolerance, + } @router.post("/extract-object") diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index a61287e..67a0a67 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -26,6 +26,9 @@ function App() { const [layers, setLayers] = useState([]); const [activeLayer, setActiveLayer] = useState('background'); const [generatedMask, setGeneratedMask] = useState(null); + const [advancedToolMode, setAdvancedToolMode] = useState(null); // 'smart-select', 'color-select', 'object-remove' + const [canvasZoom, setCanvasZoom] = useState(1); + const [externalSelection, setExternalSelection] = useState(null); // For smart-select/color-select polygon results const editsRef = useRef([]); // Create project and upload image @@ -284,9 +287,49 @@ function App() { }; // Handle mask generation from smart select / color select - const handleMaskGenerated = async (maskBlob, source) => { - setGeneratedMask({ blob: maskBlob, source }); - // The mask can be used for various operations + const handleMaskGenerated = async (maskData, source) => { + setGeneratedMask({ data: maskData, source }); + // Convert mask to selection if it contains polygon data + if (maskData && maskData.polygon && maskData.polygon.length > 0) { + // Set external selection for canvas to draw + setExternalSelection({ + polygon: maskData.polygon, + bbox: maskData.bbox, + }); + // Also set selection state for fix button + setSelection({ + type: 'polygon', + bbox: maskData.bbox, + selectionData: { points: maskData.polygon }, + }); + } + // Reset tool mode after selection + setAdvancedToolMode(null); + }; + + // Handle canvas click for advanced tools (smart select, color select) + const handleAdvancedToolClick = async (x, y, color) => { + if (!project || !advancedToolMode) return; + + try { + setIsProcessing(true); + setError(null); + + if (advancedToolMode === 'smart-select') { + const result = await toolsApi.smartSelect(project.id, x, y); + handleMaskGenerated(result, 'smart-select'); + } else if (advancedToolMode === 'color-select') { + // Color is passed from canvas click + if (color) { + const result = await toolsApi.colorSelect(project.id, color.r, color.g, color.b, 30); + handleMaskGenerated(result, 'color-select'); + } + } + } catch (err) { + setError(`${advancedToolMode} failed: ${err.message}`); + } finally { + setIsProcessing(false); + } }; // Handle flatten layers @@ -361,6 +404,11 @@ function App() { imageUrl={currentImageUrl} onSelectionChange={setSelection} selectionMode={selectionMode} + advancedToolMode={advancedToolMode} + onAdvancedToolClick={handleAdvancedToolClick} + zoom={canvasZoom} + onZoomChange={setCanvasZoom} + externalSelection={externalSelection} /> @@ -391,6 +439,8 @@ function App() { isProcessing={isProcessing} setIsProcessing={setIsProcessing} setError={setError} + activeToolMode={advancedToolMode} + setActiveToolMode={setAdvancedToolMode} /> { - const [activeToolMode, setActiveToolMode] = useState(null); const [colorTolerance, setColorTolerance] = useState(30); const handleRemoveBackground = async () => { diff --git a/frontend/src/components/ImageCanvas.css b/frontend/src/components/ImageCanvas.css index c5d03d1..d3164e0 100644 --- a/frontend/src/components/ImageCanvas.css +++ b/frontend/src/components/ImageCanvas.css @@ -43,3 +43,66 @@ z-index: 10; white-space: nowrap; } + +/* Zoom controls */ +.zoom-controls { + position: absolute; + bottom: 10px; + right: 10px; + display: flex; + align-items: center; + gap: 4px; + background-color: rgba(0, 0, 0, 0.8); + padding: 6px 10px; + border-radius: 4px; + z-index: 10; +} + +.zoom-controls button { + width: 28px; + height: 28px; + padding: 0; + font-size: 18px; + font-weight: bold; + background-color: #444; + color: white; + border: 1px solid #666; + border-radius: 4px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; +} + +.zoom-controls button:hover { + background-color: #555; +} + +.zoom-level { + color: white; + font-size: 12px; + min-width: 45px; + text-align: center; +} + +/* Tool mode indicator */ +.tool-mode-indicator { + position: absolute; + top: 10px; + left: 50%; + transform: translateX(-50%); + background-color: rgba(0, 120, 255, 0.9); + color: white; + padding: 10px 20px; + border-radius: 4px; + font-size: 14px; + font-weight: 500; + z-index: 10; + white-space: nowrap; + animation: pulse 2s infinite; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.7; } +} diff --git a/frontend/src/components/ImageCanvas.jsx b/frontend/src/components/ImageCanvas.jsx index 4d87979..ed39879 100644 --- a/frontend/src/components/ImageCanvas.jsx +++ b/frontend/src/components/ImageCanvas.jsx @@ -2,12 +2,22 @@ import React, { useEffect, useRef, useState, useCallback } from 'react'; import { fabric } from 'fabric'; import './ImageCanvas.css'; -const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => { +const ImageCanvas = ({ + imageUrl, + onSelectionChange, + selectionMode, + advancedToolMode, + onAdvancedToolClick, + zoom = 1, + onZoomChange, + externalSelection, // { polygon: [[x,y],...], bbox: {x,y,width,height} } +}) => { const canvasRef = useRef(null); const fabricCanvasRef = useRef(null); const [currentSelection, setCurrentSelection] = useState(null); const [isDrawing, setIsDrawing] = useState(false); const [isTransformMode, setIsTransformMode] = useState(false); + const [currentZoom, setCurrentZoom] = useState(zoom); const lassoPoints = useRef([]); useEffect(() => { @@ -50,11 +60,36 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => { handleResize(); window.addEventListener('resize', handleResize); + // Mouse wheel zoom + const handleWheel = (opt) => { + const e = opt.e; + e.preventDefault(); + e.stopPropagation(); + + const delta = e.deltaY; + let newZoom = canvas.getZoom(); + newZoom *= 0.999 ** delta; + + // Clamp zoom between 0.1x and 10x + if (newZoom > 10) newZoom = 10; + if (newZoom < 0.1) newZoom = 0.1; + + // Zoom to point under cursor + const pointer = canvas.getPointer(e, true); + canvas.zoomToPoint({ x: pointer.x, y: pointer.y }, newZoom); + + setCurrentZoom(newZoom); + onZoomChange?.(newZoom); + }; + + canvas.on('mouse:wheel', handleWheel); + return () => { window.removeEventListener('resize', handleResize); + canvas.off('mouse:wheel', handleWheel); canvas.dispose(); }; - }, []); + }, [onZoomChange]); // Load image when URL changes useEffect(() => { @@ -94,12 +129,128 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => { }, { crossOrigin: 'anonymous' }); }, [imageUrl]); + // Handle advanced tool mode clicks (smart-select, color-select) + useEffect(() => { + if (!fabricCanvasRef.current || !advancedToolMode) return; + + const canvas = fabricCanvasRef.current; + const bgImage = canvas.backgroundImage; + + const handleAdvancedClick = async (e) => { + if (!bgImage || !onAdvancedToolClick) return; + + const pointer = canvas.getPointer(e.e); + + // Convert canvas coordinates to image coordinates + const imgScale = bgImage.scaleX; + const imgLeft = bgImage.left; + const imgTop = bgImage.top; + + const imgX = Math.round((pointer.x - imgLeft) / imgScale); + const imgY = Math.round((pointer.y - imgTop) / imgScale); + + // Check if click is within image bounds + if (imgX < 0 || imgY < 0 || imgX > bgImage.width || imgY > bgImage.height) { + return; + } + + if (advancedToolMode === 'color-select') { + // Get pixel color at click position + const ctx = canvas.getContext('2d'); + const canvasX = pointer.x * canvas.getZoom(); + const canvasY = pointer.y * canvas.getZoom(); + + // For color picking, we need to get the color from the image + // Create a temporary canvas to read pixel color + const tempCanvas = document.createElement('canvas'); + tempCanvas.width = bgImage.width; + tempCanvas.height = bgImage.height; + const tempCtx = tempCanvas.getContext('2d'); + + // Draw the image element to temp canvas + const imgElement = bgImage.getElement(); + tempCtx.drawImage(imgElement, 0, 0); + + const pixelData = tempCtx.getImageData(imgX, imgY, 1, 1).data; + const color = { r: pixelData[0], g: pixelData[1], b: pixelData[2] }; + + onAdvancedToolClick(imgX, imgY, color); + } else { + onAdvancedToolClick(imgX, imgY, null); + } + }; + + canvas.on('mouse:down', handleAdvancedClick); + + return () => { + canvas.off('mouse:down', handleAdvancedClick); + }; + }, [advancedToolMode, onAdvancedToolClick]); + + // Handle external selection (from smart-select or color-select) + useEffect(() => { + if (!fabricCanvasRef.current || !externalSelection?.polygon?.length) return; + + const canvas = fabricCanvasRef.current; + const bgImage = canvas.backgroundImage; + + if (!bgImage) return; + + // Clear previous selection + if (currentSelection) { + canvas.remove(currentSelection); + } + + // Convert image coordinates to canvas coordinates + const imgScale = bgImage.scaleX; + const imgLeft = bgImage.left; + const imgTop = bgImage.top; + + const canvasPoints = externalSelection.polygon.map(([x, y]) => ({ + x: x * imgScale + imgLeft, + y: y * imgScale + imgTop, + })); + + // Create polygon selection + const polygon = new fabric.Polygon(canvasPoints, { + fill: 'rgba(255, 255, 255, 0.3)', + stroke: '#00ff00', + strokeWidth: 2, + selectable: true, + hasControls: true, + hasBorders: true, + lockRotation: false, + cornerColor: '#00ff00', + cornerSize: 10, + transparentCorners: false, + borderColor: '#00ff00', + borderScaleFactor: 2, + }); + + canvas.add(polygon); + canvas.setActiveObject(polygon); + setCurrentSelection(polygon); + lassoPoints.current = canvasPoints; + + // Notify parent of selection + onSelectionChange({ + type: 'polygon', + bbox: externalSelection.bbox, + selectionData: { points: externalSelection.polygon }, + }); + + canvas.renderAll(); + }, [externalSelection]); + // Handle selection mode changes useEffect(() => { if (!fabricCanvasRef.current) return; const canvas = fabricCanvasRef.current; + // Don't set up selection handlers if in advanced tool mode + if (advancedToolMode) return; + // Clear previous selection when changing modes if (currentSelection) { canvas.remove(currentSelection); @@ -123,7 +274,7 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => { } else if (selectionMode === 'lasso') { setupLassoMode(canvas); } - }, [selectionMode]); + }, [selectionMode, advancedToolMode]); const setupRectangleMode = (canvas) => { let rect, isDown, startX, startY; @@ -497,10 +648,57 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => { } }; + const handleZoomIn = () => { + if (!fabricCanvasRef.current) return; + const canvas = fabricCanvasRef.current; + let newZoom = canvas.getZoom() * 1.2; + if (newZoom > 10) newZoom = 10; + canvas.setZoom(newZoom); + setCurrentZoom(newZoom); + onZoomChange?.(newZoom); + }; + + const handleZoomOut = () => { + if (!fabricCanvasRef.current) return; + const canvas = fabricCanvasRef.current; + let newZoom = canvas.getZoom() / 1.2; + if (newZoom < 0.1) newZoom = 0.1; + canvas.setZoom(newZoom); + setCurrentZoom(newZoom); + onZoomChange?.(newZoom); + }; + + const handleZoomReset = () => { + if (!fabricCanvasRef.current) return; + const canvas = fabricCanvasRef.current; + canvas.setZoom(1); + canvas.setViewportTransform([1, 0, 0, 1, 0, 0]); + setCurrentZoom(1); + onZoomChange?.(1); + }; + return (
- {currentSelection && ( + + {/* Zoom controls */} +
+ + {Math.round(currentZoom * 100)}% + + +
+ + {/* Advanced tool mode indicator */} + {advancedToolMode && ( +
+ {advancedToolMode === 'smart-select' && 'Click on an object to select it'} + {advancedToolMode === 'color-select' && 'Click on a color to select similar pixels'} + {advancedToolMode === 'object-remove' && 'Click on an object to remove it'} +
+ )} + + {currentSelection && !advancedToolMode && ( <>
Click selection to move/resize/rotate diff --git a/frontend/src/components/Layers.jsx b/frontend/src/components/Layers.jsx index 22b50e2..f43770c 100644 --- a/frontend/src/components/Layers.jsx +++ b/frontend/src/components/Layers.jsx @@ -11,6 +11,7 @@ const Layers = ({ onLayerVisibilityChange, onFlatten, isProcessing, + onError, }) => { const [draggedLayer, setDraggedLayer] = useState(null); @@ -69,6 +70,56 @@ const Layers = ({ loadLayers(); }; + const handleNewLayer = async () => { + if (!projectId) return; + try { + // Create a new empty transparent layer + const newLayer = { + id: `layer-${Date.now()}`, + name: `Layer ${layers.length + 1}`, + visible: true, + thumbnail: null, + }; + setLayers([...layers, newLayer]); + setActiveLayer(newLayer.id); + } catch (err) { + onError?.(`Failed to create layer: ${err.message}`); + } + }; + + const handleDeleteLayer = async () => { + if (!projectId || activeLayer === 'background') return; + try { + const updatedLayers = layers.filter((l) => l.id !== activeLayer); + setLayers(updatedLayers); + setActiveLayer(updatedLayers.length > 0 ? updatedLayers[updatedLayers.length - 1].id : 'background'); + } catch (err) { + onError?.(`Failed to delete layer: ${err.message}`); + } + }; + + const handleDuplicateLayer = async () => { + if (!projectId || activeLayer === 'background') return; + try { + const layerToDuplicate = layers.find((l) => l.id === activeLayer); + if (!layerToDuplicate) return; + + const newLayer = { + ...layerToDuplicate, + id: `layer-${Date.now()}`, + name: `${layerToDuplicate.name} copy`, + }; + + const activeIndex = layers.findIndex((l) => l.id === activeLayer); + const updatedLayers = [...layers]; + updatedLayers.splice(activeIndex + 1, 0, newLayer); + setLayers(updatedLayers); + setActiveLayer(newLayer.id); + } catch (err) { + onError?.(`Failed to duplicate layer: ${err.message}`); + } + }; + return (
@@ -142,6 +193,7 @@ const Layers = ({ className="layer-action-btn" disabled={isProcessing || !projectId} title="Add new empty layer" + onClick={handleNewLayer} > + New Layer @@ -149,6 +201,7 @@ const Layers = ({ className="layer-action-btn" disabled={isProcessing || activeLayer === 'background'} title="Delete selected layer" + onClick={handleDeleteLayer} > Delete @@ -156,6 +209,7 @@ const Layers = ({ className="layer-action-btn" disabled={isProcessing || activeLayer === 'background'} title="Duplicate selected layer" + onClick={handleDuplicateLayer} > Duplicate diff --git a/frontend/src/utils/api.js b/frontend/src/utils/api.js index cea392d..164910d 100644 --- a/frontend/src/utils/api.js +++ b/frontend/src/utils/api.js @@ -171,20 +171,22 @@ export const toolsApi = { }, // Smart select object at point + // Returns { polygon, bbox, mask_base64 } smartSelect: async (projectId, x, y) => { const formData = new FormData(); formData.append('project_id', projectId); formData.append('point_x', x); formData.append('point_y', y); + formData.append('return_format', 'json'); const response = await api.post('/tools/smart-select', formData, { headers: { 'Content-Type': 'multipart/form-data' }, - responseType: 'blob', }); return response.data; }, // Select by color + // Returns { polygon, bbox, mask_base64, color, tolerance } colorSelect: async (projectId, r, g, b, tolerance = 30) => { const formData = new FormData(); formData.append('project_id', projectId); @@ -192,10 +194,10 @@ export const toolsApi = { formData.append('color_g', g); formData.append('color_b', b); formData.append('tolerance', tolerance); + formData.append('return_format', 'json'); const response = await api.post('/tools/color-select', formData, { headers: { 'Content-Type': 'multipart/form-data' }, - responseType: 'blob', }); return response.data; },